diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 981a80256..e4c0c7f26 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -9,15 +9,9 @@ body: options: - label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy). required: true - - label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions). - required: true - - label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead. - required: true - - label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead. - required: true - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion. required: true - - label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these. + - label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these. required: true - label: I did not make any code changes to lotus. required: false @@ -28,19 +22,11 @@ body: options: - label: lotus daemon - chain sync required: false - - label: lotus miner - mining and block production + - label: lotus fvm/fevm - Lotus FVM and FEVM interactions required: false - label: lotus miner/worker - sealing required: false - - label: lotus miner - proving(WindowPoSt) - required: false - - label: lotus miner/market - storage deal - required: false - - label: lotus miner/market - retrieval deal - required: false - - label: lotus miner/market - data transfer - required: false - - label: lotus client + - label: lotus miner - proving(WindowPoSt/WinningPoSt) required: false - label: lotus JSON-RPC API required: false @@ -56,22 +42,33 @@ body: description: Enter the output of `lotus version` and `lotus-miner version` if applicable. placeholder: | e.g. - Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0 - Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty + Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0 + Local: lotus-miner version 1.19.0+mainnet+git.64059ca87 validations: required: true +- type: textarea + id: ReproSteps + attributes: + label: Repro Steps + description: "Steps to reproduce the behavior" + value: | + 1. Run '...' + 2. Do '...' + 3. See error '...' + ... + validations: + required: false - type: textarea id: Description attributes: label: Describe the Bug description: | This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information: - * What you were doding when you experienced the bug? + * What you were doing when you experienced the bug? * Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas). * What is the expected behaviour? * For sealing issues, include the output of `lotus-miner sectors status --log ` for the failed sector(s). * For proving issues, include the output of `lotus-miner proving` info. - * For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question. validations: required: true - type: textarea @@ -83,18 +80,6 @@ body: Please provide debug logs of the problem, remember you can get set log level control for: * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://lotus.filecoin.io/lotus/configure/defaults/#log-level-control). * lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level - If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem. + If you don't provide detailed logs when you raise the issue it will almost certainly be the first request we make before furthur diagnosing the problem. validations: - required: true -- type: textarea - id: RepoSteps - attributes: - label: Repo Steps - description: "Steps to reproduce the behavior" - value: | - 1. Run '...' - 2. Do '...' - 3. See error '...' - ... - validations: - required: false + required: true \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..e5ae608b3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Ask a question about Lotus or get support + url: https://github.com/filecoin-project/lotus/discussions/new/choose + about: Ask a question or request support for using Lotus + - name: Filecoin protocol feature or enhancement + url: https://github.com/filecoin-project/FIPs/discussions/new/choose + about: Write a discussion in the Filecoin Improvement Proposal repo diff --git a/.github/ISSUE_TEMPLATE/enhancement.yml b/.github/ISSUE_TEMPLATE/enhancement.yml index da662688d..d367feeac 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.yml +++ b/.github/ISSUE_TEMPLATE/enhancement.yml @@ -7,13 +7,7 @@ body: label: Checklist description: Please check off the following boxes before continuing to create an improvement suggestion! options: - - label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md). - required: true - - label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead. - required: true - - label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`. - required: true - - label: I **have** a specific, actionable, and well motivated improvement to propose. + - label: I **have** a specific, actionable, and well motivated improvement to an existing lotus feature. required: true - type: checkboxes attributes: @@ -22,19 +16,11 @@ body: options: - label: lotus daemon - chain sync required: false - - label: lotus miner - mining and block production + - label: lotus fvm/fevm - Lotus FVM and FEVM interactions required: false - label: lotus miner/worker - sealing required: false - - label: lotus miner - proving(WindowPoSt) - required: false - - label: lotus miner/market - storage deal - required: false - - label: lotus miner/market - retrieval deal - required: false - - label: lotus miner/market - data transfer - required: false - - label: lotus client + - label: lotus miner - proving(WindowPoSt/WinningPoSt) required: false - label: lotus JSON-RPC API required: false @@ -45,9 +31,17 @@ body: - type: textarea id: request attributes: - label: Improvement Suggestion - description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement? - placeholder: Ex. Currently lotus... However, as a storage provider, I'd like... + label: Enhancement Suggestion + description: A clear and concise description of the suggested enhancement? + placeholder: Ex. Currently lotus... However it would be great if [enhancement] was implemented... With the ability to... + validations: + required: true +- type: textarea + id: request + attributes: + label: Use-Case + description: How would this enhancement help you? + placeholder: Ex. With the [enhancement] node operators would be able to... For Storage Providers it would enable... validations: required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 210549095..76493e9c1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -7,8 +7,6 @@ body: label: Checklist description: Please check off the following boxes before continuing to create a new feature request! options: - - label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md). - required: true - label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`. required: true - label: I **have** a specific, actionable, and well motivated feature request to propose. @@ -20,19 +18,11 @@ body: options: - label: lotus daemon - chain sync required: false - - label: lotus miner - mining and block production + - label: lotus fvm/fevm - Lotus FVM and FEVM interactions required: false - label: lotus miner/worker - sealing required: false - - label: lotus miner - proving(WindowPoSt) - required: false - - label: lotus miner/market - storage deal - required: false - - label: lotus miner/market - retrieval deal - required: false - - label: lotus miner/market - data transfer - required: false - - label: lotus client + - label: lotus miner - proving(WindowPoSt/WinningPoSt) required: false - label: lotus JSON-RPC API required: false @@ -56,7 +46,7 @@ body: validations: required: true - type: textarea - id: alternates + id: alternatives attributes: label: Describe alternatives you've considered description: A clear and concise description of any alternative solutions or features you've considered. @@ -69,4 +59,3 @@ body: description: Add any other context, design docs or screenshots about the feature request here. validations: required: false - diff --git a/.github/ISSUE_TEMPLATE/service_developer_bug_report.yml b/.github/ISSUE_TEMPLATE/service_developer_bug_report.yml new file mode 100644 index 000000000..8174d13f5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/service_developer_bug_report.yml @@ -0,0 +1,83 @@ +name: "Bug Report - developer/service provider" +description: "Bug report template about FEVM/FVM for developers/service providers" +labels: [need/triage, kind/bug, area/fevm] +body: +- type: checkboxes + attributes: + label: Checklist + description: Please check off the following boxes before continuing to file a bug report! + options: + - label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy). + required: true + - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion. + required: true + - label: I did not make any code changes to lotus. + required: false +- type: checkboxes + attributes: + label: Lotus component + description: Please select the lotus component you are filing a bug for + options: + - label: lotus Ethereum RPC + required: false + - label: lotus FVM - Lotus FVM interactions + required: false + - label: FEVM tooling + required: false + - label: Other + required: false +- type: textarea + id: version + attributes: + label: Lotus Version + render: text + description: Enter the output of `lotus version` if applicable. + placeholder: | + e.g. + Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0 + Local: lotus-miner version 1.19.0+mainnet+git.64059ca87 + validations: + required: true +- type: textarea + id: repro + attributes: + label: Repro Steps + description: "Steps to reproduce the behavior" + value: | + 1. Run '...' + 2. Do '...' + 3. See error '...' + ... + validations: + required: false +- type: textarea + id: Description + attributes: + label: Describe the Bug + description: | + This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information: + * What you were doing when you experienced the bug? What are you trying to build? + * Any *error* messages and logs you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas). + * What is the expected behaviour? Links to the actual code? + validations: + required: true +- type: textarea + id: toolingInfo + attributes: + label: Tooling + render: text + description: | + What kind of tooling are you using: + * Are you using ether.js, Alchemy, Hardhat, etc. + validations: + required: true +- type: textarea + id: extraInfo + attributes: + label: Configuration Options + render: text + description: | + Please provide your updated FEVM related configuration options, or custome enviroment variables related to Lotus FEVM + * lotus: use `lotus config updated` to get your configuration options, and copy the [FEVM] section + validations: + required: true diff --git a/api/api_gateway.go b/api/api_gateway.go index 2e877fb1a..cac28d2a0 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -15,6 +15,7 @@ import ( apitypes "github.com/filecoin-project/lotus/api/types" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) // MODIFYING THE API INTERFACE @@ -25,7 +26,7 @@ import ( // When adding / changing methods in this file: // * Do the change here // * Adjust implementation in `node/impl/` -// * Run `make gen` - this will: +// * Run `make clean && make deps && make gen` - this will: // * Generate proxy structs // * Generate mocks // * Generate markdown docs @@ -47,15 +48,18 @@ type Gateway interface { ChainReadObj(context.Context, cid.Cid) ([]byte, error) ChainGetGenesis(context.Context) (*types.TipSet, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*InvocResult, error) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error) + StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read + StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error) @@ -63,8 +67,10 @@ type Gateway interface { StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (MinerInfo, error) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) + StateNetworkName(context.Context) (dtypes.NetworkName, error) StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) + StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error) StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error) @@ -83,8 +89,6 @@ type Gateway interface { EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error) EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkOpt string) (ethtypes.EthUint64, error) EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*EthTxReceipt, error) - EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) - EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkOpt string) (ethtypes.EthBytes, error) EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error) EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam string) (ethtypes.EthBigInt, error) diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 8e958aed9..0a0470446 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -401,6 +401,10 @@ func init() { FromBlock: pstring("2301220"), Address: []ethtypes.EthAddress{ethaddr}, }) + + percent := types.Percent(123) + addExample(percent) + addExample(&percent) } func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) { diff --git a/api/proxy_gen.go b/api/proxy_gen.go index aa423c574..24b11df3f 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -692,10 +692,6 @@ type GatewayMethods struct { EthGetStorageAt func(p0 context.Context, p1 ethtypes.EthAddress, p2 ethtypes.EthBytes, p3 string) (ethtypes.EthBytes, error) `` - EthGetTransactionByBlockHashAndIndex func(p0 context.Context, p1 ethtypes.EthHash, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) `` - - EthGetTransactionByBlockNumberAndIndex func(p0 context.Context, p1 ethtypes.EthUint64, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) `` - EthGetTransactionByHash func(p0 context.Context, p1 *ethtypes.EthHash) (*ethtypes.EthTx, error) `` EthGetTransactionCount func(p0 context.Context, p1 ethtypes.EthAddress, p2 string) (ethtypes.EthUint64, error) `` @@ -724,6 +720,8 @@ type GatewayMethods struct { GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) `` + MpoolGetNonce func(p0 context.Context, p1 address.Address) (uint64, error) `` + MpoolPush func(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) `` MsigGetAvailableBalance func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (types.BigInt, error) `` @@ -740,8 +738,12 @@ type GatewayMethods struct { StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) `` + StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) `` + StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) `` + StateDecodeParams func(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) `` + StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) `` StateListMiners func(p0 context.Context, p1 types.TipSetKey) ([]address.Address, error) `` @@ -758,9 +760,11 @@ type GatewayMethods struct { StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) `` + StateNetworkName func(p0 context.Context) (dtypes.NetworkName, error) `` + StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (apitypes.NetworkVersion, error) `` - StateReadState func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) `perm:"read"` + StateReadState func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) `` StateSearchMsg func(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) `` @@ -768,6 +772,8 @@ type GatewayMethods struct { StateVerifiedClientStatus func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*abi.StoragePower, error) `` + StateVerifierStatus func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*abi.StoragePower, error) `` + StateWaitMsg func(p0 context.Context, p1 cid.Cid, p2 uint64, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) `` Version func(p0 context.Context) (APIVersion, error) `` @@ -4413,28 +4419,6 @@ func (s *GatewayStub) EthGetStorageAt(p0 context.Context, p1 ethtypes.EthAddress return *new(ethtypes.EthBytes), ErrNotSupported } -func (s *GatewayStruct) EthGetTransactionByBlockHashAndIndex(p0 context.Context, p1 ethtypes.EthHash, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) { - if s.Internal.EthGetTransactionByBlockHashAndIndex == nil { - return *new(ethtypes.EthTx), ErrNotSupported - } - return s.Internal.EthGetTransactionByBlockHashAndIndex(p0, p1, p2) -} - -func (s *GatewayStub) EthGetTransactionByBlockHashAndIndex(p0 context.Context, p1 ethtypes.EthHash, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) { - return *new(ethtypes.EthTx), ErrNotSupported -} - -func (s *GatewayStruct) EthGetTransactionByBlockNumberAndIndex(p0 context.Context, p1 ethtypes.EthUint64, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) { - if s.Internal.EthGetTransactionByBlockNumberAndIndex == nil { - return *new(ethtypes.EthTx), ErrNotSupported - } - return s.Internal.EthGetTransactionByBlockNumberAndIndex(p0, p1, p2) -} - -func (s *GatewayStub) EthGetTransactionByBlockNumberAndIndex(p0 context.Context, p1 ethtypes.EthUint64, p2 ethtypes.EthUint64) (ethtypes.EthTx, error) { - return *new(ethtypes.EthTx), ErrNotSupported -} - func (s *GatewayStruct) EthGetTransactionByHash(p0 context.Context, p1 *ethtypes.EthHash) (*ethtypes.EthTx, error) { if s.Internal.EthGetTransactionByHash == nil { return nil, ErrNotSupported @@ -4589,6 +4573,17 @@ func (s *GatewayStub) GasEstimateMessageGas(p0 context.Context, p1 *types.Messag return nil, ErrNotSupported } +func (s *GatewayStruct) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) { + if s.Internal.MpoolGetNonce == nil { + return 0, ErrNotSupported + } + return s.Internal.MpoolGetNonce(p0, p1) +} + +func (s *GatewayStub) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) { + return 0, ErrNotSupported +} + func (s *GatewayStruct) MpoolPush(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) { if s.Internal.MpoolPush == nil { return *new(cid.Cid), ErrNotSupported @@ -4677,6 +4672,17 @@ func (s *GatewayStub) StateAccountKey(p0 context.Context, p1 address.Address, p2 return *new(address.Address), ErrNotSupported } +func (s *GatewayStruct) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) { + if s.Internal.StateCall == nil { + return nil, ErrNotSupported + } + return s.Internal.StateCall(p0, p1, p2) +} + +func (s *GatewayStub) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateDealProviderCollateralBounds(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) { if s.Internal.StateDealProviderCollateralBounds == nil { return *new(DealCollateralBounds), ErrNotSupported @@ -4688,6 +4694,17 @@ func (s *GatewayStub) StateDealProviderCollateralBounds(p0 context.Context, p1 a return *new(DealCollateralBounds), ErrNotSupported } +func (s *GatewayStruct) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) { + if s.Internal.StateDecodeParams == nil { + return nil, ErrNotSupported + } + return s.Internal.StateDecodeParams(p0, p1, p2, p3, p4) +} + +func (s *GatewayStub) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateGetActor(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) { if s.Internal.StateGetActor == nil { return nil, ErrNotSupported @@ -4776,6 +4793,17 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A return nil, ErrNotSupported } +func (s *GatewayStruct) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) { + if s.Internal.StateNetworkName == nil { + return *new(dtypes.NetworkName), ErrNotSupported + } + return s.Internal.StateNetworkName(p0) +} + +func (s *GatewayStub) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) { + return *new(dtypes.NetworkName), ErrNotSupported +} + func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (apitypes.NetworkVersion, error) { if s.Internal.StateNetworkVersion == nil { return *new(apitypes.NetworkVersion), ErrNotSupported @@ -4831,6 +4859,17 @@ func (s *GatewayStub) StateVerifiedClientStatus(p0 context.Context, p1 address.A return nil, ErrNotSupported } +func (s *GatewayStruct) StateVerifierStatus(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*abi.StoragePower, error) { + if s.Internal.StateVerifierStatus == nil { + return nil, ErrNotSupported + } + return s.Internal.StateVerifierStatus(p0, p1, p2) +} + +func (s *GatewayStub) StateVerifierStatus(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*abi.StoragePower, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateWaitMsg(p0 context.Context, p1 cid.Cid, p2 uint64, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) { if s.Internal.StateWaitMsg == nil { return nil, ErrNotSupported diff --git a/api/v0api/gateway.go b/api/v0api/gateway.go index 674371c14..2a0bfb2f7 100644 --- a/api/v0api/gateway.go +++ b/api/v0api/gateway.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) // MODIFYING THE API INTERFACE @@ -44,12 +45,15 @@ type Gateway interface { ChainNotify(context.Context) (<-chan []*api.HeadChange, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) + StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) @@ -59,6 +63,7 @@ type Gateway interface { StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) + StateNetworkName(context.Context) (dtypes.NetworkName, error) StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) diff --git a/api/v0api/proxy_gen.go b/api/v0api/proxy_gen.go index bdd7af6cb..3d4a60f1c 100644 --- a/api/v0api/proxy_gen.go +++ b/api/v0api/proxy_gen.go @@ -451,6 +451,8 @@ type GatewayMethods struct { GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *api.MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) `` + MpoolGetNonce func(p0 context.Context, p1 address.Address) (uint64, error) `` + MpoolPush func(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) `` MsigGetAvailableBalance func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (types.BigInt, error) `` @@ -461,8 +463,12 @@ type GatewayMethods struct { StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) `` + StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) `` + StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (api.DealCollateralBounds, error) `` + StateDecodeParams func(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) `` + StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) `` StateGetReceipt func(p0 context.Context, p1 cid.Cid, p2 types.TipSetKey) (*types.MessageReceipt, error) `` @@ -481,6 +487,8 @@ type GatewayMethods struct { StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) `` + StateNetworkName func(p0 context.Context) (dtypes.NetworkName, error) `` + StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) `` StateSearchMsg func(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) `` @@ -2677,6 +2685,17 @@ func (s *GatewayStub) GasEstimateMessageGas(p0 context.Context, p1 *types.Messag return nil, ErrNotSupported } +func (s *GatewayStruct) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) { + if s.Internal.MpoolGetNonce == nil { + return 0, ErrNotSupported + } + return s.Internal.MpoolGetNonce(p0, p1) +} + +func (s *GatewayStub) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) { + return 0, ErrNotSupported +} + func (s *GatewayStruct) MpoolPush(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) { if s.Internal.MpoolPush == nil { return *new(cid.Cid), ErrNotSupported @@ -2732,6 +2751,17 @@ func (s *GatewayStub) StateAccountKey(p0 context.Context, p1 address.Address, p2 return *new(address.Address), ErrNotSupported } +func (s *GatewayStruct) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) { + if s.Internal.StateCall == nil { + return nil, ErrNotSupported + } + return s.Internal.StateCall(p0, p1, p2) +} + +func (s *GatewayStub) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateDealProviderCollateralBounds(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (api.DealCollateralBounds, error) { if s.Internal.StateDealProviderCollateralBounds == nil { return *new(api.DealCollateralBounds), ErrNotSupported @@ -2743,6 +2773,17 @@ func (s *GatewayStub) StateDealProviderCollateralBounds(p0 context.Context, p1 a return *new(api.DealCollateralBounds), ErrNotSupported } +func (s *GatewayStruct) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) { + if s.Internal.StateDecodeParams == nil { + return nil, ErrNotSupported + } + return s.Internal.StateDecodeParams(p0, p1, p2, p3, p4) +} + +func (s *GatewayStub) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateGetActor(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) { if s.Internal.StateGetActor == nil { return nil, ErrNotSupported @@ -2842,6 +2883,17 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A return nil, ErrNotSupported } +func (s *GatewayStruct) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) { + if s.Internal.StateNetworkName == nil { + return *new(dtypes.NetworkName), ErrNotSupported + } + return s.Internal.StateNetworkName(p0) +} + +func (s *GatewayStub) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) { + return *new(dtypes.NetworkName), ErrNotSupported +} + func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) { if s.Internal.StateNetworkVersion == nil { return *new(abinetwork.Version), ErrNotSupported diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index 3087c01f7..1b52eb548 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -601,6 +601,18 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro }) } +func (b *Blockstore) Flush(context.Context) error { + if err := b.access(); err != nil { + return err + } + defer b.viewers.Done() + + b.lockDB() + defer b.unlockDB() + + return b.db.Sync() +} + // Has implements Blockstore.Has. func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { if err := b.access(); err != nil { diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go index be1f1199f..8d16bbd50 100644 --- a/blockstore/blockstore.go +++ b/blockstore/blockstore.go @@ -18,6 +18,7 @@ type Blockstore interface { blockstore.Blockstore blockstore.Viewer BatchDeleter + Flusher } // BasicBlockstore is an alias to the original IPFS Blockstore. @@ -25,6 +26,10 @@ type BasicBlockstore = blockstore.Blockstore type Viewer = blockstore.Viewer +type Flusher interface { + Flush(context.Context) error +} + type BatchDeleter interface { DeleteMany(ctx context.Context, cids []cid.Cid) error } @@ -106,6 +111,13 @@ type adaptedBlockstore struct { var _ Blockstore = (*adaptedBlockstore)(nil) +func (a *adaptedBlockstore) Flush(ctx context.Context) error { + if flusher, canFlush := a.Blockstore.(Flusher); canFlush { + return flusher.Flush(ctx) + } + return nil +} + func (a *adaptedBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error { blk, err := a.Get(ctx, cid) if err != nil { diff --git a/blockstore/buffered.go b/blockstore/buffered.go index e089ed561..bc37b2f69 100644 --- a/blockstore/buffered.go +++ b/blockstore/buffered.go @@ -46,6 +46,8 @@ var ( _ Viewer = (*BufferedBlockstore)(nil) ) +func (bs *BufferedBlockstore) Flush(ctx context.Context) error { return bs.write.Flush(ctx) } + func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { a, err := bs.read.AllKeysChan(ctx) if err != nil { diff --git a/blockstore/discard.go b/blockstore/discard.go index 7f1a76a22..878797561 100644 --- a/blockstore/discard.go +++ b/blockstore/discard.go @@ -38,6 +38,10 @@ func (b *discardstore) View(ctx context.Context, cid cid.Cid, f func([]byte) err return b.bs.View(ctx, cid, f) } +func (b *discardstore) Flush(ctx context.Context) error { + return nil +} + func (b *discardstore) Put(ctx context.Context, blk blocks.Block) error { return nil } diff --git a/blockstore/idstore.go b/blockstore/idstore.go index a0ecec5f0..ae807076d 100644 --- a/blockstore/idstore.go +++ b/blockstore/idstore.go @@ -179,3 +179,7 @@ func (b *idstore) Close() error { } return nil } + +func (b *idstore) Flush(ctx context.Context) error { + return b.bs.Flush(ctx) +} diff --git a/blockstore/mem.go b/blockstore/mem.go index 05da287c5..5b06634de 100644 --- a/blockstore/mem.go +++ b/blockstore/mem.go @@ -17,6 +17,8 @@ func NewMemory() MemBlockstore { // To match behavior of badger blockstore we index by multihash only. type MemBlockstore map[string]blocks.Block +func (MemBlockstore) Flush(context.Context) error { return nil } + func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error { delete(m, string(k.Hash())) return nil diff --git a/blockstore/net.go b/blockstore/net.go index a6b008af2..62aceed71 100644 --- a/blockstore/net.go +++ b/blockstore/net.go @@ -410,6 +410,8 @@ func (n *NetworkStore) HashOnRead(enabled bool) { return } +func (*NetworkStore) Flush(context.Context) error { return nil } + func (n *NetworkStore) Stop(ctx context.Context) error { close(n.closing) diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index 0afb15c11..bd9efb630 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -115,6 +115,23 @@ type Config struct { // A positive value is the number of compactions before a full GC is performed; // a value of 1 will perform full GC in every compaction. HotStoreFullGCFrequency uint64 + + // HotstoreMaxSpaceTarget suggests the max allowed space the hotstore can take. + // This is not a hard limit, it is possible for the hotstore to exceed the target + // for example if state grows massively between compactions. The splitstore + // will make a best effort to avoid overflowing the target and in practice should + // never overflow. This field is used when doing GC at the end of a compaction to + // adaptively choose moving GC + HotstoreMaxSpaceTarget uint64 + + // Moving GC will be triggered when total moving size exceeds + // HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold + HotstoreMaxSpaceThreshold uint64 + + // Safety buffer to prevent moving GC from overflowing disk. + // Moving GC will not occur when total moving size exceeds + // HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer + HotstoreMaxSpaceSafetyBuffer uint64 } // ChainAccessor allows the Splitstore to access the chain. It will most likely @@ -165,6 +182,7 @@ type SplitStore struct { compactionIndex int64 pruneIndex int64 + onlineGCCnt int64 ctx context.Context cancel func() @@ -195,6 +213,17 @@ type SplitStore struct { // registered protectors protectors []func(func(cid.Cid) error) error + + // dag sizes measured during latest compaction + // logged and used for GC strategy + + // protected by compaction lock + szWalk int64 + szProtectedTxns int64 + szKeys int64 // approximate, not counting keys protected when entering critical section + + // protected by txnLk + szMarkedLiveRefs int64 } var _ bstore.Blockstore = (*SplitStore)(nil) @@ -447,6 +476,23 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { } } +func (s *SplitStore) Flush(ctx context.Context) error { + s.txnLk.RLock() + defer s.txnLk.RUnlock() + + if err := s.cold.Flush(ctx); err != nil { + return err + } + if err := s.hot.Flush(ctx); err != nil { + return err + } + if err := s.ds.Sync(ctx, dstore.Key{}); err != nil { + return err + } + + return nil +} + func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error { if isIdentiyCid(blk.Cid()) { return nil diff --git a/blockstore/splitstore/splitstore_check.go b/blockstore/splitstore/splitstore_check.go index 336515980..2645c78c5 100644 --- a/blockstore/splitstore/splitstore_check.go +++ b/blockstore/splitstore/splitstore_check.go @@ -95,7 +95,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error { } defer visitor.Close() //nolint - err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor, + size := s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor, func(c cid.Cid) error { if isUnitaryObject(c) { return errStopWalk @@ -133,7 +133,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error { return err } - log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt) + log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt, "walk size", size) write("--") write("cold: %d missing: %d", *coldCnt, *missingCnt) write("DONE") diff --git a/blockstore/splitstore/splitstore_compact.go b/blockstore/splitstore/splitstore_compact.go index 59bdd515d..12d0ff899 100644 --- a/blockstore/splitstore/splitstore_compact.go +++ b/blockstore/splitstore/splitstore_compact.go @@ -66,7 +66,8 @@ var ( ) const ( - batchSize = 16384 + batchSize = 16384 + cidKeySize = 128 ) func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error { @@ -199,9 +200,11 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) { log.Debugf("marking %d live refs", len(cids)) startMark := time.Now() + szMarked := new(int64) + count := new(int32) visitor := newConcurrentVisitor() - walkObject := func(c cid.Cid) error { + walkObject := func(c cid.Cid) (int64, error) { return s.walkObjectIncomplete(c, visitor, func(c cid.Cid) error { if isUnitaryObject(c) { @@ -228,10 +231,12 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) { // optimize the common case of single put if len(cids) == 1 { - if err := walkObject(cids[0]); err != nil { + sz, err := walkObject(cids[0]) + if err != nil { log.Errorf("error marking tipset refs: %s", err) } log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count) + atomic.AddInt64(szMarked, sz) return } @@ -243,9 +248,11 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) { worker := func() error { for c := range workch { - if err := walkObject(c); err != nil { + sz, err := walkObject(c) + if err != nil { return err } + atomic.AddInt64(szMarked, sz) } return nil @@ -268,7 +275,8 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) { log.Errorf("error marking tipset refs: %s", err) } - log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count) + log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count, "size marked", *szMarked) + s.szMarkedLiveRefs += atomic.LoadInt64(szMarked) } // transactionally protect a view @@ -361,6 +369,7 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error { log.Infow("protecting transactional references", "refs", len(txnRefs)) count := 0 + sz := new(int64) workch := make(chan cid.Cid, len(txnRefs)) startProtect := time.Now() @@ -393,10 +402,11 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error { worker := func() error { for c := range workch { - err := s.doTxnProtect(c, markSet) + szTxn, err := s.doTxnProtect(c, markSet) if err != nil { return xerrors.Errorf("error protecting transactional references to %s: %w", c, err) } + atomic.AddInt64(sz, szTxn) } return nil } @@ -409,16 +419,16 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error { if err := g.Wait(); err != nil { return err } - - log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count) + s.szProtectedTxns += atomic.LoadInt64(sz) + log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count, "protected size", sz) } } // transactionally protect a reference by walking the object and marking. // concurrent markings are short circuited by checking the markset. -func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) error { +func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) (int64, error) { if err := s.checkClosing(); err != nil { - return err + return 0, err } // Note: cold objects are deleted heaviest first, so the consituents of an object @@ -509,6 +519,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { // might be potentially inconsistent; abort compaction and notify the user to intervene. return xerrors.Errorf("checkpoint exists; aborting compaction") } + s.clearSizeMeasurements() currentEpoch := curTs.Height() boundaryEpoch := currentEpoch - CompactionBoundary @@ -598,7 +609,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { } err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{}, fHot, fCold) - if err != nil { return xerrors.Errorf("error marking: %w", err) } @@ -638,7 +648,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { defer purgew.Close() //nolint:errcheck // some stats for logging - var hotCnt, coldCnt, purgeCnt int + var hotCnt, coldCnt, purgeCnt int64 err = s.hot.ForEachKey(func(c cid.Cid) error { // was it marked? mark, err := markSet.Has(c) @@ -690,8 +700,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { log.Infow("cold collection done", "took", time.Since(startCollect)) log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "purge", purgeCnt) - stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt))) - stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt))) + s.szKeys = hotCnt * cidKeySize + stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(hotCnt)) + stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(coldCnt)) if err := s.checkClosing(); err != nil { return err @@ -773,8 +784,8 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { return xerrors.Errorf("error purging cold objects: %w", err) } log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge)) - s.endCriticalSection() + log.Infow("critical section done", "total protected size", s.szProtectedTxns, "total marked live size", s.szMarkedLiveRefs) if err := checkpoint.Close(); err != nil { log.Warnf("error closing checkpoint: %s", err) @@ -907,6 +918,7 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp copy(toWalk, ts.Cids()) walkCnt := new(int64) scanCnt := new(int64) + szWalk := new(int64) tsRef := func(blkCids []cid.Cid) (cid.Cid, error) { return types.NewTipSetKey(blkCids...).Cid() @@ -942,48 +954,64 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp if err != nil { return xerrors.Errorf("error computing cid reference to parent tipset") } - if err := s.walkObjectIncomplete(pRef, visitor, fHot, stopWalk); err != nil { + sz, err := s.walkObjectIncomplete(pRef, visitor, fHot, stopWalk) + if err != nil { return xerrors.Errorf("error walking parent tipset cid reference") } + atomic.AddInt64(szWalk, sz) // message are retained if within the inclMsgs boundary if hdr.Height >= inclMsgs && hdr.Height > 0 { if inclMsgs < inclState { // we need to use walkObjectIncomplete here, as messages/receipts may be missing early on if we // synced from snapshot and have a long HotStoreMessageRetentionPolicy. - if err := s.walkObjectIncomplete(hdr.Messages, visitor, fHot, stopWalk); err != nil { + sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fHot, stopWalk) + if err != nil { return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err) } + atomic.AddInt64(szWalk, sz) - if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk); err != nil { + sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk) + if err != nil { return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err) } + atomic.AddInt64(szWalk, sz) } else { - if err := s.walkObject(hdr.Messages, visitor, fHot); err != nil { + sz, err = s.walkObject(hdr.Messages, visitor, fHot) + if err != nil { return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err) } + atomic.AddInt64(szWalk, sz) - if err := s.walkObject(hdr.ParentMessageReceipts, visitor, fHot); err != nil { + sz, err := s.walkObject(hdr.ParentMessageReceipts, visitor, fHot) + if err != nil { return xerrors.Errorf("error walking message receipts (cid: %s): %w", hdr.ParentMessageReceipts, err) } + atomic.AddInt64(szWalk, sz) } } // messages and receipts outside of inclMsgs are included in the cold store if hdr.Height < inclMsgs && hdr.Height > 0 { - if err := s.walkObjectIncomplete(hdr.Messages, visitor, fCold, stopWalk); err != nil { + sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fCold, stopWalk) + if err != nil { return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err) } - if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fCold, stopWalk); err != nil { + atomic.AddInt64(szWalk, sz) + sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fCold, stopWalk) + if err != nil { return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err) } + atomic.AddInt64(szWalk, sz) } // state is only retained if within the inclState boundary, with the exception of genesis if hdr.Height >= inclState || hdr.Height == 0 { - if err := s.walkObject(hdr.ParentStateRoot, visitor, fHot); err != nil { + sz, err := s.walkObject(hdr.ParentStateRoot, visitor, fHot) + if err != nil { return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err) } + atomic.AddInt64(szWalk, sz) atomic.AddInt64(scanCnt, 1) } @@ -1001,9 +1029,11 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp if err != nil { return xerrors.Errorf("error computing cid reference to parent tipset") } - if err := s.walkObjectIncomplete(hRef, visitor, fHot, stopWalk); err != nil { + sz, err := s.walkObjectIncomplete(hRef, visitor, fHot, stopWalk) + if err != nil { return xerrors.Errorf("error walking parent tipset cid reference") } + atomic.AddInt64(szWalk, sz) for len(toWalk) > 0 { // walking can take a while, so check this with every opportunity @@ -1047,123 +1077,129 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp } } - log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt) - + log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt, "walk size", szWalk) + s.szWalk = atomic.LoadInt64(szWalk) return nil } -func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) error { +func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) (int64, error) { + var sz int64 visit, err := visitor.Visit(c) if err != nil { - return xerrors.Errorf("error visiting object: %w", err) + return 0, xerrors.Errorf("error visiting object: %w", err) } if !visit { - return nil + return sz, nil } if err := f(c); err != nil { if err == errStopWalk { - return nil + return sz, nil } - return err + return 0, err } if c.Prefix().Codec != cid.DagCBOR { - return nil + return sz, nil } // check this before recursing if err := s.checkClosing(); err != nil { - return err + return 0, err } var links []cid.Cid err = s.view(c, func(data []byte) error { + sz += int64(len(data)) return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) { links = append(links, c) }) }) if err != nil { - return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err) + return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err) } for _, c := range links { - err := s.walkObject(c, visitor, f) + szLink, err := s.walkObject(c, visitor, f) if err != nil { - return xerrors.Errorf("error walking link (cid: %s): %w", c, err) + return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err) } + sz += szLink } - return nil + return sz, nil } // like walkObject, but the object may be potentially incomplete (references missing) -func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) error { +func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) (int64, error) { + var sz int64 visit, err := visitor.Visit(c) if err != nil { - return xerrors.Errorf("error visiting object: %w", err) + return 0, xerrors.Errorf("error visiting object: %w", err) } if !visit { - return nil + return sz, nil } // occurs check -- only for DAGs if c.Prefix().Codec == cid.DagCBOR { has, err := s.has(c) if err != nil { - return xerrors.Errorf("error occur checking %s: %w", c, err) + return 0, xerrors.Errorf("error occur checking %s: %w", c, err) } if !has { err = missing(c) if err == errStopWalk { - return nil + return sz, nil } - return err + return 0, err } } if err := f(c); err != nil { if err == errStopWalk { - return nil + return sz, nil } - return err + return 0, err } if c.Prefix().Codec != cid.DagCBOR { - return nil + return sz, nil } // check this before recursing if err := s.checkClosing(); err != nil { - return err + return sz, err } var links []cid.Cid err = s.view(c, func(data []byte) error { + sz += int64(len(data)) return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) { links = append(links, c) }) }) if err != nil { - return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err) + return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err) } for _, c := range links { - err := s.walkObjectIncomplete(c, visitor, f, missing) + szLink, err := s.walkObjectIncomplete(c, visitor, f, missing) if err != nil { - return xerrors.Errorf("error walking link (cid: %s): %w", c, err) + return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err) } + sz += szLink } - return nil + return sz, nil } // internal version used during compaction and related operations @@ -1429,8 +1465,9 @@ func (s *SplitStore) completeCompaction() error { } s.compactType = none - // Note: at this point we can start the splitstore; a compaction should run on - // the first head change, which will trigger gc on the hotstore. + // Note: at this point we can start the splitstore; base epoch is not + // incremented here so a compaction should run on the first head + // change, which will trigger gc on the hotstore. // We don't mind the second (back-to-back) compaction as the head will // have advanced during marking and coldset accumulation. return nil @@ -1488,6 +1525,13 @@ func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint, return nil } +func (s *SplitStore) clearSizeMeasurements() { + s.szKeys = 0 + s.szMarkedLiveRefs = 0 + s.szProtectedTxns = 0 + s.szWalk = 0 +} + // I really don't like having this code, but we seem to have some occasional DAG references with // missing constituents. During testing in mainnet *some* of these references *sometimes* appeared // after a little bit. @@ -1528,7 +1572,7 @@ func (s *SplitStore) waitForMissingRefs(markSet MarkSet) { missing = make(map[cid.Cid]struct{}) for c := range towalk { - err := s.walkObjectIncomplete(c, visitor, + _, err := s.walkObjectIncomplete(c, visitor, func(c cid.Cid) error { if isUnitaryObject(c) { return errStopWalk diff --git a/blockstore/splitstore/splitstore_expose.go b/blockstore/splitstore/splitstore_expose.go index d092fbb9b..7461e338d 100644 --- a/blockstore/splitstore/splitstore_expose.go +++ b/blockstore/splitstore/splitstore_expose.go @@ -77,6 +77,10 @@ func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error return size, err } +func (es *exposedSplitStore) Flush(ctx context.Context) error { + return es.s.Flush(ctx) +} + func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error { return es.s.Put(ctx, blk) } diff --git a/blockstore/splitstore/splitstore_gc.go b/blockstore/splitstore/splitstore_gc.go index 3b53b8042..2ddb7d404 100644 --- a/blockstore/splitstore/splitstore_gc.go +++ b/blockstore/splitstore/splitstore_gc.go @@ -7,15 +7,61 @@ import ( bstore "github.com/filecoin-project/lotus/blockstore" ) +const ( + // Fraction of garbage in badger vlog for online GC traversal to collect garbage + AggressiveOnlineGCThreshold = 0.0001 +) + func (s *SplitStore) gcHotAfterCompaction() { + // Measure hotstore size, determine if we should do full GC, determine if we can do full GC. + // We should do full GC if + // FullGCFrequency is specified and compaction index matches frequency + // OR HotstoreMaxSpaceTarget is specified and total moving space is within 150 GB of target + // We can do full if + // HotstoreMaxSpaceTarget is not specified + // OR total moving space would not exceed 50 GB below target + // + // a) If we should not do full GC => online GC + // b) If we should do full GC and can => moving GC + // c) If we should do full GC and can't => aggressive online GC + getSize := func() int64 { + sizer, ok := s.hot.(bstore.BlockstoreSize) + if ok { + size, err := sizer.Size() + if err != nil { + log.Warnf("error getting hotstore size: %s, estimating empty hot store for targeting", err) + return 0 + } + return size + } + log.Errorf("Could not measure hotstore size, assuming it is 0 bytes, which it is not") + return 0 + } + hotSize := getSize() + + copySizeApprox := s.szKeys + s.szMarkedLiveRefs + s.szProtectedTxns + s.szWalk + shouldTarget := s.cfg.HotstoreMaxSpaceTarget > 0 && hotSize+copySizeApprox > int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceThreshold) + shouldFreq := s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 + shouldDoFull := shouldTarget || shouldFreq + canDoFull := s.cfg.HotstoreMaxSpaceTarget == 0 || hotSize+copySizeApprox < int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceSafetyBuffer) + log.Debugw("approximating new hot store size", "key size", s.szKeys, "marked live refs", s.szMarkedLiveRefs, "protected txns", s.szProtectedTxns, "walked DAG", s.szWalk) + log.Infof("measured hot store size: %d, approximate new size: %d, should do full %t, can do full %t", hotSize, copySizeApprox, shouldDoFull, canDoFull) + var opts []bstore.BlockstoreGCOption - if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 { + if shouldDoFull && canDoFull { opts = append(opts, bstore.WithFullGC(true)) + } else if shouldDoFull && !canDoFull { + log.Warnf("Attention! Estimated moving GC size %d is not within safety buffer %d of target max %d, performing aggressive online GC to attempt to bring hotstore size down safely", copySizeApprox, s.cfg.HotstoreMaxSpaceSafetyBuffer, s.cfg.HotstoreMaxSpaceTarget) + log.Warn("If problem continues you can 1) temporarily allocate more disk space to hotstore and 2) reflect in HotstoreMaxSpaceTarget OR trigger manual move with `lotus chain prune hot-moving`") + log.Warn("If problem continues and you do not have any more disk space you can run continue to manually trigger online GC at aggressive thresholds (< 0.01) with `lotus chain prune hot`") + + opts = append(opts, bstore.WithThreshold(AggressiveOnlineGCThreshold)) } if err := s.gcBlockstore(s.hot, opts); err != nil { log.Warnf("error garbage collecting hostore: %s", err) } + log.Infof("measured hot store size after GC: %d", getSize()) } func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error { diff --git a/blockstore/splitstore/splitstore_reify.go b/blockstore/splitstore/splitstore_reify.go index aa14f090a..07efedead 100644 --- a/blockstore/splitstore/splitstore_reify.go +++ b/blockstore/splitstore/splitstore_reify.go @@ -101,7 +101,7 @@ func (s *SplitStore) doReify(c cid.Cid) { defer s.txnLk.RUnlock() count := 0 - err := s.walkObjectIncomplete(c, newTmpVisitor(), + _, err := s.walkObjectIncomplete(c, newTmpVisitor(), func(c cid.Cid) error { if isUnitaryObject(c) { return errStopWalk diff --git a/blockstore/splitstore/splitstore_test.go b/blockstore/splitstore/splitstore_test.go index c97a9d01c..68e1bfb65 100644 --- a/blockstore/splitstore/splitstore_test.go +++ b/blockstore/splitstore/splitstore_test.go @@ -757,6 +757,8 @@ func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error { return nil } +func (b *mockStore) Flush(context.Context) error { return nil } + func (b *mockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return nil, errors.New("not implemented") } diff --git a/blockstore/sync.go b/blockstore/sync.go index 4f97027ae..652943dca 100644 --- a/blockstore/sync.go +++ b/blockstore/sync.go @@ -20,6 +20,8 @@ type SyncBlockstore struct { bs MemBlockstore // specifically use a memStore to save indirection overhead. } +func (*SyncBlockstore) Flush(context.Context) error { return nil } + func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error { m.mu.Lock() defer m.mu.Unlock() diff --git a/blockstore/timed.go b/blockstore/timed.go index dda2e1958..3deabb7b8 100644 --- a/blockstore/timed.go +++ b/blockstore/timed.go @@ -93,6 +93,16 @@ func (t *TimedCacheBlockstore) rotate() { t.mu.Unlock() } +func (t *TimedCacheBlockstore) Flush(ctx context.Context) error { + t.mu.Lock() + defer t.mu.Unlock() + + if err := t.active.Flush(ctx); err != nil { + return err + } + return t.inactive.Flush(ctx) +} + func (t *TimedCacheBlockstore) Put(ctx context.Context, b blocks.Block) error { // Don't check the inactive set here. We want to keep this block for at // least one interval. diff --git a/blockstore/union.go b/blockstore/union.go index 3372cd20c..ae6f81955 100644 --- a/blockstore/union.go +++ b/blockstore/union.go @@ -55,6 +55,15 @@ func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, er return size, err } +func (m unionBlockstore) Flush(ctx context.Context) (err error) { + for _, bs := range m { + if err = bs.Flush(ctx); err != nil { + break + } + } + return err +} + func (m unionBlockstore) Put(ctx context.Context, block blocks.Block) (err error) { for _, bs := range m { if err = bs.Put(ctx, block); err != nil { diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index bb1567be5..aec3cf7b3 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/gateway.json.gz b/build/openrpc/gateway.json.gz index 101472808..575f79a1a 100644 Binary files a/build/openrpc/gateway.json.gz and b/build/openrpc/gateway.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 9d872fbde..5ea8edb34 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 1495059f9..ab8a27e92 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/chain/badtscache.go b/chain/badtscache.go index 19b79bb9b..0f215dcdc 100644 --- a/chain/badtscache.go +++ b/chain/badtscache.go @@ -3,14 +3,14 @@ package chain import ( "fmt" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/build" ) type BadBlockCache struct { - badBlocks *lru.ARCCache + badBlocks *lru.ARCCache[cid.Cid, BadBlockReason] } type BadBlockReason struct { @@ -43,7 +43,7 @@ func (bbr BadBlockReason) String() string { } func NewBadBlockCache() *BadBlockCache { - cache, err := lru.NewARC(build.BadBlockCacheSize) + cache, err := lru.NewARC[cid.Cid, BadBlockReason](build.BadBlockCacheSize) if err != nil { panic(err) // ok } @@ -66,10 +66,5 @@ func (bts *BadBlockCache) Purge() { } func (bts *BadBlockCache) Has(c cid.Cid) (BadBlockReason, bool) { - rval, ok := bts.badBlocks.Get(c) - if !ok { - return BadBlockReason{}, false - } - - return rval.(BadBlockReason), true + return bts.badBlocks.Get(c) } diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index a8ce9f4b7..9b62a7928 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -12,7 +12,7 @@ import ( dlog "github.com/drand/drand/log" gclient "github.com/drand/drand/lp2p/client" "github.com/drand/kyber" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/zap" @@ -61,7 +61,7 @@ type DrandBeacon struct { filGenTime uint64 filRoundTime uint64 - localCache *lru.Cache + localCache *lru.Cache[uint64, *types.BeaconEntry] } // DrandHTTPClient interface overrides the user agent used by drand @@ -119,7 +119,7 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config dtypes return nil, xerrors.Errorf("creating drand client: %w", err) } - lc, err := lru.New(1024) + lc, err := lru.New[uint64, *types.BeaconEntry](1024) if err != nil { return nil, err } @@ -169,16 +169,12 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re return out } func (db *DrandBeacon) cacheValue(e types.BeaconEntry) { - db.localCache.Add(e.Round, e) + db.localCache.Add(e.Round, &e) } func (db *DrandBeacon) getCachedValue(round uint64) *types.BeaconEntry { - v, ok := db.localCache.Get(round) - if !ok { - return nil - } - e, _ := v.(types.BeaconEntry) - return &e + v, _ := db.localCache.Get(round) + return v } func (db *DrandBeacon) VerifyEntry(curr types.BeaconEntry, prev types.BeaconEntry) error { diff --git a/chain/block_receipt_tracker.go b/chain/block_receipt_tracker.go index 58de71e19..9c1e035a2 100644 --- a/chain/block_receipt_tracker.go +++ b/chain/block_receipt_tracker.go @@ -5,7 +5,7 @@ import ( "sync" "time" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/libp2p/go-libp2p/core/peer" "github.com/filecoin-project/lotus/build" @@ -17,7 +17,7 @@ type blockReceiptTracker struct { // using an LRU cache because i don't want to handle all the edge cases for // manual cleanup and maintenance of a fixed size set - cache *lru.Cache + cache *lru.Cache[types.TipSetKey, *peerSet] } type peerSet struct { @@ -25,7 +25,7 @@ type peerSet struct { } func newBlockReceiptTracker() *blockReceiptTracker { - c, _ := lru.New(512) + c, _ := lru.New[types.TipSetKey, *peerSet](512) return &blockReceiptTracker{ cache: c, } @@ -46,20 +46,18 @@ func (brt *blockReceiptTracker) Add(p peer.ID, ts *types.TipSet) { return } - val.(*peerSet).peers[p] = build.Clock.Now() + val.peers[p] = build.Clock.Now() } func (brt *blockReceiptTracker) GetPeers(ts *types.TipSet) []peer.ID { brt.lk.Lock() defer brt.lk.Unlock() - val, ok := brt.cache.Get(ts.Key()) + ps, ok := brt.cache.Get(ts.Key()) if !ok { return nil } - ps := val.(*peerSet) - out := make([]peer.ID, 0, len(ps.peers)) for p := range ps.peers { out = append(out, p) diff --git a/chain/consensus/compute_state.go b/chain/consensus/compute_state.go index cfb2cdf31..e627a62d2 100644 --- a/chain/consensus/compute_state.go +++ b/chain/consensus/compute_state.go @@ -3,6 +3,7 @@ package consensus import ( "context" "sync/atomic" + "time" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -113,6 +114,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, return sm.VMConstructor()(ctx, vmopt) } + var cronGas int64 + runCron := func(vmCron vm.Interface, epoch abi.ChainEpoch) error { cronMsg := &types.Message{ To: cron.Address, @@ -130,6 +133,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, return xerrors.Errorf("running cron: %w", err) } + cronGas += ret.GasUsed + if em != nil { if err := em.MessageApplied(ctx, ts, cronMsg.Cid(), cronMsg, ret, true); err != nil { return xerrors.Errorf("callback failed on cron message: %w", err) @@ -181,7 +186,9 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, } } - partDone() + vmEarly := partDone() + earlyCronGas := cronGas + cronGas = 0 partDone = metrics.Timer(ctx, metrics.VMApplyMessages) vmi, err := makeVm(pstate, epoch, ts.MinTimestamp()) @@ -196,6 +203,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, processedMsgs = make(map[cid.Cid]struct{}) ) + var msgGas int64 + for _, b := range bms { penalty := types.NewInt(0) gasReward := big.Zero() @@ -210,6 +219,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, return cid.Undef, cid.Undef, err } + msgGas += r.GasUsed + receipts = append(receipts, &r.MessageReceipt) gasReward = big.Add(gasReward, r.GasCosts.MinerTip) penalty = big.Add(penalty, r.GasCosts.MinerPenalty) @@ -239,14 +250,14 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, } } - partDone() + vmMsg := partDone() partDone = metrics.Timer(ctx, metrics.VMApplyCron) if err := runCron(vmi, epoch); err != nil { return cid.Cid{}, cid.Cid{}, err } - partDone() + vmCron := partDone() partDone = metrics.Timer(ctx, metrics.VMApplyFlush) rectarr := blockadt.MakeEmptyArray(sm.ChainStore().ActorStore(ctx)) @@ -282,6 +293,11 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, return cid.Undef, cid.Undef, xerrors.Errorf("vm flush failed: %w", err) } + vmFlush := partDone() + partDone = func() time.Duration { return time.Duration(0) } + + log.Infow("ApplyBlocks stats", "early", vmEarly, "earlyCronGas", earlyCronGas, "vmMsg", vmMsg, "msgGas", msgGas, "vmCron", vmCron, "cronGas", cronGas, "vmFlush", vmFlush, "epoch", epoch, "tsk", ts.Key()) + stats.Record(ctx, metrics.VMSends.M(int64(atomic.LoadUint64(&vm.StatSends))), metrics.VMApplied.M(int64(atomic.LoadUint64(&vm.StatApplied)))) diff --git a/chain/events/message_cache.go b/chain/events/message_cache.go index 81b79cb38..d47d3a168 100644 --- a/chain/events/message_cache.go +++ b/chain/events/message_cache.go @@ -4,7 +4,7 @@ import ( "context" "sync" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/api" @@ -14,14 +14,14 @@ type messageCache struct { api EventAPI blockMsgLk sync.Mutex - blockMsgCache *lru.ARCCache + blockMsgCache *lru.ARCCache[cid.Cid, *api.BlockMessages] } -func newMessageCache(api EventAPI) *messageCache { - blsMsgCache, _ := lru.NewARC(500) +func newMessageCache(a EventAPI) *messageCache { + blsMsgCache, _ := lru.NewARC[cid.Cid, *api.BlockMessages](500) return &messageCache{ - api: api, + api: a, blockMsgCache: blsMsgCache, } } @@ -30,14 +30,14 @@ func (c *messageCache) ChainGetBlockMessages(ctx context.Context, blkCid cid.Cid c.blockMsgLk.Lock() defer c.blockMsgLk.Unlock() - msgsI, ok := c.blockMsgCache.Get(blkCid) + msgs, ok := c.blockMsgCache.Get(blkCid) var err error if !ok { - msgsI, err = c.api.ChainGetBlockMessages(ctx, blkCid) + msgs, err = c.api.ChainGetBlockMessages(ctx, blkCid) if err != nil { return nil, err } - c.blockMsgCache.Add(blkCid, msgsI) + c.blockMsgCache.Add(blkCid, msgs) } - return msgsI.(*api.BlockMessages), nil + return msgs, nil } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 673a299e3..de2df97c2 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -256,7 +256,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS //return nil, xerrors.Errorf("creating drand beacon: %w", err) //} - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), sys, us, beac) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), sys, us, beac, ds) if err != nil { return nil, xerrors.Errorf("initing stmgr: %w", err) } diff --git a/chain/messagepool/config.go b/chain/messagepool/config.go index 5b7e2c590..72b7d9567 100644 --- a/chain/messagepool/config.go +++ b/chain/messagepool/config.go @@ -13,7 +13,11 @@ import ( ) var ( - ReplaceByFeeRatioDefault = 1.25 + ReplaceByFeePercentageMinimum types.Percent = 110 + ReplaceByFeePercentageDefault types.Percent = 125 +) + +var ( MemPoolSizeLimitHiDefault = 30000 MemPoolSizeLimitLoDefault = 20000 PruneCooldownDefault = time.Minute @@ -60,9 +64,9 @@ func (mp *MessagePool) getConfig() *types.MpoolConfig { } func validateConfg(cfg *types.MpoolConfig) error { - if cfg.ReplaceByFeeRatio < ReplaceByFeeRatioDefault { - return fmt.Errorf("'ReplaceByFeeRatio' is less than required %f < %f", - cfg.ReplaceByFeeRatio, ReplaceByFeeRatioDefault) + if cfg.ReplaceByFeeRatio < ReplaceByFeePercentageMinimum { + return fmt.Errorf("'ReplaceByFeeRatio' is less than required %s < %s", + cfg.ReplaceByFeeRatio, ReplaceByFeePercentageMinimum) } if cfg.GasLimitOverestimation < 1 { return fmt.Errorf("'GasLimitOverestimation' cannot be less than 1") @@ -91,7 +95,7 @@ func DefaultConfig() *types.MpoolConfig { return &types.MpoolConfig{ SizeLimitHigh: MemPoolSizeLimitHiDefault, SizeLimitLow: MemPoolSizeLimitLoDefault, - ReplaceByFeeRatio: ReplaceByFeeRatioDefault, + ReplaceByFeeRatio: ReplaceByFeePercentageDefault, PruneCooldown: PruneCooldownDefault, GasLimitOverestimation: GasLimitOverestimation, } diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 9abe38648..0d787bd50 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -12,7 +12,7 @@ import ( "time" "github.com/hashicorp/go-multierror" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -47,10 +47,8 @@ var log = logging.Logger("messagepool") var futureDebug = false -var rbfNumBig = types.NewInt(uint64((ReplaceByFeeRatioDefault - 1) * RbfDenom)) -var rbfDenomBig = types.NewInt(RbfDenom) - -const RbfDenom = 256 +var rbfNumBig = types.NewInt(uint64(ReplaceByFeePercentageMinimum)) +var rbfDenomBig = types.NewInt(100) var RepublishInterval = time.Duration(10*build.BlockDelaySecs+build.PropagationDelaySecs) * time.Second @@ -161,7 +159,7 @@ type MessagePool struct { // pruneCooldown is a channel used to allow a cooldown time between prunes pruneCooldown chan struct{} - blsSigCache *lru.TwoQueueCache + blsSigCache *lru.TwoQueueCache[cid.Cid, crypto.Signature] changes *lps.PubSub @@ -169,9 +167,9 @@ type MessagePool struct { netName dtypes.NetworkName - sigValCache *lru.TwoQueueCache + sigValCache *lru.TwoQueueCache[string, struct{}] - nonceCache *lru.Cache + nonceCache *lru.Cache[nonceCacheKey, uint64] evtTypes [3]journal.EventType journal journal.Journal @@ -197,7 +195,13 @@ func newMsgSet(nonce uint64) *msgSet { } func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount { - minPrice := types.BigAdd(curPrem, types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig)) + minPrice := types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig) + return types.BigAdd(minPrice, types.NewInt(1)) +} + +func ComputeRBF(curPrem abi.TokenAmount, replaceByFeeRatio types.Percent) abi.TokenAmount { + rbfNumBig := types.NewInt(uint64(replaceByFeeRatio)) + minPrice := types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig) return types.BigAdd(minPrice, types.NewInt(1)) } @@ -365,9 +369,9 @@ func (ms *msgSet) toSlice() []*types.SignedMessage { } func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) { - cache, _ := lru.New2Q(build.BlsSignatureCacheSize) - verifcache, _ := lru.New2Q(build.VerifSigCacheSize) - noncecache, _ := lru.New(256) + cache, _ := lru.New2Q[cid.Cid, crypto.Signature](build.BlsSignatureCacheSize) + verifcache, _ := lru.New2Q[string, struct{}](build.VerifSigCacheSize) + noncecache, _ := lru.New[nonceCacheKey, uint64](256) cfg, err := loadConfig(ctx, ds) if err != nil { @@ -1049,7 +1053,7 @@ func (mp *MessagePool) getStateNonce(ctx context.Context, addr address.Address, n, ok := mp.nonceCache.Get(nk) if ok { - return n.(uint64), nil + return n, nil } act, err := mp.api.GetActorAfter(addr, ts) @@ -1469,15 +1473,10 @@ func (mp *MessagePool) MessagesForBlocks(ctx context.Context, blks []*types.Bloc } func (mp *MessagePool) RecoverSig(msg *types.Message) *types.SignedMessage { - val, ok := mp.blsSigCache.Get(msg.Cid()) + sig, ok := mp.blsSigCache.Get(msg.Cid()) if !ok { return nil } - sig, ok := val.(crypto.Signature) - if !ok { - log.Errorf("value in signature cache was not a signature (got %T)", val) - return nil - } return &types.SignedMessage{ Message: *msg, diff --git a/chain/stmgr/execute.go b/chain/stmgr/execute.go index e67398299..f85ff7c04 100644 --- a/chain/stmgr/execute.go +++ b/chain/stmgr/execute.go @@ -8,6 +8,7 @@ import ( "go.opencensus.io/trace" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) @@ -52,6 +53,22 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c sm.stlk.Unlock() + if ts.Height() == 0 { + // NB: This is here because the process that executes blocks requires that the + // block miner reference a valid miner in the state tree. Unless we create some + // magical genesis miner, this won't work properly, so we short circuit here + // This avoids the question of 'who gets paid the genesis block reward'. + // This also makes us not attempt to lookup the tipset state with + // tryLookupTipsetState, which would cause a very long, very slow walk. + return ts.Blocks()[0].ParentStateRoot, ts.Blocks()[0].ParentMessageReceipts, nil + } + + // First, try to find the tipset in the current chain. If found, we can avoid re-executing + // it. + if st, rec, found := tryLookupTipsetState(ctx, sm.cs, ts); found { + return st, rec, nil + } + st, rec, err = sm.tsExec.ExecuteTipSet(ctx, sm, ts, sm.tsExecMonitor, false) if err != nil { return cid.Undef, cid.Undef, err @@ -60,6 +77,51 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c return st, rec, nil } +// Try to lookup a state & receipt CID for a given tipset by walking the chain instead of executing +// it. This will only successfully return the state/receipt CIDs if they're found in the state +// store. +// +// NOTE: This _won't_ recursively walk the receipt/state trees. It assumes that having the root +// implies having the rest of the tree. However, lotus generally makes that assumption anyways. +func tryLookupTipsetState(ctx context.Context, cs *store.ChainStore, ts *types.TipSet) (cid.Cid, cid.Cid, bool) { + nextTs, err := cs.GetTipsetByHeight(ctx, ts.Height()+1, nil, false) + if err != nil { + // Nothing to see here. The requested height may be beyond the current head. + return cid.Undef, cid.Undef, false + } + + // Make sure we're on the correct fork. + if nextTs.Parents() != ts.Key() { + // Also nothing to see here. This just means that the requested tipset is on a + // different fork. + return cid.Undef, cid.Undef, false + } + + stateCid := nextTs.ParentState() + receiptCid := nextTs.ParentMessageReceipts() + + // Make sure we have the parent state. + if hasState, err := cs.StateBlockstore().Has(ctx, stateCid); err != nil { + log.Errorw("failed to lookup state-root in blockstore", "cid", stateCid, "error", err) + return cid.Undef, cid.Undef, false + } else if !hasState { + // We have the chain but don't have the state. It looks like we need to try + // executing? + return cid.Undef, cid.Undef, false + } + + // Make sure we have the receipts. + if hasReceipts, err := cs.ChainBlockstore().Has(ctx, receiptCid); err != nil { + log.Errorw("failed to lookup receipts in blockstore", "cid", receiptCid, "error", err) + return cid.Undef, cid.Undef, false + } else if !hasReceipts { + // If we don't have the receipts, re-execute and try again. + return cid.Undef, cid.Undef, false + } + + return stateCid, receiptCid, true +} + func (sm *StateManager) ExecutionTraceWithMonitor(ctx context.Context, ts *types.TipSet, em ExecMonitor) (cid.Cid, error) { st, _, err := sm.tsExec.ExecuteTipSet(ctx, sm, ts, em, true) return st, err diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 7d7e234b1..1f9977d96 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -174,9 +174,16 @@ func (us UpgradeSchedule) GetNtwkVersion(e abi.ChainEpoch) (network.Version, err func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecMonitor, ts *types.TipSet) (cid.Cid, error) { retCid := root - var err error u := sm.stateMigrations[height] if u != nil && u.upgrade != nil { + migCid, ok, err := u.migrationResultCache.Get(ctx, root) + if err == nil && ok { + log.Infow("CACHED migration", "height", height, "from", root, "to", migCid) + return migCid, nil + } else if err != nil { + log.Errorw("failed to lookup previous migration result", "err", err) + } + startTime := time.Now() log.Warnw("STARTING migration", "height", height, "from", root) // Yes, we clone the cache, even for the final upgrade epoch. Why? Reverts. We may @@ -197,6 +204,11 @@ func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, heig "to", retCid, "duration", time.Since(startTime), ) + + // Only set if migration ran, we do not want a root => root mapping + if err := u.migrationResultCache.Store(ctx, root, retCid); err != nil { + log.Errorw("failed to store migration result", "err", err) + } } return retCid, nil diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 52cfe5fce..a904172cd 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" ipldcbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" @@ -35,6 +36,7 @@ import ( "github.com/filecoin-project/lotus/chain/consensus" "github.com/filecoin-project/lotus/chain/consensus/filcns" "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/stmgr" . "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" @@ -166,7 +168,7 @@ func TestForkHeightTriggers(t *testing.T) { } return st.Flush(ctx) - }}}, cg.BeaconSchedule()) + }}}, cg.BeaconSchedule(), datastore.NewMapDatastore()) if err != nil { t.Fatal(err) } @@ -284,7 +286,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) { root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { migrationCount++ return root, nil - }}}, cg.BeaconSchedule()) + }}}, cg.BeaconSchedule(), datastore.NewMapDatastore()) if err != nil { t.Fatal(err) } @@ -502,7 +504,7 @@ func TestForkPreMigration(t *testing.T) { return nil }, }}}, - }, cg.BeaconSchedule()) + }, cg.BeaconSchedule(), datastore.NewMapDatastore()) if err != nil { t.Fatal(err) } @@ -576,6 +578,7 @@ func TestDisablePreMigration(t *testing.T) { }}}, }, cg.BeaconSchedule(), + datastore.NewMapDatastore(), ) require.NoError(t, err) require.NoError(t, sm.Start(context.Background())) @@ -603,3 +606,102 @@ func TestDisablePreMigration(t *testing.T) { require.Equal(t, 1, len(counter)) } + +func TestMigrtionCache(t *testing.T) { + logging.SetAllLoggers(logging.LevelInfo) + + cg, err := gen.NewGenerator() + require.NoError(t, err) + + counter := make(chan struct{}, 10) + metadataDs := datastore.NewMapDatastore() + + sm, err := NewStateManager( + cg.ChainStore(), + consensus.NewTipSetExecutor(filcns.RewardFunc), + cg.StateManager().VMSys(), + UpgradeSchedule{{ + Network: network.Version1, + Height: testForkHeight, + Migration: func(_ context.Context, _ *StateManager, _ MigrationCache, _ ExecMonitor, + root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) { + + counter <- struct{}{} + + return root, nil + }}, + }, + cg.BeaconSchedule(), + metadataDs, + ) + require.NoError(t, err) + require.NoError(t, sm.Start(context.Background())) + defer func() { + require.NoError(t, sm.Stop(context.Background())) + }() + + inv := consensus.NewActorRegistry() + registry := builtin.MakeRegistryLegacy([]rtt.VMActor{testActor{}}) + inv.Register(actorstypes.Version0, nil, registry) + + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) { + nvm, err := vm.NewLegacyVM(ctx, vmopt) + require.NoError(t, err) + nvm.SetInvoker(inv) + return nvm, nil + }) + + cg.SetStateManager(sm) + + for i := 0; i < 50; i++ { + _, err := cg.NextTipSet() + require.NoError(t, err) + } + + ts, err := cg.ChainStore().GetTipsetByHeight(context.Background(), testForkHeight, nil, false) + require.NoError(t, err) + + root, _, err := stmgr.ComputeState(context.Background(), sm, testForkHeight+1, []*types.Message{}, ts) + require.NoError(t, err) + t.Log(root) + + require.Equal(t, 1, len(counter)) + + { + sm, err := NewStateManager( + cg.ChainStore(), + consensus.NewTipSetExecutor(filcns.RewardFunc), + cg.StateManager().VMSys(), + UpgradeSchedule{{ + Network: network.Version1, + Height: testForkHeight, + Migration: func(_ context.Context, _ *StateManager, _ MigrationCache, _ ExecMonitor, + root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) { + + counter <- struct{}{} + + return root, nil + }}, + }, + cg.BeaconSchedule(), + metadataDs, + ) + require.NoError(t, err) + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) { + nvm, err := vm.NewLegacyVM(ctx, vmopt) + require.NoError(t, err) + nvm.SetInvoker(inv) + return nvm, nil + }) + + ctx := context.Background() + + base, _, err := sm.ExecutionTrace(ctx, ts) + require.NoError(t, err) + _, err = sm.HandleStateForks(context.Background(), base, ts.Height(), nil, ts) + require.NoError(t, err) + + // Should not have increased as we should be using the cached results in the metadataDs + require.Equal(t, 1, len(counter)) + } +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ee9338e63..b9f8d81bf 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -2,10 +2,13 @@ package stmgr import ( "context" + "fmt" "sync" "github.com/ipfs/go-cid" + dstore "github.com/ipfs/go-datastore" cbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" @@ -51,9 +54,47 @@ type versionSpec struct { } type migration struct { - upgrade MigrationFunc - preMigrations []PreMigration - cache *nv16.MemMigrationCache + upgrade MigrationFunc + preMigrations []PreMigration + cache *nv16.MemMigrationCache + migrationResultCache *migrationResultCache +} + +type migrationResultCache struct { + ds dstore.Batching + keyPrefix string +} + +func (m *migrationResultCache) keyForMigration(root cid.Cid) dstore.Key { + kStr := fmt.Sprintf("%s/%s", m.keyPrefix, root) + return dstore.NewKey(kStr) +} + +func (m *migrationResultCache) Get(ctx context.Context, root cid.Cid) (cid.Cid, bool, error) { + k := m.keyForMigration(root) + + bs, err := m.ds.Get(ctx, k) + if ipld.IsNotFound(err) { + return cid.Undef, false, nil + } else if err != nil { + return cid.Undef, false, xerrors.Errorf("error loading migration result: %w", err) + } + + c, err := cid.Parse(bs) + if err != nil { + return cid.Undef, false, xerrors.Errorf("error parsing migration result: %w", err) + } + + return c, true, nil +} + +func (m *migrationResultCache) Store(ctx context.Context, root cid.Cid, resultCid cid.Cid) error { + k := m.keyForMigration(root) + if err := m.ds.Put(ctx, k, resultCid.Bytes()); err != nil { + return err + } + + return nil } type Executor interface { @@ -103,7 +144,7 @@ type treeCache struct { tree *state.StateTree } -func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, beacon beacon.Schedule) (*StateManager, error) { +func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, beacon beacon.Schedule, metadataDs dstore.Batching) (*StateManager, error) { // If we have upgrades, make sure they're in-order and make sense. if err := us.Validate(); err != nil { return nil, err @@ -122,12 +163,18 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, upgrade: upgrade.Migration, preMigrations: upgrade.PreMigrations, cache: nv16.NewMemMigrationCache(), + migrationResultCache: &migrationResultCache{ + keyPrefix: fmt.Sprintf("/migration-cache/nv%d", upgrade.Network), + ds: metadataDs, + }, } + stateMigrations[upgrade.Height] = migration } if upgrade.Expensive { expensiveUpgrades[upgrade.Height] = struct{}{} } + networkVersions = append(networkVersions, versionSpec{ networkVersion: lastVersion, atOrBelow: upgrade.Height, @@ -155,8 +202,8 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, }, nil } -func NewStateManagerWithUpgradeScheduleAndMonitor(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, b beacon.Schedule, em ExecMonitor) (*StateManager, error) { - sm, err := NewStateManager(cs, exec, sys, us, b) +func NewStateManagerWithUpgradeScheduleAndMonitor(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, b beacon.Schedule, em ExecMonitor, metadataDs dstore.Batching) (*StateManager, error) { + sm, err := NewStateManager(cs, exec, sys, us, b, metadataDs) if err != nil { return nil, err } diff --git a/chain/store/index.go b/chain/store/index.go index fe8f399ee..620cb2dee 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -4,8 +4,8 @@ import ( "context" "os" "strconv" + "sync" - lru "github.com/hashicorp/golang-lru" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -var DefaultChainIndexCacheSize = 32 << 10 +var DefaultChainIndexCacheSize = 32 << 15 func init() { if s := os.Getenv("LOTUS_CHAIN_INDEX_CACHE"); s != "" { @@ -27,7 +27,8 @@ func init() { } type ChainIndex struct { - skipCache *lru.ARCCache + indexCacheLk sync.Mutex + indexCache map[types.TipSetKey]*lbEntry loadTipSet loadTipSetFunc @@ -36,17 +37,14 @@ type ChainIndex struct { type loadTipSetFunc func(context.Context, types.TipSetKey) (*types.TipSet, error) func NewChainIndex(lts loadTipSetFunc) *ChainIndex { - sc, _ := lru.NewARC(DefaultChainIndexCacheSize) return &ChainIndex{ - skipCache: sc, + indexCache: make(map[types.TipSetKey]*lbEntry, DefaultChainIndexCacheSize), loadTipSet: lts, skipLength: 20, } } type lbEntry struct { - ts *types.TipSet - parentHeight abi.ChainEpoch targetHeight abi.ChainEpoch target types.TipSetKey } @@ -58,25 +56,36 @@ func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet, rounded, err := ci.roundDown(ctx, from) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to round down: %w", err) } + ci.indexCacheLk.Lock() + defer ci.indexCacheLk.Unlock() cur := rounded.Key() for { - cval, ok := ci.skipCache.Get(cur) + lbe, ok := ci.indexCache[cur] if !ok { fc, err := ci.fillCache(ctx, cur) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to fill cache: %w", err) } - cval = fc + lbe = fc } - lbe := cval.(*lbEntry) - if lbe.ts.Height() == to || lbe.parentHeight < to { - return lbe.ts, nil - } else if to > lbe.targetHeight { - return ci.walkBack(ctx, lbe.ts, to) + if to == lbe.targetHeight { + ts, err := ci.loadTipSet(ctx, lbe.target) + if err != nil { + return nil, xerrors.Errorf("failed to load tipset: %w", err) + } + + return ts, nil + } + if to > lbe.targetHeight { + ts, err := ci.loadTipSet(ctx, cur) + if err != nil { + return nil, xerrors.Errorf("failed to load tipset: %w", err) + } + return ci.walkBack(ctx, ts, to) } cur = lbe.target @@ -87,16 +96,17 @@ func (ci *ChainIndex) GetTipsetByHeightWithoutCache(ctx context.Context, from *t return ci.walkBack(ctx, from, to) } +// Caller must hold indexCacheLk func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEntry, error) { ts, err := ci.loadTipSet(ctx, tsk) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load tipset: %w", err) } if ts.Height() == 0 { return &lbEntry{ - ts: ts, - parentHeight: 0, + targetHeight: 0, + target: tsk, }, nil } @@ -124,12 +134,10 @@ func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEn } lbe := &lbEntry{ - ts: ts, - parentHeight: parent.Height(), targetHeight: skipTarget.Height(), target: skipTarget.Key(), } - ci.skipCache.Add(tsk, lbe) + ci.indexCache[tsk] = lbe return lbe, nil } @@ -144,7 +152,7 @@ func (ci *ChainIndex) roundDown(ctx context.Context, ts *types.TipSet) (*types.T rounded, err := ci.walkBack(ctx, ts, target) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to walk back: %w", err) } return rounded, nil @@ -164,7 +172,7 @@ func (ci *ChainIndex) walkBack(ctx context.Context, from *types.TipSet, to abi.C for { pts, err := ci.loadTipSet(ctx, ts.Parents()) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load tipset: %w", err) } if to > pts.Height() { diff --git a/chain/store/messages.go b/chain/store/messages.go index 5ac62d394..c39cb3f9b 100644 --- a/chain/store/messages.go +++ b/chain/store/messages.go @@ -207,9 +207,7 @@ type mmCids struct { } func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) { - o, ok := cs.mmCache.Get(mmc) - if ok { - mmcids := o.(*mmCids) + if mmcids, ok := cs.mmCache.Get(mmc); ok { return mmcids.bls, mmcids.secpk, nil } @@ -229,7 +227,7 @@ func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.C return nil, nil, xerrors.Errorf("loading secpk message cids for block: %w", err) } - cs.mmCache.Add(mmc, &mmCids{ + cs.mmCache.Add(mmc, mmCids{ bls: blscids, secpk: secpkcids, }) @@ -237,6 +235,26 @@ func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.C return blscids, secpkcids, nil } +func (cs *ChainStore) ReadReceipts(ctx context.Context, root cid.Cid) ([]types.MessageReceipt, error) { + a, err := blockadt.AsArray(cs.ActorStore(ctx), root) + if err != nil { + return nil, err + } + + receipts := make([]types.MessageReceipt, 0, a.Length()) + var rcpt types.MessageReceipt + if err := a.ForEach(&rcpt, func(i int64) error { + if int64(len(receipts)) != i { + return xerrors.Errorf("missing receipt %d", i) + } + receipts = append(receipts, rcpt) + return nil + }); err != nil { + return nil, err + } + return receipts, nil +} + func (cs *ChainStore) MessagesForBlock(ctx context.Context, b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) { blscids, secpkcids, err := cs.ReadMsgMetaCids(ctx, b.Messages) if err != nil { diff --git a/chain/store/store.go b/chain/store/store.go index 754e3a123..d7188a7bf 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -11,7 +11,7 @@ import ( "sync" "time" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" dstore "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" @@ -120,8 +120,8 @@ type ChainStore struct { reorgCh chan<- reorg reorgNotifeeCh chan ReorgNotifee - mmCache *lru.ARCCache // msg meta cache (mh.Messages -> secp, bls []cid) - tsCache *lru.ARCCache + mmCache *lru.ARCCache[cid.Cid, mmCids] + tsCache *lru.ARCCache[types.TipSetKey, *types.TipSet] evtTypes [1]journal.EventType journal journal.Journal @@ -133,8 +133,8 @@ type ChainStore struct { } func NewChainStore(chainBs bstore.Blockstore, stateBs bstore.Blockstore, ds dstore.Batching, weight WeightFunc, j journal.Journal) *ChainStore { - c, _ := lru.NewARC(DefaultMsgMetaCacheSize) - tsc, _ := lru.NewARC(DefaultTipSetCacheSize) + c, _ := lru.NewARC[cid.Cid, mmCids](DefaultMsgMetaCacheSize) + tsc, _ := lru.NewARC[types.TipSetKey, *types.TipSet](DefaultTipSetCacheSize) if j == nil { j = journal.NilJournal() } @@ -818,9 +818,8 @@ func (cs *ChainStore) GetBlock(ctx context.Context, c cid.Cid) (*types.BlockHead } func (cs *ChainStore) LoadTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - v, ok := cs.tsCache.Get(tsk) - if ok { - return v.(*types.TipSet), nil + if ts, ok := cs.tsCache.Get(tsk); ok { + return ts, nil } // Fetch tipset block headers from blockstore in parallel diff --git a/chain/store/store_test.go b/chain/store/store_test.go index f5765fddc..cc72acc95 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -196,7 +196,8 @@ func TestChainExportImportFull(t *testing.T) { } nbs := blockstore.NewMemorySync() - cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), filcns.Weight, nil) + ds := datastore.NewMapDatastore() + cs := store.NewChainStore(nbs, nbs, ds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck root, err := cs.Import(context.TODO(), buf) @@ -213,7 +214,7 @@ func TestChainExportImportFull(t *testing.T) { t.Fatal("imported chain differed from exported chain") } - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), nil, filcns.DefaultUpgradeSchedule(), cg.BeaconSchedule()) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), nil, filcns.DefaultUpgradeSchedule(), cg.BeaconSchedule(), ds) if err != nil { t.Fatal(err) } diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 5006ad0a8..f641f0ff9 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -7,7 +7,7 @@ import ( "sync" "time" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" blocks "github.com/ipfs/go-libipfs/blocks" @@ -217,7 +217,7 @@ func fetchCids( type BlockValidator struct { self peer.ID - peers *lru.TwoQueueCache + peers *lru.TwoQueueCache[peer.ID, int] killThresh int @@ -231,7 +231,7 @@ type BlockValidator struct { } func NewBlockValidator(self peer.ID, chain *store.ChainStore, cns consensus.Consensus, blacklist func(peer.ID)) *BlockValidator { - p, _ := lru.New2Q(4096) + p, _ := lru.New2Q[peer.ID, int](4096) return &BlockValidator{ self: self, peers: p, @@ -244,21 +244,19 @@ func NewBlockValidator(self peer.ID, chain *store.ChainStore, cns consensus.Cons } func (bv *BlockValidator) flagPeer(p peer.ID) { - v, ok := bv.peers.Get(p) + val, ok := bv.peers.Get(p) if !ok { - bv.peers.Add(p, int(1)) + bv.peers.Add(p, 1) return } - val := v.(int) - if val >= bv.killThresh { log.Warnf("blacklisting peer %s", p) bv.blacklist(p) return } - bv.peers.Add(p, v.(int)+1) + bv.peers.Add(p, val+1) } func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) (res pubsub.ValidationResult) { @@ -293,11 +291,11 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub } type blockReceiptCache struct { - blocks *lru.TwoQueueCache + blocks *lru.TwoQueueCache[cid.Cid, int] } func newBlockReceiptCache() *blockReceiptCache { - c, _ := lru.New2Q(8192) + c, _ := lru.New2Q[cid.Cid, int](8192) return &blockReceiptCache{ blocks: c, @@ -307,12 +305,12 @@ func newBlockReceiptCache() *blockReceiptCache { func (brc *blockReceiptCache) add(bcid cid.Cid) int { val, ok := brc.blocks.Get(bcid) if !ok { - brc.blocks.Add(bcid, int(1)) + brc.blocks.Add(bcid, 1) return 0 } - brc.blocks.Add(bcid, val.(int)+1) - return val.(int) + brc.blocks.Add(bcid, val+1) + return val } type MessageValidator struct { @@ -466,13 +464,13 @@ type peerMsgInfo struct { type IndexerMessageValidator struct { self peer.ID - peerCache *lru.TwoQueueCache + peerCache *lru.TwoQueueCache[address.Address, *peerMsgInfo] chainApi full.ChainModuleAPI stateApi full.StateModuleAPI } func NewIndexerMessageValidator(self peer.ID, chainApi full.ChainModuleAPI, stateApi full.StateModuleAPI) *IndexerMessageValidator { - peerCache, _ := lru.New2Q(8192) + peerCache, _ := lru.New2Q[address.Address, *peerMsgInfo](8192) return &IndexerMessageValidator{ self: self, @@ -515,15 +513,12 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg return pubsub.ValidationReject } - minerID := minerAddr.String() msgCid := idxrMsg.Cid var msgInfo *peerMsgInfo - val, ok := v.peerCache.Get(minerID) + msgInfo, ok := v.peerCache.Get(minerAddr) if !ok { msgInfo = &peerMsgInfo{} - } else { - msgInfo = val.(*peerMsgInfo) } // Lock this peer's message info. @@ -544,7 +539,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg // Check that the miner ID maps to the peer that sent the message. err = v.authenticateMessage(ctx, minerAddr, originPeer) if err != nil { - log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerID) + log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerAddr) stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1)) return pubsub.ValidationReject } @@ -554,7 +549,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg // messages from the same peer are handled concurrently, there is a // small chance that one msgInfo could replace the other here when // the info is first cached. This is OK, so no need to prevent it. - v.peerCache.Add(minerID, msgInfo) + v.peerCache.Add(minerAddr, msgInfo) } } diff --git a/chain/types/ethtypes/eth_transactions.go b/chain/types/ethtypes/eth_transactions.go index 7afde4bd2..6c13c5bf6 100644 --- a/chain/types/ethtypes/eth_transactions.go +++ b/chain/types/ethtypes/eth_transactions.go @@ -44,14 +44,6 @@ type EthTx struct { S EthBigInt `json:"s"` } -func (tx *EthTx) Reward(blkBaseFee big.Int) EthBigInt { - availablePriorityFee := big.Sub(big.Int(tx.MaxFeePerGas), blkBaseFee) - if big.Cmp(big.Int(tx.MaxPriorityFeePerGas), availablePriorityFee) <= 0 { - return tx.MaxPriorityFeePerGas - } - return EthBigInt(availablePriorityFee) -} - type EthTxArgs struct { ChainID int `json:"chainId"` Nonce int `json:"nonce"` diff --git a/chain/types/ethtypes/eth_types.go b/chain/types/ethtypes/eth_types.go index ddda91ea0..f157c7f94 100644 --- a/chain/types/ethtypes/eth_types.go +++ b/chain/types/ethtypes/eth_types.go @@ -295,17 +295,21 @@ func EthAddressFromPubKey(pubk []byte) ([]byte, error) { return ethAddr, nil } +var maskedIDPrefix = [20 - 8]byte{0xff} + func IsEthAddress(addr address.Address) bool { if addr.Protocol() != address.Delegated { return false } payload := addr.Payload() - namespace, _, err := varint.FromUvarint(payload) + namespace, offset, err := varint.FromUvarint(payload) if err != nil { return false } - return namespace == builtintypes.EthereumAddressManagerActorID + payload = payload[offset:] + + return namespace == builtintypes.EthereumAddressManagerActorID && len(payload) == 20 && !bytes.HasPrefix(payload, maskedIDPrefix[:]) } func EthAddressFromFilecoinAddress(addr address.Address) (EthAddress, error) { @@ -326,9 +330,17 @@ func EthAddressFromFilecoinAddress(addr address.Address) (EthAddress, error) { return EthAddress{}, xerrors.Errorf("invalid delegated address namespace in: %s", addr) } payload = payload[n:] - if namespace == builtintypes.EthereumAddressManagerActorID { - return CastEthAddress(payload) + if namespace != builtintypes.EthereumAddressManagerActorID { + return EthAddress{}, ErrInvalidAddress } + ethAddr, err := CastEthAddress(payload) + if err != nil { + return EthAddress{}, err + } + if ethAddr.IsMaskedID() { + return EthAddress{}, xerrors.Errorf("f410f addresses cannot embed masked-ID payloads: %s", ethAddr) + } + return ethAddr, nil } return EthAddress{}, ErrInvalidAddress } @@ -376,8 +388,7 @@ func (ea *EthAddress) UnmarshalJSON(b []byte) error { } func (ea EthAddress) IsMaskedID() bool { - idmask := [12]byte{0xff} - return bytes.Equal(ea[:12], idmask[:]) + return bytes.HasPrefix(ea[:], maskedIDPrefix[:]) } func (ea EthAddress) ToFilecoinAddress() (address.Address, error) { diff --git a/chain/types/ethtypes/eth_types_test.go b/chain/types/ethtypes/eth_types_test.go index 118fbc901..4a73184c2 100644 --- a/chain/types/ethtypes/eth_types_test.go +++ b/chain/types/ethtypes/eth_types_test.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/builtin" ) type TestCase struct { @@ -178,6 +179,20 @@ func TestParseEthAddr(t *testing.T) { } } +func TestMaskedIDInF4(t *testing.T) { + addr, err := address.NewIDAddress(100) + require.NoError(t, err) + + eaddr, err := EthAddressFromFilecoinAddress(addr) + require.NoError(t, err) + + badaddr, err := address.NewDelegatedAddress(builtin.EthereumAddressManagerActorID, eaddr[:]) + require.NoError(t, err) + + _, err = EthAddressFromFilecoinAddress(badaddr) + require.Error(t, err) +} + func TestUnmarshalEthCall(t *testing.T) { data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":""}` diff --git a/chain/types/message.go b/chain/types/message.go index becd2c010..4304ba659 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -219,4 +219,17 @@ func (m *Message) ValidForBlockInclusion(minGas int64, version network.Version) return nil } +// EffectiveGasPremium returns the effective gas premium claimable by the miner +// given the supplied base fee. +// +// Filecoin clamps the gas premium at GasFeeCap - BaseFee, if lower than the +// specified premium. +func (m *Message) EffectiveGasPremium(baseFee abi.TokenAmount) abi.TokenAmount { + available := big.Sub(m.GasFeeCap, baseFee) + if big.Cmp(m.GasPremium, available) <= 0 { + return m.GasPremium + } + return available +} + const TestGasLimit = 100e6 diff --git a/chain/types/mpool.go b/chain/types/mpool.go index cf08177e9..497d5c590 100644 --- a/chain/types/mpool.go +++ b/chain/types/mpool.go @@ -10,7 +10,7 @@ type MpoolConfig struct { PriorityAddrs []address.Address SizeLimitHigh int SizeLimitLow int - ReplaceByFeeRatio float64 + ReplaceByFeeRatio Percent PruneCooldown time.Duration GasLimitOverestimation float64 } diff --git a/chain/types/percent.go b/chain/types/percent.go new file mode 100644 index 000000000..858d9a2e3 --- /dev/null +++ b/chain/types/percent.go @@ -0,0 +1,39 @@ +package types + +import ( + "fmt" + "math" + "strconv" + + "golang.org/x/xerrors" +) + +// Percent stores a signed percentage as an int64. When converted to a string (or json), it's stored +// as a decimal with two places (e.g., 100% -> 1.00). +type Percent int64 + +func (p Percent) String() string { + abs := p + sign := "" + if abs < 0 { + abs = -abs + sign = "-" + } + return fmt.Sprintf(`%s%d.%d`, sign, abs/100, abs%100) +} + +func (p Percent) MarshalJSON() ([]byte, error) { + return []byte(p.String()), nil +} + +func (p *Percent) UnmarshalJSON(b []byte) error { + flt, err := strconv.ParseFloat(string(b)+"e2", 64) + if err != nil { + return xerrors.Errorf("unable to parse ratio %s: %w", string(b), err) + } + if math.Trunc(flt) != flt { + return xerrors.Errorf("ratio may only have two decimals: %s", string(b)) + } + *p = Percent(flt) + return nil +} diff --git a/chain/types/percent_test.go b/chain/types/percent_test.go new file mode 100644 index 000000000..7364c2447 --- /dev/null +++ b/chain/types/percent_test.go @@ -0,0 +1,34 @@ +package types + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPercent(t *testing.T) { + for _, tc := range []struct { + p Percent + s string + }{ + {100, "1.0"}, + {111, "1.11"}, + {12, "0.12"}, + {-12, "-0.12"}, + {1012, "10.12"}, + {-1012, "-10.12"}, + {0, "0.0"}, + } { + tc := tc + t.Run(fmt.Sprintf("%d <> %s", tc.p, tc.s), func(t *testing.T) { + m, err := tc.p.MarshalJSON() + require.NoError(t, err) + require.Equal(t, tc.s, string(m)) + var p Percent + require.NoError(t, p.UnmarshalJSON([]byte(tc.s))) + require.Equal(t, tc.p, p) + }) + } + +} diff --git a/chain/types/tipset.go b/chain/types/tipset.go index c1aa90fc9..047a1c00e 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -234,6 +234,10 @@ func (ts *TipSet) MinTicketBlock() *BlockHeader { return min } +func (ts *TipSet) ParentMessageReceipts() cid.Cid { + return ts.blks[0].ParentMessageReceipts +} + func (ts *TipSet) ParentState() cid.Cid { return ts.blks[0].ParentStateRoot } diff --git a/cli/evm.go b/cli/evm.go index d153e7212..84cbf8c61 100644 --- a/cli/evm.go +++ b/cli/evm.go @@ -35,6 +35,7 @@ var EvmCmd = &cli.Command{ EvmGetInfoCmd, EvmCallSimulateCmd, EvmGetContractAddress, + EvmGetBytecode, }, } @@ -486,3 +487,51 @@ func ethAddrFromFilecoinAddress(ctx context.Context, addr address.Address, fnapi return ethAddr, faddr, nil } + +var EvmGetBytecode = &cli.Command{ + Name: "bytecode", + Usage: "Write the bytecode of a smart contract to a file", + ArgsUsage: "[contract-address] [file-name]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "bin", + Usage: "write the bytecode as raw binary and don't hex-encode", + }, + }, + Action: func(cctx *cli.Context) error { + + if cctx.NArg() != 2 { + return IncorrectNumArgs(cctx) + } + + contractAddr, err := ethtypes.ParseEthAddress(cctx.Args().Get(0)) + if err != nil { + return err + } + + fileName := cctx.Args().Get(1) + + api, closer, err := GetFullNodeAPIV1(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + code, err := api.EthGetCode(ctx, contractAddr, "latest") + if err != nil { + return err + } + if !cctx.Bool("bin") { + newCode := make([]byte, hex.EncodedLen(len(code))) + hex.Encode(newCode, code) + code = newCode + } + if err := os.WriteFile(fileName, code, 0o666); err != nil { + return xerrors.Errorf("failed to write bytecode to file %s: %w", fileName, err) + } + + fmt.Printf("Code for %s written to %s\n", contractAddr, fileName) + return nil + }, +} diff --git a/cli/mpool.go b/cli/mpool.go index e098806cb..c83fb4b61 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -461,7 +461,12 @@ var MpoolReplaceCmd = &cli.Command{ msg := found.Message if cctx.Bool("auto") { - minRBF := messagepool.ComputeMinRBF(msg.GasPremium) + cfg, err := api.MpoolGetConfig(ctx) + if err != nil { + return xerrors.Errorf("failed to lookup the message pool config: %w", err) + } + + defaultRBF := messagepool.ComputeRBF(msg.GasPremium, cfg.ReplaceByFeeRatio) var mss *lapi.MessageSendSpec if cctx.IsSet("fee-limit") { @@ -482,7 +487,7 @@ var MpoolReplaceCmd = &cli.Command{ return xerrors.Errorf("failed to estimate gas values: %w", err) } - msg.GasPremium = big.Max(retm.GasPremium, minRBF) + msg.GasPremium = big.Max(retm.GasPremium, defaultRBF) msg.GasFeeCap = big.Max(retm.GasFeeCap, msg.GasPremium) mff := func() (abi.TokenAmount, error) { diff --git a/cli/mpool_test.go b/cli/mpool_test.go index 01b49d4b3..0aa055ba3 100644 --- a/cli/mpool_test.go +++ b/cli/mpool_test.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" @@ -298,6 +299,7 @@ func TestReplace(t *testing.T) { mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil), mockApi.EXPECT().ChainHead(ctx).Return(nil, nil), mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil), + mockApi.EXPECT().MpoolGetConfig(ctx).Return(messagepool.DefaultConfig(), nil), // use gomock.any to match the message in expected api calls // since the replace function modifies the message between calls, it would be pointless to try to match the exact argument mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil), @@ -342,6 +344,7 @@ func TestReplace(t *testing.T) { gomock.InOrder( mockApi.EXPECT().ChainHead(ctx).Return(nil, nil), mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil), + mockApi.EXPECT().MpoolGetConfig(ctx).Return(messagepool.DefaultConfig(), nil), // use gomock.any to match the message in expected api calls // since the replace function modifies the message between calls, it would be pointless to try to match the exact argument mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil), @@ -538,7 +541,7 @@ func TestConfig(t *testing.T) { t.Fatal(err) } - mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: 0.25} + mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: types.Percent(25)} gomock.InOrder( mockApi.EXPECT().MpoolGetConfig(ctx).Return(mpoolCfg, nil), ) @@ -566,7 +569,7 @@ func TestConfig(t *testing.T) { t.Fatal(err) } - mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: 0.33} + mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: types.Percent(33)} gomock.InOrder( mockApi.EXPECT().MpoolSetConfig(ctx, mpoolCfg).Return(nil), ) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 5a97be9e5..51c567d90 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -229,7 +229,7 @@ var importBenchCmd = &cli.Command{ defer cs.Close() //nolint:errcheck // TODO: We need to supply the actual beacon after v14 - stm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(verifier), filcns.DefaultUpgradeSchedule(), nil) + stm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(verifier), filcns.DefaultUpgradeSchedule(), nil, metadataDs) if err != nil { return err } diff --git a/cmd/lotus-miner/proving.go b/cmd/lotus-miner/proving.go index 9ae8bdd48..d58ed1250 100644 --- a/cmd/lotus-miner/proving.go +++ b/cmd/lotus-miner/proving.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/build" @@ -640,7 +641,42 @@ It will not send any messages to the chain.`, if err != nil { return err } - jr, err := json.Marshal(res) + + //convert sector information into easily readable information + type PoStPartition struct { + Index uint64 + Skipped []uint64 + } + type SubmitWindowedPoStParams struct { + Deadline uint64 + Partitions []PoStPartition + Proofs []proof.PoStProof + ChainCommitEpoch abi.ChainEpoch + ChainCommitRand abi.Randomness + } + var postParams []SubmitWindowedPoStParams + for _, i := range res { + var postParam SubmitWindowedPoStParams + postParam.Deadline = i.Deadline + for id, part := range i.Partitions { + postParam.Partitions[id].Index = part.Index + count, err := part.Skipped.Count() + if err != nil { + return err + } + sectors, err := part.Skipped.All(count) + if err != nil { + return err + } + postParam.Partitions[id].Skipped = sectors + } + postParam.Proofs = i.Proofs + postParam.ChainCommitEpoch = i.ChainCommitEpoch + postParam.ChainCommitRand = i.ChainCommitRand + postParams = append(postParams, postParam) + } + + jr, err := json.MarshalIndent(postParams, "", " ") if err != nil { return err } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index d678c7977..bae281583 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -513,7 +513,7 @@ var chainBalanceStateCmd = &cli.Command{ cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } @@ -737,7 +737,7 @@ var chainPledgeCmd = &cli.Command{ cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } diff --git a/cmd/lotus-shed/gas-estimation.go b/cmd/lotus-shed/gas-estimation.go index 26c4663bf..fe8428d1e 100644 --- a/cmd/lotus-shed/gas-estimation.go +++ b/cmd/lotus-shed/gas-estimation.go @@ -111,7 +111,7 @@ var gasTraceCmd = &cli.Command{ cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds) if err != nil { return err } @@ -212,7 +212,7 @@ var replayOfflineCmd = &cli.Command{ cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds) if err != nil { return err } diff --git a/cmd/lotus-shed/invariants.go b/cmd/lotus-shed/invariants.go index 9798d111a..b759e2c2c 100644 --- a/cmd/lotus-shed/invariants.go +++ b/cmd/lotus-shed/invariants.go @@ -90,7 +90,7 @@ var invariantsCmd = &cli.Command{ cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index e31957395..b3d5da387 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" "github.com/urfave/cli/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -121,7 +122,8 @@ var migrationsCmd = &cli.Command{ cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + // Note: we use a map datastore for the metadata to avoid writing / using cached migration results in the metadata store + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, datastore.NewMapDatastore()) if err != nil { return err } diff --git a/cmd/lotus-shed/state-stats.go b/cmd/lotus-shed/state-stats.go index 8ec4a0ff4..28fa79778 100644 --- a/cmd/lotus-shed/state-stats.go +++ b/cmd/lotus-shed/state-stats.go @@ -9,7 +9,7 @@ import ( "sync" "github.com/docker/go-units" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" offline "github.com/ipfs/go-ipfs-exchange-offline" @@ -50,13 +50,13 @@ type fieldItem struct { type cacheNodeGetter struct { ds format.NodeGetter - cache *lru.TwoQueueCache + cache *lru.TwoQueueCache[cid.Cid, format.Node] } func newCacheNodeGetter(d format.NodeGetter, size int) (*cacheNodeGetter, error) { cng := &cacheNodeGetter{ds: d} - cache, err := lru.New2Q(size) + cache, err := lru.New2Q[cid.Cid, format.Node](size) if err != nil { return nil, err } @@ -68,7 +68,7 @@ func newCacheNodeGetter(d format.NodeGetter, size int) (*cacheNodeGetter, error) func (cng *cacheNodeGetter) Get(ctx context.Context, c cid.Cid) (format.Node, error) { if n, ok := cng.cache.Get(c); ok { - return n.(format.Node), nil + return n, nil } n, err := cng.ds.Get(ctx, c) @@ -308,7 +308,7 @@ to reduce the number of decode operations performed by caching the decoded objec } tsExec := consensus.NewTipSetExecutor(filcns.RewardFunc) - sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } diff --git a/cmd/lotus-sim/simulation/node.go b/cmd/lotus-sim/simulation/node.go index 06e837f8f..9b37da6c8 100644 --- a/cmd/lotus-sim/simulation/node.go +++ b/cmd/lotus-sim/simulation/node.go @@ -106,7 +106,7 @@ func (nd *Node) LoadSim(ctx context.Context, name string) (*Simulation, error) { if err != nil { return nil, xerrors.Errorf("failed to create upgrade schedule for simulation %s: %w", name, err) } - sim.StateManager, err = stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), us, nil) + sim.StateManager, err = stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), us, nil, nd.MetadataDS) if err != nil { return nil, xerrors.Errorf("failed to create state manager for simulation %s: %w", name, err) } @@ -125,7 +125,7 @@ func (nd *Node) CreateSim(ctx context.Context, name string, head *types.TipSet) if err != nil { return nil, err } - sm, err := stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), filcns.DefaultUpgradeSchedule(), nil, nd.MetadataDS) if err != nil { return nil, xerrors.Errorf("creating state manager: %w", err) } diff --git a/cmd/lotus-sim/simulation/simulation.go b/cmd/lotus-sim/simulation/simulation.go index 19ab391dc..294f4cfbc 100644 --- a/cmd/lotus-sim/simulation/simulation.go +++ b/cmd/lotus-sim/simulation/simulation.go @@ -201,7 +201,7 @@ func (sim *Simulation) SetUpgradeHeight(nv network.Version, epoch abi.ChainEpoch if err != nil { return err } - sm, err := stmgr.NewStateManager(sim.Node.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), newUpgradeSchedule, nil) + sm, err := stmgr.NewStateManager(sim.Node.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), newUpgradeSchedule, nil, sim.Node.MetadataDS) if err != nil { return err } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 4a4a691ee..b3341bd79 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -540,7 +540,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) } // TODO: We need to supply the actual beacon after v14 - stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } diff --git a/conformance/driver.go b/conformance/driver.go index 3a32c7604..2680a7154 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -108,7 +108,7 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, params cs = store.NewChainStore(bs, bs, ds, filcns.Weight, nil) tse = consensus.NewTipSetExecutor(filcns.RewardFunc) - sm, err = stmgr.NewStateManager(cs, tse, syscalls, filcns.DefaultUpgradeSchedule(), nil) + sm, err = stmgr.NewStateManager(cs, tse, syscalls, filcns.DefaultUpgradeSchedule(), nil, ds) ) if err != nil { return nil, err diff --git a/documentation/en/api-v0-methods.md b/documentation/en/api-v0-methods.md index d254c14e7..b73244fa8 100644 --- a/documentation/en/api-v0-methods.md +++ b/documentation/en/api-v0-methods.md @@ -2879,7 +2879,7 @@ Response: ], "SizeLimitHigh": 123, "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, + "ReplaceByFeeRatio": 1.23, "PruneCooldown": 60000000000, "GasLimitOverestimation": 12.3 } @@ -3167,7 +3167,7 @@ Inputs: ], "SizeLimitHigh": 123, "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, + "ReplaceByFeeRatio": 1.23, "PruneCooldown": 60000000000, "GasLimitOverestimation": 12.3 } diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 5516acb44..aedfbfce0 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -3882,7 +3882,7 @@ Response: ], "SizeLimitHigh": 123, "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, + "ReplaceByFeeRatio": 1.23, "PruneCooldown": 60000000000, "GasLimitOverestimation": 12.3 } @@ -4170,7 +4170,7 @@ Inputs: ], "SizeLimitHigh": 123, "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, + "ReplaceByFeeRatio": 1.23, "PruneCooldown": 60000000000, "GasLimitOverestimation": 12.3 } diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 4458599ab..325b2738d 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -2647,6 +2647,7 @@ COMMANDS: stat Print eth/filecoin addrs and code cid call Simulate an eth contract call contract-address Generate contract address from smart contract code + bytecode Write the bytecode of a smart contract to a file help, h Shows a list of commands or help for one command OPTIONS: @@ -2721,6 +2722,19 @@ OPTIONS: ``` +### lotus evm bytecode +``` +NAME: + lotus evm bytecode - Write the bytecode of a smart contract to a file + +USAGE: + lotus evm bytecode [command options] [contract-address] [file-name] + +OPTIONS: + --bin write the bytecode as raw binary and don't hex-encode (default: false) + +``` + ## lotus net ``` NAME: diff --git a/documentation/en/default-lotus-config.toml b/documentation/en/default-lotus-config.toml index 41d7e6aca..c51321714 100644 --- a/documentation/en/default-lotus-config.toml +++ b/documentation/en/default-lotus-config.toml @@ -191,7 +191,7 @@ [Chainstore] # type: bool # env var: LOTUS_CHAINSTORE_ENABLESPLITSTORE - #EnableSplitstore = false + #EnableSplitstore = true [Chainstore.Splitstore] # ColdStoreType specifies the type of the coldstore. @@ -199,7 +199,7 @@ # # type: string # env var: LOTUS_CHAINSTORE_SPLITSTORE_COLDSTORETYPE - #ColdStoreType = "messages" + #ColdStoreType = "discard" # HotStoreType specifies the type of the hotstore. # Only currently supported value is "badger". @@ -230,6 +230,35 @@ # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREFULLGCFREQUENCY #HotStoreFullGCFrequency = 20 + # HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC + # will run moving GC if disk utilization gets within a threshold (150 GB) of the target. + # Splitstore GC will NOT run moving GC if the total size of the move would get + # within 50 GB of the target, and instead will run a more aggressive online GC. + # If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore + # GC will trigger moving GC if either configuration condition is met. + # A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer. + # At this minimum size moving GC happens every time, any smaller and moving GC won't + # be able to run. In spring 2023 this minimum is ~550 GB. + # + # type: uint64 + # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACETARGET + #HotStoreMaxSpaceTarget = 0 + + # When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size + # exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold + # + # type: uint64 + # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACETHRESHOLD + #HotStoreMaxSpaceThreshold = 150000000000 + + # Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget + # is set. Moving GC will not occur when total moving size exceeds + # HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer + # + # type: uint64 + # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACESAFETYBUFFER + #HotstoreMaxSpaceSafetyBuffer = 50000000000 + [Cluster] # EXPERIMENTAL. config to enabled node cluster with raft consensus diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 88dc4c4ce..28e3cd44d 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 88dc4c4ceb322337818b6508c8e9c23948f36cb1 +Subproject commit 28e3cd44d91681c074aba362d1e5c954db110ad9 diff --git a/gateway/node.go b/gateway/node.go index 90a6812b5..494aa2dc8 100644 --- a/gateway/node.go +++ b/gateway/node.go @@ -27,6 +27,7 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) const ( @@ -59,18 +60,22 @@ type TargetAPI interface { ChainPutObj(context.Context, blocks.Block) error ChainGetGenesis(context.Context) (*types.TipSet, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error) MsigGetPending(ctx context.Context, addr address.Address, ts types.TipSetKey) ([]*api.MsigTransaction, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) + StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) + StateNetworkName(context.Context) (dtypes.NetworkName, error) StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) @@ -85,6 +90,7 @@ type TargetAPI interface { StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) + StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) WalletBalance(context.Context, address.Address) (types.BigInt, error) @@ -188,6 +194,9 @@ func (gw *Node) checkTipset(ts *types.TipSet) error { } func (gw *Node) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error { + if h > ts.Height() { + return fmt.Errorf("tipset height in future") + } tsBlock := ts.Blocks()[0] heightDelta := time.Duration(uint64(tsBlock.Height-h)*build.BlockDelaySecs) * time.Second timeAtHeight := time.Unix(int64(tsBlock.Timestamp), 0).Add(-heightDelta) diff --git a/gateway/proxy_eth.go b/gateway/proxy_eth.go index 838a9ba3f..e6f266d55 100644 --- a/gateway/proxy_eth.go +++ b/gateway/proxy_eth.go @@ -21,7 +21,7 @@ import ( ) func (gw *Node) Web3ClientVersion(ctx context.Context) (string, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + if err := gw.limit(ctx, basicRateLimitTokens); err != nil { return "", err } @@ -34,7 +34,7 @@ func (gw *Node) EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error) } func (gw *Node) EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + if err := gw.limit(ctx, chainRateLimitTokens); err != nil { return 0, err } @@ -80,40 +80,40 @@ func (gw *Node) checkBlkHash(ctx context.Context, blkHash ethtypes.EthHash) erro return gw.checkTipsetKey(ctx, tsk) } -func (gw *Node) checkBlkParam(ctx context.Context, blkParam string) error { +func (gw *Node) checkBlkParam(ctx context.Context, blkParam string, lookback ethtypes.EthUint64) error { if blkParam == "earliest" { // also not supported in node impl return fmt.Errorf("block param \"earliest\" is not supported") } - switch blkParam { - case "pending", "latest": - // those will be recent enough, so we don't need to check - return nil - default: - var num ethtypes.EthUint64 - err := num.UnmarshalJSON([]byte(`"` + blkParam + `"`)) - if err != nil { - return fmt.Errorf("cannot parse block number: %v", err) - } - head, err := gw.target.ChainHead(ctx) - if err != nil { - return err - } - if err := gw.checkTipsetHeight(head, abi.ChainEpoch(num)); err != nil { - return err - } + head, err := gw.target.ChainHead(ctx) + if err != nil { + return err } - return nil + var num ethtypes.EthUint64 + switch blkParam { + case "pending", "latest": + // Head is always ok. + if lookback == 0 { + return nil + } + // Can't look beyond 0 anyways. + if lookback > ethtypes.EthUint64(head.Height()) { + break + } + num = ethtypes.EthUint64(head.Height()) - lookback + default: + if err := num.UnmarshalJSON([]byte(`"` + blkParam + `"`)); err != nil { + return fmt.Errorf("cannot parse block number: %v", err) + } + + } + return gw.checkTipsetHeight(head, abi.ChainEpoch(num)) } func (gw *Node) EthGetBlockTransactionCountByHash(ctx context.Context, blkHash ethtypes.EthHash) (ethtypes.EthUint64, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { - return 0, err - } - - if err := gw.checkBlkHash(ctx, blkHash); err != nil { + if err := gw.limit(ctx, chainRateLimitTokens); err != nil { return 0, err } @@ -137,7 +137,7 @@ func (gw *Node) EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxIn return ethtypes.EthBlock{}, err } - if err := gw.checkBlkParam(ctx, blkNum); err != nil { + if err := gw.checkBlkParam(ctx, blkNum, 0); err != nil { return ethtypes.EthBlock{}, err } @@ -161,7 +161,7 @@ func (gw *Node) EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*e } func (gw *Node) EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + if err := gw.limit(ctx, chainRateLimitTokens); err != nil { return nil, err } @@ -173,7 +173,7 @@ func (gw *Node) EthGetTransactionCount(ctx context.Context, sender ethtypes.EthA return 0, err } - if err := gw.checkBlkParam(ctx, blkOpt); err != nil { + if err := gw.checkBlkParam(ctx, blkOpt, 0); err != nil { return 0, err } @@ -188,40 +188,12 @@ func (gw *Node) EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.Et return gw.target.EthGetTransactionReceipt(ctx, txHash) } -func (gw *Node) EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { - return ethtypes.EthTx{}, err - } - - if err := gw.checkBlkHash(ctx, blkHash); err != nil { - return ethtypes.EthTx{}, err - } - - return gw.target.EthGetTransactionByBlockHashAndIndex(ctx, blkHash, txIndex) -} - -func (gw *Node) EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { - return ethtypes.EthTx{}, err - } - - head, err := gw.target.ChainHead(ctx) - if err != nil { - return ethtypes.EthTx{}, err - } - if err := gw.checkTipsetHeight(head, abi.ChainEpoch(blkNum)); err != nil { - return ethtypes.EthTx{}, err - } - - return gw.target.EthGetTransactionByBlockNumberAndIndex(ctx, blkNum, txIndex) -} - func (gw *Node) EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkOpt string) (ethtypes.EthBytes, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return nil, err } - if err := gw.checkBlkParam(ctx, blkOpt); err != nil { + if err := gw.checkBlkParam(ctx, blkOpt, 0); err != nil { return nil, err } @@ -233,7 +205,7 @@ func (gw *Node) EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress return nil, err } - if err := gw.checkBlkParam(ctx, blkParam); err != nil { + if err := gw.checkBlkParam(ctx, blkParam, 0); err != nil { return nil, err } @@ -245,7 +217,7 @@ func (gw *Node) EthGetBalance(ctx context.Context, address ethtypes.EthAddress, return ethtypes.EthBigInt(big.Zero()), err } - if err := gw.checkBlkParam(ctx, blkParam); err != nil { + if err := gw.checkBlkParam(ctx, blkParam, 0); err != nil { return ethtypes.EthBigInt(big.Zero()), err } @@ -253,7 +225,7 @@ func (gw *Node) EthGetBalance(ctx context.Context, address ethtypes.EthAddress, } func (gw *Node) EthChainId(ctx context.Context) (ethtypes.EthUint64, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + if err := gw.limit(ctx, basicRateLimitTokens); err != nil { return 0, err } @@ -269,7 +241,7 @@ func (gw *Node) NetVersion(ctx context.Context) (string, error) { } func (gw *Node) NetListening(ctx context.Context) (bool, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + if err := gw.limit(ctx, basicRateLimitTokens); err != nil { return false, err } @@ -285,7 +257,7 @@ func (gw *Node) EthProtocolVersion(ctx context.Context) (ethtypes.EthUint64, err } func (gw *Node) EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + if err := gw.limit(ctx, chainRateLimitTokens); err != nil { return ethtypes.EthBigInt(big.Zero()), err } @@ -304,7 +276,7 @@ func (gw *Node) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtype return ethtypes.EthFeeHistory{}, err } - if err := gw.checkBlkParam(ctx, params.NewestBlkNum); err != nil { + if err := gw.checkBlkParam(ctx, params.NewestBlkNum, params.BlkCount); err != nil { return ethtypes.EthFeeHistory{}, err } @@ -316,7 +288,7 @@ func (gw *Node) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtype } func (gw *Node) EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error) { - if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + if err := gw.limit(ctx, chainRateLimitTokens); err != nil { return ethtypes.EthBigInt(big.Zero()), err } @@ -337,7 +309,7 @@ func (gw *Node) EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam strin return nil, err } - if err := gw.checkBlkParam(ctx, blkParam); err != nil { + if err := gw.checkBlkParam(ctx, blkParam, 0); err != nil { return nil, err } @@ -359,12 +331,12 @@ func (gw *Node) EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) } if filter.FromBlock != nil { - if err := gw.checkBlkParam(ctx, *filter.FromBlock); err != nil { + if err := gw.checkBlkParam(ctx, *filter.FromBlock, 0); err != nil { return nil, err } } if filter.ToBlock != nil { - if err := gw.checkBlkParam(ctx, *filter.ToBlock); err != nil { + if err := gw.checkBlkParam(ctx, *filter.ToBlock, 0); err != nil { return nil, err } } diff --git a/gateway/proxy_fil.go b/gateway/proxy_fil.go index 1f6ee2ccc..8007e1e7b 100644 --- a/gateway/proxy_fil.go +++ b/gateway/proxy_fil.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) func (gw *Node) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) { @@ -187,6 +188,13 @@ func (gw *Node) GasEstimateMessageGas(ctx context.Context, msg *types.Message, s return gw.target.GasEstimateMessageGas(ctx, msg, spec, tsk) } +func (gw *Node) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return 0, err + } + return gw.target.MpoolGetNonce(ctx, addr) +} + func (gw *Node) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return cid.Cid{}, err @@ -209,9 +217,6 @@ func (gw *Node) MsigGetVested(ctx context.Context, addr address.Address, start t if err := gw.limit(ctx, walletRateLimitTokens); err != nil { return types.BigInt{}, err } - if err := gw.checkTipsetKey(ctx, start); err != nil { - return types.NewInt(0), err - } if err := gw.checkTipsetKey(ctx, end); err != nil { return types.NewInt(0), err } @@ -248,6 +253,16 @@ func (gw *Node) StateAccountKey(ctx context.Context, addr address.Address, tsk t return gw.target.StateAccountKey(ctx, addr, tsk) } +func (gw *Node) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return nil, err + } + if err := gw.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return gw.target.StateCall(ctx, msg, tsk) +} + func (gw *Node) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return api.DealCollateralBounds{}, err @@ -258,6 +273,16 @@ func (gw *Node) StateDealProviderCollateralBounds(ctx context.Context, size abi. return gw.target.StateDealProviderCollateralBounds(ctx, size, verified, tsk) } +func (gw *Node) StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return nil, err + } + if err := gw.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return gw.target.StateDecodeParams(ctx, toAddr, method, params, tsk) +} + func (gw *Node) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return nil, err @@ -308,6 +333,13 @@ func (gw *Node) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, t return gw.target.StateMarketStorageDeal(ctx, dealId, tsk) } +func (gw *Node) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return *new(dtypes.NetworkName), err + } + return gw.target.StateNetworkName(ctx) +} + func (gw *Node) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return network.VersionMax, err @@ -457,6 +489,16 @@ func (gw *Node) StateVerifiedClientStatus(ctx context.Context, addr address.Addr return gw.target.StateVerifiedClientStatus(ctx, addr, tsk) } +func (gw *Node) StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return nil, err + } + if err := gw.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return gw.target.StateVerifierStatus(ctx, addr, tsk) +} + func (gw *Node) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return api.CirculatingSupply{}, err diff --git a/go.mod b/go.mod index d60d2f34d..2b8e8e143 100644 --- a/go.mod +++ b/go.mod @@ -69,12 +69,13 @@ require ( github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e github.com/hashicorp/go-hclog v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/golang-lru v0.5.4 + github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/hashicorp/raft v1.3.10 github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab github.com/ipfs/bbloom v0.0.4 + github.com/ipfs/go-block-format v0.1.1 github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-cidutil v0.1.0 @@ -227,11 +228,10 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect + github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/iancoleman/orderedmap v0.1.0 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect - github.com/ipfs/go-block-format v0.1.1 // indirect github.com/ipfs/go-filestore v1.2.0 // indirect github.com/ipfs/go-ipfs-cmds v0.8.2 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect diff --git a/go.sum b/go.sum index 199db1bf2..26a17091d 100644 --- a/go.sum +++ b/go.sum @@ -633,10 +633,11 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= +github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= diff --git a/itests/eth_api_test.go b/itests/eth_api_test.go index c24b32416..43b4b5266 100644 --- a/itests/eth_api_test.go +++ b/itests/eth_api_test.go @@ -2,6 +2,7 @@ package itests import ( "context" + "strconv" "testing" "time" @@ -9,6 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" "github.com/filecoin-project/lotus/chain/wallet/key" @@ -109,3 +111,16 @@ func TestEthGetGenesis(t *testing.T) { require.NoError(t, err) require.Equal(t, ethBlk.Hash, genesisHash) } + +func TestNetVersion(t *testing.T) { + blockTime := 100 * time.Millisecond + client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC()) + ens.InterconnectAll().BeginMining(blockTime) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + version, err := client.NetVersion(ctx) + require.NoError(t, err) + require.Equal(t, strconv.Itoa(build.Eip155ChainId), version) +} diff --git a/itests/eth_block_hash_test.go b/itests/eth_block_hash_test.go index ac6506bb2..db21375c5 100644 --- a/itests/eth_block_hash_test.go +++ b/itests/eth_block_hash_test.go @@ -3,6 +3,7 @@ package itests import ( "context" "fmt" + "strings" "testing" "time" @@ -48,13 +49,14 @@ func TestEthBlockHashesCorrect_MultiBlockTipset(t *testing.T) { // let the chain run a little bit longer to minimise the chance of reorgs n2.WaitTillChain(ctx, kit.HeightAtLeast(head.Height()+50)) - head, err = n2.ChainHead(context.Background()) - require.NoError(t, err) - for i := 1; i <= int(head.Height()); i++ { hex := fmt.Sprintf("0x%x", i) ethBlockA, err := n2.EthGetBlockByNumber(ctx, hex, true) + // Cannot use static ErrFullRound error for comparison since it gets reserialized as a JSON RPC error. + if err != nil && strings.Contains(err.Error(), "null round") { + continue + } require.NoError(t, err) ethBlockB, err := n2.EthGetBlockByHash(ctx, ethBlockA.Hash, true) diff --git a/itests/eth_conformance_test.go b/itests/eth_conformance_test.go index 8a367d6b1..4d8f5c3dd 100644 --- a/itests/eth_conformance_test.go +++ b/itests/eth_conformance_test.go @@ -236,14 +236,6 @@ func TestEthOpenRPCConformance(t *testing.T) { skipReason: "earliest block is not supported", }, - { - method: "eth_getBlockByNumber", - variant: "pending", - call: func(a *ethAPIRaw) (json.RawMessage, error) { - return ethapi.EthGetBlockByNumber(context.Background(), "pending", true) - }, - }, - { method: "eth_getBlockByNumber", call: func(a *ethAPIRaw) (json.RawMessage, error) { diff --git a/itests/eth_fee_history_test.go b/itests/eth_fee_history_test.go index 72302f298..a792c7f0e 100644 --- a/itests/eth_fee_history_test.go +++ b/itests/eth_fee_history_test.go @@ -3,18 +3,42 @@ package itests import ( "context" "encoding/json" + "sort" "testing" "time" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/lib/result" + "github.com/filecoin-project/lotus/node/impl/full" ) +// calculateExpectations calculates the expected number of items to be included in the response +// of eth_feeHistory. It takes care of null rounds by finding the closet tipset with height +// smaller than startHeight, and then looks back at requestAmount of items. It also considers +// scenarios where there are not enough items to look back. +func calculateExpectations(tsHeights []int, requestAmount, startHeight int) (count, oldestHeight int) { + latestIdx := sort.SearchInts(tsHeights, startHeight) + // SearchInts returns the index of the number that's larger than the target if the target + // doesn't exist. However, we're looking for the closet number that's smaller that the target + for tsHeights[latestIdx] > startHeight { + latestIdx-- + } + cnt := requestAmount + oldestIdx := latestIdx - requestAmount + 1 + if oldestIdx < 0 { + cnt = latestIdx + 1 + oldestIdx = 0 + } + return cnt, tsHeights[oldestIdx] +} + func TestEthFeeHistory(t *testing.T) { require := require.New(t) @@ -22,70 +46,136 @@ func TestEthFeeHistory(t *testing.T) { blockTime := 100 * time.Millisecond client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC()) - ens.InterconnectAll().BeginMining(blockTime) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - // Wait for the network to create 20 blocks + heads, err := client.ChainNotify(ctx) + require.NoError(err) + + // Save the full view of the tipsets to calculate the answer when there are null rounds + tsHeights := []int{1} + go func() { + for chg := range heads { + for _, c := range chg { + tsHeights = append(tsHeights, int(c.Val.Height())) + } + } + }() + + miner := ens.InterconnectAll().BeginMining(blockTime) + + client.WaitTillChain(ctx, kit.HeightAtLeast(7)) + miner[0].InjectNulls(abi.ChainEpoch(5)) + + // Wait for the network to create at least 20 tipsets client.WaitTillChain(ctx, kit.HeightAtLeast(20)) + for _, m := range miner { + m.Pause() + } + + ch, err := client.ChainNotify(ctx) + require.NoError(err) + + // Wait for 5 seconds of inactivity + func() { + for { + select { + case <-ch: + continue + case <-time.After(5 * time.Second): + return + } + } + }() + + sort.Ints(tsHeights) + + // because of the deferred execution, the last tipset is not executed yet, + // and the one before the last one is the last executed tipset, + // which corresponds to the "latest" tag in EthGetBlockByNumber + latestBlk := ethtypes.EthUint64(tsHeights[len(tsHeights)-2]) + blk, err := client.EthGetBlockByNumber(ctx, "latest", false) + require.NoError(err) + require.Equal(blk.Number, latestBlk) + + assertHistory := func(history *ethtypes.EthFeeHistory, requestAmount, startHeight int) { + amount, oldest := calculateExpectations(tsHeights, requestAmount, startHeight) + require.Equal(amount+1, len(history.BaseFeePerGas)) + require.Equal(amount, len(history.GasUsedRatio)) + require.Equal(ethtypes.EthUint64(oldest), history.OldestBlock) + } history, err := client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "0x10"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock) + assertHistory(&history, 5, 16) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{"5", "0x10"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock) + assertHistory(&history, 5, 16) + require.Nil(history.Reward) + + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( + json.Marshal([]interface{}{5, "latest"}), + ).Assert(require.NoError)) + require.NoError(err) + assertHistory(&history, 5, int(latestBlk)) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{"0x10", "0x12"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(17, len(history.BaseFeePerGas)) - require.Equal(16, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(18-16+1), history.OldestBlock) + assertHistory(&history, 16, 18) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "0x10"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock) + assertHistory(&history, 5, 16) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "10"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(10-5+1), history.OldestBlock) + assertHistory(&history, 5, 10) + require.Nil(history.Reward) + + // test when the requested number of blocks is longer than chain length + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( + json.Marshal([]interface{}{"0x30", "latest"}), + ).Assert(require.NoError)) + require.NoError(err) + assertHistory(&history, 48, int(latestBlk)) + require.Nil(history.Reward) + + // test when the requested number of blocks is longer than chain length + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( + json.Marshal([]interface{}{"0x30", "10"}), + ).Assert(require.NoError)) + require.NoError(err) + assertHistory(&history, 48, 10) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "10", &[]float64{25, 50, 75}}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(10-5+1), history.OldestBlock) + assertHistory(&history, 5, 10) require.NotNil(history.Reward) require.Equal(5, len(*history.Reward)) for _, arr := range *history.Reward { require.Equal(3, len(arr)) + for _, item := range arr { + require.Equal(ethtypes.EthBigInt(types.NewInt(full.MinGasPremium)), item) + } } history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( @@ -93,6 +183,11 @@ func TestEthFeeHistory(t *testing.T) { ).Assert(require.NoError)) require.Error(err) + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( + json.Marshal([]interface{}{5, "10", &[]float64{75, 50}}), + ).Assert(require.NoError)) + require.Error(err) + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "10", &[]float64{}}), ).Assert(require.NoError)) diff --git a/itests/eth_transactions_test.go b/itests/eth_transactions_test.go index 9afeb7bd5..8d0df0433 100644 --- a/itests/eth_transactions_test.go +++ b/itests/eth_transactions_test.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" "github.com/filecoin-project/lotus/itests/kit" @@ -270,6 +271,57 @@ func TestContractInvocation(t *testing.T) { require.EqualValues(t, ethtypes.EthUint64(0x1), receipt.Status) } +func TestGetBlockByNumber(t *testing.T) { + blockTime := 100 * time.Millisecond + client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC()) + + bms := ens.InterconnectAll().BeginMining(blockTime) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + // create a new Ethereum account + _, ethAddr, filAddr := client.EVM().NewAccount() + // send some funds to the f410 address + kit.SendFunds(ctx, t, client, filAddr, types.FromFil(10)) + + latest, err := client.EthBlockNumber(ctx) + require.NoError(t, err) + + // can get the latest block + _, err = client.EthGetBlockByNumber(ctx, latest.Hex(), true) + require.NoError(t, err) + + // fail to get a future block + _, err = client.EthGetBlockByNumber(ctx, (latest + 10000).Hex(), true) + require.Error(t, err) + + // inject 10 null rounds + bms[0].InjectNulls(10) + + // wait until we produce blocks again + tctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + ch, err := client.ChainNotify(tctx) + require.NoError(t, err) + <-ch // current + hc := <-ch // wait for next block + require.Equal(t, store.HCApply, hc[0].Type) + + afterNullHeight := hc[0].Val.Height() + + // Fail when trying to fetch a null round. + _, err = client.EthGetBlockByNumber(ctx, (ethtypes.EthUint64(afterNullHeight - 1)).Hex(), true) + require.Error(t, err) + + // Fetch balance on a null round; should not fail and should return previous balance. + // Should be lower than original balance. + bal, err := client.EthGetBalance(ctx, ethAddr, (ethtypes.EthUint64(afterNullHeight - 1)).Hex()) + require.NoError(t, err) + require.NotEqual(t, big.Zero(), bal) + require.Equal(t, types.FromFil(10).Int, bal.Int) +} + func deployContractTx(ctx context.Context, client *kit.TestFullNode, ethAddr ethtypes.EthAddress, contract []byte) (*ethtypes.EthTxArgs, error) { gaslimit, err := client.EthEstimateGas(ctx, ethtypes.EthCall{ From: ðAddr, diff --git a/itests/kit/blockminer.go b/itests/kit/blockminer.go index 9dd83ee47..4cd0cc671 100644 --- a/itests/kit/blockminer.go +++ b/itests/kit/blockminer.go @@ -192,12 +192,15 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur reportSuccessFn := func(success bool, epoch abi.ChainEpoch, err error) { // if api shuts down before mining, we may get an error which we should probably just ignore // (fixing it will require rewriting most of the mining loop) - if err != nil && !strings.Contains(err.Error(), "websocket connection closed") && !api.ErrorIsIn(err, []error{new(jsonrpc.RPCConnectionError)}) { + if err != nil && ctx.Err() == nil && !strings.Contains(err.Error(), "websocket connection closed") && !api.ErrorIsIn(err, []error{new(jsonrpc.RPCConnectionError)}) { require.NoError(bm.t, err) } target = epoch - wait <- success + select { + case wait <- success: + case <-ctx.Done(): + } } var success bool diff --git a/lib/rpcenc/reader.go b/lib/rpcenc/reader.go index 64dcc4b49..34b9fcfb4 100644 --- a/lib/rpcenc/reader.go +++ b/lib/rpcenc/reader.go @@ -221,7 +221,7 @@ type RpcReader struct { res chan readRes beginOnce *sync.Once - closeOnce sync.Once + closeOnce *sync.Once } var ErrHasBody = errors.New("RPCReader has body, either already read from or from a client with no redirect support") @@ -265,6 +265,7 @@ func (w *RpcReader) beginPost() { w.postBody = nr.postBody w.res = nr.res w.beginOnce = nr.beginOnce + w.closeOnce = nr.closeOnce } } @@ -355,6 +356,7 @@ func ReaderParamDecoder() (http.HandlerFunc, jsonrpc.ServerOption) { res: make(chan readRes), next: ch, beginOnce: &sync.Once{}, + closeOnce: &sync.Once{}, } switch req.Method { diff --git a/lib/rpcenc/reader_test.go b/lib/rpcenc/reader_test.go index da436068f..3a554a0ca 100644 --- a/lib/rpcenc/reader_test.go +++ b/lib/rpcenc/reader_test.go @@ -3,8 +3,8 @@ package rpcenc import ( "context" + "fmt" "io" - "io/ioutil" "net/http/httptest" "strings" "sync" @@ -77,7 +77,12 @@ func (h *ReaderHandler) CloseReader(ctx context.Context, r io.Reader) error { } func (h *ReaderHandler) ReadAll(ctx context.Context, r io.Reader) ([]byte, error) { - return ioutil.ReadAll(r) + b, err := io.ReadAll(r) + if err != nil { + return nil, xerrors.Errorf("readall: %w", err) + } + + return b, nil } func (h *ReaderHandler) ReadNullLen(ctx context.Context, r io.Reader) (int64, error) { @@ -219,9 +224,15 @@ func TestReaderRedirect(t *testing.T) { } func TestReaderRedirectDrop(t *testing.T) { + for i := 0; i < 10; i++ { + t.Run(fmt.Sprintf("test %d", i), testReaderRedirectDrop) + } +} + +func testReaderRedirectDrop(t *testing.T) { // lower timeout so that the dangling connection between client and reader is dropped quickly // after the test. Otherwise httptest.Close is blocked. - Timeout = 200 * time.Millisecond + Timeout = 90 * time.Millisecond var allClient struct { ReadAll func(ctx context.Context, r io.Reader) ([]byte, error) @@ -294,6 +305,8 @@ func TestReaderRedirectDrop(t *testing.T) { done.Wait() + fmt.Println("---------------------") + // Redir client drops before subcall done.Add(1) @@ -322,5 +335,9 @@ func TestReaderRedirectDrop(t *testing.T) { // wait for subcall to finish <-contCh - require.ErrorContains(t, allServerHandler.subErr, "decoding params for 'ReaderHandler.ReadAll' (param: 0; custom decoder): context canceled") + estr := allServerHandler.subErr.Error() + + require.True(t, + strings.Contains(estr, "decoding params for 'ReaderHandler.ReadAll' (param: 0; custom decoder): context canceled") || + strings.Contains(estr, "readall: unexpected EOF"), "unexpected error: %s", estr) } diff --git a/metrics/metrics.go b/metrics/metrics.go index 149195faf..ca638ac27 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -775,9 +775,10 @@ func SinceInMilliseconds(startTime time.Time) float64 { // Timer is a function stopwatch, calling it starts the timer, // calling the returned function will record the duration. -func Timer(ctx context.Context, m *stats.Float64Measure) func() { +func Timer(ctx context.Context, m *stats.Float64Measure) func() time.Duration { start := time.Now() - return func() { + return func() time.Duration { stats.Record(ctx, m.M(SinceInMilliseconds(start))) + return time.Since(start) } } diff --git a/miner/miner.go b/miner/miner.go index f954352ae..0d3e12de8 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -10,7 +10,7 @@ import ( "sync" "time" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -61,7 +61,7 @@ func randTimeOffset(width time.Duration) time.Duration { // NewMiner instantiates a miner with a concrete WinningPoStProver and a miner // address (which can be different from the worker's address). func NewMiner(api v1api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter, j journal.Journal) *Miner { - arc, err := lru.NewARC(10000) + arc, err := lru.NewARC[abi.ChainEpoch, bool](10000) if err != nil { panic(err) } @@ -122,7 +122,7 @@ type Miner struct { // minedBlockHeights is a safeguard that caches the last heights we mined. // It is consulted before publishing a newly mined block, for a sanity check // intended to avoid slashings in case of a bug. - minedBlockHeights *lru.ARCCache + minedBlockHeights *lru.ARCCache[abi.ChainEpoch, bool] evtTypes [1]journal.EventType journal journal.Journal @@ -331,13 +331,12 @@ minerLoop: } } - blkKey := fmt.Sprintf("%d", b.Header.Height) - if _, ok := m.minedBlockHeights.Get(blkKey); ok { + if _, ok := m.minedBlockHeights.Get(b.Header.Height); ok { log.Warnw("Created a block at the same height as another block we've created", "height", b.Header.Height, "miner", b.Header.Miner, "parents", b.Header.Parents) continue } - m.minedBlockHeights.Add(blkKey, true) + m.minedBlockHeights.Add(b.Header.Height, true) if err := m.api.SyncSubmitBlock(ctx, b); err != nil { log.Errorf("failed to submit newly mined block: %+v", err) diff --git a/miner/testminer.go b/miner/testminer.go index 7f29a7ae0..deda89f42 100644 --- a/miner/testminer.go +++ b/miner/testminer.go @@ -3,7 +3,7 @@ package miner import ( "context" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" ds "github.com/ipfs/go-datastore" "github.com/filecoin-project/go-address" @@ -22,7 +22,7 @@ type MineReq struct { func NewTestMiner(nextCh <-chan MineReq, addr address.Address) func(v1api.FullNode, gen.WinningPoStProver) *Miner { return func(api v1api.FullNode, epp gen.WinningPoStProver) *Miner { - arc, err := lru.NewARC(10000) + arc, err := lru.NewARC[abi.ChainEpoch, bool](10000) if err != nil { panic(err) } diff --git a/node/config/def.go b/node/config/def.go index f50e9575a..2617ec5ba 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -89,13 +89,15 @@ func DefaultFullNode() *FullNode { SimultaneousTransfersForRetrieval: DefaultSimultaneousTransfers, }, Chainstore: Chainstore{ - EnableSplitstore: false, + EnableSplitstore: true, Splitstore: Splitstore{ - ColdStoreType: "messages", + ColdStoreType: "discard", HotStoreType: "badger", MarkSetType: "badger", - HotStoreFullGCFrequency: 20, + HotStoreFullGCFrequency: 20, + HotStoreMaxSpaceThreshold: 150_000_000_000, + HotstoreMaxSpaceSafetyBuffer: 50_000_000_000, }, }, Cluster: *DefaultUserRaftConfig(), diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 8b79bed4f..c62084708 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -1286,6 +1286,35 @@ the compaction boundary; default is 0.`, A value of 0 disables, while a value 1 will do full GC in every compaction. Default is 20 (about once a week).`, }, + { + Name: "HotStoreMaxSpaceTarget", + Type: "uint64", + + Comment: `HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC +will run moving GC if disk utilization gets within a threshold (150 GB) of the target. +Splitstore GC will NOT run moving GC if the total size of the move would get +within 50 GB of the target, and instead will run a more aggressive online GC. +If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore +GC will trigger moving GC if either configuration condition is met. +A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer. +At this minimum size moving GC happens every time, any smaller and moving GC won't +be able to run. In spring 2023 this minimum is ~550 GB.`, + }, + { + Name: "HotStoreMaxSpaceThreshold", + Type: "uint64", + + Comment: `When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size +exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold`, + }, + { + Name: "HotstoreMaxSpaceSafetyBuffer", + Type: "uint64", + + Comment: `Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget +is set. Moving GC will not occur when total moving size exceeds +HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`, + }, }, "StorageMiner": []DocField{ { diff --git a/node/config/types.go b/node/config/types.go index 690e8caee..5b952d35e 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -601,6 +601,25 @@ type Splitstore struct { // A value of 0 disables, while a value 1 will do full GC in every compaction. // Default is 20 (about once a week). HotStoreFullGCFrequency uint64 + // HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC + // will run moving GC if disk utilization gets within a threshold (150 GB) of the target. + // Splitstore GC will NOT run moving GC if the total size of the move would get + // within 50 GB of the target, and instead will run a more aggressive online GC. + // If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore + // GC will trigger moving GC if either configuration condition is met. + // A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer. + // At this minimum size moving GC happens every time, any smaller and moving GC won't + // be able to run. In spring 2023 this minimum is ~550 GB. + HotStoreMaxSpaceTarget uint64 + + // When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size + // exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold + HotStoreMaxSpaceThreshold uint64 + + // Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget + // is set. Moving GC will not occur when total moving size exceeds + // HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer + HotstoreMaxSpaceSafetyBuffer uint64 } // // Full Node diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 465b680ef..d0f36ce48 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -193,25 +193,14 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([] return nil, nil } - // TODO: need to get the number of messages better than this - pts, err := a.Chain.LoadTipSet(ctx, types.NewTipSetKey(b.Parents...)) + receipts, err := a.Chain.ReadReceipts(ctx, b.ParentMessageReceipts) if err != nil { return nil, err } - cm, err := a.Chain.MessagesForTipset(ctx, pts) - if err != nil { - return nil, err - } - - var out []*types.MessageReceipt - for i := 0; i < len(cm); i++ { - r, err := a.Chain.GetParentReceipt(ctx, b, i) - if err != nil { - return nil, err - } - - out = append(out, r) + out := make([]*types.MessageReceipt, len(receipts)) + for i := range receipts { + out[i] = &receipts[i] } return out, nil diff --git a/node/impl/full/eth.go b/node/impl/full/eth.go index 907adf8cd..b4a5c75d6 100644 --- a/node/impl/full/eth.go +++ b/node/impl/full/eth.go @@ -39,9 +39,12 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" + "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/node/modules/dtypes" ) +var ErrUnsupported = errors.New("unsupported method") + type EthModuleAPI interface { EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error) EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error) @@ -54,8 +57,6 @@ type EthModuleAPI interface { EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*ethtypes.EthHash, error) EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkOpt string) (ethtypes.EthUint64, error) EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*api.EthTxReceipt, error) - EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) - EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkOpt string) (ethtypes.EthBytes, error) EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error) EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam string) (ethtypes.EthBigInt, error) @@ -153,6 +154,8 @@ type EthAPI struct { EthEventAPI } +var ErrNullRound = errors.New("requested epoch was a null round") + func (a *EthModule) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) { return stmgr.GetNetworkName(ctx, a.StateManager, a.Chain.GetHeaviestTipSet().ParentState()) } @@ -231,15 +234,14 @@ func (a *EthModule) EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthH return newEthBlockFromFilecoinTipSet(ctx, ts, fullTxInfo, a.Chain, a.StateAPI) } -func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string) (tipset *types.TipSet, err error) { - if blkParam == "earliest" { - return nil, fmt.Errorf("block param \"earliest\" is not supported") +func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string, strict bool) (tipset *types.TipSet, err error) { + switch blkParam { + case "earliest", "pending": + return nil, fmt.Errorf("block param %q is not supported", blkParam) } head := a.Chain.GetHeaviestTipSet() switch blkParam { - case "pending": - return head, nil case "latest": parent, err := a.Chain.GetTipSetFromKey(ctx, head.Parents()) if err != nil { @@ -252,16 +254,22 @@ func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string) (tipset if err != nil { return nil, fmt.Errorf("cannot parse block number: %v", err) } - ts, err := a.Chain.GetTipsetByHeight(ctx, abi.ChainEpoch(num), nil, false) + if abi.ChainEpoch(num) > head.Height()-1 { + return nil, fmt.Errorf("requested a future epoch (beyond 'latest')") + } + ts, err := a.ChainAPI.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(num), head.Key()) if err != nil { return nil, fmt.Errorf("cannot get tipset at height: %v", num) } + if strict && ts.Height() != abi.ChainEpoch(num) { + return nil, ErrNullRound + } return ts, nil } } func (a *EthModule) EthGetBlockByNumber(ctx context.Context, blkParam string, fullTxInfo bool) (ethtypes.EthBlock, error) { - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, true) if err != nil { return ethtypes.EthBlock{}, err } @@ -335,13 +343,13 @@ func (a *EthModule) EthGetMessageCidByTransactionHash(ctx context.Context, txHas c = txHash.ToCid() } - _, err = a.StateAPI.Chain.GetSignedMessage(ctx, c) + _, err = a.Chain.GetSignedMessage(ctx, c) if err == nil { // This is an Eth Tx, Secp message, Or BLS message in the mpool return &c, nil } - _, err = a.StateAPI.Chain.GetMessage(ctx, c) + _, err = a.Chain.GetMessage(ctx, c) if err == nil { // This is a BLS message return &c, nil @@ -367,7 +375,7 @@ func (a *EthModule) EthGetTransactionCount(ctx context.Context, sender ethtypes. return ethtypes.EthUint64(0), nil } - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return ethtypes.EthUint64(0), xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -420,20 +428,15 @@ func (a *EthModule) EthGetTransactionReceipt(ctx context.Context, txHash ethtype return nil, nil } - replay, err := a.StateAPI.StateReplay(ctx, types.EmptyTSK, c) - if err != nil { - return nil, nil - } - var events []types.Event - if rct := replay.MsgRct; rct != nil && rct.EventsRoot != nil { + if rct := msgLookup.Receipt; rct.EventsRoot != nil { events, err = a.ChainAPI.ChainGetEvents(ctx, *rct.EventsRoot) if err != nil { return nil, nil } } - receipt, err := newEthTxReceipt(ctx, tx, msgLookup, replay, events, a.StateAPI) + receipt, err := newEthTxReceipt(ctx, tx, msgLookup, events, a.Chain, a.StateAPI) if err != nil { return nil, nil } @@ -441,12 +444,12 @@ func (a *EthModule) EthGetTransactionReceipt(ctx context.Context, txHash ethtype return &receipt, nil } -func (a *EthModule) EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) { - return ethtypes.EthTx{}, nil +func (a *EthAPI) EthGetTransactionByBlockHashAndIndex(context.Context, ethtypes.EthHash, ethtypes.EthUint64) (ethtypes.EthTx, error) { + return ethtypes.EthTx{}, ErrUnsupported } -func (a *EthModule) EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) { - return ethtypes.EthTx{}, nil +func (a *EthAPI) EthGetTransactionByBlockNumberAndIndex(context.Context, ethtypes.EthUint64, ethtypes.EthUint64) (ethtypes.EthTx, error) { + return ethtypes.EthTx{}, ErrUnsupported } // EthGetCode returns string value of the compiled bytecode @@ -456,7 +459,7 @@ func (a *EthModule) EthGetCode(ctx context.Context, ethAddr ethtypes.EthAddress, return nil, xerrors.Errorf("cannot get Filecoin address: %w", err) } - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return nil, xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -535,7 +538,7 @@ func (a *EthModule) EthGetCode(ctx context.Context, ethAddr ethtypes.EthAddress, } func (a *EthModule) EthGetStorageAt(ctx context.Context, ethAddr ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error) { - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return nil, xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -631,7 +634,7 @@ func (a *EthModule) EthGetBalance(ctx context.Context, address ethtypes.EthAddre return ethtypes.EthBigInt{}, err } - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return ethtypes.EthBigInt{}, xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -676,57 +679,48 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth } } - ts, err := a.parseBlkParam(ctx, params.NewestBlkNum) + ts, err := a.parseBlkParam(ctx, params.NewestBlkNum, false) if err != nil { return ethtypes.EthFeeHistory{}, fmt.Errorf("bad block parameter %s: %s", params.NewestBlkNum, err) } - // Deal with the case that the chain is shorter than the number of requested blocks. - oldestBlkHeight := uint64(1) - if abi.ChainEpoch(params.BlkCount) <= ts.Height() { - oldestBlkHeight = uint64(ts.Height()) - uint64(params.BlkCount) + 1 - } + var ( + basefee = ts.Blocks()[0].ParentBaseFee + oldestBlkHeight = uint64(1) - // NOTE: baseFeePerGas should include the next block after the newest of the returned range, - // because the next base fee can be inferred from the messages in the newest block. - // However, this is NOT the case in Filecoin due to deferred execution, so the best - // we can do is duplicate the last value. - baseFeeArray := []ethtypes.EthBigInt{ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee)} - gasUsedRatioArray := []float64{} - rewardsArray := make([][]ethtypes.EthBigInt, 0) + // NOTE: baseFeePerGas should include the next block after the newest of the returned range, + // because the next base fee can be inferred from the messages in the newest block. + // However, this is NOT the case in Filecoin due to deferred execution, so the best + // we can do is duplicate the last value. + baseFeeArray = []ethtypes.EthBigInt{ethtypes.EthBigInt(basefee)} + rewardsArray = make([][]ethtypes.EthBigInt, 0) + gasUsedRatioArray = []float64{} + blocksIncluded int + ) - for ts.Height() >= abi.ChainEpoch(oldestBlkHeight) { - // Unfortunately we need to rebuild the full message view so we can - // totalize gas used in the tipset. - msgs, err := a.Chain.MessagesForTipset(ctx, ts) + for blocksIncluded < int(params.BlkCount) && ts.Height() > 0 { + msgs, rcpts, err := messagesAndReceipts(ctx, ts, a.Chain, a.StateAPI) if err != nil { - return ethtypes.EthFeeHistory{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err) + return ethtypes.EthFeeHistory{}, xerrors.Errorf("failed to retrieve messages and receipts for height %d: %w", ts.Height(), err) } txGasRewards := gasRewardSorter{} - for txIdx, msg := range msgs { - msgLookup, err := a.StateAPI.StateSearchMsg(ctx, types.EmptyTSK, msg.Cid(), api.LookbackNoLimit, false) - if err != nil || msgLookup == nil { - return ethtypes.EthFeeHistory{}, nil - } - - tx, err := newEthTxFromMessageLookup(ctx, msgLookup, txIdx, a.Chain, a.StateAPI) - if err != nil { - return ethtypes.EthFeeHistory{}, nil - } - + for i, msg := range msgs { + effectivePremium := msg.VMMessage().EffectiveGasPremium(basefee) txGasRewards = append(txGasRewards, gasRewardTuple{ - reward: tx.Reward(ts.Blocks()[0].ParentBaseFee), - gas: uint64(msgLookup.Receipt.GasUsed), + premium: effectivePremium, + gasUsed: rcpts[i].GasUsed, }) } rewards, totalGasUsed := calculateRewardsAndGasUsed(rewardPercentiles, txGasRewards) // arrays should be reversed at the end - baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee)) + baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(basefee)) gasUsedRatioArray = append(gasUsedRatioArray, float64(totalGasUsed)/float64(build.BlockGasLimit)) rewardsArray = append(rewardsArray, rewards) + oldestBlkHeight = uint64(ts.Height()) + blocksIncluded++ parentTsKey := ts.Parents() ts, err = a.Chain.LoadTipSet(ctx, parentTsKey) @@ -757,13 +751,8 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth return ret, nil } -func (a *EthModule) NetVersion(ctx context.Context) (string, error) { - // Note that networkId is not encoded in hex - nv, err := a.StateNetworkVersion(ctx, types.EmptyTSK) - if err != nil { - return "", err - } - return strconv.FormatUint(uint64(nv), 10), nil +func (a *EthModule) NetVersion(_ context.Context) (string, error) { + return strconv.FormatInt(build.Eip155ChainId, 10), nil } func (a *EthModule) NetListening(ctx context.Context) (bool, error) { @@ -1066,7 +1055,7 @@ func (a *EthModule) EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam s return nil, xerrors.Errorf("failed to convert ethcall to filecoin message: %w", err) } - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return nil, xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -1783,37 +1772,37 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx return ethtypes.EthBlock{}, err } - msgs, err := cs.MessagesForTipset(ctx, ts) + msgs, rcpts, err := messagesAndReceipts(ctx, ts, cs, sa) if err != nil { - return ethtypes.EthBlock{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err) + return ethtypes.EthBlock{}, xerrors.Errorf("failed to retrieve messages and receipts: %w", err) } block := ethtypes.NewEthBlock(len(msgs) > 0) gasUsed := int64(0) - compOutput, err := sa.StateCompute(ctx, ts.Height(), nil, ts.Key()) - if err != nil { - return ethtypes.EthBlock{}, xerrors.Errorf("failed to compute state: %w", err) - } - - for txIdx, msg := range compOutput.Trace { - // skip system messages like reward application and cron - if msg.Msg.From == builtintypes.SystemActorAddr { - continue + for i, msg := range msgs { + rcpt := rcpts[i] + ti := ethtypes.EthUint64(i) + gasUsed += rcpt.GasUsed + var smsg *types.SignedMessage + switch msg := msg.(type) { + case *types.SignedMessage: + smsg = msg + case *types.Message: + smsg = &types.SignedMessage{ + Message: *msg, + Signature: crypto.Signature{ + Type: crypto.SigTypeBLS, + }, + } + default: + return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.Cid(), err) } - - gasUsed += msg.MsgRct.GasUsed - smsgCid, err := getSignedMessage(ctx, cs, msg.MsgCid) - if err != nil { - return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.MsgCid, err) - } - tx, err := newEthTxFromSignedMessage(ctx, smsgCid, sa) + tx, err := newEthTxFromSignedMessage(ctx, smsg, sa) if err != nil { return ethtypes.EthBlock{}, xerrors.Errorf("failed to convert msg to ethTx: %w", err) } - ti := ethtypes.EthUint64(txIdx) - tx.ChainID = ethtypes.EthUint64(build.Eip155ChainId) tx.BlockHash = &blkHash tx.BlockNumber = &bn @@ -1835,6 +1824,29 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx return block, nil } +func messagesAndReceipts(ctx context.Context, ts *types.TipSet, cs *store.ChainStore, sa StateAPI) ([]types.ChainMsg, []types.MessageReceipt, error) { + msgs, err := cs.MessagesForTipset(ctx, ts) + if err != nil { + return nil, nil, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err) + } + + _, rcptRoot, err := sa.StateManager.TipSetState(ctx, ts) + if err != nil { + return nil, nil, xerrors.Errorf("failed to compute state: %w", err) + } + + rcpts, err := cs.ReadReceipts(ctx, rcptRoot) + if err != nil { + return nil, nil, xerrors.Errorf("error loading receipts for tipset: %v: %w", ts, err) + } + + if len(msgs) != len(rcpts) { + return nil, nil, xerrors.Errorf("receipts and message array lengths didn't match for tipset: %v: %w", ts, err) + } + + return msgs, rcpts, nil +} + // lookupEthAddress makes its best effort at finding the Ethereum address for a // Filecoin address. It does the following: // @@ -2030,7 +2042,7 @@ func newEthTxFromMessageLookup(ctx context.Context, msgLookup *api.MsgLookup, tx return tx, nil } -func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLookup, replay *api.InvocResult, events []types.Event, sa StateAPI) (api.EthTxReceipt, error) { +func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLookup, events []types.Event, cs *store.ChainStore, sa StateAPI) (api.EthTxReceipt, error) { var ( transactionIndex ethtypes.EthUint64 blockHash ethtypes.EthHash @@ -2061,8 +2073,7 @@ func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLook if lookup.Receipt.ExitCode.IsSuccess() { receipt.Status = 1 - } - if lookup.Receipt.ExitCode.IsError() { + } else { receipt.Status = 0 } @@ -2071,7 +2082,17 @@ func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLook // TODO: handle CumulativeGasUsed receipt.CumulativeGasUsed = ethtypes.EmptyEthInt - effectiveGasPrice := big.Div(replay.GasCost.TotalCost, big.NewInt(lookup.Receipt.GasUsed)) + // TODO: avoid loading the tipset twice (once here, once when we convert the message to a txn) + ts, err := cs.GetTipSetFromKey(ctx, lookup.TipSet) + if err != nil { + return api.EthTxReceipt{}, xerrors.Errorf("failed to lookup tipset %s when constructing the eth txn receipt: %w", lookup.TipSet, err) + } + + baseFee := ts.Blocks()[0].ParentBaseFee + gasOutputs := vm.ComputeGasOutputs(lookup.Receipt.GasUsed, int64(tx.Gas), baseFee, big.Int(tx.MaxFeePerGas), big.Int(tx.MaxPriorityFeePerGas), true) + totalSpent := big.Sum(gasOutputs.BaseFeeBurn, gasOutputs.MinerTip, gasOutputs.OverEstimationBurn) + + effectiveGasPrice := big.Div(totalSpent, big.NewInt(lookup.Receipt.GasUsed)) receipt.EffectiveGasPrice = ethtypes.EthBigInt(effectiveGasPrice) if receipt.To == nil && lookup.Receipt.ExitCode.IsSuccess() { @@ -2335,35 +2356,35 @@ func parseEthRevert(ret []byte) string { return ethtypes.EthBytes(cbytes).String() } -func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, uint64) { - var totalGasUsed uint64 +func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, int64) { + var gasUsedTotal int64 for _, tx := range txGasRewards { - totalGasUsed += tx.gas + gasUsedTotal += tx.gasUsed } rewards := make([]ethtypes.EthBigInt, len(rewardPercentiles)) for i := range rewards { - rewards[i] = ethtypes.EthBigIntZero + rewards[i] = ethtypes.EthBigInt(types.NewInt(MinGasPremium)) } if len(txGasRewards) == 0 { - return rewards, totalGasUsed + return rewards, gasUsedTotal } sort.Stable(txGasRewards) var idx int - var sum uint64 + var sum int64 for i, percentile := range rewardPercentiles { - threshold := uint64(float64(totalGasUsed) * percentile / 100) + threshold := int64(float64(gasUsedTotal) * percentile / 100) for sum < threshold && idx < len(txGasRewards)-1 { - sum += txGasRewards[idx].gas + sum += txGasRewards[idx].gasUsed idx++ } - rewards[i] = txGasRewards[idx].reward + rewards[i] = ethtypes.EthBigInt(txGasRewards[idx].premium) } - return rewards, totalGasUsed + return rewards, gasUsedTotal } func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid) (*types.SignedMessage, error) { @@ -2386,8 +2407,8 @@ func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid) } type gasRewardTuple struct { - gas uint64 - reward ethtypes.EthBigInt + gasUsed int64 + premium abi.TokenAmount } // sorted in ascending order @@ -2398,5 +2419,5 @@ func (g gasRewardSorter) Swap(i, j int) { g[i], g[j] = g[j], g[i] } func (g gasRewardSorter) Less(i, j int) bool { - return g[i].reward.Int.Cmp(g[j].reward.Int) == -1 + return g[i].premium.Int.Cmp(g[j].premium.Int) == -1 } diff --git a/node/impl/full/eth_test.go b/node/impl/full/eth_test.go index 67a8b0500..87c0852fb 100644 --- a/node/impl/full/eth_test.go +++ b/node/impl/full/eth_test.go @@ -117,11 +117,8 @@ func TestReward(t *testing.T) { {maxFeePerGas: big.NewInt(50), maxPriorityFeePerGas: big.NewInt(200), answer: big.NewInt(-50)}, } for _, tc := range testcases { - tx := ethtypes.EthTx{ - MaxFeePerGas: ethtypes.EthBigInt(tc.maxFeePerGas), - MaxPriorityFeePerGas: ethtypes.EthBigInt(tc.maxPriorityFeePerGas), - } - reward := tx.Reward(baseFee) + msg := &types.Message{GasFeeCap: tc.maxFeePerGas, GasPremium: tc.maxPriorityFeePerGas} + reward := msg.EffectiveGasPremium(baseFee) require.Equal(t, 0, reward.Int.Cmp(tc.answer.Int), reward, tc.answer) } } @@ -135,25 +132,25 @@ func TestRewardPercentiles(t *testing.T) { { percentiles: []float64{25, 50, 75}, txGasRewards: []gasRewardTuple{}, - answer: []int64{0, 0, 0}, + answer: []int64{MinGasPremium, MinGasPremium, MinGasPremium}, }, { percentiles: []float64{25, 50, 75, 100}, txGasRewards: []gasRewardTuple{ - {gas: uint64(0), reward: ethtypes.EthBigInt(big.NewInt(300))}, - {gas: uint64(100), reward: ethtypes.EthBigInt(big.NewInt(200))}, - {gas: uint64(350), reward: ethtypes.EthBigInt(big.NewInt(100))}, - {gas: uint64(500), reward: ethtypes.EthBigInt(big.NewInt(600))}, - {gas: uint64(300), reward: ethtypes.EthBigInt(big.NewInt(700))}, + {gasUsed: int64(0), premium: big.NewInt(300)}, + {gasUsed: int64(100), premium: big.NewInt(200)}, + {gasUsed: int64(350), premium: big.NewInt(100)}, + {gasUsed: int64(500), premium: big.NewInt(600)}, + {gasUsed: int64(300), premium: big.NewInt(700)}, }, answer: []int64{200, 700, 700, 700}, }, } for _, tc := range testcases { rewards, totalGasUsed := calculateRewardsAndGasUsed(tc.percentiles, tc.txGasRewards) - gasUsed := uint64(0) + var gasUsed int64 for _, tx := range tc.txGasRewards { - gasUsed += tx.gas + gasUsed += tx.gasUsed } ans := []ethtypes.EthBigInt{} for _, bi := range tc.answer { diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index cd3cb0156..24e1b3fbc 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -6,7 +6,7 @@ import ( "math/rand" "sort" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/fx" "golang.org/x/xerrors" @@ -61,7 +61,7 @@ type GasAPI struct { func NewGasPriceCache() *GasPriceCache { // 50 because we usually won't access more than 40 - c, err := lru.New2Q(50) + c, err := lru.New2Q[types.TipSetKey, []GasMeta](50) if err != nil { // err only if parameter is bad panic(err) @@ -73,7 +73,7 @@ func NewGasPriceCache() *GasPriceCache { } type GasPriceCache struct { - c *lru.TwoQueueCache + c *lru.TwoQueueCache[types.TipSetKey, []GasMeta] } type GasMeta struct { @@ -84,7 +84,7 @@ type GasMeta struct { func (g *GasPriceCache) GetTSGasStats(ctx context.Context, cstore *store.ChainStore, ts *types.TipSet) ([]GasMeta, error) { i, has := g.c.Get(ts.Key()) if has { - return i.([]GasMeta), nil + return i, nil } var prices []GasMeta diff --git a/node/modules/blockstore.go b/node/modules/blockstore.go index 90b7b6183..f96fd0db4 100644 --- a/node/modules/blockstore.go +++ b/node/modules/blockstore.go @@ -82,11 +82,14 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked } cfg := &splitstore.Config{ - MarkSetType: cfg.Splitstore.MarkSetType, - DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard", - UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal", - HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention, - HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency, + MarkSetType: cfg.Splitstore.MarkSetType, + DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard", + UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal", + HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention, + HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency, + HotstoreMaxSpaceTarget: cfg.Splitstore.HotStoreMaxSpaceTarget, + HotstoreMaxSpaceThreshold: cfg.Splitstore.HotStoreMaxSpaceThreshold, + HotstoreMaxSpaceSafetyBuffer: cfg.Splitstore.HotstoreMaxSpaceSafetyBuffer, } ss, err := splitstore.Open(path, ds, hot, cold, cfg) if err != nil { diff --git a/node/modules/chain.go b/node/modules/chain.go index f304ab135..0c3bad2c7 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -123,7 +123,7 @@ func NetworkName(mctx helpers.MetricsCtx, ctx := helpers.LifecycleCtx(mctx, lc) - sm, err := stmgr.NewStateManager(cs, tsexec, syscalls, us, nil) + sm, err := stmgr.NewStateManager(cs, tsexec, syscalls, us, nil, nil) if err != nil { return "", err } diff --git a/node/modules/stmgr.go b/node/modules/stmgr.go index dd3d90f10..b8f6f4776 100644 --- a/node/modules/stmgr.go +++ b/node/modules/stmgr.go @@ -7,10 +7,11 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func StateManager(lc fx.Lifecycle, cs *store.ChainStore, exec stmgr.Executor, sys vm.SyscallBuilder, us stmgr.UpgradeSchedule, b beacon.Schedule) (*stmgr.StateManager, error) { - sm, err := stmgr.NewStateManager(cs, exec, sys, us, b) +func StateManager(lc fx.Lifecycle, cs *store.ChainStore, exec stmgr.Executor, sys vm.SyscallBuilder, us stmgr.UpgradeSchedule, b beacon.Schedule, metadataDs dtypes.MetadataDS) (*stmgr.StateManager, error) { + sm, err := stmgr.NewStateManager(cs, exec, sys, us, b, metadataDs) if err != nil { return nil, err } diff --git a/storage/paths/localstorage_cached.go b/storage/paths/localstorage_cached.go index cac0a44b6..af43d1696 100644 --- a/storage/paths/localstorage_cached.go +++ b/storage/paths/localstorage_cached.go @@ -4,7 +4,7 @@ import ( "sync" "time" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/filecoin-project/lotus/storage/sealer/fsutil" "github.com/filecoin-project/lotus/storage/sealer/storiface" @@ -17,13 +17,13 @@ type cachedLocalStorage struct { base LocalStorage statLk sync.Mutex - stats *lru.Cache // path -> statEntry - pathDUs *lru.Cache // path -> *diskUsageEntry + stats *lru.Cache[string, statEntry] + pathDUs *lru.Cache[string, *diskUsageEntry] } func newCachedLocalStorage(ls LocalStorage) *cachedLocalStorage { - statCache, _ := lru.New(1024) - duCache, _ := lru.New(1024) + statCache, _ := lru.New[string, statEntry](1024) + duCache, _ := lru.New[string, *diskUsageEntry](1024) return &cachedLocalStorage{ base: ls, @@ -60,8 +60,8 @@ func (c *cachedLocalStorage) Stat(path string) (fsutil.FsStat, error) { c.statLk.Lock() defer c.statLk.Unlock() - if v, ok := c.stats.Get(path); ok && time.Now().Sub(v.(statEntry).time) < StatTimeout { - return v.(statEntry).stat, nil + if v, ok := c.stats.Get(path); ok && time.Now().Sub(v.time) < StatTimeout { + return v.stat, nil } // if we don't, get the stat @@ -83,7 +83,7 @@ func (c *cachedLocalStorage) DiskUsage(path string) (int64, error) { var entry *diskUsageEntry if v, ok := c.pathDUs.Get(path); ok { - entry = v.(*diskUsageEntry) + entry = v // if we have recent cached entry, use that if time.Now().Sub(entry.last.time) < StatTimeout { diff --git a/storage/pipeline/fsm.go b/storage/pipeline/fsm.go index 25fd6fcef..8ae18a9fd 100644 --- a/storage/pipeline/fsm.go +++ b/storage/pipeline/fsm.go @@ -6,8 +6,10 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/http" + "os" "reflect" "time" @@ -19,9 +21,15 @@ import ( "github.com/filecoin-project/lotus/api" ) +var errSectorRemoved = errors.New("sector removed") + func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) { next, processed, err := m.plan(events, user.(*SectorInfo)) if err != nil || next == nil { + if err == errSectorRemoved && os.Getenv("LOTUS_KEEP_REMOVED_FSM_ACTIVE") != "1" { + return nil, processed, statemachine.ErrTerminated + } + l := Log{ Timestamp: uint64(time.Now().Unix()), Message: fmt.Sprintf("state machine error: %s", err), @@ -601,7 +609,7 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta case Removing: return m.handleRemoving, processed, nil case Removed: - return nil, processed, nil + return nil, processed, errSectorRemoved case RemoveFailed: return m.handleRemoveFailed, processed, nil @@ -615,13 +623,14 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta // Fatal errors case UndefinedSectorState: log.Error("sector update with undefined state!") + return nil, processed, xerrors.Errorf("sector update with undefined state") case FailedUnrecoverable: log.Errorf("sector %d failed unrecoverably", state.SectorNumber) + return nil, processed, xerrors.Errorf("sector %d failed unrecoverably", state.SectorNumber) default: log.Errorf("unexpected sector update state: %s", state.State) + return nil, processed, xerrors.Errorf("unexpected sector update state: %s", state.State) } - - return nil, processed, nil } func (m *Sealing) onUpdateSector(ctx context.Context, state *SectorInfo) error { diff --git a/storage/sealer/manager.go b/storage/sealer/manager.go index 336664ca8..db5f9a589 100644 --- a/storage/sealer/manager.go +++ b/storage/sealer/manager.go @@ -289,14 +289,20 @@ func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.remoteHnd.ServeHTTP(w, r) } -func schedNop(context.Context, Worker) error { - return nil +var schedNop = PrepareAction{ + Action: func(ctx context.Context, w Worker) error { + return nil + }, + PrepType: sealtasks.TTNoop, } -func (m *Manager) schedFetch(sector storiface.SectorRef, ft storiface.SectorFileType, ptype storiface.PathType, am storiface.AcquireMode) func(context.Context, Worker) error { - return func(ctx context.Context, worker Worker) error { - _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, ft, ptype, am)) - return err +func (m *Manager) schedFetch(sector storiface.SectorRef, ft storiface.SectorFileType, ptype storiface.PathType, am storiface.AcquireMode) PrepareAction { + return PrepareAction{ + Action: func(ctx context.Context, worker Worker) error { + _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, ft, ptype, am)) + return err + }, + PrepType: sealtasks.TTFetch, } } @@ -315,16 +321,19 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storiface.Secto // if the selected worker does NOT have the sealed files for the sector, instruct it to fetch it from a worker that has them and // put it in the sealing scratch space. - sealFetch := func(ctx context.Context, worker Worker) error { - log.Debugf("copy sealed/cache sector data for sector %d", sector.ID) - _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy)) - _, err2 := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy)) + sealFetch := PrepareAction{ + Action: func(ctx context.Context, worker Worker) error { + log.Debugf("copy sealed/cache sector data for sector %d", sector.ID) + _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy)) + _, err2 := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy)) - if err != nil && err2 != nil { - return xerrors.Errorf("cannot unseal piece. error fetching sealed data: %w. error fetching replica data: %w", err, err2) - } + if err != nil && err2 != nil { + return xerrors.Errorf("cannot unseal piece. error fetching sealed data: %w. error fetching replica data: %w", err, err2) + } - return nil + return nil + }, + PrepType: sealtasks.TTFetch, } if unsealed == nil { diff --git a/storage/sealer/sched.go b/storage/sealer/sched.go index c2b7d6a2d..c0ac11bcf 100644 --- a/storage/sealer/sched.go +++ b/storage/sealer/sched.go @@ -42,6 +42,10 @@ func WithPriority(ctx context.Context, priority int) context.Context { const mib = 1 << 20 type WorkerAction func(ctx context.Context, w Worker) error +type PrepareAction struct { + Action WorkerAction + PrepType sealtasks.TaskType +} type SchedWorker interface { TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) @@ -130,7 +134,7 @@ type WorkerRequest struct { Sel WorkerSelector SchedId uuid.UUID - prepare WorkerAction + prepare PrepareAction work WorkerAction start time.Time @@ -157,7 +161,15 @@ func newScheduler(ctx context.Context, assigner string) (*Scheduler, error) { case "", "utilization": a = NewLowestUtilizationAssigner() case "spread": - a = NewSpreadAssigner() + a = NewSpreadAssigner(false) + case "experiment-spread-qcount": + a = NewSpreadAssigner(true) + case "experiment-spread-tasks": + a = NewSpreadTasksAssigner(false) + case "experiment-spread-tasks-qcount": + a = NewSpreadTasksAssigner(true) + case "experiment-random": + a = NewRandomAssigner() default: return nil, xerrors.Errorf("unknown assigner '%s'", assigner) } @@ -189,7 +201,7 @@ func newScheduler(ctx context.Context, assigner string) (*Scheduler, error) { }, nil } -func (sh *Scheduler) Schedule(ctx context.Context, sector storiface.SectorRef, taskType sealtasks.TaskType, sel WorkerSelector, prepare WorkerAction, work WorkerAction) error { +func (sh *Scheduler) Schedule(ctx context.Context, sector storiface.SectorRef, taskType sealtasks.TaskType, sel WorkerSelector, prepare PrepareAction, work WorkerAction) error { ret := make(chan workerResponse) select { @@ -239,6 +251,13 @@ func (r *WorkerRequest) SealTask() sealtasks.SealTaskType { } } +func (r *WorkerRequest) PrepSealTask() sealtasks.SealTaskType { + return sealtasks.SealTaskType{ + TaskType: r.prepare.PrepType, + RegisteredSealProof: r.Sector.ProofType, + } +} + type SchedDiagRequestInfo struct { Sector abi.SectorID TaskType sealtasks.TaskType diff --git a/storage/sealer/sched_assigner_common.go b/storage/sealer/sched_assigner_common.go index bf92dbf15..d676d410d 100644 --- a/storage/sealer/sched_assigner_common.go +++ b/storage/sealer/sched_assigner_common.go @@ -58,7 +58,7 @@ func (a *AssignerCommon) TrySched(sh *Scheduler) { windows := make([]SchedWindow, windowsLen) for i := range windows { - windows[i].Allocated = *NewActiveResources() + windows[i].Allocated = *NewActiveResources(newTaskCounter()) } acceptableWindows := make([][]int, queueLen) // QueueIndex -> []OpenWindowIndex diff --git a/storage/sealer/sched_assigner_darts.go b/storage/sealer/sched_assigner_darts.go new file mode 100644 index 000000000..e28b70e78 --- /dev/null +++ b/storage/sealer/sched_assigner_darts.go @@ -0,0 +1,88 @@ +package sealer + +import ( + "math/rand" + + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +func NewRandomAssigner() Assigner { + return &AssignerCommon{ + WindowSel: RandomWS, + } +} + +func RandomWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + scheduled := 0 + rmQueue := make([]int, 0, queueLen) + + for sqi := 0; sqi < queueLen; sqi++ { + task := (*sh.SchedQueue)[sqi] + + //bestAssigned := math.MaxInt // smaller = better + + type choice struct { + selectedWindow int + needRes storiface.Resources + info storiface.WorkerInfo + bestWid storiface.WorkerID + } + choices := make([]choice, 0, len(acceptableWindows[task.IndexHeap])) + + for i, wnd := range acceptableWindows[task.IndexHeap] { + wid := sh.OpenWindows[wnd].Worker + w := sh.Workers[wid] + + res := w.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) + + log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) + + if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + continue + } + + choices = append(choices, choice{ + selectedWindow: wnd, + needRes: res, + info: w.Info, + bestWid: wid, + }) + + } + + if len(choices) == 0 { + // all windows full + continue + } + + // chose randomly + randIndex := rand.Intn(len(choices)) + selectedWindow := choices[randIndex].selectedWindow + needRes := choices[randIndex].needRes + info := choices[randIndex].info + bestWid := choices[randIndex].bestWid + + log.Debugw("SCHED ASSIGNED", + "assigner", "darts", + "sqi", sqi, + "sector", task.Sector.ID.Number, + "task", task.TaskType, + "window", selectedWindow, + "worker", bestWid, + "choices", len(choices)) + + windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) + + rmQueue = append(rmQueue, sqi) + scheduled++ + } + + if len(rmQueue) > 0 { + for i := len(rmQueue) - 1; i >= 0; i-- { + sh.SchedQueue.Remove(rmQueue[i]) + } + } + + return scheduled +} diff --git a/storage/sealer/sched_assigner_spread.go b/storage/sealer/sched_assigner_spread.go index f00d24d82..0a62b7406 100644 --- a/storage/sealer/sched_assigner_spread.go +++ b/storage/sealer/sched_assigner_spread.go @@ -6,76 +6,84 @@ import ( "github.com/filecoin-project/lotus/storage/sealer/storiface" ) -func NewSpreadAssigner() Assigner { +func NewSpreadAssigner(queued bool) Assigner { return &AssignerCommon{ - WindowSel: SpreadWS, + WindowSel: SpreadWS(queued), } } -func SpreadWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { - scheduled := 0 - rmQueue := make([]int, 0, queueLen) - workerAssigned := map[storiface.WorkerID]int{} +func SpreadWS(queued bool) func(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + return func(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + scheduled := 0 + rmQueue := make([]int, 0, queueLen) + workerAssigned := map[storiface.WorkerID]int{} - for sqi := 0; sqi < queueLen; sqi++ { - task := (*sh.SchedQueue)[sqi] + for sqi := 0; sqi < queueLen; sqi++ { + task := (*sh.SchedQueue)[sqi] - selectedWindow := -1 - var needRes storiface.Resources - var info storiface.WorkerInfo - var bestWid storiface.WorkerID - bestAssigned := math.MaxInt // smaller = better + selectedWindow := -1 + var needRes storiface.Resources + var info storiface.WorkerInfo + var bestWid storiface.WorkerID + bestAssigned := math.MaxInt // smaller = better - for i, wnd := range acceptableWindows[task.IndexHeap] { - wid := sh.OpenWindows[wnd].Worker - w := sh.Workers[wid] + for i, wnd := range acceptableWindows[task.IndexHeap] { + wid := sh.OpenWindows[wnd].Worker + w := sh.Workers[wid] - res := w.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) + res := w.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) - log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) + log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) - if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + continue + } + + wu, found := workerAssigned[wid] + if !found && queued { + wu = w.TaskCounts() + workerAssigned[wid] = wu + } + if wu >= bestAssigned { + continue + } + + info = w.Info + needRes = res + bestWid = wid + selectedWindow = wnd + bestAssigned = wu + } + + if selectedWindow < 0 { + // all windows full continue } - wu, _ := workerAssigned[wid] - if wu >= bestAssigned { - continue + log.Debugw("SCHED ASSIGNED", + "assigner", "spread", + "spread-queued", queued, + "sqi", sqi, + "sector", task.Sector.ID.Number, + "task", task.TaskType, + "window", selectedWindow, + "worker", bestWid, + "assigned", bestAssigned) + + workerAssigned[bestWid]++ + windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) + + rmQueue = append(rmQueue, sqi) + scheduled++ + } + + if len(rmQueue) > 0 { + for i := len(rmQueue) - 1; i >= 0; i-- { + sh.SchedQueue.Remove(rmQueue[i]) } - - info = w.Info - needRes = res - bestWid = wid - selectedWindow = wnd - bestAssigned = wu } - if selectedWindow < 0 { - // all windows full - continue - } - - log.Debugw("SCHED ASSIGNED", - "sqi", sqi, - "sector", task.Sector.ID.Number, - "task", task.TaskType, - "window", selectedWindow, - "worker", bestWid, - "assigned", bestAssigned) - - workerAssigned[bestWid]++ - windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) - windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) - - rmQueue = append(rmQueue, sqi) - scheduled++ + return scheduled } - - if len(rmQueue) > 0 { - for i := len(rmQueue) - 1; i >= 0; i-- { - sh.SchedQueue.Remove(rmQueue[i]) - } - } - - return scheduled } diff --git a/storage/sealer/sched_assigner_spread_tasks.go b/storage/sealer/sched_assigner_spread_tasks.go new file mode 100644 index 000000000..09cf98046 --- /dev/null +++ b/storage/sealer/sched_assigner_spread_tasks.go @@ -0,0 +1,98 @@ +package sealer + +import ( + "math" + + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +func NewSpreadTasksAssigner(queued bool) Assigner { + return &AssignerCommon{ + WindowSel: SpreadTasksWS(queued), + } +} + +type widTask struct { + wid storiface.WorkerID + tt sealtasks.TaskType +} + +func SpreadTasksWS(queued bool) func(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + return func(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + scheduled := 0 + rmQueue := make([]int, 0, queueLen) + workerAssigned := map[widTask]int{} + + for sqi := 0; sqi < queueLen; sqi++ { + task := (*sh.SchedQueue)[sqi] + + selectedWindow := -1 + var needRes storiface.Resources + var info storiface.WorkerInfo + var bestWid widTask + bestAssigned := math.MaxInt // smaller = better + + for i, wnd := range acceptableWindows[task.IndexHeap] { + wid := sh.OpenWindows[wnd].Worker + w := sh.Workers[wid] + + res := w.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) + + log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) + + if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + continue + } + + wt := widTask{wid: wid, tt: task.TaskType} + + wu, found := workerAssigned[wt] + if !found && queued { + st := task.SealTask() + wu = w.TaskCount(&st) + workerAssigned[wt] = wu + } + if wu >= bestAssigned { + continue + } + + info = w.Info + needRes = res + bestWid = wt + selectedWindow = wnd + bestAssigned = wu + } + + if selectedWindow < 0 { + // all windows full + continue + } + + log.Debugw("SCHED ASSIGNED", + "assigner", "spread-tasks", + "spread-queued", queued, + "sqi", sqi, + "sector", task.Sector.ID.Number, + "task", task.TaskType, + "window", selectedWindow, + "worker", bestWid, + "assigned", bestAssigned) + + workerAssigned[bestWid]++ + windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) + + rmQueue = append(rmQueue, sqi) + scheduled++ + } + + if len(rmQueue) > 0 { + for i := len(rmQueue) - 1; i >= 0; i-- { + sh.SchedQueue.Remove(rmQueue[i]) + } + } + + return scheduled + } +} diff --git a/storage/sealer/sched_assigner_utilization.go b/storage/sealer/sched_assigner_utilization.go index 2d051d000..1e75d904a 100644 --- a/storage/sealer/sched_assigner_utilization.go +++ b/storage/sealer/sched_assigner_utilization.go @@ -74,6 +74,7 @@ func LowestUtilizationWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, } log.Debugw("SCHED ASSIGNED", + "assigner", "util", "sqi", sqi, "sector", task.Sector.ID.Number, "task", task.TaskType, diff --git a/storage/sealer/sched_resources.go b/storage/sealer/sched_resources.go index 487e294a2..597f36dbe 100644 --- a/storage/sealer/sched_resources.go +++ b/storage/sealer/sched_resources.go @@ -13,18 +13,68 @@ type ActiveResources struct { gpuUsed float64 cpuUse uint64 - taskCounters map[sealtasks.SealTaskType]int + taskCounters *taskCounter cond *sync.Cond waiting int } -func NewActiveResources() *ActiveResources { - return &ActiveResources{ +type taskCounter struct { + taskCounters map[sealtasks.SealTaskType]int + + // this lock is technically redundant, as ActiveResources is always accessed + // with the worker lock, but let's not panic if we ever change that + lk sync.Mutex +} + +func newTaskCounter() *taskCounter { + return &taskCounter{ taskCounters: map[sealtasks.SealTaskType]int{}, } } +func (tc *taskCounter) Add(tt sealtasks.SealTaskType) { + tc.lk.Lock() + defer tc.lk.Unlock() + tc.taskCounters[tt]++ +} + +func (tc *taskCounter) Free(tt sealtasks.SealTaskType) { + tc.lk.Lock() + defer tc.lk.Unlock() + tc.taskCounters[tt]-- +} + +func (tc *taskCounter) Get(tt sealtasks.SealTaskType) int { + tc.lk.Lock() + defer tc.lk.Unlock() + return tc.taskCounters[tt] +} + +func (tc *taskCounter) Sum() int { + tc.lk.Lock() + defer tc.lk.Unlock() + sum := 0 + for _, v := range tc.taskCounters { + sum += v + } + return sum +} + +func (tc *taskCounter) ForEach(cb func(tt sealtasks.SealTaskType, count int)) { + tc.lk.Lock() + defer tc.lk.Unlock() + for tt, count := range tc.taskCounters { + cb(tt, count) + } +} + +func NewActiveResources(tc *taskCounter) *ActiveResources { + return &ActiveResources{ + taskCounters: tc, + } +} + func (a *ActiveResources) withResources(id storiface.WorkerID, wr storiface.WorkerInfo, tt sealtasks.SealTaskType, r storiface.Resources, locker sync.Locker, cb func() error) error { for !a.CanHandleRequest(tt, r, id, "withResources", wr) { if a.cond == nil { @@ -59,7 +109,7 @@ func (a *ActiveResources) Add(tt sealtasks.SealTaskType, wr storiface.WorkerReso a.cpuUse += r.Threads(wr.CPUs, len(wr.GPUs)) a.memUsedMin += r.MinMemory a.memUsedMax += r.MaxMemory - a.taskCounters[tt]++ + a.taskCounters.Add(tt) return a.utilization(wr) - startUtil } @@ -71,7 +121,7 @@ func (a *ActiveResources) Free(tt sealtasks.SealTaskType, wr storiface.WorkerRes a.cpuUse -= r.Threads(wr.CPUs, len(wr.GPUs)) a.memUsedMin -= r.MinMemory a.memUsedMax -= r.MaxMemory - a.taskCounters[tt]-- + a.taskCounters.Free(tt) if a.cond != nil { a.cond.Broadcast() @@ -82,8 +132,8 @@ func (a *ActiveResources) Free(tt sealtasks.SealTaskType, wr storiface.WorkerRes // handle the request. func (a *ActiveResources) CanHandleRequest(tt sealtasks.SealTaskType, needRes storiface.Resources, wid storiface.WorkerID, caller string, info storiface.WorkerInfo) bool { if needRes.MaxConcurrent > 0 { - if a.taskCounters[tt] >= needRes.MaxConcurrent { - log.Debugf("sched: not scheduling on worker %s for %s; at task limit tt=%s, curcount=%d", wid, caller, tt, a.taskCounters[tt]) + if a.taskCounters.Get(tt) >= needRes.MaxConcurrent { + log.Debugf("sched: not scheduling on worker %s for %s; at task limit tt=%s, curcount=%d", wid, caller, tt, a.taskCounters.Get(tt)) return false } } @@ -170,6 +220,15 @@ func (a *ActiveResources) utilization(wr storiface.WorkerResources) float64 { // return max } +func (a *ActiveResources) taskCount(tt *sealtasks.SealTaskType) int { + // nil means all tasks + if tt == nil { + return a.taskCounters.Sum() + } + + return a.taskCounters.Get(*tt) +} + func (wh *WorkerHandle) Utilization() float64 { wh.lk.Lock() u := wh.active.utilization(wh.Info.Resources) @@ -183,3 +242,31 @@ func (wh *WorkerHandle) Utilization() float64 { return u } + +func (wh *WorkerHandle) TaskCounts() int { + wh.lk.Lock() + u := wh.active.taskCount(nil) + u += wh.preparing.taskCount(nil) + wh.lk.Unlock() + wh.wndLk.Lock() + for _, window := range wh.activeWindows { + u += window.Allocated.taskCount(nil) + } + wh.wndLk.Unlock() + + return u +} + +func (wh *WorkerHandle) TaskCount(tt *sealtasks.SealTaskType) int { + wh.lk.Lock() + u := wh.active.taskCount(tt) + u += wh.preparing.taskCount(tt) + wh.lk.Unlock() + wh.wndLk.Lock() + for _, window := range wh.activeWindows { + u += window.Allocated.taskCount(tt) + } + wh.wndLk.Unlock() + + return u +} diff --git a/storage/sealer/sched_test.go b/storage/sealer/sched_test.go index 2eed1ce73..07731e934 100644 --- a/storage/sealer/sched_test.go +++ b/storage/sealer/sched_test.go @@ -288,25 +288,30 @@ func TestSched(t *testing.T) { ProofType: spt, } - err := sched.Schedule(ctx, sectorRef, taskType, sel, func(ctx context.Context, w Worker) error { - wi, err := w.Info(ctx) - require.NoError(t, err) + prep := PrepareAction{ + Action: func(ctx context.Context, w Worker) error { + wi, err := w.Info(ctx) + require.NoError(t, err) - require.Equal(t, expectWorker, wi.Hostname) + require.Equal(t, expectWorker, wi.Hostname) - log.Info("IN ", taskName) + log.Info("IN ", taskName) - for { - _, ok := <-done - if !ok { - break + for { + _, ok := <-done + if !ok { + break + } } - } - log.Info("OUT ", taskName) + log.Info("OUT ", taskName) - return nil - }, noopAction) + return nil + }, + PrepType: taskType, + } + + err := sched.Schedule(ctx, sectorRef, taskType, sel, prep, noopAction) if err != context.Canceled { require.NoError(t, err, fmt.Sprint(l, l2)) } @@ -639,8 +644,8 @@ func BenchmarkTrySched(b *testing.B) { Resources: decentWorkerResources, }, Enabled: true, - preparing: NewActiveResources(), - active: NewActiveResources(), + preparing: NewActiveResources(newTaskCounter()), + active: NewActiveResources(newTaskCounter()), } for i := 0; i < windows; i++ { @@ -685,7 +690,7 @@ func TestWindowCompact(t *testing.T) { for _, windowTasks := range start { window := &SchedWindow{ - Allocated: *NewActiveResources(), + Allocated: *NewActiveResources(newTaskCounter()), } for _, task := range windowTasks { @@ -708,7 +713,7 @@ func TestWindowCompact(t *testing.T) { require.Equal(t, len(start)-len(expect), -sw.windowsRequested) for wi, tasks := range expect { - expectRes := NewActiveResources() + expectRes := NewActiveResources(newTaskCounter()) for ti, task := range tasks { require.Equal(t, task, wh.activeWindows[wi].Todo[ti].TaskType, "%d, %d", wi, ti) diff --git a/storage/sealer/sched_worker.go b/storage/sealer/sched_worker.go index e6e1f62da..b6efc851a 100644 --- a/storage/sealer/sched_worker.go +++ b/storage/sealer/sched_worker.go @@ -30,12 +30,14 @@ func newWorkerHandle(ctx context.Context, w Worker) (*WorkerHandle, error) { return nil, xerrors.Errorf("getting worker info: %w", err) } + tc := newTaskCounter() + worker := &WorkerHandle{ workerRpc: w, Info: info, - preparing: NewActiveResources(), - active: NewActiveResources(), + preparing: NewActiveResources(tc), + active: NewActiveResources(tc), Enabled: true, closingMgr: make(chan struct{}), @@ -352,8 +354,8 @@ assignLoop: worker.lk.Lock() for t, todo := range firstWindow.Todo { - needRes := worker.Info.Resources.ResourceSpec(todo.Sector.ProofType, todo.TaskType) - if worker.preparing.CanHandleRequest(todo.SealTask(), needRes, sw.wid, "startPreparing", worker.Info) { + needResPrep := worker.Info.Resources.PrepResourceSpec(todo.Sector.ProofType, todo.TaskType, todo.prepare.PrepType) + if worker.preparing.CanHandleRequest(todo.PrepSealTask(), needResPrep, sw.wid, "startPreparing", worker.Info) { tidx = t break } @@ -452,20 +454,21 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { w, sh := sw.worker, sw.sched needRes := w.Info.Resources.ResourceSpec(req.Sector.ProofType, req.TaskType) + needResPrep := w.Info.Resources.PrepResourceSpec(req.Sector.ProofType, req.TaskType, req.prepare.PrepType) w.lk.Lock() - w.preparing.Add(req.SealTask(), w.Info.Resources, needRes) + w.preparing.Add(req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() go func() { // first run the prepare step (e.g. fetching sector data from other worker) tw := sh.workTracker.worker(sw.wid, w.Info, w.workerRpc) tw.start() - err := req.prepare(req.Ctx, tw) + err := req.prepare.Action(req.Ctx, tw) w.lk.Lock() if err != nil { - w.preparing.Free(req.SealTask(), w.Info.Resources, needRes) + w.preparing.Free(req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() select { @@ -495,7 +498,7 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { // wait (if needed) for resources in the 'active' window err = w.active.withResources(sw.wid, w.Info, req.SealTask(), needRes, &w.lk, func() error { - w.preparing.Free(req.SealTask(), w.Info.Resources, needRes) + w.preparing.Free(req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() defer w.lk.Lock() // we MUST return locked from this function diff --git a/storage/sealer/sealtasks/task.go b/storage/sealer/sealtasks/task.go index bbf33b159..2e134f3d3 100644 --- a/storage/sealer/sealtasks/task.go +++ b/storage/sealer/sealtasks/task.go @@ -36,6 +36,8 @@ const ( TTGenerateWindowPoSt TaskType = "post/v0/windowproof" TTGenerateWinningPoSt TaskType = "post/v0/winningproof" + + TTNoop TaskType = "" ) var order = map[TaskType]int{ diff --git a/storage/sealer/stats.go b/storage/sealer/stats.go index f34109d78..90b6287c3 100644 --- a/storage/sealer/stats.go +++ b/storage/sealer/stats.go @@ -43,9 +43,9 @@ func (m *Manager) WorkerStats(ctx context.Context) map[uuid.UUID]storiface.Worke TaskCounts: map[string]int{}, } - for tt, count := range handle.active.taskCounters { + handle.active.taskCounters.ForEach(func(tt sealtasks.SealTaskType, count int) { out[uuid.UUID(id)].TaskCounts[tt.String()] = count - } + }) handle.lk.Unlock() } diff --git a/storage/sealer/storiface/worker.go b/storage/sealer/storiface/worker.go index 3cbf9f737..2badad292 100644 --- a/storage/sealer/storiface/worker.go +++ b/storage/sealer/storiface/worker.go @@ -65,6 +65,20 @@ func (wr WorkerResources) ResourceSpec(spt abi.RegisteredSealProof, tt sealtasks return res } +// PrepResourceSpec is like ResourceSpec, but meant for use limiting parallel preparing +// tasks. +func (wr WorkerResources) PrepResourceSpec(spt abi.RegisteredSealProof, tt, prepTT sealtasks.TaskType) Resources { + res := wr.ResourceSpec(spt, tt) + + if prepTT != tt && prepTT != sealtasks.TTNoop { + prepRes := wr.ResourceSpec(spt, prepTT) + res.MaxConcurrent = prepRes.MaxConcurrent + } + + // otherwise, use the default resource table + return res +} + type WorkerStats struct { Info WorkerInfo Tasks []sealtasks.TaskType diff --git a/tools/stats/ipldstore/ipldstore.go b/tools/stats/ipldstore/ipldstore.go index e172d2af2..2b96fb0cd 100644 --- a/tools/stats/ipldstore/ipldstore.go +++ b/tools/stats/ipldstore/ipldstore.go @@ -5,7 +5,7 @@ import ( "context" "fmt" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "go.opencensus.io/stats" @@ -16,7 +16,7 @@ import ( type ApiIpldStore struct { ctx context.Context api apiIpldStoreApi - cache *lru.TwoQueueCache + cache *lru.TwoQueueCache[cid.Cid, []byte] cacheSize int } @@ -31,7 +31,7 @@ func NewApiIpldStore(ctx context.Context, api apiIpldStoreApi, cacheSize int) (* cacheSize: cacheSize, } - cache, err := lru.New2Q(store.cacheSize) + cache, err := lru.New2Q[cid.Cid, []byte](store.cacheSize) if err != nil { return nil, err } @@ -64,7 +64,7 @@ func (ht *ApiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) err if a, ok := ht.cache.Get(c); ok { stats.Record(ctx, metrics.IpldStoreCacheHit.M(1)) - raw = a.([]byte) + raw = a } else { bs, err := ht.read(ctx, c) if err != nil { diff --git a/tools/stats/points/collect.go b/tools/stats/points/collect.go index 98c124b2d..8b8669574 100644 --- a/tools/stats/points/collect.go +++ b/tools/stats/points/collect.go @@ -8,7 +8,7 @@ import ( "strings" "time" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" client "github.com/influxdata/influxdb1-client/v2" "github.com/ipfs/go-cid" "go.opencensus.io/stats" @@ -41,11 +41,11 @@ type ChainPointCollector struct { ctx context.Context api LotusApi store adt.Store - actorDigestCache *lru.TwoQueueCache + actorDigestCache *lru.TwoQueueCache[address.Address, string] } func NewChainPointCollector(ctx context.Context, store adt.Store, api LotusApi) (*ChainPointCollector, error) { - actorDigestCache, err := lru.New2Q(2 << 15) + actorDigestCache, err := lru.New2Q[address.Address, string](2 << 15) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func NewChainPointCollector(ctx context.Context, store adt.Store, api LotusApi) func (c *ChainPointCollector) actorDigest(ctx context.Context, addr address.Address, tipset *types.TipSet) (string, error) { if code, ok := c.actorDigestCache.Get(addr); ok { - return code.(string), nil + return code, nil } actor, err := c.api.StateGetActor(ctx, addr, tipset.Key())