diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 981a80256..e4c0c7f26 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -9,15 +9,9 @@ body: options: - label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy). required: true - - label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions). - required: true - - label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead. - required: true - - label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead. - required: true - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion. required: true - - label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these. + - label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these. required: true - label: I did not make any code changes to lotus. required: false @@ -28,19 +22,11 @@ body: options: - label: lotus daemon - chain sync required: false - - label: lotus miner - mining and block production + - label: lotus fvm/fevm - Lotus FVM and FEVM interactions required: false - label: lotus miner/worker - sealing required: false - - label: lotus miner - proving(WindowPoSt) - required: false - - label: lotus miner/market - storage deal - required: false - - label: lotus miner/market - retrieval deal - required: false - - label: lotus miner/market - data transfer - required: false - - label: lotus client + - label: lotus miner - proving(WindowPoSt/WinningPoSt) required: false - label: lotus JSON-RPC API required: false @@ -56,22 +42,33 @@ body: description: Enter the output of `lotus version` and `lotus-miner version` if applicable. placeholder: | e.g. - Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0 - Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty + Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0 + Local: lotus-miner version 1.19.0+mainnet+git.64059ca87 validations: required: true +- type: textarea + id: ReproSteps + attributes: + label: Repro Steps + description: "Steps to reproduce the behavior" + value: | + 1. Run '...' + 2. Do '...' + 3. See error '...' + ... + validations: + required: false - type: textarea id: Description attributes: label: Describe the Bug description: | This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information: - * What you were doding when you experienced the bug? + * What you were doing when you experienced the bug? * Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas). * What is the expected behaviour? * For sealing issues, include the output of `lotus-miner sectors status --log ` for the failed sector(s). * For proving issues, include the output of `lotus-miner proving` info. - * For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question. validations: required: true - type: textarea @@ -83,18 +80,6 @@ body: Please provide debug logs of the problem, remember you can get set log level control for: * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://lotus.filecoin.io/lotus/configure/defaults/#log-level-control). * lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level - If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem. + If you don't provide detailed logs when you raise the issue it will almost certainly be the first request we make before furthur diagnosing the problem. validations: - required: true -- type: textarea - id: RepoSteps - attributes: - label: Repo Steps - description: "Steps to reproduce the behavior" - value: | - 1. Run '...' - 2. Do '...' - 3. See error '...' - ... - validations: - required: false + required: true \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..e5ae608b3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Ask a question about Lotus or get support + url: https://github.com/filecoin-project/lotus/discussions/new/choose + about: Ask a question or request support for using Lotus + - name: Filecoin protocol feature or enhancement + url: https://github.com/filecoin-project/FIPs/discussions/new/choose + about: Write a discussion in the Filecoin Improvement Proposal repo diff --git a/.github/ISSUE_TEMPLATE/enhancement.yml b/.github/ISSUE_TEMPLATE/enhancement.yml index da662688d..d367feeac 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.yml +++ b/.github/ISSUE_TEMPLATE/enhancement.yml @@ -7,13 +7,7 @@ body: label: Checklist description: Please check off the following boxes before continuing to create an improvement suggestion! options: - - label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md). - required: true - - label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead. - required: true - - label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`. - required: true - - label: I **have** a specific, actionable, and well motivated improvement to propose. + - label: I **have** a specific, actionable, and well motivated improvement to an existing lotus feature. required: true - type: checkboxes attributes: @@ -22,19 +16,11 @@ body: options: - label: lotus daemon - chain sync required: false - - label: lotus miner - mining and block production + - label: lotus fvm/fevm - Lotus FVM and FEVM interactions required: false - label: lotus miner/worker - sealing required: false - - label: lotus miner - proving(WindowPoSt) - required: false - - label: lotus miner/market - storage deal - required: false - - label: lotus miner/market - retrieval deal - required: false - - label: lotus miner/market - data transfer - required: false - - label: lotus client + - label: lotus miner - proving(WindowPoSt/WinningPoSt) required: false - label: lotus JSON-RPC API required: false @@ -45,9 +31,17 @@ body: - type: textarea id: request attributes: - label: Improvement Suggestion - description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement? - placeholder: Ex. Currently lotus... However, as a storage provider, I'd like... + label: Enhancement Suggestion + description: A clear and concise description of the suggested enhancement? + placeholder: Ex. Currently lotus... However it would be great if [enhancement] was implemented... With the ability to... + validations: + required: true +- type: textarea + id: request + attributes: + label: Use-Case + description: How would this enhancement help you? + placeholder: Ex. With the [enhancement] node operators would be able to... For Storage Providers it would enable... validations: required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 210549095..76493e9c1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -7,8 +7,6 @@ body: label: Checklist description: Please check off the following boxes before continuing to create a new feature request! options: - - label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md). - required: true - label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`. required: true - label: I **have** a specific, actionable, and well motivated feature request to propose. @@ -20,19 +18,11 @@ body: options: - label: lotus daemon - chain sync required: false - - label: lotus miner - mining and block production + - label: lotus fvm/fevm - Lotus FVM and FEVM interactions required: false - label: lotus miner/worker - sealing required: false - - label: lotus miner - proving(WindowPoSt) - required: false - - label: lotus miner/market - storage deal - required: false - - label: lotus miner/market - retrieval deal - required: false - - label: lotus miner/market - data transfer - required: false - - label: lotus client + - label: lotus miner - proving(WindowPoSt/WinningPoSt) required: false - label: lotus JSON-RPC API required: false @@ -56,7 +46,7 @@ body: validations: required: true - type: textarea - id: alternates + id: alternatives attributes: label: Describe alternatives you've considered description: A clear and concise description of any alternative solutions or features you've considered. @@ -69,4 +59,3 @@ body: description: Add any other context, design docs or screenshots about the feature request here. validations: required: false - diff --git a/.github/ISSUE_TEMPLATE/service_developer_bug_report.yml b/.github/ISSUE_TEMPLATE/service_developer_bug_report.yml new file mode 100644 index 000000000..8174d13f5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/service_developer_bug_report.yml @@ -0,0 +1,83 @@ +name: "Bug Report - developer/service provider" +description: "Bug report template about FEVM/FVM for developers/service providers" +labels: [need/triage, kind/bug, area/fevm] +body: +- type: checkboxes + attributes: + label: Checklist + description: Please check off the following boxes before continuing to file a bug report! + options: + - label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy). + required: true + - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion. + required: true + - label: I did not make any code changes to lotus. + required: false +- type: checkboxes + attributes: + label: Lotus component + description: Please select the lotus component you are filing a bug for + options: + - label: lotus Ethereum RPC + required: false + - label: lotus FVM - Lotus FVM interactions + required: false + - label: FEVM tooling + required: false + - label: Other + required: false +- type: textarea + id: version + attributes: + label: Lotus Version + render: text + description: Enter the output of `lotus version` if applicable. + placeholder: | + e.g. + Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0 + Local: lotus-miner version 1.19.0+mainnet+git.64059ca87 + validations: + required: true +- type: textarea + id: repro + attributes: + label: Repro Steps + description: "Steps to reproduce the behavior" + value: | + 1. Run '...' + 2. Do '...' + 3. See error '...' + ... + validations: + required: false +- type: textarea + id: Description + attributes: + label: Describe the Bug + description: | + This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information: + * What you were doing when you experienced the bug? What are you trying to build? + * Any *error* messages and logs you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas). + * What is the expected behaviour? Links to the actual code? + validations: + required: true +- type: textarea + id: toolingInfo + attributes: + label: Tooling + render: text + description: | + What kind of tooling are you using: + * Are you using ether.js, Alchemy, Hardhat, etc. + validations: + required: true +- type: textarea + id: extraInfo + attributes: + label: Configuration Options + render: text + description: | + Please provide your updated FEVM related configuration options, or custome enviroment variables related to Lotus FEVM + * lotus: use `lotus config updated` to get your configuration options, and copy the [FEVM] section + validations: + required: true diff --git a/api/api_gateway.go b/api/api_gateway.go index f04a86a63..7efccea6a 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -15,6 +15,7 @@ import ( apitypes "github.com/filecoin-project/lotus/api/types" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) // MODIFYING THE API INTERFACE @@ -25,7 +26,7 @@ import ( // When adding / changing methods in this file: // * Do the change here // * Adjust implementation in `node/impl/` -// * Run `make gen` - this will: +// * Run `make clean && make deps && make gen` - this will: // * Generate proxy structs // * Generate mocks // * Generate markdown docs @@ -47,15 +48,18 @@ type Gateway interface { ChainReadObj(context.Context, cid.Cid) ([]byte, error) ChainGetGenesis(context.Context) (*types.TipSet, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*InvocResult, error) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error) + StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read + StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error) @@ -63,6 +67,7 @@ type Gateway interface { StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (MinerInfo, error) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) + StateNetworkName(context.Context) (dtypes.NetworkName, error) StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 3793f2fc2..10e9c2fce 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -401,6 +401,10 @@ func init() { FromBlock: pstring("2301220"), Address: []ethtypes.EthAddress{ethaddr}, }) + + percent := types.Percent(123) + addExample(percent) + addExample(&percent) } func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) { diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 9aa9a9c7b..6c3264ac3 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -720,6 +720,8 @@ type GatewayMethods struct { GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) `` + MpoolGetNonce func(p0 context.Context, p1 address.Address) (uint64, error) `` + MpoolPush func(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) `` MsigGetAvailableBalance func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (types.BigInt, error) `` @@ -736,8 +738,12 @@ type GatewayMethods struct { StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) `` + StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) `` + StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) `` + StateDecodeParams func(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) `` + StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) `` StateListMiners func(p0 context.Context, p1 types.TipSetKey) ([]address.Address, error) `` @@ -754,9 +760,11 @@ type GatewayMethods struct { StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) `` + StateNetworkName func(p0 context.Context) (dtypes.NetworkName, error) `` + StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (apitypes.NetworkVersion, error) `` - StateReadState func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) `perm:"read"` + StateReadState func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) `` StateSearchMsg func(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) `` @@ -4563,6 +4571,17 @@ func (s *GatewayStub) GasEstimateMessageGas(p0 context.Context, p1 *types.Messag return nil, ErrNotSupported } +func (s *GatewayStruct) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) { + if s.Internal.MpoolGetNonce == nil { + return 0, ErrNotSupported + } + return s.Internal.MpoolGetNonce(p0, p1) +} + +func (s *GatewayStub) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) { + return 0, ErrNotSupported +} + func (s *GatewayStruct) MpoolPush(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) { if s.Internal.MpoolPush == nil { return *new(cid.Cid), ErrNotSupported @@ -4651,6 +4670,17 @@ func (s *GatewayStub) StateAccountKey(p0 context.Context, p1 address.Address, p2 return *new(address.Address), ErrNotSupported } +func (s *GatewayStruct) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) { + if s.Internal.StateCall == nil { + return nil, ErrNotSupported + } + return s.Internal.StateCall(p0, p1, p2) +} + +func (s *GatewayStub) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*InvocResult, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateDealProviderCollateralBounds(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) { if s.Internal.StateDealProviderCollateralBounds == nil { return *new(DealCollateralBounds), ErrNotSupported @@ -4662,6 +4692,17 @@ func (s *GatewayStub) StateDealProviderCollateralBounds(p0 context.Context, p1 a return *new(DealCollateralBounds), ErrNotSupported } +func (s *GatewayStruct) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) { + if s.Internal.StateDecodeParams == nil { + return nil, ErrNotSupported + } + return s.Internal.StateDecodeParams(p0, p1, p2, p3, p4) +} + +func (s *GatewayStub) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateGetActor(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) { if s.Internal.StateGetActor == nil { return nil, ErrNotSupported @@ -4750,6 +4791,17 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A return nil, ErrNotSupported } +func (s *GatewayStruct) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) { + if s.Internal.StateNetworkName == nil { + return *new(dtypes.NetworkName), ErrNotSupported + } + return s.Internal.StateNetworkName(p0) +} + +func (s *GatewayStub) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) { + return *new(dtypes.NetworkName), ErrNotSupported +} + func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (apitypes.NetworkVersion, error) { if s.Internal.StateNetworkVersion == nil { return *new(apitypes.NetworkVersion), ErrNotSupported diff --git a/api/v0api/gateway.go b/api/v0api/gateway.go index 674371c14..2a0bfb2f7 100644 --- a/api/v0api/gateway.go +++ b/api/v0api/gateway.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) // MODIFYING THE API INTERFACE @@ -44,12 +45,15 @@ type Gateway interface { ChainNotify(context.Context) (<-chan []*api.HeadChange, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) + StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) @@ -59,6 +63,7 @@ type Gateway interface { StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) + StateNetworkName(context.Context) (dtypes.NetworkName, error) StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) diff --git a/api/v0api/proxy_gen.go b/api/v0api/proxy_gen.go index 5fa0d949c..17a1ae84a 100644 --- a/api/v0api/proxy_gen.go +++ b/api/v0api/proxy_gen.go @@ -451,6 +451,8 @@ type GatewayMethods struct { GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *api.MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) `` + MpoolGetNonce func(p0 context.Context, p1 address.Address) (uint64, error) `` + MpoolPush func(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) `` MsigGetAvailableBalance func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (types.BigInt, error) `` @@ -461,8 +463,12 @@ type GatewayMethods struct { StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) `` + StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) `` + StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (api.DealCollateralBounds, error) `` + StateDecodeParams func(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) `` + StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) `` StateGetReceipt func(p0 context.Context, p1 cid.Cid, p2 types.TipSetKey) (*types.MessageReceipt, error) `` @@ -481,6 +487,8 @@ type GatewayMethods struct { StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) `` + StateNetworkName func(p0 context.Context) (dtypes.NetworkName, error) `` + StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) `` StateSearchMsg func(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) `` @@ -2677,6 +2685,17 @@ func (s *GatewayStub) GasEstimateMessageGas(p0 context.Context, p1 *types.Messag return nil, ErrNotSupported } +func (s *GatewayStruct) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) { + if s.Internal.MpoolGetNonce == nil { + return 0, ErrNotSupported + } + return s.Internal.MpoolGetNonce(p0, p1) +} + +func (s *GatewayStub) MpoolGetNonce(p0 context.Context, p1 address.Address) (uint64, error) { + return 0, ErrNotSupported +} + func (s *GatewayStruct) MpoolPush(p0 context.Context, p1 *types.SignedMessage) (cid.Cid, error) { if s.Internal.MpoolPush == nil { return *new(cid.Cid), ErrNotSupported @@ -2732,6 +2751,17 @@ func (s *GatewayStub) StateAccountKey(p0 context.Context, p1 address.Address, p2 return *new(address.Address), ErrNotSupported } +func (s *GatewayStruct) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) { + if s.Internal.StateCall == nil { + return nil, ErrNotSupported + } + return s.Internal.StateCall(p0, p1, p2) +} + +func (s *GatewayStub) StateCall(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateDealProviderCollateralBounds(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (api.DealCollateralBounds, error) { if s.Internal.StateDealProviderCollateralBounds == nil { return *new(api.DealCollateralBounds), ErrNotSupported @@ -2743,6 +2773,17 @@ func (s *GatewayStub) StateDealProviderCollateralBounds(p0 context.Context, p1 a return *new(api.DealCollateralBounds), ErrNotSupported } +func (s *GatewayStruct) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) { + if s.Internal.StateDecodeParams == nil { + return nil, ErrNotSupported + } + return s.Internal.StateDecodeParams(p0, p1, p2, p3, p4) +} + +func (s *GatewayStub) StateDecodeParams(p0 context.Context, p1 address.Address, p2 abi.MethodNum, p3 []byte, p4 types.TipSetKey) (interface{}, error) { + return nil, ErrNotSupported +} + func (s *GatewayStruct) StateGetActor(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) { if s.Internal.StateGetActor == nil { return nil, ErrNotSupported @@ -2842,6 +2883,17 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A return nil, ErrNotSupported } +func (s *GatewayStruct) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) { + if s.Internal.StateNetworkName == nil { + return *new(dtypes.NetworkName), ErrNotSupported + } + return s.Internal.StateNetworkName(p0) +} + +func (s *GatewayStub) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) { + return *new(dtypes.NetworkName), ErrNotSupported +} + func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) { if s.Internal.StateNetworkVersion == nil { return *new(abinetwork.Version), ErrNotSupported diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index 0afb15c11..410cc50df 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -115,6 +115,23 @@ type Config struct { // A positive value is the number of compactions before a full GC is performed; // a value of 1 will perform full GC in every compaction. HotStoreFullGCFrequency uint64 + + // HotstoreMaxSpaceTarget suggests the max allowed space the hotstore can take. + // This is not a hard limit, it is possible for the hotstore to exceed the target + // for example if state grows massively between compactions. The splitstore + // will make a best effort to avoid overflowing the target and in practice should + // never overflow. This field is used when doing GC at the end of a compaction to + // adaptively choose moving GC + HotstoreMaxSpaceTarget uint64 + + // Moving GC will be triggered when total moving size exceeds + // HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold + HotstoreMaxSpaceThreshold uint64 + + // Safety buffer to prevent moving GC from overflowing disk. + // Moving GC will not occur when total moving size exceeds + // HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer + HotstoreMaxSpaceSafetyBuffer uint64 } // ChainAccessor allows the Splitstore to access the chain. It will most likely @@ -165,6 +182,7 @@ type SplitStore struct { compactionIndex int64 pruneIndex int64 + onlineGCCnt int64 ctx context.Context cancel func() @@ -195,6 +213,17 @@ type SplitStore struct { // registered protectors protectors []func(func(cid.Cid) error) error + + // dag sizes measured during latest compaction + // logged and used for GC strategy + + // protected by compaction lock + szWalk int64 + szProtectedTxns int64 + szKeys int64 // approximate, not counting keys protected when entering critical section + + // protected by txnLk + szMarkedLiveRefs int64 } var _ bstore.Blockstore = (*SplitStore)(nil) diff --git a/blockstore/splitstore/splitstore_check.go b/blockstore/splitstore/splitstore_check.go index 336515980..2645c78c5 100644 --- a/blockstore/splitstore/splitstore_check.go +++ b/blockstore/splitstore/splitstore_check.go @@ -95,7 +95,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error { } defer visitor.Close() //nolint - err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor, + size := s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor, func(c cid.Cid) error { if isUnitaryObject(c) { return errStopWalk @@ -133,7 +133,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error { return err } - log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt) + log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt, "walk size", size) write("--") write("cold: %d missing: %d", *coldCnt, *missingCnt) write("DONE") diff --git a/blockstore/splitstore/splitstore_compact.go b/blockstore/splitstore/splitstore_compact.go index 59bdd515d..12d0ff899 100644 --- a/blockstore/splitstore/splitstore_compact.go +++ b/blockstore/splitstore/splitstore_compact.go @@ -66,7 +66,8 @@ var ( ) const ( - batchSize = 16384 + batchSize = 16384 + cidKeySize = 128 ) func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error { @@ -199,9 +200,11 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) { log.Debugf("marking %d live refs", len(cids)) startMark := time.Now() + szMarked := new(int64) + count := new(int32) visitor := newConcurrentVisitor() - walkObject := func(c cid.Cid) error { + walkObject := func(c cid.Cid) (int64, error) { return s.walkObjectIncomplete(c, visitor, func(c cid.Cid) error { if isUnitaryObject(c) { @@ -228,10 +231,12 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) { // optimize the common case of single put if len(cids) == 1 { - if err := walkObject(cids[0]); err != nil { + sz, err := walkObject(cids[0]) + if err != nil { log.Errorf("error marking tipset refs: %s", err) } log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count) + atomic.AddInt64(szMarked, sz) return } @@ -243,9 +248,11 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) { worker := func() error { for c := range workch { - if err := walkObject(c); err != nil { + sz, err := walkObject(c) + if err != nil { return err } + atomic.AddInt64(szMarked, sz) } return nil @@ -268,7 +275,8 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) { log.Errorf("error marking tipset refs: %s", err) } - log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count) + log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count, "size marked", *szMarked) + s.szMarkedLiveRefs += atomic.LoadInt64(szMarked) } // transactionally protect a view @@ -361,6 +369,7 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error { log.Infow("protecting transactional references", "refs", len(txnRefs)) count := 0 + sz := new(int64) workch := make(chan cid.Cid, len(txnRefs)) startProtect := time.Now() @@ -393,10 +402,11 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error { worker := func() error { for c := range workch { - err := s.doTxnProtect(c, markSet) + szTxn, err := s.doTxnProtect(c, markSet) if err != nil { return xerrors.Errorf("error protecting transactional references to %s: %w", c, err) } + atomic.AddInt64(sz, szTxn) } return nil } @@ -409,16 +419,16 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error { if err := g.Wait(); err != nil { return err } - - log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count) + s.szProtectedTxns += atomic.LoadInt64(sz) + log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count, "protected size", sz) } } // transactionally protect a reference by walking the object and marking. // concurrent markings are short circuited by checking the markset. -func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) error { +func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) (int64, error) { if err := s.checkClosing(); err != nil { - return err + return 0, err } // Note: cold objects are deleted heaviest first, so the consituents of an object @@ -509,6 +519,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { // might be potentially inconsistent; abort compaction and notify the user to intervene. return xerrors.Errorf("checkpoint exists; aborting compaction") } + s.clearSizeMeasurements() currentEpoch := curTs.Height() boundaryEpoch := currentEpoch - CompactionBoundary @@ -598,7 +609,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { } err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{}, fHot, fCold) - if err != nil { return xerrors.Errorf("error marking: %w", err) } @@ -638,7 +648,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { defer purgew.Close() //nolint:errcheck // some stats for logging - var hotCnt, coldCnt, purgeCnt int + var hotCnt, coldCnt, purgeCnt int64 err = s.hot.ForEachKey(func(c cid.Cid) error { // was it marked? mark, err := markSet.Has(c) @@ -690,8 +700,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { log.Infow("cold collection done", "took", time.Since(startCollect)) log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "purge", purgeCnt) - stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt))) - stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt))) + s.szKeys = hotCnt * cidKeySize + stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(hotCnt)) + stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(coldCnt)) if err := s.checkClosing(); err != nil { return err @@ -773,8 +784,8 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { return xerrors.Errorf("error purging cold objects: %w", err) } log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge)) - s.endCriticalSection() + log.Infow("critical section done", "total protected size", s.szProtectedTxns, "total marked live size", s.szMarkedLiveRefs) if err := checkpoint.Close(); err != nil { log.Warnf("error closing checkpoint: %s", err) @@ -907,6 +918,7 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp copy(toWalk, ts.Cids()) walkCnt := new(int64) scanCnt := new(int64) + szWalk := new(int64) tsRef := func(blkCids []cid.Cid) (cid.Cid, error) { return types.NewTipSetKey(blkCids...).Cid() @@ -942,48 +954,64 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp if err != nil { return xerrors.Errorf("error computing cid reference to parent tipset") } - if err := s.walkObjectIncomplete(pRef, visitor, fHot, stopWalk); err != nil { + sz, err := s.walkObjectIncomplete(pRef, visitor, fHot, stopWalk) + if err != nil { return xerrors.Errorf("error walking parent tipset cid reference") } + atomic.AddInt64(szWalk, sz) // message are retained if within the inclMsgs boundary if hdr.Height >= inclMsgs && hdr.Height > 0 { if inclMsgs < inclState { // we need to use walkObjectIncomplete here, as messages/receipts may be missing early on if we // synced from snapshot and have a long HotStoreMessageRetentionPolicy. - if err := s.walkObjectIncomplete(hdr.Messages, visitor, fHot, stopWalk); err != nil { + sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fHot, stopWalk) + if err != nil { return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err) } + atomic.AddInt64(szWalk, sz) - if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk); err != nil { + sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk) + if err != nil { return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err) } + atomic.AddInt64(szWalk, sz) } else { - if err := s.walkObject(hdr.Messages, visitor, fHot); err != nil { + sz, err = s.walkObject(hdr.Messages, visitor, fHot) + if err != nil { return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err) } + atomic.AddInt64(szWalk, sz) - if err := s.walkObject(hdr.ParentMessageReceipts, visitor, fHot); err != nil { + sz, err := s.walkObject(hdr.ParentMessageReceipts, visitor, fHot) + if err != nil { return xerrors.Errorf("error walking message receipts (cid: %s): %w", hdr.ParentMessageReceipts, err) } + atomic.AddInt64(szWalk, sz) } } // messages and receipts outside of inclMsgs are included in the cold store if hdr.Height < inclMsgs && hdr.Height > 0 { - if err := s.walkObjectIncomplete(hdr.Messages, visitor, fCold, stopWalk); err != nil { + sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fCold, stopWalk) + if err != nil { return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err) } - if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fCold, stopWalk); err != nil { + atomic.AddInt64(szWalk, sz) + sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fCold, stopWalk) + if err != nil { return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err) } + atomic.AddInt64(szWalk, sz) } // state is only retained if within the inclState boundary, with the exception of genesis if hdr.Height >= inclState || hdr.Height == 0 { - if err := s.walkObject(hdr.ParentStateRoot, visitor, fHot); err != nil { + sz, err := s.walkObject(hdr.ParentStateRoot, visitor, fHot) + if err != nil { return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err) } + atomic.AddInt64(szWalk, sz) atomic.AddInt64(scanCnt, 1) } @@ -1001,9 +1029,11 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp if err != nil { return xerrors.Errorf("error computing cid reference to parent tipset") } - if err := s.walkObjectIncomplete(hRef, visitor, fHot, stopWalk); err != nil { + sz, err := s.walkObjectIncomplete(hRef, visitor, fHot, stopWalk) + if err != nil { return xerrors.Errorf("error walking parent tipset cid reference") } + atomic.AddInt64(szWalk, sz) for len(toWalk) > 0 { // walking can take a while, so check this with every opportunity @@ -1047,123 +1077,129 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp } } - log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt) - + log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt, "walk size", szWalk) + s.szWalk = atomic.LoadInt64(szWalk) return nil } -func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) error { +func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) (int64, error) { + var sz int64 visit, err := visitor.Visit(c) if err != nil { - return xerrors.Errorf("error visiting object: %w", err) + return 0, xerrors.Errorf("error visiting object: %w", err) } if !visit { - return nil + return sz, nil } if err := f(c); err != nil { if err == errStopWalk { - return nil + return sz, nil } - return err + return 0, err } if c.Prefix().Codec != cid.DagCBOR { - return nil + return sz, nil } // check this before recursing if err := s.checkClosing(); err != nil { - return err + return 0, err } var links []cid.Cid err = s.view(c, func(data []byte) error { + sz += int64(len(data)) return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) { links = append(links, c) }) }) if err != nil { - return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err) + return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err) } for _, c := range links { - err := s.walkObject(c, visitor, f) + szLink, err := s.walkObject(c, visitor, f) if err != nil { - return xerrors.Errorf("error walking link (cid: %s): %w", c, err) + return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err) } + sz += szLink } - return nil + return sz, nil } // like walkObject, but the object may be potentially incomplete (references missing) -func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) error { +func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) (int64, error) { + var sz int64 visit, err := visitor.Visit(c) if err != nil { - return xerrors.Errorf("error visiting object: %w", err) + return 0, xerrors.Errorf("error visiting object: %w", err) } if !visit { - return nil + return sz, nil } // occurs check -- only for DAGs if c.Prefix().Codec == cid.DagCBOR { has, err := s.has(c) if err != nil { - return xerrors.Errorf("error occur checking %s: %w", c, err) + return 0, xerrors.Errorf("error occur checking %s: %w", c, err) } if !has { err = missing(c) if err == errStopWalk { - return nil + return sz, nil } - return err + return 0, err } } if err := f(c); err != nil { if err == errStopWalk { - return nil + return sz, nil } - return err + return 0, err } if c.Prefix().Codec != cid.DagCBOR { - return nil + return sz, nil } // check this before recursing if err := s.checkClosing(); err != nil { - return err + return sz, err } var links []cid.Cid err = s.view(c, func(data []byte) error { + sz += int64(len(data)) return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) { links = append(links, c) }) }) if err != nil { - return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err) + return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err) } for _, c := range links { - err := s.walkObjectIncomplete(c, visitor, f, missing) + szLink, err := s.walkObjectIncomplete(c, visitor, f, missing) if err != nil { - return xerrors.Errorf("error walking link (cid: %s): %w", c, err) + return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err) } + sz += szLink } - return nil + return sz, nil } // internal version used during compaction and related operations @@ -1429,8 +1465,9 @@ func (s *SplitStore) completeCompaction() error { } s.compactType = none - // Note: at this point we can start the splitstore; a compaction should run on - // the first head change, which will trigger gc on the hotstore. + // Note: at this point we can start the splitstore; base epoch is not + // incremented here so a compaction should run on the first head + // change, which will trigger gc on the hotstore. // We don't mind the second (back-to-back) compaction as the head will // have advanced during marking and coldset accumulation. return nil @@ -1488,6 +1525,13 @@ func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint, return nil } +func (s *SplitStore) clearSizeMeasurements() { + s.szKeys = 0 + s.szMarkedLiveRefs = 0 + s.szProtectedTxns = 0 + s.szWalk = 0 +} + // I really don't like having this code, but we seem to have some occasional DAG references with // missing constituents. During testing in mainnet *some* of these references *sometimes* appeared // after a little bit. @@ -1528,7 +1572,7 @@ func (s *SplitStore) waitForMissingRefs(markSet MarkSet) { missing = make(map[cid.Cid]struct{}) for c := range towalk { - err := s.walkObjectIncomplete(c, visitor, + _, err := s.walkObjectIncomplete(c, visitor, func(c cid.Cid) error { if isUnitaryObject(c) { return errStopWalk diff --git a/blockstore/splitstore/splitstore_gc.go b/blockstore/splitstore/splitstore_gc.go index 3b53b8042..2ddb7d404 100644 --- a/blockstore/splitstore/splitstore_gc.go +++ b/blockstore/splitstore/splitstore_gc.go @@ -7,15 +7,61 @@ import ( bstore "github.com/filecoin-project/lotus/blockstore" ) +const ( + // Fraction of garbage in badger vlog for online GC traversal to collect garbage + AggressiveOnlineGCThreshold = 0.0001 +) + func (s *SplitStore) gcHotAfterCompaction() { + // Measure hotstore size, determine if we should do full GC, determine if we can do full GC. + // We should do full GC if + // FullGCFrequency is specified and compaction index matches frequency + // OR HotstoreMaxSpaceTarget is specified and total moving space is within 150 GB of target + // We can do full if + // HotstoreMaxSpaceTarget is not specified + // OR total moving space would not exceed 50 GB below target + // + // a) If we should not do full GC => online GC + // b) If we should do full GC and can => moving GC + // c) If we should do full GC and can't => aggressive online GC + getSize := func() int64 { + sizer, ok := s.hot.(bstore.BlockstoreSize) + if ok { + size, err := sizer.Size() + if err != nil { + log.Warnf("error getting hotstore size: %s, estimating empty hot store for targeting", err) + return 0 + } + return size + } + log.Errorf("Could not measure hotstore size, assuming it is 0 bytes, which it is not") + return 0 + } + hotSize := getSize() + + copySizeApprox := s.szKeys + s.szMarkedLiveRefs + s.szProtectedTxns + s.szWalk + shouldTarget := s.cfg.HotstoreMaxSpaceTarget > 0 && hotSize+copySizeApprox > int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceThreshold) + shouldFreq := s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 + shouldDoFull := shouldTarget || shouldFreq + canDoFull := s.cfg.HotstoreMaxSpaceTarget == 0 || hotSize+copySizeApprox < int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceSafetyBuffer) + log.Debugw("approximating new hot store size", "key size", s.szKeys, "marked live refs", s.szMarkedLiveRefs, "protected txns", s.szProtectedTxns, "walked DAG", s.szWalk) + log.Infof("measured hot store size: %d, approximate new size: %d, should do full %t, can do full %t", hotSize, copySizeApprox, shouldDoFull, canDoFull) + var opts []bstore.BlockstoreGCOption - if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 { + if shouldDoFull && canDoFull { opts = append(opts, bstore.WithFullGC(true)) + } else if shouldDoFull && !canDoFull { + log.Warnf("Attention! Estimated moving GC size %d is not within safety buffer %d of target max %d, performing aggressive online GC to attempt to bring hotstore size down safely", copySizeApprox, s.cfg.HotstoreMaxSpaceSafetyBuffer, s.cfg.HotstoreMaxSpaceTarget) + log.Warn("If problem continues you can 1) temporarily allocate more disk space to hotstore and 2) reflect in HotstoreMaxSpaceTarget OR trigger manual move with `lotus chain prune hot-moving`") + log.Warn("If problem continues and you do not have any more disk space you can run continue to manually trigger online GC at aggressive thresholds (< 0.01) with `lotus chain prune hot`") + + opts = append(opts, bstore.WithThreshold(AggressiveOnlineGCThreshold)) } if err := s.gcBlockstore(s.hot, opts); err != nil { log.Warnf("error garbage collecting hostore: %s", err) } + log.Infof("measured hot store size after GC: %d", getSize()) } func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error { diff --git a/blockstore/splitstore/splitstore_reify.go b/blockstore/splitstore/splitstore_reify.go index aa14f090a..07efedead 100644 --- a/blockstore/splitstore/splitstore_reify.go +++ b/blockstore/splitstore/splitstore_reify.go @@ -101,7 +101,7 @@ func (s *SplitStore) doReify(c cid.Cid) { defer s.txnLk.RUnlock() count := 0 - err := s.walkObjectIncomplete(c, newTmpVisitor(), + _, err := s.walkObjectIncomplete(c, newTmpVisitor(), func(c cid.Cid) error { if isUnitaryObject(c) { return errStopWalk diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 718eed997..cea575685 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/gateway.json.gz b/build/openrpc/gateway.json.gz index 03453c045..950a111a1 100644 Binary files a/build/openrpc/gateway.json.gz and b/build/openrpc/gateway.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 6e70721a6..20916fd24 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 88b2118ad..ffb4befd3 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 673a299e3..de2df97c2 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -256,7 +256,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS //return nil, xerrors.Errorf("creating drand beacon: %w", err) //} - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), sys, us, beac) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), sys, us, beac, ds) if err != nil { return nil, xerrors.Errorf("initing stmgr: %w", err) } diff --git a/chain/messagepool/config.go b/chain/messagepool/config.go index 5b7e2c590..72b7d9567 100644 --- a/chain/messagepool/config.go +++ b/chain/messagepool/config.go @@ -13,7 +13,11 @@ import ( ) var ( - ReplaceByFeeRatioDefault = 1.25 + ReplaceByFeePercentageMinimum types.Percent = 110 + ReplaceByFeePercentageDefault types.Percent = 125 +) + +var ( MemPoolSizeLimitHiDefault = 30000 MemPoolSizeLimitLoDefault = 20000 PruneCooldownDefault = time.Minute @@ -60,9 +64,9 @@ func (mp *MessagePool) getConfig() *types.MpoolConfig { } func validateConfg(cfg *types.MpoolConfig) error { - if cfg.ReplaceByFeeRatio < ReplaceByFeeRatioDefault { - return fmt.Errorf("'ReplaceByFeeRatio' is less than required %f < %f", - cfg.ReplaceByFeeRatio, ReplaceByFeeRatioDefault) + if cfg.ReplaceByFeeRatio < ReplaceByFeePercentageMinimum { + return fmt.Errorf("'ReplaceByFeeRatio' is less than required %s < %s", + cfg.ReplaceByFeeRatio, ReplaceByFeePercentageMinimum) } if cfg.GasLimitOverestimation < 1 { return fmt.Errorf("'GasLimitOverestimation' cannot be less than 1") @@ -91,7 +95,7 @@ func DefaultConfig() *types.MpoolConfig { return &types.MpoolConfig{ SizeLimitHigh: MemPoolSizeLimitHiDefault, SizeLimitLow: MemPoolSizeLimitLoDefault, - ReplaceByFeeRatio: ReplaceByFeeRatioDefault, + ReplaceByFeeRatio: ReplaceByFeePercentageDefault, PruneCooldown: PruneCooldownDefault, GasLimitOverestimation: GasLimitOverestimation, } diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 9abe38648..c9704403e 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -47,10 +47,8 @@ var log = logging.Logger("messagepool") var futureDebug = false -var rbfNumBig = types.NewInt(uint64((ReplaceByFeeRatioDefault - 1) * RbfDenom)) -var rbfDenomBig = types.NewInt(RbfDenom) - -const RbfDenom = 256 +var rbfNumBig = types.NewInt(uint64(ReplaceByFeePercentageMinimum)) +var rbfDenomBig = types.NewInt(100) var RepublishInterval = time.Duration(10*build.BlockDelaySecs+build.PropagationDelaySecs) * time.Second @@ -197,7 +195,13 @@ func newMsgSet(nonce uint64) *msgSet { } func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount { - minPrice := types.BigAdd(curPrem, types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig)) + minPrice := types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig) + return types.BigAdd(minPrice, types.NewInt(1)) +} + +func ComputeRBF(curPrem abi.TokenAmount, replaceByFeeRatio types.Percent) abi.TokenAmount { + rbfNumBig := types.NewInt(uint64(replaceByFeeRatio)) + minPrice := types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig) return types.BigAdd(minPrice, types.NewInt(1)) } diff --git a/chain/stmgr/execute.go b/chain/stmgr/execute.go index e67398299..57ff205b9 100644 --- a/chain/stmgr/execute.go +++ b/chain/stmgr/execute.go @@ -8,6 +8,7 @@ import ( "go.opencensus.io/trace" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) @@ -52,6 +53,12 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c sm.stlk.Unlock() + // First, try to find the tipset in the current chain. If found, we can avoid re-executing + // it. + if st, rec, found := tryLookupTipsetState(ctx, sm.cs, ts); found { + return st, rec, nil + } + st, rec, err = sm.tsExec.ExecuteTipSet(ctx, sm, ts, sm.tsExecMonitor, false) if err != nil { return cid.Undef, cid.Undef, err @@ -60,6 +67,51 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c return st, rec, nil } +// Try to lookup a state & receipt CID for a given tipset by walking the chain instead of executing +// it. This will only successfully return the state/receipt CIDs if they're found in the state +// store. +// +// NOTE: This _won't_ recursively walk the receipt/state trees. It assumes that having the root +// implies having the rest of the tree. However, lotus generally makes that assumption anyways. +func tryLookupTipsetState(ctx context.Context, cs *store.ChainStore, ts *types.TipSet) (cid.Cid, cid.Cid, bool) { + nextTs, err := cs.GetTipsetByHeight(ctx, ts.Height()+1, nil, false) + if err != nil { + // Nothing to see here. The requested height may be beyond the current head. + return cid.Undef, cid.Undef, false + } + + // Make sure we're on the correct fork. + if nextTs.Parents() != ts.Key() { + // Also nothing to see here. This just means that the requested tipset is on a + // different fork. + return cid.Undef, cid.Undef, false + } + + stateCid := nextTs.ParentState() + receiptCid := nextTs.ParentMessageReceipts() + + // Make sure we have the parent state. + if hasState, err := cs.StateBlockstore().Has(ctx, stateCid); err != nil { + log.Errorw("failed to lookup state-root in blockstore", "cid", stateCid, "error", err) + return cid.Undef, cid.Undef, false + } else if !hasState { + // We have the chain but don't have the state. It looks like we need to try + // executing? + return cid.Undef, cid.Undef, false + } + + // Make sure we have the receipts. + if hasReceipts, err := cs.ChainBlockstore().Has(ctx, receiptCid); err != nil { + log.Errorw("failed to lookup receipts in blockstore", "cid", receiptCid, "error", err) + return cid.Undef, cid.Undef, false + } else if !hasReceipts { + // If we don't have the receipts, re-execute and try again. + return cid.Undef, cid.Undef, false + } + + return stateCid, receiptCid, true +} + func (sm *StateManager) ExecutionTraceWithMonitor(ctx context.Context, ts *types.TipSet, em ExecMonitor) (cid.Cid, error) { st, _, err := sm.tsExec.ExecuteTipSet(ctx, sm, ts, em, true) return st, err diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 7d7e234b1..1f9977d96 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -174,9 +174,16 @@ func (us UpgradeSchedule) GetNtwkVersion(e abi.ChainEpoch) (network.Version, err func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecMonitor, ts *types.TipSet) (cid.Cid, error) { retCid := root - var err error u := sm.stateMigrations[height] if u != nil && u.upgrade != nil { + migCid, ok, err := u.migrationResultCache.Get(ctx, root) + if err == nil && ok { + log.Infow("CACHED migration", "height", height, "from", root, "to", migCid) + return migCid, nil + } else if err != nil { + log.Errorw("failed to lookup previous migration result", "err", err) + } + startTime := time.Now() log.Warnw("STARTING migration", "height", height, "from", root) // Yes, we clone the cache, even for the final upgrade epoch. Why? Reverts. We may @@ -197,6 +204,11 @@ func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, heig "to", retCid, "duration", time.Since(startTime), ) + + // Only set if migration ran, we do not want a root => root mapping + if err := u.migrationResultCache.Store(ctx, root, retCid); err != nil { + log.Errorw("failed to store migration result", "err", err) + } } return retCid, nil diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 52cfe5fce..a904172cd 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" ipldcbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" @@ -35,6 +36,7 @@ import ( "github.com/filecoin-project/lotus/chain/consensus" "github.com/filecoin-project/lotus/chain/consensus/filcns" "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/stmgr" . "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" @@ -166,7 +168,7 @@ func TestForkHeightTriggers(t *testing.T) { } return st.Flush(ctx) - }}}, cg.BeaconSchedule()) + }}}, cg.BeaconSchedule(), datastore.NewMapDatastore()) if err != nil { t.Fatal(err) } @@ -284,7 +286,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) { root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { migrationCount++ return root, nil - }}}, cg.BeaconSchedule()) + }}}, cg.BeaconSchedule(), datastore.NewMapDatastore()) if err != nil { t.Fatal(err) } @@ -502,7 +504,7 @@ func TestForkPreMigration(t *testing.T) { return nil }, }}}, - }, cg.BeaconSchedule()) + }, cg.BeaconSchedule(), datastore.NewMapDatastore()) if err != nil { t.Fatal(err) } @@ -576,6 +578,7 @@ func TestDisablePreMigration(t *testing.T) { }}}, }, cg.BeaconSchedule(), + datastore.NewMapDatastore(), ) require.NoError(t, err) require.NoError(t, sm.Start(context.Background())) @@ -603,3 +606,102 @@ func TestDisablePreMigration(t *testing.T) { require.Equal(t, 1, len(counter)) } + +func TestMigrtionCache(t *testing.T) { + logging.SetAllLoggers(logging.LevelInfo) + + cg, err := gen.NewGenerator() + require.NoError(t, err) + + counter := make(chan struct{}, 10) + metadataDs := datastore.NewMapDatastore() + + sm, err := NewStateManager( + cg.ChainStore(), + consensus.NewTipSetExecutor(filcns.RewardFunc), + cg.StateManager().VMSys(), + UpgradeSchedule{{ + Network: network.Version1, + Height: testForkHeight, + Migration: func(_ context.Context, _ *StateManager, _ MigrationCache, _ ExecMonitor, + root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) { + + counter <- struct{}{} + + return root, nil + }}, + }, + cg.BeaconSchedule(), + metadataDs, + ) + require.NoError(t, err) + require.NoError(t, sm.Start(context.Background())) + defer func() { + require.NoError(t, sm.Stop(context.Background())) + }() + + inv := consensus.NewActorRegistry() + registry := builtin.MakeRegistryLegacy([]rtt.VMActor{testActor{}}) + inv.Register(actorstypes.Version0, nil, registry) + + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) { + nvm, err := vm.NewLegacyVM(ctx, vmopt) + require.NoError(t, err) + nvm.SetInvoker(inv) + return nvm, nil + }) + + cg.SetStateManager(sm) + + for i := 0; i < 50; i++ { + _, err := cg.NextTipSet() + require.NoError(t, err) + } + + ts, err := cg.ChainStore().GetTipsetByHeight(context.Background(), testForkHeight, nil, false) + require.NoError(t, err) + + root, _, err := stmgr.ComputeState(context.Background(), sm, testForkHeight+1, []*types.Message{}, ts) + require.NoError(t, err) + t.Log(root) + + require.Equal(t, 1, len(counter)) + + { + sm, err := NewStateManager( + cg.ChainStore(), + consensus.NewTipSetExecutor(filcns.RewardFunc), + cg.StateManager().VMSys(), + UpgradeSchedule{{ + Network: network.Version1, + Height: testForkHeight, + Migration: func(_ context.Context, _ *StateManager, _ MigrationCache, _ ExecMonitor, + root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) { + + counter <- struct{}{} + + return root, nil + }}, + }, + cg.BeaconSchedule(), + metadataDs, + ) + require.NoError(t, err) + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) { + nvm, err := vm.NewLegacyVM(ctx, vmopt) + require.NoError(t, err) + nvm.SetInvoker(inv) + return nvm, nil + }) + + ctx := context.Background() + + base, _, err := sm.ExecutionTrace(ctx, ts) + require.NoError(t, err) + _, err = sm.HandleStateForks(context.Background(), base, ts.Height(), nil, ts) + require.NoError(t, err) + + // Should not have increased as we should be using the cached results in the metadataDs + require.Equal(t, 1, len(counter)) + } +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ee9338e63..b9f8d81bf 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -2,10 +2,13 @@ package stmgr import ( "context" + "fmt" "sync" "github.com/ipfs/go-cid" + dstore "github.com/ipfs/go-datastore" cbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" @@ -51,9 +54,47 @@ type versionSpec struct { } type migration struct { - upgrade MigrationFunc - preMigrations []PreMigration - cache *nv16.MemMigrationCache + upgrade MigrationFunc + preMigrations []PreMigration + cache *nv16.MemMigrationCache + migrationResultCache *migrationResultCache +} + +type migrationResultCache struct { + ds dstore.Batching + keyPrefix string +} + +func (m *migrationResultCache) keyForMigration(root cid.Cid) dstore.Key { + kStr := fmt.Sprintf("%s/%s", m.keyPrefix, root) + return dstore.NewKey(kStr) +} + +func (m *migrationResultCache) Get(ctx context.Context, root cid.Cid) (cid.Cid, bool, error) { + k := m.keyForMigration(root) + + bs, err := m.ds.Get(ctx, k) + if ipld.IsNotFound(err) { + return cid.Undef, false, nil + } else if err != nil { + return cid.Undef, false, xerrors.Errorf("error loading migration result: %w", err) + } + + c, err := cid.Parse(bs) + if err != nil { + return cid.Undef, false, xerrors.Errorf("error parsing migration result: %w", err) + } + + return c, true, nil +} + +func (m *migrationResultCache) Store(ctx context.Context, root cid.Cid, resultCid cid.Cid) error { + k := m.keyForMigration(root) + if err := m.ds.Put(ctx, k, resultCid.Bytes()); err != nil { + return err + } + + return nil } type Executor interface { @@ -103,7 +144,7 @@ type treeCache struct { tree *state.StateTree } -func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, beacon beacon.Schedule) (*StateManager, error) { +func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, beacon beacon.Schedule, metadataDs dstore.Batching) (*StateManager, error) { // If we have upgrades, make sure they're in-order and make sense. if err := us.Validate(); err != nil { return nil, err @@ -122,12 +163,18 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, upgrade: upgrade.Migration, preMigrations: upgrade.PreMigrations, cache: nv16.NewMemMigrationCache(), + migrationResultCache: &migrationResultCache{ + keyPrefix: fmt.Sprintf("/migration-cache/nv%d", upgrade.Network), + ds: metadataDs, + }, } + stateMigrations[upgrade.Height] = migration } if upgrade.Expensive { expensiveUpgrades[upgrade.Height] = struct{}{} } + networkVersions = append(networkVersions, versionSpec{ networkVersion: lastVersion, atOrBelow: upgrade.Height, @@ -155,8 +202,8 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, }, nil } -func NewStateManagerWithUpgradeScheduleAndMonitor(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, b beacon.Schedule, em ExecMonitor) (*StateManager, error) { - sm, err := NewStateManager(cs, exec, sys, us, b) +func NewStateManagerWithUpgradeScheduleAndMonitor(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, b beacon.Schedule, em ExecMonitor, metadataDs dstore.Batching) (*StateManager, error) { + sm, err := NewStateManager(cs, exec, sys, us, b, metadataDs) if err != nil { return nil, err } diff --git a/chain/store/index.go b/chain/store/index.go index fe8f399ee..620cb2dee 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -4,8 +4,8 @@ import ( "context" "os" "strconv" + "sync" - lru "github.com/hashicorp/golang-lru" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -var DefaultChainIndexCacheSize = 32 << 10 +var DefaultChainIndexCacheSize = 32 << 15 func init() { if s := os.Getenv("LOTUS_CHAIN_INDEX_CACHE"); s != "" { @@ -27,7 +27,8 @@ func init() { } type ChainIndex struct { - skipCache *lru.ARCCache + indexCacheLk sync.Mutex + indexCache map[types.TipSetKey]*lbEntry loadTipSet loadTipSetFunc @@ -36,17 +37,14 @@ type ChainIndex struct { type loadTipSetFunc func(context.Context, types.TipSetKey) (*types.TipSet, error) func NewChainIndex(lts loadTipSetFunc) *ChainIndex { - sc, _ := lru.NewARC(DefaultChainIndexCacheSize) return &ChainIndex{ - skipCache: sc, + indexCache: make(map[types.TipSetKey]*lbEntry, DefaultChainIndexCacheSize), loadTipSet: lts, skipLength: 20, } } type lbEntry struct { - ts *types.TipSet - parentHeight abi.ChainEpoch targetHeight abi.ChainEpoch target types.TipSetKey } @@ -58,25 +56,36 @@ func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet, rounded, err := ci.roundDown(ctx, from) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to round down: %w", err) } + ci.indexCacheLk.Lock() + defer ci.indexCacheLk.Unlock() cur := rounded.Key() for { - cval, ok := ci.skipCache.Get(cur) + lbe, ok := ci.indexCache[cur] if !ok { fc, err := ci.fillCache(ctx, cur) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to fill cache: %w", err) } - cval = fc + lbe = fc } - lbe := cval.(*lbEntry) - if lbe.ts.Height() == to || lbe.parentHeight < to { - return lbe.ts, nil - } else if to > lbe.targetHeight { - return ci.walkBack(ctx, lbe.ts, to) + if to == lbe.targetHeight { + ts, err := ci.loadTipSet(ctx, lbe.target) + if err != nil { + return nil, xerrors.Errorf("failed to load tipset: %w", err) + } + + return ts, nil + } + if to > lbe.targetHeight { + ts, err := ci.loadTipSet(ctx, cur) + if err != nil { + return nil, xerrors.Errorf("failed to load tipset: %w", err) + } + return ci.walkBack(ctx, ts, to) } cur = lbe.target @@ -87,16 +96,17 @@ func (ci *ChainIndex) GetTipsetByHeightWithoutCache(ctx context.Context, from *t return ci.walkBack(ctx, from, to) } +// Caller must hold indexCacheLk func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEntry, error) { ts, err := ci.loadTipSet(ctx, tsk) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load tipset: %w", err) } if ts.Height() == 0 { return &lbEntry{ - ts: ts, - parentHeight: 0, + targetHeight: 0, + target: tsk, }, nil } @@ -124,12 +134,10 @@ func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEn } lbe := &lbEntry{ - ts: ts, - parentHeight: parent.Height(), targetHeight: skipTarget.Height(), target: skipTarget.Key(), } - ci.skipCache.Add(tsk, lbe) + ci.indexCache[tsk] = lbe return lbe, nil } @@ -144,7 +152,7 @@ func (ci *ChainIndex) roundDown(ctx context.Context, ts *types.TipSet) (*types.T rounded, err := ci.walkBack(ctx, ts, target) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to walk back: %w", err) } return rounded, nil @@ -164,7 +172,7 @@ func (ci *ChainIndex) walkBack(ctx context.Context, from *types.TipSet, to abi.C for { pts, err := ci.loadTipSet(ctx, ts.Parents()) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load tipset: %w", err) } if to > pts.Height() { diff --git a/chain/store/messages.go b/chain/store/messages.go index 5ac62d394..33d0c579a 100644 --- a/chain/store/messages.go +++ b/chain/store/messages.go @@ -237,6 +237,26 @@ func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.C return blscids, secpkcids, nil } +func (cs *ChainStore) ReadReceipts(ctx context.Context, root cid.Cid) ([]types.MessageReceipt, error) { + a, err := blockadt.AsArray(cs.ActorStore(ctx), root) + if err != nil { + return nil, err + } + + receipts := make([]types.MessageReceipt, 0, a.Length()) + var rcpt types.MessageReceipt + if err := a.ForEach(&rcpt, func(i int64) error { + if int64(len(receipts)) != i { + return xerrors.Errorf("missing receipt %d", i) + } + receipts = append(receipts, rcpt) + return nil + }); err != nil { + return nil, err + } + return receipts, nil +} + func (cs *ChainStore) MessagesForBlock(ctx context.Context, b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) { blscids, secpkcids, err := cs.ReadMsgMetaCids(ctx, b.Messages) if err != nil { diff --git a/chain/store/store_test.go b/chain/store/store_test.go index f5765fddc..cc72acc95 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -196,7 +196,8 @@ func TestChainExportImportFull(t *testing.T) { } nbs := blockstore.NewMemorySync() - cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), filcns.Weight, nil) + ds := datastore.NewMapDatastore() + cs := store.NewChainStore(nbs, nbs, ds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck root, err := cs.Import(context.TODO(), buf) @@ -213,7 +214,7 @@ func TestChainExportImportFull(t *testing.T) { t.Fatal("imported chain differed from exported chain") } - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), nil, filcns.DefaultUpgradeSchedule(), cg.BeaconSchedule()) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), nil, filcns.DefaultUpgradeSchedule(), cg.BeaconSchedule(), ds) if err != nil { t.Fatal(err) } diff --git a/chain/types/ethtypes/eth_transactions.go b/chain/types/ethtypes/eth_transactions.go index 7afde4bd2..6c13c5bf6 100644 --- a/chain/types/ethtypes/eth_transactions.go +++ b/chain/types/ethtypes/eth_transactions.go @@ -44,14 +44,6 @@ type EthTx struct { S EthBigInt `json:"s"` } -func (tx *EthTx) Reward(blkBaseFee big.Int) EthBigInt { - availablePriorityFee := big.Sub(big.Int(tx.MaxFeePerGas), blkBaseFee) - if big.Cmp(big.Int(tx.MaxPriorityFeePerGas), availablePriorityFee) <= 0 { - return tx.MaxPriorityFeePerGas - } - return EthBigInt(availablePriorityFee) -} - type EthTxArgs struct { ChainID int `json:"chainId"` Nonce int `json:"nonce"` diff --git a/chain/types/ethtypes/eth_types.go b/chain/types/ethtypes/eth_types.go index ddda91ea0..f157c7f94 100644 --- a/chain/types/ethtypes/eth_types.go +++ b/chain/types/ethtypes/eth_types.go @@ -295,17 +295,21 @@ func EthAddressFromPubKey(pubk []byte) ([]byte, error) { return ethAddr, nil } +var maskedIDPrefix = [20 - 8]byte{0xff} + func IsEthAddress(addr address.Address) bool { if addr.Protocol() != address.Delegated { return false } payload := addr.Payload() - namespace, _, err := varint.FromUvarint(payload) + namespace, offset, err := varint.FromUvarint(payload) if err != nil { return false } - return namespace == builtintypes.EthereumAddressManagerActorID + payload = payload[offset:] + + return namespace == builtintypes.EthereumAddressManagerActorID && len(payload) == 20 && !bytes.HasPrefix(payload, maskedIDPrefix[:]) } func EthAddressFromFilecoinAddress(addr address.Address) (EthAddress, error) { @@ -326,9 +330,17 @@ func EthAddressFromFilecoinAddress(addr address.Address) (EthAddress, error) { return EthAddress{}, xerrors.Errorf("invalid delegated address namespace in: %s", addr) } payload = payload[n:] - if namespace == builtintypes.EthereumAddressManagerActorID { - return CastEthAddress(payload) + if namespace != builtintypes.EthereumAddressManagerActorID { + return EthAddress{}, ErrInvalidAddress } + ethAddr, err := CastEthAddress(payload) + if err != nil { + return EthAddress{}, err + } + if ethAddr.IsMaskedID() { + return EthAddress{}, xerrors.Errorf("f410f addresses cannot embed masked-ID payloads: %s", ethAddr) + } + return ethAddr, nil } return EthAddress{}, ErrInvalidAddress } @@ -376,8 +388,7 @@ func (ea *EthAddress) UnmarshalJSON(b []byte) error { } func (ea EthAddress) IsMaskedID() bool { - idmask := [12]byte{0xff} - return bytes.Equal(ea[:12], idmask[:]) + return bytes.HasPrefix(ea[:], maskedIDPrefix[:]) } func (ea EthAddress) ToFilecoinAddress() (address.Address, error) { diff --git a/chain/types/ethtypes/eth_types_test.go b/chain/types/ethtypes/eth_types_test.go index 118fbc901..4a73184c2 100644 --- a/chain/types/ethtypes/eth_types_test.go +++ b/chain/types/ethtypes/eth_types_test.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/builtin" ) type TestCase struct { @@ -178,6 +179,20 @@ func TestParseEthAddr(t *testing.T) { } } +func TestMaskedIDInF4(t *testing.T) { + addr, err := address.NewIDAddress(100) + require.NoError(t, err) + + eaddr, err := EthAddressFromFilecoinAddress(addr) + require.NoError(t, err) + + badaddr, err := address.NewDelegatedAddress(builtin.EthereumAddressManagerActorID, eaddr[:]) + require.NoError(t, err) + + _, err = EthAddressFromFilecoinAddress(badaddr) + require.Error(t, err) +} + func TestUnmarshalEthCall(t *testing.T) { data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":""}` diff --git a/chain/types/message.go b/chain/types/message.go index becd2c010..4304ba659 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -219,4 +219,17 @@ func (m *Message) ValidForBlockInclusion(minGas int64, version network.Version) return nil } +// EffectiveGasPremium returns the effective gas premium claimable by the miner +// given the supplied base fee. +// +// Filecoin clamps the gas premium at GasFeeCap - BaseFee, if lower than the +// specified premium. +func (m *Message) EffectiveGasPremium(baseFee abi.TokenAmount) abi.TokenAmount { + available := big.Sub(m.GasFeeCap, baseFee) + if big.Cmp(m.GasPremium, available) <= 0 { + return m.GasPremium + } + return available +} + const TestGasLimit = 100e6 diff --git a/chain/types/mpool.go b/chain/types/mpool.go index cf08177e9..497d5c590 100644 --- a/chain/types/mpool.go +++ b/chain/types/mpool.go @@ -10,7 +10,7 @@ type MpoolConfig struct { PriorityAddrs []address.Address SizeLimitHigh int SizeLimitLow int - ReplaceByFeeRatio float64 + ReplaceByFeeRatio Percent PruneCooldown time.Duration GasLimitOverestimation float64 } diff --git a/chain/types/percent.go b/chain/types/percent.go new file mode 100644 index 000000000..858d9a2e3 --- /dev/null +++ b/chain/types/percent.go @@ -0,0 +1,39 @@ +package types + +import ( + "fmt" + "math" + "strconv" + + "golang.org/x/xerrors" +) + +// Percent stores a signed percentage as an int64. When converted to a string (or json), it's stored +// as a decimal with two places (e.g., 100% -> 1.00). +type Percent int64 + +func (p Percent) String() string { + abs := p + sign := "" + if abs < 0 { + abs = -abs + sign = "-" + } + return fmt.Sprintf(`%s%d.%d`, sign, abs/100, abs%100) +} + +func (p Percent) MarshalJSON() ([]byte, error) { + return []byte(p.String()), nil +} + +func (p *Percent) UnmarshalJSON(b []byte) error { + flt, err := strconv.ParseFloat(string(b)+"e2", 64) + if err != nil { + return xerrors.Errorf("unable to parse ratio %s: %w", string(b), err) + } + if math.Trunc(flt) != flt { + return xerrors.Errorf("ratio may only have two decimals: %s", string(b)) + } + *p = Percent(flt) + return nil +} diff --git a/chain/types/percent_test.go b/chain/types/percent_test.go new file mode 100644 index 000000000..7364c2447 --- /dev/null +++ b/chain/types/percent_test.go @@ -0,0 +1,34 @@ +package types + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPercent(t *testing.T) { + for _, tc := range []struct { + p Percent + s string + }{ + {100, "1.0"}, + {111, "1.11"}, + {12, "0.12"}, + {-12, "-0.12"}, + {1012, "10.12"}, + {-1012, "-10.12"}, + {0, "0.0"}, + } { + tc := tc + t.Run(fmt.Sprintf("%d <> %s", tc.p, tc.s), func(t *testing.T) { + m, err := tc.p.MarshalJSON() + require.NoError(t, err) + require.Equal(t, tc.s, string(m)) + var p Percent + require.NoError(t, p.UnmarshalJSON([]byte(tc.s))) + require.Equal(t, tc.p, p) + }) + } + +} diff --git a/chain/types/tipset.go b/chain/types/tipset.go index c1aa90fc9..047a1c00e 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -234,6 +234,10 @@ func (ts *TipSet) MinTicketBlock() *BlockHeader { return min } +func (ts *TipSet) ParentMessageReceipts() cid.Cid { + return ts.blks[0].ParentMessageReceipts +} + func (ts *TipSet) ParentState() cid.Cid { return ts.blks[0].ParentStateRoot } diff --git a/cli/evm.go b/cli/evm.go index d153e7212..84cbf8c61 100644 --- a/cli/evm.go +++ b/cli/evm.go @@ -35,6 +35,7 @@ var EvmCmd = &cli.Command{ EvmGetInfoCmd, EvmCallSimulateCmd, EvmGetContractAddress, + EvmGetBytecode, }, } @@ -486,3 +487,51 @@ func ethAddrFromFilecoinAddress(ctx context.Context, addr address.Address, fnapi return ethAddr, faddr, nil } + +var EvmGetBytecode = &cli.Command{ + Name: "bytecode", + Usage: "Write the bytecode of a smart contract to a file", + ArgsUsage: "[contract-address] [file-name]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "bin", + Usage: "write the bytecode as raw binary and don't hex-encode", + }, + }, + Action: func(cctx *cli.Context) error { + + if cctx.NArg() != 2 { + return IncorrectNumArgs(cctx) + } + + contractAddr, err := ethtypes.ParseEthAddress(cctx.Args().Get(0)) + if err != nil { + return err + } + + fileName := cctx.Args().Get(1) + + api, closer, err := GetFullNodeAPIV1(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + code, err := api.EthGetCode(ctx, contractAddr, "latest") + if err != nil { + return err + } + if !cctx.Bool("bin") { + newCode := make([]byte, hex.EncodedLen(len(code))) + hex.Encode(newCode, code) + code = newCode + } + if err := os.WriteFile(fileName, code, 0o666); err != nil { + return xerrors.Errorf("failed to write bytecode to file %s: %w", fileName, err) + } + + fmt.Printf("Code for %s written to %s\n", contractAddr, fileName) + return nil + }, +} diff --git a/cli/mpool.go b/cli/mpool.go index e098806cb..c83fb4b61 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -461,7 +461,12 @@ var MpoolReplaceCmd = &cli.Command{ msg := found.Message if cctx.Bool("auto") { - minRBF := messagepool.ComputeMinRBF(msg.GasPremium) + cfg, err := api.MpoolGetConfig(ctx) + if err != nil { + return xerrors.Errorf("failed to lookup the message pool config: %w", err) + } + + defaultRBF := messagepool.ComputeRBF(msg.GasPremium, cfg.ReplaceByFeeRatio) var mss *lapi.MessageSendSpec if cctx.IsSet("fee-limit") { @@ -482,7 +487,7 @@ var MpoolReplaceCmd = &cli.Command{ return xerrors.Errorf("failed to estimate gas values: %w", err) } - msg.GasPremium = big.Max(retm.GasPremium, minRBF) + msg.GasPremium = big.Max(retm.GasPremium, defaultRBF) msg.GasFeeCap = big.Max(retm.GasFeeCap, msg.GasPremium) mff := func() (abi.TokenAmount, error) { diff --git a/cli/mpool_test.go b/cli/mpool_test.go index 01b49d4b3..0aa055ba3 100644 --- a/cli/mpool_test.go +++ b/cli/mpool_test.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" @@ -298,6 +299,7 @@ func TestReplace(t *testing.T) { mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil), mockApi.EXPECT().ChainHead(ctx).Return(nil, nil), mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil), + mockApi.EXPECT().MpoolGetConfig(ctx).Return(messagepool.DefaultConfig(), nil), // use gomock.any to match the message in expected api calls // since the replace function modifies the message between calls, it would be pointless to try to match the exact argument mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil), @@ -342,6 +344,7 @@ func TestReplace(t *testing.T) { gomock.InOrder( mockApi.EXPECT().ChainHead(ctx).Return(nil, nil), mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil), + mockApi.EXPECT().MpoolGetConfig(ctx).Return(messagepool.DefaultConfig(), nil), // use gomock.any to match the message in expected api calls // since the replace function modifies the message between calls, it would be pointless to try to match the exact argument mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil), @@ -538,7 +541,7 @@ func TestConfig(t *testing.T) { t.Fatal(err) } - mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: 0.25} + mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: types.Percent(25)} gomock.InOrder( mockApi.EXPECT().MpoolGetConfig(ctx).Return(mpoolCfg, nil), ) @@ -566,7 +569,7 @@ func TestConfig(t *testing.T) { t.Fatal(err) } - mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: 0.33} + mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: types.Percent(33)} gomock.InOrder( mockApi.EXPECT().MpoolSetConfig(ctx, mpoolCfg).Return(nil), ) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 5a97be9e5..51c567d90 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -229,7 +229,7 @@ var importBenchCmd = &cli.Command{ defer cs.Close() //nolint:errcheck // TODO: We need to supply the actual beacon after v14 - stm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(verifier), filcns.DefaultUpgradeSchedule(), nil) + stm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(verifier), filcns.DefaultUpgradeSchedule(), nil, metadataDs) if err != nil { return err } diff --git a/cmd/lotus-miner/proving.go b/cmd/lotus-miner/proving.go index 9ae8bdd48..d58ed1250 100644 --- a/cmd/lotus-miner/proving.go +++ b/cmd/lotus-miner/proving.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/build" @@ -640,7 +641,42 @@ It will not send any messages to the chain.`, if err != nil { return err } - jr, err := json.Marshal(res) + + //convert sector information into easily readable information + type PoStPartition struct { + Index uint64 + Skipped []uint64 + } + type SubmitWindowedPoStParams struct { + Deadline uint64 + Partitions []PoStPartition + Proofs []proof.PoStProof + ChainCommitEpoch abi.ChainEpoch + ChainCommitRand abi.Randomness + } + var postParams []SubmitWindowedPoStParams + for _, i := range res { + var postParam SubmitWindowedPoStParams + postParam.Deadline = i.Deadline + for id, part := range i.Partitions { + postParam.Partitions[id].Index = part.Index + count, err := part.Skipped.Count() + if err != nil { + return err + } + sectors, err := part.Skipped.All(count) + if err != nil { + return err + } + postParam.Partitions[id].Skipped = sectors + } + postParam.Proofs = i.Proofs + postParam.ChainCommitEpoch = i.ChainCommitEpoch + postParam.ChainCommitRand = i.ChainCommitRand + postParams = append(postParams, postParam) + } + + jr, err := json.MarshalIndent(postParams, "", " ") if err != nil { return err } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index d678c7977..bae281583 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -513,7 +513,7 @@ var chainBalanceStateCmd = &cli.Command{ cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } @@ -737,7 +737,7 @@ var chainPledgeCmd = &cli.Command{ cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } diff --git a/cmd/lotus-shed/gas-estimation.go b/cmd/lotus-shed/gas-estimation.go index 26c4663bf..fe8428d1e 100644 --- a/cmd/lotus-shed/gas-estimation.go +++ b/cmd/lotus-shed/gas-estimation.go @@ -111,7 +111,7 @@ var gasTraceCmd = &cli.Command{ cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds) if err != nil { return err } @@ -212,7 +212,7 @@ var replayOfflineCmd = &cli.Command{ cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds) if err != nil { return err } diff --git a/cmd/lotus-shed/invariants.go b/cmd/lotus-shed/invariants.go index 9798d111a..b759e2c2c 100644 --- a/cmd/lotus-shed/invariants.go +++ b/cmd/lotus-shed/invariants.go @@ -90,7 +90,7 @@ var invariantsCmd = &cli.Command{ cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index e31957395..b3d5da387 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" "github.com/urfave/cli/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -121,7 +122,8 @@ var migrationsCmd = &cli.Command{ cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) defer cs.Close() //nolint:errcheck - sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + // Note: we use a map datastore for the metadata to avoid writing / using cached migration results in the metadata store + sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, datastore.NewMapDatastore()) if err != nil { return err } diff --git a/cmd/lotus-shed/state-stats.go b/cmd/lotus-shed/state-stats.go index 8ec4a0ff4..521e32c79 100644 --- a/cmd/lotus-shed/state-stats.go +++ b/cmd/lotus-shed/state-stats.go @@ -308,7 +308,7 @@ to reduce the number of decode operations performed by caching the decoded objec } tsExec := consensus.NewTipSetExecutor(filcns.RewardFunc) - sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } diff --git a/cmd/lotus-sim/simulation/node.go b/cmd/lotus-sim/simulation/node.go index 06e837f8f..9b37da6c8 100644 --- a/cmd/lotus-sim/simulation/node.go +++ b/cmd/lotus-sim/simulation/node.go @@ -106,7 +106,7 @@ func (nd *Node) LoadSim(ctx context.Context, name string) (*Simulation, error) { if err != nil { return nil, xerrors.Errorf("failed to create upgrade schedule for simulation %s: %w", name, err) } - sim.StateManager, err = stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), us, nil) + sim.StateManager, err = stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), us, nil, nd.MetadataDS) if err != nil { return nil, xerrors.Errorf("failed to create state manager for simulation %s: %w", name, err) } @@ -125,7 +125,7 @@ func (nd *Node) CreateSim(ctx context.Context, name string, head *types.TipSet) if err != nil { return nil, err } - sm, err := stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), filcns.DefaultUpgradeSchedule(), nil) + sm, err := stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), filcns.DefaultUpgradeSchedule(), nil, nd.MetadataDS) if err != nil { return nil, xerrors.Errorf("creating state manager: %w", err) } diff --git a/cmd/lotus-sim/simulation/simulation.go b/cmd/lotus-sim/simulation/simulation.go index 19ab391dc..294f4cfbc 100644 --- a/cmd/lotus-sim/simulation/simulation.go +++ b/cmd/lotus-sim/simulation/simulation.go @@ -201,7 +201,7 @@ func (sim *Simulation) SetUpgradeHeight(nv network.Version, epoch abi.ChainEpoch if err != nil { return err } - sm, err := stmgr.NewStateManager(sim.Node.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), newUpgradeSchedule, nil) + sm, err := stmgr.NewStateManager(sim.Node.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), newUpgradeSchedule, nil, sim.Node.MetadataDS) if err != nil { return err } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 4a4a691ee..b3341bd79 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -540,7 +540,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) } // TODO: We need to supply the actual beacon after v14 - stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds) if err != nil { return err } diff --git a/conformance/driver.go b/conformance/driver.go index 3a32c7604..2680a7154 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -108,7 +108,7 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, params cs = store.NewChainStore(bs, bs, ds, filcns.Weight, nil) tse = consensus.NewTipSetExecutor(filcns.RewardFunc) - sm, err = stmgr.NewStateManager(cs, tse, syscalls, filcns.DefaultUpgradeSchedule(), nil) + sm, err = stmgr.NewStateManager(cs, tse, syscalls, filcns.DefaultUpgradeSchedule(), nil, ds) ) if err != nil { return nil, err diff --git a/documentation/en/api-v0-methods.md b/documentation/en/api-v0-methods.md index d254c14e7..b73244fa8 100644 --- a/documentation/en/api-v0-methods.md +++ b/documentation/en/api-v0-methods.md @@ -2879,7 +2879,7 @@ Response: ], "SizeLimitHigh": 123, "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, + "ReplaceByFeeRatio": 1.23, "PruneCooldown": 60000000000, "GasLimitOverestimation": 12.3 } @@ -3167,7 +3167,7 @@ Inputs: ], "SizeLimitHigh": 123, "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, + "ReplaceByFeeRatio": 1.23, "PruneCooldown": 60000000000, "GasLimitOverestimation": 12.3 } diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 5516acb44..aedfbfce0 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -3882,7 +3882,7 @@ Response: ], "SizeLimitHigh": 123, "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, + "ReplaceByFeeRatio": 1.23, "PruneCooldown": 60000000000, "GasLimitOverestimation": 12.3 } @@ -4170,7 +4170,7 @@ Inputs: ], "SizeLimitHigh": 123, "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, + "ReplaceByFeeRatio": 1.23, "PruneCooldown": 60000000000, "GasLimitOverestimation": 12.3 } diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 4458599ab..325b2738d 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -2647,6 +2647,7 @@ COMMANDS: stat Print eth/filecoin addrs and code cid call Simulate an eth contract call contract-address Generate contract address from smart contract code + bytecode Write the bytecode of a smart contract to a file help, h Shows a list of commands or help for one command OPTIONS: @@ -2721,6 +2722,19 @@ OPTIONS: ``` +### lotus evm bytecode +``` +NAME: + lotus evm bytecode - Write the bytecode of a smart contract to a file + +USAGE: + lotus evm bytecode [command options] [contract-address] [file-name] + +OPTIONS: + --bin write the bytecode as raw binary and don't hex-encode (default: false) + +``` + ## lotus net ``` NAME: diff --git a/documentation/en/default-lotus-config.toml b/documentation/en/default-lotus-config.toml index 41d7e6aca..c51321714 100644 --- a/documentation/en/default-lotus-config.toml +++ b/documentation/en/default-lotus-config.toml @@ -191,7 +191,7 @@ [Chainstore] # type: bool # env var: LOTUS_CHAINSTORE_ENABLESPLITSTORE - #EnableSplitstore = false + #EnableSplitstore = true [Chainstore.Splitstore] # ColdStoreType specifies the type of the coldstore. @@ -199,7 +199,7 @@ # # type: string # env var: LOTUS_CHAINSTORE_SPLITSTORE_COLDSTORETYPE - #ColdStoreType = "messages" + #ColdStoreType = "discard" # HotStoreType specifies the type of the hotstore. # Only currently supported value is "badger". @@ -230,6 +230,35 @@ # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREFULLGCFREQUENCY #HotStoreFullGCFrequency = 20 + # HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC + # will run moving GC if disk utilization gets within a threshold (150 GB) of the target. + # Splitstore GC will NOT run moving GC if the total size of the move would get + # within 50 GB of the target, and instead will run a more aggressive online GC. + # If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore + # GC will trigger moving GC if either configuration condition is met. + # A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer. + # At this minimum size moving GC happens every time, any smaller and moving GC won't + # be able to run. In spring 2023 this minimum is ~550 GB. + # + # type: uint64 + # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACETARGET + #HotStoreMaxSpaceTarget = 0 + + # When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size + # exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold + # + # type: uint64 + # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACETHRESHOLD + #HotStoreMaxSpaceThreshold = 150000000000 + + # Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget + # is set. Moving GC will not occur when total moving size exceeds + # HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer + # + # type: uint64 + # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACESAFETYBUFFER + #HotstoreMaxSpaceSafetyBuffer = 50000000000 + [Cluster] # EXPERIMENTAL. config to enabled node cluster with raft consensus diff --git a/gateway/node.go b/gateway/node.go index 90a6812b5..a97778eac 100644 --- a/gateway/node.go +++ b/gateway/node.go @@ -27,6 +27,7 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) const ( @@ -59,18 +60,22 @@ type TargetAPI interface { ChainPutObj(context.Context, blocks.Block) error ChainGetGenesis(context.Context) (*types.TipSet, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error) MsigGetPending(ctx context.Context, addr address.Address, ts types.TipSetKey) ([]*api.MsigTransaction, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) + StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) + StateNetworkName(context.Context) (dtypes.NetworkName, error) StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) diff --git a/gateway/proxy_fil.go b/gateway/proxy_fil.go index 1f6ee2ccc..bf6c2dff8 100644 --- a/gateway/proxy_fil.go +++ b/gateway/proxy_fil.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) func (gw *Node) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) { @@ -187,6 +188,13 @@ func (gw *Node) GasEstimateMessageGas(ctx context.Context, msg *types.Message, s return gw.target.GasEstimateMessageGas(ctx, msg, spec, tsk) } +func (gw *Node) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return 0, err + } + return gw.target.MpoolGetNonce(ctx, addr) +} + func (gw *Node) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return cid.Cid{}, err @@ -248,6 +256,16 @@ func (gw *Node) StateAccountKey(ctx context.Context, addr address.Address, tsk t return gw.target.StateAccountKey(ctx, addr, tsk) } +func (gw *Node) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return nil, err + } + if err := gw.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return gw.target.StateCall(ctx, msg, tsk) +} + func (gw *Node) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return api.DealCollateralBounds{}, err @@ -258,6 +276,16 @@ func (gw *Node) StateDealProviderCollateralBounds(ctx context.Context, size abi. return gw.target.StateDealProviderCollateralBounds(ctx, size, verified, tsk) } +func (gw *Node) StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return nil, err + } + if err := gw.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return gw.target.StateDecodeParams(ctx, toAddr, method, params, tsk) +} + func (gw *Node) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return nil, err @@ -308,6 +336,13 @@ func (gw *Node) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, t return gw.target.StateMarketStorageDeal(ctx, dealId, tsk) } +func (gw *Node) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) { + if err := gw.limit(ctx, stateRateLimitTokens); err != nil { + return *new(dtypes.NetworkName), err + } + return gw.target.StateNetworkName(ctx) +} + func (gw *Node) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { if err := gw.limit(ctx, stateRateLimitTokens); err != nil { return network.VersionMax, err diff --git a/itests/eth_block_hash_test.go b/itests/eth_block_hash_test.go index ac6506bb2..db21375c5 100644 --- a/itests/eth_block_hash_test.go +++ b/itests/eth_block_hash_test.go @@ -3,6 +3,7 @@ package itests import ( "context" "fmt" + "strings" "testing" "time" @@ -48,13 +49,14 @@ func TestEthBlockHashesCorrect_MultiBlockTipset(t *testing.T) { // let the chain run a little bit longer to minimise the chance of reorgs n2.WaitTillChain(ctx, kit.HeightAtLeast(head.Height()+50)) - head, err = n2.ChainHead(context.Background()) - require.NoError(t, err) - for i := 1; i <= int(head.Height()); i++ { hex := fmt.Sprintf("0x%x", i) ethBlockA, err := n2.EthGetBlockByNumber(ctx, hex, true) + // Cannot use static ErrFullRound error for comparison since it gets reserialized as a JSON RPC error. + if err != nil && strings.Contains(err.Error(), "null round") { + continue + } require.NoError(t, err) ethBlockB, err := n2.EthGetBlockByHash(ctx, ethBlockA.Hash, true) diff --git a/itests/eth_conformance_test.go b/itests/eth_conformance_test.go index 8a367d6b1..4d8f5c3dd 100644 --- a/itests/eth_conformance_test.go +++ b/itests/eth_conformance_test.go @@ -236,14 +236,6 @@ func TestEthOpenRPCConformance(t *testing.T) { skipReason: "earliest block is not supported", }, - { - method: "eth_getBlockByNumber", - variant: "pending", - call: func(a *ethAPIRaw) (json.RawMessage, error) { - return ethapi.EthGetBlockByNumber(context.Background(), "pending", true) - }, - }, - { method: "eth_getBlockByNumber", call: func(a *ethAPIRaw) (json.RawMessage, error) { diff --git a/itests/eth_fee_history_test.go b/itests/eth_fee_history_test.go index 72302f298..a792c7f0e 100644 --- a/itests/eth_fee_history_test.go +++ b/itests/eth_fee_history_test.go @@ -3,18 +3,42 @@ package itests import ( "context" "encoding/json" + "sort" "testing" "time" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/lib/result" + "github.com/filecoin-project/lotus/node/impl/full" ) +// calculateExpectations calculates the expected number of items to be included in the response +// of eth_feeHistory. It takes care of null rounds by finding the closet tipset with height +// smaller than startHeight, and then looks back at requestAmount of items. It also considers +// scenarios where there are not enough items to look back. +func calculateExpectations(tsHeights []int, requestAmount, startHeight int) (count, oldestHeight int) { + latestIdx := sort.SearchInts(tsHeights, startHeight) + // SearchInts returns the index of the number that's larger than the target if the target + // doesn't exist. However, we're looking for the closet number that's smaller that the target + for tsHeights[latestIdx] > startHeight { + latestIdx-- + } + cnt := requestAmount + oldestIdx := latestIdx - requestAmount + 1 + if oldestIdx < 0 { + cnt = latestIdx + 1 + oldestIdx = 0 + } + return cnt, tsHeights[oldestIdx] +} + func TestEthFeeHistory(t *testing.T) { require := require.New(t) @@ -22,70 +46,136 @@ func TestEthFeeHistory(t *testing.T) { blockTime := 100 * time.Millisecond client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC()) - ens.InterconnectAll().BeginMining(blockTime) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - // Wait for the network to create 20 blocks + heads, err := client.ChainNotify(ctx) + require.NoError(err) + + // Save the full view of the tipsets to calculate the answer when there are null rounds + tsHeights := []int{1} + go func() { + for chg := range heads { + for _, c := range chg { + tsHeights = append(tsHeights, int(c.Val.Height())) + } + } + }() + + miner := ens.InterconnectAll().BeginMining(blockTime) + + client.WaitTillChain(ctx, kit.HeightAtLeast(7)) + miner[0].InjectNulls(abi.ChainEpoch(5)) + + // Wait for the network to create at least 20 tipsets client.WaitTillChain(ctx, kit.HeightAtLeast(20)) + for _, m := range miner { + m.Pause() + } + + ch, err := client.ChainNotify(ctx) + require.NoError(err) + + // Wait for 5 seconds of inactivity + func() { + for { + select { + case <-ch: + continue + case <-time.After(5 * time.Second): + return + } + } + }() + + sort.Ints(tsHeights) + + // because of the deferred execution, the last tipset is not executed yet, + // and the one before the last one is the last executed tipset, + // which corresponds to the "latest" tag in EthGetBlockByNumber + latestBlk := ethtypes.EthUint64(tsHeights[len(tsHeights)-2]) + blk, err := client.EthGetBlockByNumber(ctx, "latest", false) + require.NoError(err) + require.Equal(blk.Number, latestBlk) + + assertHistory := func(history *ethtypes.EthFeeHistory, requestAmount, startHeight int) { + amount, oldest := calculateExpectations(tsHeights, requestAmount, startHeight) + require.Equal(amount+1, len(history.BaseFeePerGas)) + require.Equal(amount, len(history.GasUsedRatio)) + require.Equal(ethtypes.EthUint64(oldest), history.OldestBlock) + } history, err := client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "0x10"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock) + assertHistory(&history, 5, 16) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{"5", "0x10"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock) + assertHistory(&history, 5, 16) + require.Nil(history.Reward) + + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( + json.Marshal([]interface{}{5, "latest"}), + ).Assert(require.NoError)) + require.NoError(err) + assertHistory(&history, 5, int(latestBlk)) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{"0x10", "0x12"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(17, len(history.BaseFeePerGas)) - require.Equal(16, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(18-16+1), history.OldestBlock) + assertHistory(&history, 16, 18) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "0x10"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock) + assertHistory(&history, 5, 16) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "10"}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(10-5+1), history.OldestBlock) + assertHistory(&history, 5, 10) + require.Nil(history.Reward) + + // test when the requested number of blocks is longer than chain length + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( + json.Marshal([]interface{}{"0x30", "latest"}), + ).Assert(require.NoError)) + require.NoError(err) + assertHistory(&history, 48, int(latestBlk)) + require.Nil(history.Reward) + + // test when the requested number of blocks is longer than chain length + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( + json.Marshal([]interface{}{"0x30", "10"}), + ).Assert(require.NoError)) + require.NoError(err) + assertHistory(&history, 48, 10) require.Nil(history.Reward) history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "10", &[]float64{25, 50, 75}}), ).Assert(require.NoError)) require.NoError(err) - require.Equal(6, len(history.BaseFeePerGas)) - require.Equal(5, len(history.GasUsedRatio)) - require.Equal(ethtypes.EthUint64(10-5+1), history.OldestBlock) + assertHistory(&history, 5, 10) require.NotNil(history.Reward) require.Equal(5, len(*history.Reward)) for _, arr := range *history.Reward { require.Equal(3, len(arr)) + for _, item := range arr { + require.Equal(ethtypes.EthBigInt(types.NewInt(full.MinGasPremium)), item) + } } history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( @@ -93,6 +183,11 @@ func TestEthFeeHistory(t *testing.T) { ).Assert(require.NoError)) require.Error(err) + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( + json.Marshal([]interface{}{5, "10", &[]float64{75, 50}}), + ).Assert(require.NoError)) + require.Error(err) + history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams]( json.Marshal([]interface{}{5, "10", &[]float64{}}), ).Assert(require.NoError)) diff --git a/itests/eth_transactions_test.go b/itests/eth_transactions_test.go index 9afeb7bd5..8d0df0433 100644 --- a/itests/eth_transactions_test.go +++ b/itests/eth_transactions_test.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" "github.com/filecoin-project/lotus/itests/kit" @@ -270,6 +271,57 @@ func TestContractInvocation(t *testing.T) { require.EqualValues(t, ethtypes.EthUint64(0x1), receipt.Status) } +func TestGetBlockByNumber(t *testing.T) { + blockTime := 100 * time.Millisecond + client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC()) + + bms := ens.InterconnectAll().BeginMining(blockTime) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + // create a new Ethereum account + _, ethAddr, filAddr := client.EVM().NewAccount() + // send some funds to the f410 address + kit.SendFunds(ctx, t, client, filAddr, types.FromFil(10)) + + latest, err := client.EthBlockNumber(ctx) + require.NoError(t, err) + + // can get the latest block + _, err = client.EthGetBlockByNumber(ctx, latest.Hex(), true) + require.NoError(t, err) + + // fail to get a future block + _, err = client.EthGetBlockByNumber(ctx, (latest + 10000).Hex(), true) + require.Error(t, err) + + // inject 10 null rounds + bms[0].InjectNulls(10) + + // wait until we produce blocks again + tctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + ch, err := client.ChainNotify(tctx) + require.NoError(t, err) + <-ch // current + hc := <-ch // wait for next block + require.Equal(t, store.HCApply, hc[0].Type) + + afterNullHeight := hc[0].Val.Height() + + // Fail when trying to fetch a null round. + _, err = client.EthGetBlockByNumber(ctx, (ethtypes.EthUint64(afterNullHeight - 1)).Hex(), true) + require.Error(t, err) + + // Fetch balance on a null round; should not fail and should return previous balance. + // Should be lower than original balance. + bal, err := client.EthGetBalance(ctx, ethAddr, (ethtypes.EthUint64(afterNullHeight - 1)).Hex()) + require.NoError(t, err) + require.NotEqual(t, big.Zero(), bal) + require.Equal(t, types.FromFil(10).Int, bal.Int) +} + func deployContractTx(ctx context.Context, client *kit.TestFullNode, ethAddr ethtypes.EthAddress, contract []byte) (*ethtypes.EthTxArgs, error) { gaslimit, err := client.EthEstimateGas(ctx, ethtypes.EthCall{ From: ðAddr, diff --git a/lib/rpcenc/reader.go b/lib/rpcenc/reader.go index 64dcc4b49..34b9fcfb4 100644 --- a/lib/rpcenc/reader.go +++ b/lib/rpcenc/reader.go @@ -221,7 +221,7 @@ type RpcReader struct { res chan readRes beginOnce *sync.Once - closeOnce sync.Once + closeOnce *sync.Once } var ErrHasBody = errors.New("RPCReader has body, either already read from or from a client with no redirect support") @@ -265,6 +265,7 @@ func (w *RpcReader) beginPost() { w.postBody = nr.postBody w.res = nr.res w.beginOnce = nr.beginOnce + w.closeOnce = nr.closeOnce } } @@ -355,6 +356,7 @@ func ReaderParamDecoder() (http.HandlerFunc, jsonrpc.ServerOption) { res: make(chan readRes), next: ch, beginOnce: &sync.Once{}, + closeOnce: &sync.Once{}, } switch req.Method { diff --git a/lib/rpcenc/reader_test.go b/lib/rpcenc/reader_test.go index da436068f..3a554a0ca 100644 --- a/lib/rpcenc/reader_test.go +++ b/lib/rpcenc/reader_test.go @@ -3,8 +3,8 @@ package rpcenc import ( "context" + "fmt" "io" - "io/ioutil" "net/http/httptest" "strings" "sync" @@ -77,7 +77,12 @@ func (h *ReaderHandler) CloseReader(ctx context.Context, r io.Reader) error { } func (h *ReaderHandler) ReadAll(ctx context.Context, r io.Reader) ([]byte, error) { - return ioutil.ReadAll(r) + b, err := io.ReadAll(r) + if err != nil { + return nil, xerrors.Errorf("readall: %w", err) + } + + return b, nil } func (h *ReaderHandler) ReadNullLen(ctx context.Context, r io.Reader) (int64, error) { @@ -219,9 +224,15 @@ func TestReaderRedirect(t *testing.T) { } func TestReaderRedirectDrop(t *testing.T) { + for i := 0; i < 10; i++ { + t.Run(fmt.Sprintf("test %d", i), testReaderRedirectDrop) + } +} + +func testReaderRedirectDrop(t *testing.T) { // lower timeout so that the dangling connection between client and reader is dropped quickly // after the test. Otherwise httptest.Close is blocked. - Timeout = 200 * time.Millisecond + Timeout = 90 * time.Millisecond var allClient struct { ReadAll func(ctx context.Context, r io.Reader) ([]byte, error) @@ -294,6 +305,8 @@ func TestReaderRedirectDrop(t *testing.T) { done.Wait() + fmt.Println("---------------------") + // Redir client drops before subcall done.Add(1) @@ -322,5 +335,9 @@ func TestReaderRedirectDrop(t *testing.T) { // wait for subcall to finish <-contCh - require.ErrorContains(t, allServerHandler.subErr, "decoding params for 'ReaderHandler.ReadAll' (param: 0; custom decoder): context canceled") + estr := allServerHandler.subErr.Error() + + require.True(t, + strings.Contains(estr, "decoding params for 'ReaderHandler.ReadAll' (param: 0; custom decoder): context canceled") || + strings.Contains(estr, "readall: unexpected EOF"), "unexpected error: %s", estr) } diff --git a/node/config/def.go b/node/config/def.go index f50e9575a..2617ec5ba 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -89,13 +89,15 @@ func DefaultFullNode() *FullNode { SimultaneousTransfersForRetrieval: DefaultSimultaneousTransfers, }, Chainstore: Chainstore{ - EnableSplitstore: false, + EnableSplitstore: true, Splitstore: Splitstore{ - ColdStoreType: "messages", + ColdStoreType: "discard", HotStoreType: "badger", MarkSetType: "badger", - HotStoreFullGCFrequency: 20, + HotStoreFullGCFrequency: 20, + HotStoreMaxSpaceThreshold: 150_000_000_000, + HotstoreMaxSpaceSafetyBuffer: 50_000_000_000, }, }, Cluster: *DefaultUserRaftConfig(), diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 8b79bed4f..c62084708 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -1286,6 +1286,35 @@ the compaction boundary; default is 0.`, A value of 0 disables, while a value 1 will do full GC in every compaction. Default is 20 (about once a week).`, }, + { + Name: "HotStoreMaxSpaceTarget", + Type: "uint64", + + Comment: `HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC +will run moving GC if disk utilization gets within a threshold (150 GB) of the target. +Splitstore GC will NOT run moving GC if the total size of the move would get +within 50 GB of the target, and instead will run a more aggressive online GC. +If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore +GC will trigger moving GC if either configuration condition is met. +A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer. +At this minimum size moving GC happens every time, any smaller and moving GC won't +be able to run. In spring 2023 this minimum is ~550 GB.`, + }, + { + Name: "HotStoreMaxSpaceThreshold", + Type: "uint64", + + Comment: `When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size +exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold`, + }, + { + Name: "HotstoreMaxSpaceSafetyBuffer", + Type: "uint64", + + Comment: `Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget +is set. Moving GC will not occur when total moving size exceeds +HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`, + }, }, "StorageMiner": []DocField{ { diff --git a/node/config/types.go b/node/config/types.go index 690e8caee..5b952d35e 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -601,6 +601,25 @@ type Splitstore struct { // A value of 0 disables, while a value 1 will do full GC in every compaction. // Default is 20 (about once a week). HotStoreFullGCFrequency uint64 + // HotStoreMaxSpaceTarget sets a target max disk size for the hotstore. Splitstore GC + // will run moving GC if disk utilization gets within a threshold (150 GB) of the target. + // Splitstore GC will NOT run moving GC if the total size of the move would get + // within 50 GB of the target, and instead will run a more aggressive online GC. + // If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore + // GC will trigger moving GC if either configuration condition is met. + // A reasonable minimum is 2x fully GCed hotstore size + 50 G buffer. + // At this minimum size moving GC happens every time, any smaller and moving GC won't + // be able to run. In spring 2023 this minimum is ~550 GB. + HotStoreMaxSpaceTarget uint64 + + // When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size + // exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold + HotStoreMaxSpaceThreshold uint64 + + // Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget + // is set. Moving GC will not occur when total moving size exceeds + // HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer + HotstoreMaxSpaceSafetyBuffer uint64 } // // Full Node diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 465b680ef..d0f36ce48 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -193,25 +193,14 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([] return nil, nil } - // TODO: need to get the number of messages better than this - pts, err := a.Chain.LoadTipSet(ctx, types.NewTipSetKey(b.Parents...)) + receipts, err := a.Chain.ReadReceipts(ctx, b.ParentMessageReceipts) if err != nil { return nil, err } - cm, err := a.Chain.MessagesForTipset(ctx, pts) - if err != nil { - return nil, err - } - - var out []*types.MessageReceipt - for i := 0; i < len(cm); i++ { - r, err := a.Chain.GetParentReceipt(ctx, b, i) - if err != nil { - return nil, err - } - - out = append(out, r) + out := make([]*types.MessageReceipt, len(receipts)) + for i := range receipts { + out[i] = &receipts[i] } return out, nil diff --git a/node/impl/full/eth.go b/node/impl/full/eth.go index 721439502..a1652da28 100644 --- a/node/impl/full/eth.go +++ b/node/impl/full/eth.go @@ -153,6 +153,8 @@ type EthAPI struct { EthEventAPI } +var ErrNullRound = errors.New("requested epoch was a null round") + func (a *EthModule) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) { return stmgr.GetNetworkName(ctx, a.StateManager, a.Chain.GetHeaviestTipSet().ParentState()) } @@ -231,15 +233,14 @@ func (a *EthModule) EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthH return newEthBlockFromFilecoinTipSet(ctx, ts, fullTxInfo, a.Chain, a.StateAPI) } -func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string) (tipset *types.TipSet, err error) { - if blkParam == "earliest" { - return nil, fmt.Errorf("block param \"earliest\" is not supported") +func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string, strict bool) (tipset *types.TipSet, err error) { + switch blkParam { + case "earliest", "pending": + return nil, fmt.Errorf("block param %q is not supported", blkParam) } head := a.Chain.GetHeaviestTipSet() switch blkParam { - case "pending": - return head, nil case "latest": parent, err := a.Chain.GetTipSetFromKey(ctx, head.Parents()) if err != nil { @@ -252,16 +253,22 @@ func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string) (tipset if err != nil { return nil, fmt.Errorf("cannot parse block number: %v", err) } - ts, err := a.Chain.GetTipsetByHeight(ctx, abi.ChainEpoch(num), nil, false) + if abi.ChainEpoch(num) > head.Height()-1 { + return nil, fmt.Errorf("requested a future epoch (beyond 'latest')") + } + ts, err := a.ChainAPI.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(num), head.Key()) if err != nil { return nil, fmt.Errorf("cannot get tipset at height: %v", num) } + if strict && ts.Height() != abi.ChainEpoch(num) { + return nil, ErrNullRound + } return ts, nil } } func (a *EthModule) EthGetBlockByNumber(ctx context.Context, blkParam string, fullTxInfo bool) (ethtypes.EthBlock, error) { - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, true) if err != nil { return ethtypes.EthBlock{}, err } @@ -367,7 +374,7 @@ func (a *EthModule) EthGetTransactionCount(ctx context.Context, sender ethtypes. return ethtypes.EthUint64(0), nil } - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return ethtypes.EthUint64(0), xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -433,7 +440,7 @@ func (a *EthModule) EthGetTransactionReceipt(ctx context.Context, txHash ethtype } } - receipt, err := newEthTxReceipt(ctx, tx, msgLookup, replay, events, a.StateAPI) + receipt, err := newEthTxReceipt(ctx, tx, replay, events, a.StateAPI) if err != nil { return nil, nil } @@ -456,7 +463,7 @@ func (a *EthModule) EthGetCode(ctx context.Context, ethAddr ethtypes.EthAddress, return nil, xerrors.Errorf("cannot get Filecoin address: %w", err) } - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return nil, xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -535,7 +542,7 @@ func (a *EthModule) EthGetCode(ctx context.Context, ethAddr ethtypes.EthAddress, } func (a *EthModule) EthGetStorageAt(ctx context.Context, ethAddr ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error) { - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return nil, xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -631,7 +638,7 @@ func (a *EthModule) EthGetBalance(ctx context.Context, address ethtypes.EthAddre return ethtypes.EthBigInt{}, err } - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return ethtypes.EthBigInt{}, xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -676,57 +683,48 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth } } - ts, err := a.parseBlkParam(ctx, params.NewestBlkNum) + ts, err := a.parseBlkParam(ctx, params.NewestBlkNum, false) if err != nil { return ethtypes.EthFeeHistory{}, fmt.Errorf("bad block parameter %s: %s", params.NewestBlkNum, err) } - // Deal with the case that the chain is shorter than the number of requested blocks. - oldestBlkHeight := uint64(1) - if abi.ChainEpoch(params.BlkCount) <= ts.Height() { - oldestBlkHeight = uint64(ts.Height()) - uint64(params.BlkCount) + 1 - } + var ( + basefee = ts.Blocks()[0].ParentBaseFee + oldestBlkHeight = uint64(1) - // NOTE: baseFeePerGas should include the next block after the newest of the returned range, - // because the next base fee can be inferred from the messages in the newest block. - // However, this is NOT the case in Filecoin due to deferred execution, so the best - // we can do is duplicate the last value. - baseFeeArray := []ethtypes.EthBigInt{ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee)} - gasUsedRatioArray := []float64{} - rewardsArray := make([][]ethtypes.EthBigInt, 0) + // NOTE: baseFeePerGas should include the next block after the newest of the returned range, + // because the next base fee can be inferred from the messages in the newest block. + // However, this is NOT the case in Filecoin due to deferred execution, so the best + // we can do is duplicate the last value. + baseFeeArray = []ethtypes.EthBigInt{ethtypes.EthBigInt(basefee)} + rewardsArray = make([][]ethtypes.EthBigInt, 0) + gasUsedRatioArray = []float64{} + blocksIncluded int + ) - for ts.Height() >= abi.ChainEpoch(oldestBlkHeight) { - // Unfortunately we need to rebuild the full message view so we can - // totalize gas used in the tipset. - msgs, err := a.Chain.MessagesForTipset(ctx, ts) + for blocksIncluded < int(params.BlkCount) && ts.Height() > 0 { + msgs, rcpts, err := messagesAndReceipts(ctx, ts, a.Chain, a.StateAPI) if err != nil { - return ethtypes.EthFeeHistory{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err) + return ethtypes.EthFeeHistory{}, xerrors.Errorf("failed to retrieve messages and receipts for height %d: %w", ts.Height(), err) } txGasRewards := gasRewardSorter{} - for txIdx, msg := range msgs { - msgLookup, err := a.StateAPI.StateSearchMsg(ctx, types.EmptyTSK, msg.Cid(), api.LookbackNoLimit, false) - if err != nil || msgLookup == nil { - return ethtypes.EthFeeHistory{}, nil - } - - tx, err := newEthTxFromMessageLookup(ctx, msgLookup, txIdx, a.Chain, a.StateAPI) - if err != nil { - return ethtypes.EthFeeHistory{}, nil - } - + for i, msg := range msgs { + effectivePremium := msg.VMMessage().EffectiveGasPremium(basefee) txGasRewards = append(txGasRewards, gasRewardTuple{ - reward: tx.Reward(ts.Blocks()[0].ParentBaseFee), - gas: uint64(msgLookup.Receipt.GasUsed), + premium: effectivePremium, + gasUsed: rcpts[i].GasUsed, }) } rewards, totalGasUsed := calculateRewardsAndGasUsed(rewardPercentiles, txGasRewards) // arrays should be reversed at the end - baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee)) + baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(basefee)) gasUsedRatioArray = append(gasUsedRatioArray, float64(totalGasUsed)/float64(build.BlockGasLimit)) rewardsArray = append(rewardsArray, rewards) + oldestBlkHeight = uint64(ts.Height()) + blocksIncluded++ parentTsKey := ts.Parents() ts, err = a.Chain.LoadTipSet(ctx, parentTsKey) @@ -1066,7 +1064,7 @@ func (a *EthModule) EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam s return nil, xerrors.Errorf("failed to convert ethcall to filecoin message: %w", err) } - ts, err := a.parseBlkParam(ctx, blkParam) + ts, err := a.parseBlkParam(ctx, blkParam, false) if err != nil { return nil, xerrors.Errorf("cannot parse block param: %s", blkParam) } @@ -1783,37 +1781,37 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx return ethtypes.EthBlock{}, err } - msgs, err := cs.MessagesForTipset(ctx, ts) + msgs, rcpts, err := messagesAndReceipts(ctx, ts, cs, sa) if err != nil { - return ethtypes.EthBlock{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err) + return ethtypes.EthBlock{}, xerrors.Errorf("failed to retrieve messages and receipts: %w", err) } block := ethtypes.NewEthBlock(len(msgs) > 0) gasUsed := int64(0) - compOutput, err := sa.StateCompute(ctx, ts.Height(), nil, ts.Key()) - if err != nil { - return ethtypes.EthBlock{}, xerrors.Errorf("failed to compute state: %w", err) - } - - for txIdx, msg := range compOutput.Trace { - // skip system messages like reward application and cron - if msg.Msg.From == builtintypes.SystemActorAddr { - continue + for i, msg := range msgs { + rcpt := rcpts[i] + ti := ethtypes.EthUint64(i) + gasUsed += rcpt.GasUsed + var smsg *types.SignedMessage + switch msg := msg.(type) { + case *types.SignedMessage: + smsg = msg + case *types.Message: + smsg = &types.SignedMessage{ + Message: *msg, + Signature: crypto.Signature{ + Type: crypto.SigTypeBLS, + }, + } + default: + return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.Cid(), err) } - - gasUsed += msg.MsgRct.GasUsed - smsgCid, err := getSignedMessage(ctx, cs, msg.MsgCid) - if err != nil { - return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.MsgCid, err) - } - tx, err := newEthTxFromSignedMessage(ctx, smsgCid, sa) + tx, err := newEthTxFromSignedMessage(ctx, smsg, sa) if err != nil { return ethtypes.EthBlock{}, xerrors.Errorf("failed to convert msg to ethTx: %w", err) } - ti := ethtypes.EthUint64(txIdx) - tx.ChainID = ethtypes.EthUint64(build.Eip155ChainId) tx.BlockHash = &blkHash tx.BlockNumber = &bn @@ -1835,6 +1833,29 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx return block, nil } +func messagesAndReceipts(ctx context.Context, ts *types.TipSet, cs *store.ChainStore, sa StateAPI) ([]types.ChainMsg, []types.MessageReceipt, error) { + msgs, err := cs.MessagesForTipset(ctx, ts) + if err != nil { + return nil, nil, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err) + } + + _, rcptRoot, err := sa.StateManager.TipSetState(ctx, ts) + if err != nil { + return nil, nil, xerrors.Errorf("failed to compute state: %w", err) + } + + rcpts, err := cs.ReadReceipts(ctx, rcptRoot) + if err != nil { + return nil, nil, xerrors.Errorf("error loading receipts for tipset: %v: %w", ts, err) + } + + if len(msgs) != len(rcpts) { + return nil, nil, xerrors.Errorf("receipts and message array lengths didn't match for tipset: %v: %w", ts, err) + } + + return msgs, rcpts, nil +} + // lookupEthAddress makes its best effort at finding the Ethereum address for a // Filecoin address. It does the following: // @@ -2030,7 +2051,7 @@ func newEthTxFromMessageLookup(ctx context.Context, msgLookup *api.MsgLookup, tx return tx, nil } -func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLookup, replay *api.InvocResult, events []types.Event, sa StateAPI) (api.EthTxReceipt, error) { +func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, replay *api.InvocResult, events []types.Event, sa StateAPI) (api.EthTxReceipt, error) { var ( transactionIndex ethtypes.EthUint64 blockHash ethtypes.EthHash @@ -2059,25 +2080,25 @@ func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLook LogsBloom: ethtypes.EmptyEthBloom[:], } - if lookup.Receipt.ExitCode.IsSuccess() { + if replay.MsgRct.ExitCode.IsSuccess() { receipt.Status = 1 } - if lookup.Receipt.ExitCode.IsError() { + if replay.MsgRct.ExitCode.IsError() { receipt.Status = 0 } - receipt.GasUsed = ethtypes.EthUint64(lookup.Receipt.GasUsed) + receipt.GasUsed = ethtypes.EthUint64(replay.MsgRct.GasUsed) // TODO: handle CumulativeGasUsed receipt.CumulativeGasUsed = ethtypes.EmptyEthInt - effectiveGasPrice := big.Div(replay.GasCost.TotalCost, big.NewInt(lookup.Receipt.GasUsed)) + effectiveGasPrice := big.Div(replay.GasCost.TotalCost, big.NewInt(replay.MsgRct.GasUsed)) receipt.EffectiveGasPrice = ethtypes.EthBigInt(effectiveGasPrice) - if receipt.To == nil && lookup.Receipt.ExitCode.IsSuccess() { + if receipt.To == nil && replay.MsgRct.ExitCode.IsSuccess() { // Create and Create2 return the same things. var ret eam.CreateExternalReturn - if err := ret.UnmarshalCBOR(bytes.NewReader(lookup.Receipt.Return)); err != nil { + if err := ret.UnmarshalCBOR(bytes.NewReader(replay.MsgRct.Return)); err != nil { return api.EthTxReceipt{}, xerrors.Errorf("failed to parse contract creation result: %w", err) } addr := ethtypes.EthAddress(ret.EthAddress) @@ -2335,35 +2356,35 @@ func parseEthRevert(ret []byte) string { return ethtypes.EthBytes(cbytes).String() } -func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, uint64) { - var totalGasUsed uint64 +func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, int64) { + var gasUsedTotal int64 for _, tx := range txGasRewards { - totalGasUsed += tx.gas + gasUsedTotal += tx.gasUsed } rewards := make([]ethtypes.EthBigInt, len(rewardPercentiles)) for i := range rewards { - rewards[i] = ethtypes.EthBigIntZero + rewards[i] = ethtypes.EthBigInt(types.NewInt(MinGasPremium)) } if len(txGasRewards) == 0 { - return rewards, totalGasUsed + return rewards, gasUsedTotal } sort.Stable(txGasRewards) var idx int - var sum uint64 + var sum int64 for i, percentile := range rewardPercentiles { - threshold := uint64(float64(totalGasUsed) * percentile / 100) + threshold := int64(float64(gasUsedTotal) * percentile / 100) for sum < threshold && idx < len(txGasRewards)-1 { - sum += txGasRewards[idx].gas + sum += txGasRewards[idx].gasUsed idx++ } - rewards[i] = txGasRewards[idx].reward + rewards[i] = ethtypes.EthBigInt(txGasRewards[idx].premium) } - return rewards, totalGasUsed + return rewards, gasUsedTotal } func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid) (*types.SignedMessage, error) { @@ -2386,8 +2407,8 @@ func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid) } type gasRewardTuple struct { - gas uint64 - reward ethtypes.EthBigInt + gasUsed int64 + premium abi.TokenAmount } // sorted in ascending order @@ -2398,5 +2419,5 @@ func (g gasRewardSorter) Swap(i, j int) { g[i], g[j] = g[j], g[i] } func (g gasRewardSorter) Less(i, j int) bool { - return g[i].reward.Int.Cmp(g[j].reward.Int) == -1 + return g[i].premium.Int.Cmp(g[j].premium.Int) == -1 } diff --git a/node/impl/full/eth_test.go b/node/impl/full/eth_test.go index 67a8b0500..87c0852fb 100644 --- a/node/impl/full/eth_test.go +++ b/node/impl/full/eth_test.go @@ -117,11 +117,8 @@ func TestReward(t *testing.T) { {maxFeePerGas: big.NewInt(50), maxPriorityFeePerGas: big.NewInt(200), answer: big.NewInt(-50)}, } for _, tc := range testcases { - tx := ethtypes.EthTx{ - MaxFeePerGas: ethtypes.EthBigInt(tc.maxFeePerGas), - MaxPriorityFeePerGas: ethtypes.EthBigInt(tc.maxPriorityFeePerGas), - } - reward := tx.Reward(baseFee) + msg := &types.Message{GasFeeCap: tc.maxFeePerGas, GasPremium: tc.maxPriorityFeePerGas} + reward := msg.EffectiveGasPremium(baseFee) require.Equal(t, 0, reward.Int.Cmp(tc.answer.Int), reward, tc.answer) } } @@ -135,25 +132,25 @@ func TestRewardPercentiles(t *testing.T) { { percentiles: []float64{25, 50, 75}, txGasRewards: []gasRewardTuple{}, - answer: []int64{0, 0, 0}, + answer: []int64{MinGasPremium, MinGasPremium, MinGasPremium}, }, { percentiles: []float64{25, 50, 75, 100}, txGasRewards: []gasRewardTuple{ - {gas: uint64(0), reward: ethtypes.EthBigInt(big.NewInt(300))}, - {gas: uint64(100), reward: ethtypes.EthBigInt(big.NewInt(200))}, - {gas: uint64(350), reward: ethtypes.EthBigInt(big.NewInt(100))}, - {gas: uint64(500), reward: ethtypes.EthBigInt(big.NewInt(600))}, - {gas: uint64(300), reward: ethtypes.EthBigInt(big.NewInt(700))}, + {gasUsed: int64(0), premium: big.NewInt(300)}, + {gasUsed: int64(100), premium: big.NewInt(200)}, + {gasUsed: int64(350), premium: big.NewInt(100)}, + {gasUsed: int64(500), premium: big.NewInt(600)}, + {gasUsed: int64(300), premium: big.NewInt(700)}, }, answer: []int64{200, 700, 700, 700}, }, } for _, tc := range testcases { rewards, totalGasUsed := calculateRewardsAndGasUsed(tc.percentiles, tc.txGasRewards) - gasUsed := uint64(0) + var gasUsed int64 for _, tx := range tc.txGasRewards { - gasUsed += tx.gas + gasUsed += tx.gasUsed } ans := []ethtypes.EthBigInt{} for _, bi := range tc.answer { diff --git a/node/modules/blockstore.go b/node/modules/blockstore.go index 90b7b6183..f96fd0db4 100644 --- a/node/modules/blockstore.go +++ b/node/modules/blockstore.go @@ -82,11 +82,14 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked } cfg := &splitstore.Config{ - MarkSetType: cfg.Splitstore.MarkSetType, - DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard", - UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal", - HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention, - HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency, + MarkSetType: cfg.Splitstore.MarkSetType, + DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard", + UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal", + HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention, + HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency, + HotstoreMaxSpaceTarget: cfg.Splitstore.HotStoreMaxSpaceTarget, + HotstoreMaxSpaceThreshold: cfg.Splitstore.HotStoreMaxSpaceThreshold, + HotstoreMaxSpaceSafetyBuffer: cfg.Splitstore.HotstoreMaxSpaceSafetyBuffer, } ss, err := splitstore.Open(path, ds, hot, cold, cfg) if err != nil { diff --git a/node/modules/chain.go b/node/modules/chain.go index f304ab135..0c3bad2c7 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -123,7 +123,7 @@ func NetworkName(mctx helpers.MetricsCtx, ctx := helpers.LifecycleCtx(mctx, lc) - sm, err := stmgr.NewStateManager(cs, tsexec, syscalls, us, nil) + sm, err := stmgr.NewStateManager(cs, tsexec, syscalls, us, nil, nil) if err != nil { return "", err } diff --git a/node/modules/stmgr.go b/node/modules/stmgr.go index dd3d90f10..b8f6f4776 100644 --- a/node/modules/stmgr.go +++ b/node/modules/stmgr.go @@ -7,10 +7,11 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func StateManager(lc fx.Lifecycle, cs *store.ChainStore, exec stmgr.Executor, sys vm.SyscallBuilder, us stmgr.UpgradeSchedule, b beacon.Schedule) (*stmgr.StateManager, error) { - sm, err := stmgr.NewStateManager(cs, exec, sys, us, b) +func StateManager(lc fx.Lifecycle, cs *store.ChainStore, exec stmgr.Executor, sys vm.SyscallBuilder, us stmgr.UpgradeSchedule, b beacon.Schedule, metadataDs dtypes.MetadataDS) (*stmgr.StateManager, error) { + sm, err := stmgr.NewStateManager(cs, exec, sys, us, b, metadataDs) if err != nil { return nil, err } diff --git a/storage/pipeline/fsm.go b/storage/pipeline/fsm.go index 25fd6fcef..8ae18a9fd 100644 --- a/storage/pipeline/fsm.go +++ b/storage/pipeline/fsm.go @@ -6,8 +6,10 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/http" + "os" "reflect" "time" @@ -19,9 +21,15 @@ import ( "github.com/filecoin-project/lotus/api" ) +var errSectorRemoved = errors.New("sector removed") + func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) { next, processed, err := m.plan(events, user.(*SectorInfo)) if err != nil || next == nil { + if err == errSectorRemoved && os.Getenv("LOTUS_KEEP_REMOVED_FSM_ACTIVE") != "1" { + return nil, processed, statemachine.ErrTerminated + } + l := Log{ Timestamp: uint64(time.Now().Unix()), Message: fmt.Sprintf("state machine error: %s", err), @@ -601,7 +609,7 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta case Removing: return m.handleRemoving, processed, nil case Removed: - return nil, processed, nil + return nil, processed, errSectorRemoved case RemoveFailed: return m.handleRemoveFailed, processed, nil @@ -615,13 +623,14 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta // Fatal errors case UndefinedSectorState: log.Error("sector update with undefined state!") + return nil, processed, xerrors.Errorf("sector update with undefined state") case FailedUnrecoverable: log.Errorf("sector %d failed unrecoverably", state.SectorNumber) + return nil, processed, xerrors.Errorf("sector %d failed unrecoverably", state.SectorNumber) default: log.Errorf("unexpected sector update state: %s", state.State) + return nil, processed, xerrors.Errorf("unexpected sector update state: %s", state.State) } - - return nil, processed, nil } func (m *Sealing) onUpdateSector(ctx context.Context, state *SectorInfo) error { diff --git a/storage/sealer/manager.go b/storage/sealer/manager.go index 336664ca8..db5f9a589 100644 --- a/storage/sealer/manager.go +++ b/storage/sealer/manager.go @@ -289,14 +289,20 @@ func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.remoteHnd.ServeHTTP(w, r) } -func schedNop(context.Context, Worker) error { - return nil +var schedNop = PrepareAction{ + Action: func(ctx context.Context, w Worker) error { + return nil + }, + PrepType: sealtasks.TTNoop, } -func (m *Manager) schedFetch(sector storiface.SectorRef, ft storiface.SectorFileType, ptype storiface.PathType, am storiface.AcquireMode) func(context.Context, Worker) error { - return func(ctx context.Context, worker Worker) error { - _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, ft, ptype, am)) - return err +func (m *Manager) schedFetch(sector storiface.SectorRef, ft storiface.SectorFileType, ptype storiface.PathType, am storiface.AcquireMode) PrepareAction { + return PrepareAction{ + Action: func(ctx context.Context, worker Worker) error { + _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, ft, ptype, am)) + return err + }, + PrepType: sealtasks.TTFetch, } } @@ -315,16 +321,19 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storiface.Secto // if the selected worker does NOT have the sealed files for the sector, instruct it to fetch it from a worker that has them and // put it in the sealing scratch space. - sealFetch := func(ctx context.Context, worker Worker) error { - log.Debugf("copy sealed/cache sector data for sector %d", sector.ID) - _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy)) - _, err2 := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy)) + sealFetch := PrepareAction{ + Action: func(ctx context.Context, worker Worker) error { + log.Debugf("copy sealed/cache sector data for sector %d", sector.ID) + _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy)) + _, err2 := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy)) - if err != nil && err2 != nil { - return xerrors.Errorf("cannot unseal piece. error fetching sealed data: %w. error fetching replica data: %w", err, err2) - } + if err != nil && err2 != nil { + return xerrors.Errorf("cannot unseal piece. error fetching sealed data: %w. error fetching replica data: %w", err, err2) + } - return nil + return nil + }, + PrepType: sealtasks.TTFetch, } if unsealed == nil { diff --git a/storage/sealer/sched.go b/storage/sealer/sched.go index c2b7d6a2d..c0ac11bcf 100644 --- a/storage/sealer/sched.go +++ b/storage/sealer/sched.go @@ -42,6 +42,10 @@ func WithPriority(ctx context.Context, priority int) context.Context { const mib = 1 << 20 type WorkerAction func(ctx context.Context, w Worker) error +type PrepareAction struct { + Action WorkerAction + PrepType sealtasks.TaskType +} type SchedWorker interface { TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) @@ -130,7 +134,7 @@ type WorkerRequest struct { Sel WorkerSelector SchedId uuid.UUID - prepare WorkerAction + prepare PrepareAction work WorkerAction start time.Time @@ -157,7 +161,15 @@ func newScheduler(ctx context.Context, assigner string) (*Scheduler, error) { case "", "utilization": a = NewLowestUtilizationAssigner() case "spread": - a = NewSpreadAssigner() + a = NewSpreadAssigner(false) + case "experiment-spread-qcount": + a = NewSpreadAssigner(true) + case "experiment-spread-tasks": + a = NewSpreadTasksAssigner(false) + case "experiment-spread-tasks-qcount": + a = NewSpreadTasksAssigner(true) + case "experiment-random": + a = NewRandomAssigner() default: return nil, xerrors.Errorf("unknown assigner '%s'", assigner) } @@ -189,7 +201,7 @@ func newScheduler(ctx context.Context, assigner string) (*Scheduler, error) { }, nil } -func (sh *Scheduler) Schedule(ctx context.Context, sector storiface.SectorRef, taskType sealtasks.TaskType, sel WorkerSelector, prepare WorkerAction, work WorkerAction) error { +func (sh *Scheduler) Schedule(ctx context.Context, sector storiface.SectorRef, taskType sealtasks.TaskType, sel WorkerSelector, prepare PrepareAction, work WorkerAction) error { ret := make(chan workerResponse) select { @@ -239,6 +251,13 @@ func (r *WorkerRequest) SealTask() sealtasks.SealTaskType { } } +func (r *WorkerRequest) PrepSealTask() sealtasks.SealTaskType { + return sealtasks.SealTaskType{ + TaskType: r.prepare.PrepType, + RegisteredSealProof: r.Sector.ProofType, + } +} + type SchedDiagRequestInfo struct { Sector abi.SectorID TaskType sealtasks.TaskType diff --git a/storage/sealer/sched_assigner_common.go b/storage/sealer/sched_assigner_common.go index bf92dbf15..d676d410d 100644 --- a/storage/sealer/sched_assigner_common.go +++ b/storage/sealer/sched_assigner_common.go @@ -58,7 +58,7 @@ func (a *AssignerCommon) TrySched(sh *Scheduler) { windows := make([]SchedWindow, windowsLen) for i := range windows { - windows[i].Allocated = *NewActiveResources() + windows[i].Allocated = *NewActiveResources(newTaskCounter()) } acceptableWindows := make([][]int, queueLen) // QueueIndex -> []OpenWindowIndex diff --git a/storage/sealer/sched_assigner_darts.go b/storage/sealer/sched_assigner_darts.go new file mode 100644 index 000000000..e28b70e78 --- /dev/null +++ b/storage/sealer/sched_assigner_darts.go @@ -0,0 +1,88 @@ +package sealer + +import ( + "math/rand" + + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +func NewRandomAssigner() Assigner { + return &AssignerCommon{ + WindowSel: RandomWS, + } +} + +func RandomWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + scheduled := 0 + rmQueue := make([]int, 0, queueLen) + + for sqi := 0; sqi < queueLen; sqi++ { + task := (*sh.SchedQueue)[sqi] + + //bestAssigned := math.MaxInt // smaller = better + + type choice struct { + selectedWindow int + needRes storiface.Resources + info storiface.WorkerInfo + bestWid storiface.WorkerID + } + choices := make([]choice, 0, len(acceptableWindows[task.IndexHeap])) + + for i, wnd := range acceptableWindows[task.IndexHeap] { + wid := sh.OpenWindows[wnd].Worker + w := sh.Workers[wid] + + res := w.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) + + log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) + + if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + continue + } + + choices = append(choices, choice{ + selectedWindow: wnd, + needRes: res, + info: w.Info, + bestWid: wid, + }) + + } + + if len(choices) == 0 { + // all windows full + continue + } + + // chose randomly + randIndex := rand.Intn(len(choices)) + selectedWindow := choices[randIndex].selectedWindow + needRes := choices[randIndex].needRes + info := choices[randIndex].info + bestWid := choices[randIndex].bestWid + + log.Debugw("SCHED ASSIGNED", + "assigner", "darts", + "sqi", sqi, + "sector", task.Sector.ID.Number, + "task", task.TaskType, + "window", selectedWindow, + "worker", bestWid, + "choices", len(choices)) + + windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) + + rmQueue = append(rmQueue, sqi) + scheduled++ + } + + if len(rmQueue) > 0 { + for i := len(rmQueue) - 1; i >= 0; i-- { + sh.SchedQueue.Remove(rmQueue[i]) + } + } + + return scheduled +} diff --git a/storage/sealer/sched_assigner_spread.go b/storage/sealer/sched_assigner_spread.go index f00d24d82..0a62b7406 100644 --- a/storage/sealer/sched_assigner_spread.go +++ b/storage/sealer/sched_assigner_spread.go @@ -6,76 +6,84 @@ import ( "github.com/filecoin-project/lotus/storage/sealer/storiface" ) -func NewSpreadAssigner() Assigner { +func NewSpreadAssigner(queued bool) Assigner { return &AssignerCommon{ - WindowSel: SpreadWS, + WindowSel: SpreadWS(queued), } } -func SpreadWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { - scheduled := 0 - rmQueue := make([]int, 0, queueLen) - workerAssigned := map[storiface.WorkerID]int{} +func SpreadWS(queued bool) func(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + return func(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + scheduled := 0 + rmQueue := make([]int, 0, queueLen) + workerAssigned := map[storiface.WorkerID]int{} - for sqi := 0; sqi < queueLen; sqi++ { - task := (*sh.SchedQueue)[sqi] + for sqi := 0; sqi < queueLen; sqi++ { + task := (*sh.SchedQueue)[sqi] - selectedWindow := -1 - var needRes storiface.Resources - var info storiface.WorkerInfo - var bestWid storiface.WorkerID - bestAssigned := math.MaxInt // smaller = better + selectedWindow := -1 + var needRes storiface.Resources + var info storiface.WorkerInfo + var bestWid storiface.WorkerID + bestAssigned := math.MaxInt // smaller = better - for i, wnd := range acceptableWindows[task.IndexHeap] { - wid := sh.OpenWindows[wnd].Worker - w := sh.Workers[wid] + for i, wnd := range acceptableWindows[task.IndexHeap] { + wid := sh.OpenWindows[wnd].Worker + w := sh.Workers[wid] - res := w.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) + res := w.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) - log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) + log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) - if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + continue + } + + wu, found := workerAssigned[wid] + if !found && queued { + wu = w.TaskCounts() + workerAssigned[wid] = wu + } + if wu >= bestAssigned { + continue + } + + info = w.Info + needRes = res + bestWid = wid + selectedWindow = wnd + bestAssigned = wu + } + + if selectedWindow < 0 { + // all windows full continue } - wu, _ := workerAssigned[wid] - if wu >= bestAssigned { - continue + log.Debugw("SCHED ASSIGNED", + "assigner", "spread", + "spread-queued", queued, + "sqi", sqi, + "sector", task.Sector.ID.Number, + "task", task.TaskType, + "window", selectedWindow, + "worker", bestWid, + "assigned", bestAssigned) + + workerAssigned[bestWid]++ + windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) + + rmQueue = append(rmQueue, sqi) + scheduled++ + } + + if len(rmQueue) > 0 { + for i := len(rmQueue) - 1; i >= 0; i-- { + sh.SchedQueue.Remove(rmQueue[i]) } - - info = w.Info - needRes = res - bestWid = wid - selectedWindow = wnd - bestAssigned = wu } - if selectedWindow < 0 { - // all windows full - continue - } - - log.Debugw("SCHED ASSIGNED", - "sqi", sqi, - "sector", task.Sector.ID.Number, - "task", task.TaskType, - "window", selectedWindow, - "worker", bestWid, - "assigned", bestAssigned) - - workerAssigned[bestWid]++ - windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) - windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) - - rmQueue = append(rmQueue, sqi) - scheduled++ + return scheduled } - - if len(rmQueue) > 0 { - for i := len(rmQueue) - 1; i >= 0; i-- { - sh.SchedQueue.Remove(rmQueue[i]) - } - } - - return scheduled } diff --git a/storage/sealer/sched_assigner_spread_tasks.go b/storage/sealer/sched_assigner_spread_tasks.go new file mode 100644 index 000000000..09cf98046 --- /dev/null +++ b/storage/sealer/sched_assigner_spread_tasks.go @@ -0,0 +1,98 @@ +package sealer + +import ( + "math" + + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +func NewSpreadTasksAssigner(queued bool) Assigner { + return &AssignerCommon{ + WindowSel: SpreadTasksWS(queued), + } +} + +type widTask struct { + wid storiface.WorkerID + tt sealtasks.TaskType +} + +func SpreadTasksWS(queued bool) func(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + return func(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows []SchedWindow) int { + scheduled := 0 + rmQueue := make([]int, 0, queueLen) + workerAssigned := map[widTask]int{} + + for sqi := 0; sqi < queueLen; sqi++ { + task := (*sh.SchedQueue)[sqi] + + selectedWindow := -1 + var needRes storiface.Resources + var info storiface.WorkerInfo + var bestWid widTask + bestAssigned := math.MaxInt // smaller = better + + for i, wnd := range acceptableWindows[task.IndexHeap] { + wid := sh.OpenWindows[wnd].Worker + w := sh.Workers[wid] + + res := w.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) + + log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) + + if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + continue + } + + wt := widTask{wid: wid, tt: task.TaskType} + + wu, found := workerAssigned[wt] + if !found && queued { + st := task.SealTask() + wu = w.TaskCount(&st) + workerAssigned[wt] = wu + } + if wu >= bestAssigned { + continue + } + + info = w.Info + needRes = res + bestWid = wt + selectedWindow = wnd + bestAssigned = wu + } + + if selectedWindow < 0 { + // all windows full + continue + } + + log.Debugw("SCHED ASSIGNED", + "assigner", "spread-tasks", + "spread-queued", queued, + "sqi", sqi, + "sector", task.Sector.ID.Number, + "task", task.TaskType, + "window", selectedWindow, + "worker", bestWid, + "assigned", bestAssigned) + + workerAssigned[bestWid]++ + windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) + + rmQueue = append(rmQueue, sqi) + scheduled++ + } + + if len(rmQueue) > 0 { + for i := len(rmQueue) - 1; i >= 0; i-- { + sh.SchedQueue.Remove(rmQueue[i]) + } + } + + return scheduled + } +} diff --git a/storage/sealer/sched_assigner_utilization.go b/storage/sealer/sched_assigner_utilization.go index 2d051d000..1e75d904a 100644 --- a/storage/sealer/sched_assigner_utilization.go +++ b/storage/sealer/sched_assigner_utilization.go @@ -74,6 +74,7 @@ func LowestUtilizationWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, } log.Debugw("SCHED ASSIGNED", + "assigner", "util", "sqi", sqi, "sector", task.Sector.ID.Number, "task", task.TaskType, diff --git a/storage/sealer/sched_resources.go b/storage/sealer/sched_resources.go index 487e294a2..597f36dbe 100644 --- a/storage/sealer/sched_resources.go +++ b/storage/sealer/sched_resources.go @@ -13,18 +13,68 @@ type ActiveResources struct { gpuUsed float64 cpuUse uint64 - taskCounters map[sealtasks.SealTaskType]int + taskCounters *taskCounter cond *sync.Cond waiting int } -func NewActiveResources() *ActiveResources { - return &ActiveResources{ +type taskCounter struct { + taskCounters map[sealtasks.SealTaskType]int + + // this lock is technically redundant, as ActiveResources is always accessed + // with the worker lock, but let's not panic if we ever change that + lk sync.Mutex +} + +func newTaskCounter() *taskCounter { + return &taskCounter{ taskCounters: map[sealtasks.SealTaskType]int{}, } } +func (tc *taskCounter) Add(tt sealtasks.SealTaskType) { + tc.lk.Lock() + defer tc.lk.Unlock() + tc.taskCounters[tt]++ +} + +func (tc *taskCounter) Free(tt sealtasks.SealTaskType) { + tc.lk.Lock() + defer tc.lk.Unlock() + tc.taskCounters[tt]-- +} + +func (tc *taskCounter) Get(tt sealtasks.SealTaskType) int { + tc.lk.Lock() + defer tc.lk.Unlock() + return tc.taskCounters[tt] +} + +func (tc *taskCounter) Sum() int { + tc.lk.Lock() + defer tc.lk.Unlock() + sum := 0 + for _, v := range tc.taskCounters { + sum += v + } + return sum +} + +func (tc *taskCounter) ForEach(cb func(tt sealtasks.SealTaskType, count int)) { + tc.lk.Lock() + defer tc.lk.Unlock() + for tt, count := range tc.taskCounters { + cb(tt, count) + } +} + +func NewActiveResources(tc *taskCounter) *ActiveResources { + return &ActiveResources{ + taskCounters: tc, + } +} + func (a *ActiveResources) withResources(id storiface.WorkerID, wr storiface.WorkerInfo, tt sealtasks.SealTaskType, r storiface.Resources, locker sync.Locker, cb func() error) error { for !a.CanHandleRequest(tt, r, id, "withResources", wr) { if a.cond == nil { @@ -59,7 +109,7 @@ func (a *ActiveResources) Add(tt sealtasks.SealTaskType, wr storiface.WorkerReso a.cpuUse += r.Threads(wr.CPUs, len(wr.GPUs)) a.memUsedMin += r.MinMemory a.memUsedMax += r.MaxMemory - a.taskCounters[tt]++ + a.taskCounters.Add(tt) return a.utilization(wr) - startUtil } @@ -71,7 +121,7 @@ func (a *ActiveResources) Free(tt sealtasks.SealTaskType, wr storiface.WorkerRes a.cpuUse -= r.Threads(wr.CPUs, len(wr.GPUs)) a.memUsedMin -= r.MinMemory a.memUsedMax -= r.MaxMemory - a.taskCounters[tt]-- + a.taskCounters.Free(tt) if a.cond != nil { a.cond.Broadcast() @@ -82,8 +132,8 @@ func (a *ActiveResources) Free(tt sealtasks.SealTaskType, wr storiface.WorkerRes // handle the request. func (a *ActiveResources) CanHandleRequest(tt sealtasks.SealTaskType, needRes storiface.Resources, wid storiface.WorkerID, caller string, info storiface.WorkerInfo) bool { if needRes.MaxConcurrent > 0 { - if a.taskCounters[tt] >= needRes.MaxConcurrent { - log.Debugf("sched: not scheduling on worker %s for %s; at task limit tt=%s, curcount=%d", wid, caller, tt, a.taskCounters[tt]) + if a.taskCounters.Get(tt) >= needRes.MaxConcurrent { + log.Debugf("sched: not scheduling on worker %s for %s; at task limit tt=%s, curcount=%d", wid, caller, tt, a.taskCounters.Get(tt)) return false } } @@ -170,6 +220,15 @@ func (a *ActiveResources) utilization(wr storiface.WorkerResources) float64 { // return max } +func (a *ActiveResources) taskCount(tt *sealtasks.SealTaskType) int { + // nil means all tasks + if tt == nil { + return a.taskCounters.Sum() + } + + return a.taskCounters.Get(*tt) +} + func (wh *WorkerHandle) Utilization() float64 { wh.lk.Lock() u := wh.active.utilization(wh.Info.Resources) @@ -183,3 +242,31 @@ func (wh *WorkerHandle) Utilization() float64 { return u } + +func (wh *WorkerHandle) TaskCounts() int { + wh.lk.Lock() + u := wh.active.taskCount(nil) + u += wh.preparing.taskCount(nil) + wh.lk.Unlock() + wh.wndLk.Lock() + for _, window := range wh.activeWindows { + u += window.Allocated.taskCount(nil) + } + wh.wndLk.Unlock() + + return u +} + +func (wh *WorkerHandle) TaskCount(tt *sealtasks.SealTaskType) int { + wh.lk.Lock() + u := wh.active.taskCount(tt) + u += wh.preparing.taskCount(tt) + wh.lk.Unlock() + wh.wndLk.Lock() + for _, window := range wh.activeWindows { + u += window.Allocated.taskCount(tt) + } + wh.wndLk.Unlock() + + return u +} diff --git a/storage/sealer/sched_test.go b/storage/sealer/sched_test.go index 2eed1ce73..07731e934 100644 --- a/storage/sealer/sched_test.go +++ b/storage/sealer/sched_test.go @@ -288,25 +288,30 @@ func TestSched(t *testing.T) { ProofType: spt, } - err := sched.Schedule(ctx, sectorRef, taskType, sel, func(ctx context.Context, w Worker) error { - wi, err := w.Info(ctx) - require.NoError(t, err) + prep := PrepareAction{ + Action: func(ctx context.Context, w Worker) error { + wi, err := w.Info(ctx) + require.NoError(t, err) - require.Equal(t, expectWorker, wi.Hostname) + require.Equal(t, expectWorker, wi.Hostname) - log.Info("IN ", taskName) + log.Info("IN ", taskName) - for { - _, ok := <-done - if !ok { - break + for { + _, ok := <-done + if !ok { + break + } } - } - log.Info("OUT ", taskName) + log.Info("OUT ", taskName) - return nil - }, noopAction) + return nil + }, + PrepType: taskType, + } + + err := sched.Schedule(ctx, sectorRef, taskType, sel, prep, noopAction) if err != context.Canceled { require.NoError(t, err, fmt.Sprint(l, l2)) } @@ -639,8 +644,8 @@ func BenchmarkTrySched(b *testing.B) { Resources: decentWorkerResources, }, Enabled: true, - preparing: NewActiveResources(), - active: NewActiveResources(), + preparing: NewActiveResources(newTaskCounter()), + active: NewActiveResources(newTaskCounter()), } for i := 0; i < windows; i++ { @@ -685,7 +690,7 @@ func TestWindowCompact(t *testing.T) { for _, windowTasks := range start { window := &SchedWindow{ - Allocated: *NewActiveResources(), + Allocated: *NewActiveResources(newTaskCounter()), } for _, task := range windowTasks { @@ -708,7 +713,7 @@ func TestWindowCompact(t *testing.T) { require.Equal(t, len(start)-len(expect), -sw.windowsRequested) for wi, tasks := range expect { - expectRes := NewActiveResources() + expectRes := NewActiveResources(newTaskCounter()) for ti, task := range tasks { require.Equal(t, task, wh.activeWindows[wi].Todo[ti].TaskType, "%d, %d", wi, ti) diff --git a/storage/sealer/sched_worker.go b/storage/sealer/sched_worker.go index e6e1f62da..b6efc851a 100644 --- a/storage/sealer/sched_worker.go +++ b/storage/sealer/sched_worker.go @@ -30,12 +30,14 @@ func newWorkerHandle(ctx context.Context, w Worker) (*WorkerHandle, error) { return nil, xerrors.Errorf("getting worker info: %w", err) } + tc := newTaskCounter() + worker := &WorkerHandle{ workerRpc: w, Info: info, - preparing: NewActiveResources(), - active: NewActiveResources(), + preparing: NewActiveResources(tc), + active: NewActiveResources(tc), Enabled: true, closingMgr: make(chan struct{}), @@ -352,8 +354,8 @@ assignLoop: worker.lk.Lock() for t, todo := range firstWindow.Todo { - needRes := worker.Info.Resources.ResourceSpec(todo.Sector.ProofType, todo.TaskType) - if worker.preparing.CanHandleRequest(todo.SealTask(), needRes, sw.wid, "startPreparing", worker.Info) { + needResPrep := worker.Info.Resources.PrepResourceSpec(todo.Sector.ProofType, todo.TaskType, todo.prepare.PrepType) + if worker.preparing.CanHandleRequest(todo.PrepSealTask(), needResPrep, sw.wid, "startPreparing", worker.Info) { tidx = t break } @@ -452,20 +454,21 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { w, sh := sw.worker, sw.sched needRes := w.Info.Resources.ResourceSpec(req.Sector.ProofType, req.TaskType) + needResPrep := w.Info.Resources.PrepResourceSpec(req.Sector.ProofType, req.TaskType, req.prepare.PrepType) w.lk.Lock() - w.preparing.Add(req.SealTask(), w.Info.Resources, needRes) + w.preparing.Add(req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() go func() { // first run the prepare step (e.g. fetching sector data from other worker) tw := sh.workTracker.worker(sw.wid, w.Info, w.workerRpc) tw.start() - err := req.prepare(req.Ctx, tw) + err := req.prepare.Action(req.Ctx, tw) w.lk.Lock() if err != nil { - w.preparing.Free(req.SealTask(), w.Info.Resources, needRes) + w.preparing.Free(req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() select { @@ -495,7 +498,7 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { // wait (if needed) for resources in the 'active' window err = w.active.withResources(sw.wid, w.Info, req.SealTask(), needRes, &w.lk, func() error { - w.preparing.Free(req.SealTask(), w.Info.Resources, needRes) + w.preparing.Free(req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() defer w.lk.Lock() // we MUST return locked from this function diff --git a/storage/sealer/sealtasks/task.go b/storage/sealer/sealtasks/task.go index bbf33b159..2e134f3d3 100644 --- a/storage/sealer/sealtasks/task.go +++ b/storage/sealer/sealtasks/task.go @@ -36,6 +36,8 @@ const ( TTGenerateWindowPoSt TaskType = "post/v0/windowproof" TTGenerateWinningPoSt TaskType = "post/v0/winningproof" + + TTNoop TaskType = "" ) var order = map[TaskType]int{ diff --git a/storage/sealer/stats.go b/storage/sealer/stats.go index f34109d78..90b6287c3 100644 --- a/storage/sealer/stats.go +++ b/storage/sealer/stats.go @@ -43,9 +43,9 @@ func (m *Manager) WorkerStats(ctx context.Context) map[uuid.UUID]storiface.Worke TaskCounts: map[string]int{}, } - for tt, count := range handle.active.taskCounters { + handle.active.taskCounters.ForEach(func(tt sealtasks.SealTaskType, count int) { out[uuid.UUID(id)].TaskCounts[tt.String()] = count - } + }) handle.lk.Unlock() } diff --git a/storage/sealer/storiface/worker.go b/storage/sealer/storiface/worker.go index 3cbf9f737..2badad292 100644 --- a/storage/sealer/storiface/worker.go +++ b/storage/sealer/storiface/worker.go @@ -65,6 +65,20 @@ func (wr WorkerResources) ResourceSpec(spt abi.RegisteredSealProof, tt sealtasks return res } +// PrepResourceSpec is like ResourceSpec, but meant for use limiting parallel preparing +// tasks. +func (wr WorkerResources) PrepResourceSpec(spt abi.RegisteredSealProof, tt, prepTT sealtasks.TaskType) Resources { + res := wr.ResourceSpec(spt, tt) + + if prepTT != tt && prepTT != sealtasks.TTNoop { + prepRes := wr.ResourceSpec(spt, prepTT) + res.MaxConcurrent = prepRes.MaxConcurrent + } + + // otherwise, use the default resource table + return res +} + type WorkerStats struct { Info WorkerInfo Tasks []sealtasks.TaskType