diff --git a/.gitignore b/.gitignore index 05f762d8d..fd51881b7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ /lotus-bench /lotus-gateway /lotus-pcr +/lotus-wallet /bench.json /lotuspond/front/node_modules /lotuspond/front/build diff --git a/CHANGELOG.md b/CHANGELOG.md index 6de6ddc2c..d397762a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,118 @@ # Lotus changelog +# 0.9.1 / 2020-10-10 + +This release fixes an issue which may cause the actors v2 migration to compute the state incorrectly when more than one migration is running in parallel. + +## Changes + +- Make concurrent actor migrations safe (https://github.com/filecoin-project/lotus/pull/4293) +- Remote wallet backends (https://github.com/filecoin-project/lotus/pull/3583) +- Track funds in FundMgr correctly in case of AddFunds failing (https://github.com/filecoin-project/lotus/pull/4273) +- Partial lite-node mode (https://github.com/filecoin-project/lotus/pull/4095) +- Fix potential infinite loop in GetBestMiningCandidate (https://github.com/filecoin-project/lotus/pull/3444) +- sync wait: Handle processed message offset (https://github.com/filecoin-project/lotus/pull/4253) +- Add some new endpoints for querying Msig info (https://github.com/filecoin-project/lotus/pull/4250) +- Update markets v0.7.1 (https://github.com/filecoin-project/lotus/pull/4254) +- Optimize SearchForMessage and GetReceipt (https://github.com/filecoin-project/lotus/pull/4246) +- Use FIL instead of attoFIL in CLI more consistently (https://github.com/filecoin-project/lotus/pull/4249) +- fix: clash between daemon --api flag and cli tests (https://github.com/filecoin-project/lotus/pull/4241) +- add more info to chain sync lookback failure (https://github.com/filecoin-project/lotus/pull/4245) +- Add message counts to inspect chain output (https://github.com/filecoin-project/lotus/pull/4230) + +# 0.9.0 / 2020-10-07 + +This consensus-breaking release of Lotus upgrades the actors version to v2.0.0. This requires migrating actor state from v0 to v2. The changes that break consensus are: + +- Introducing v2 actors and its migration (https://github.com/filecoin-project/lotus/pull/3936) +- Runtime's Receiver() should only return ID addresses (https://github.com/filecoin-project/lotus/pull/3589) +- Update miner eligibility checks for v2 actors (https://github.com/filecoin-project/lotus/pull/4188) +- Add funds that have left FilReserve to circ supply (https://github.com/filecoin-project/lotus/pull/4160) +- Set WinningPoStSectorSetLookback to finality post-v2 actors (https://github.com/filecoin-project/lotus/pull/4190) +- fix: error when actor panics directly (https://github.com/filecoin-project/lotus/pull/3697) + +## Changes + +#### Dependencies + +- Update go-bitfield (https://github.com/filecoin-project/lotus/pull/4171) +- update the AMT implementation (https://github.com/filecoin-project/lotus/pull/4194) +- Update to actors v0.2.1 (https://github.com/filecoin-project/lotus/pull/4199) + +#### Core Lotus + +- Paych: fix voucher amount verification (https://github.com/filecoin-project/lotus/pull/3821) +- Cap market provider messages (https://github.com/filecoin-project/lotus/pull/4141) +- Run fork function after cron for null block safety (https://github.com/filecoin-project/lotus/pull/4114) +- use bitswap sessions when fetching messages, and cancel them (https://github.com/filecoin-project/lotus/pull/4142) +- relax pubsub IPColocationFactorThreshold to 5 (https://github.com/filecoin-project/lotus/pull/4183) +- Support addresses with mainnet prefixes (https://github.com/filecoin-project/lotus/pull/4186) +- fix: make message signer nonce generation transactional (https://github.com/filecoin-project/lotus/pull/4165) +- build: Env var to keep test address output (https://github.com/filecoin-project/lotus/pull/4213) +- make vm.EnableGasTracing public (https://github.com/filecoin-project/lotus/pull/4214) +- introduce separate state-tree versions (https://github.com/filecoin-project/lotus/pull/4197) +- reject explicit "calls" at the upgrade height (https://github.com/filecoin-project/lotus/pull/4231) +- return an illegal actor error when we see an unsupported actor version (https://github.com/filecoin-project/lotus/pull/4232) +- Set head should unmark blocks as valid (https://gist.github.com/travisperson/3c7cddd77a33979a519ccef4e6515f20) + +#### Mining + +- Increased ExpectedSealDuration and and WaitDealsDelay (https://github.com/filecoin-project/lotus/pull/3743) +- Miner backup/restore commands (https://github.com/filecoin-project/lotus/pull/4133) +- lotus-miner: add more help text to storage / attach (https://github.com/filecoin-project/lotus/pull/3961) +- Reject deals that are > 7 days in the future in the BasicDealFilter (https://github.com/filecoin-project/lotus/pull/4173) +- feat(miner): add miner deadline diffing logic (https://github.com/filecoin-project/lotus/pull/4178) + +#### UX + +- Improve the UX for replacing messages (https://github.com/filecoin-project/lotus/pull/4134) +- Add verified flag to interactive deal creation (https://github.com/filecoin-project/lotus/pull/4145) +- Add command to (slowly) prune lotus chain datastore (https://github.com/filecoin-project/lotus/pull/3876) +- Some helpers for verifreg work (https://github.com/filecoin-project/lotus/pull/4124) +- Always use default 720h for setask duration and hide the duration param option (https://github.com/filecoin-project/lotus/pull/4077) +- Convert ID addresses to key addresses before checking wallet (https://github.com/filecoin-project/lotus/pull/4122) +- add a command to view block space utilization (https://github.com/filecoin-project/lotus/pull/4176) +- allow usage inspection on a chain segment (https://github.com/filecoin-project/lotus/pull/4177) +- Add mpool stats for base fee (https://github.com/filecoin-project/lotus/pull/4170) +- Add verified status to api.DealInfo (https://github.com/filecoin-project/lotus/pull/4153) +- Add a CLI command to set a miner's owner address (https://github.com/filecoin-project/lotus/pull/4189) + +#### Tooling and validation + +- Lotus-pcr: add recover-miners command (https://github.com/filecoin-project/lotus/pull/3714) +- MpoolPushUntrusted API for gateway (https://github.com/filecoin-project/lotus/pull/3915) +- Test lotus-miner info all (https://github.com/filecoin-project/lotus/pull/4166) +- chain export: Error with unfinished exports (https://github.com/filecoin-project/lotus/pull/4179) +- add printf in TestWindowPost (https://github.com/filecoin-project/lotus/pull/4043) +- add trace wdpost (https://github.com/filecoin-project/lotus/pull/4020) +- Fix noncefix (https://github.com/filecoin-project/lotus/pull/4202) +- Lotus-pcr: Limit the fee cap of messages we will process, refund gas fees for windowed post and storage deals (https://github.com/filecoin-project/lotus/pull/4198) +- Fix pond (https://github.com/filecoin-project/lotus/pull/4203) +- allow manual setting of noncefix fee cap (https://github.com/filecoin-project/lotus/pull/4205) +- implement command to get execution traces of any message (https://github.com/filecoin-project/lotus/pull/4200) +- conformance: minor driver refactors (https://github.com/filecoin-project/lotus/pull/4211) +- lotus-pcr: ignore all other messages (https://github.com/filecoin-project/lotus/pull/4218) +- lotus-pcr: zero refund (https://github.com/filecoin-project/lotus/pull/4229) + +## Contributors + +The following contributors had 5 or more commits go into this release. +We are grateful for every contribution! + +| Contributor | Commits | Lines ± | +|--------------------|---------|---------------| +| Stebalien | 84 | +3425/-2287 | +| magik6k | 41 | +2121/-506 | +| arajasek | 39 | +2467/-424 | +| Kubuxu | 25 | +2344/-775 | +| raulk | 21 | +287/-196 | +| whyrusleeping | 13 | +727/-71 | +| hsanjuan | 13 | +5886/-7956 | +| dirkmc | 11 | +2634/-576 | +| travisperson | 8 | +923/-202 | +| ribasushi | 6 | +188/-128 | +| zgfzgf | 5 | +21/-17 | + # 0.8.1 / 2020-09-30 This optional release of Lotus introduces a new version of markets which switches to CBOR-map encodings, and allows datastore migrations. The release also introduces several improvements to the mining process, a few performance optimizations, and a battery of UX additions and enhancements. @@ -117,7 +230,7 @@ We are grateful for every contribution! | vyzo | 22 | +287/-196 | | alanshaw | 15 | +761/-146 | | whyrusleeping | 15 | +736/-52 | -| hannahhoward | 14 | +1237/837- | +| hannahhoward | 14 | +1237/-837 | | anton | 6 | +32/-8 | | travisperson | 5 | +502/-6 | | Frank | 5 | +78/-39 | diff --git a/Makefile b/Makefile index 56ab361ec..79f7fa81e 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,12 @@ lotus-health: .PHONY: lotus-health BINS+=lotus-health +lotus-wallet: + rm -f lotus-wallet + go build -o lotus-wallet ./cmd/lotus-wallet +.PHONY: lotus-wallet +BINS+=lotus-wallet + testground: go build -tags testground -o /dev/null ./cmd/lotus .PHONY: testground diff --git a/api/api_full.go b/api/api_full.go index b6ae77f77..a2fe94ee9 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -172,6 +172,9 @@ type FullNode interface { // SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error + // SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad + SyncUnmarkAllBad(ctx context.Context) error + // SyncCheckBad checks if a block was marked as bad, and if it was, returns // the reason. SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) @@ -367,6 +370,10 @@ type FullNode interface { // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) + // StateWaitMsgLimited looks back up to limit epochs in the chain for a message. + // If not found, it blocks until the message arrives on chain, and gets to the + // indicated confidence depth. + StateWaitMsgLimited(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch) (*MsgLookup, error) // StateListMiners returns the addresses of every miner that has claimed power in the Power Actor StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) // StateListActors returns the addresses of every actor in the state @@ -418,6 +425,8 @@ type FullNode interface { // MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + // MsigGetVestingSchedule returns the vesting details of a given multisig. + MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (MsigVesting, error) // MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. // It takes the following params: , , MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) @@ -429,12 +438,21 @@ type FullNode interface { // It takes the following params: , , , // , , MsigPropose(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) - // MsigApprove approves a previously-proposed multisig message + + // MsigApprove approves a previously-proposed multisig message by transaction ID + // It takes the following params: , + MsigApprove(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) + + // MsigApproveTxnHash approves a previously-proposed multisig message, specified + // using both transaction ID and a hash of the parameters used in the + // proposal. This method of approval can be used to ensure you only approve + // exactly the transaction you think you are. // It takes the following params: , , , , , // , , - MsigApprove(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) + MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) + // MsigCancel cancels a previously-proposed multisig message - // It takes the following params: , , , , + // It takes the following params: , , , , // , , MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) // MsigAddPropose proposes adding a signer in the multisig @@ -462,6 +480,13 @@ type FullNode interface { // , MsigSwapCancel(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) + // MsigRemoveSigner proposes the removal of a signer from the multisig. + // It accepts the multisig to make the change on, the proposer address to + // send the message from, the address to be removed, and a boolean + // indicating whether or not the signing threshold should be lowered by one + // along with the address removal. + MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) + MarketEnsureAvailable(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) // MarketFreeBalance @@ -871,3 +896,15 @@ type Fault struct { Miner address.Address Epoch abi.ChainEpoch } + +var EmptyVesting = MsigVesting{ + InitialBalance: types.EmptyInt, + StartEpoch: -1, + UnlockDuration: -1, +} + +type MsigVesting struct { + InitialBalance abi.TokenAmount + StartEpoch abi.ChainEpoch + UnlockDuration abi.ChainEpoch +} diff --git a/api/api_gateway.go b/api/api_gateway.go new file mode 100644 index 000000000..95d28887d --- /dev/null +++ b/api/api_gateway.go @@ -0,0 +1,24 @@ +package api + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +type GatewayAPI interface { + ChainHead(ctx context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error) +} diff --git a/api/api_wallet.go b/api/api_wallet.go new file mode 100644 index 000000000..1213ffc1d --- /dev/null +++ b/api/api_wallet.go @@ -0,0 +1,47 @@ +package api + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/types" +) + +type MsgType string + +const ( + MTUnknown = "unknown" + + // Signing message CID. MsgMeta.Extra contains raw cbor message bytes + MTChainMsg = "message" + + // Signing a blockheader. signing raw cbor block bytes (MsgMeta.Extra is empty) + MTBlock = "block" + + // Signing a deal proposal. signing raw cbor proposal bytes (MsgMeta.Extra is empty) + MTDealProposal = "dealproposal" + + // TODO: Deals, Vouchers, VRF +) + +type MsgMeta struct { + Type MsgType + + // Additional data related to what is signed. Should be verifiable with the + // signed bytes (e.g. CID(Extra).Bytes() == toSign) + Extra []byte +} + +type WalletAPI interface { + WalletNew(context.Context, crypto.SigType) (address.Address, error) + WalletHas(context.Context, address.Address) (bool, error) + WalletList(context.Context) ([]address.Address, error) + + WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta MsgMeta) (*crypto.Signature, error) + + WalletExport(context.Context, address.Address) (*types.KeyInfo, error) + WalletImport(context.Context, *types.KeyInfo) (address.Address, error) + WalletDelete(context.Context, address.Address) error +} diff --git a/api/apistruct/permissioned.go b/api/apistruct/permissioned.go index c93662733..86902d31b 100644 --- a/api/apistruct/permissioned.go +++ b/api/apistruct/permissioned.go @@ -36,3 +36,9 @@ func PermissionedWorkerAPI(a api.WorkerAPI) api.WorkerAPI { auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal) return &out } + +func PermissionedWalletAPI(a api.WalletAPI) api.WalletAPI { + var out WalletStruct + auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal) + return &out +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a09700eb9..22d50e726 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -111,6 +111,7 @@ type FullNodeStruct struct { SyncCheckpoint func(ctx context.Context, key types.TipSetKey) error `perm:"admin"` SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncUnmarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` + SyncUnmarkAllBad func(ctx context.Context) error `perm:"admin"` SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"` SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"` @@ -190,6 +191,7 @@ type FullNodeStruct struct { StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` + StateWaitMsgLimited func(context.Context, cid.Cid, uint64, abi.ChainEpoch) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -212,10 +214,12 @@ type FullNodeStruct struct { StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` + MsigGetVestingSchedule func(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error) `perm:"read"` MsigGetVested func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) `perm:"read"` MsigCreate func(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"` MsigPropose func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` - MsigApprove func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` + MsigApprove func(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) `perm:"sign"` + MsigApproveTxnHash func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` MsigCancel func(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` MsigAddPropose func(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error) `perm:"sign"` MsigAddApprove func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error) `perm:"sign"` @@ -223,6 +227,7 @@ type FullNodeStruct struct { MsigSwapPropose func(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` MsigSwapApprove func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` MsigSwapCancel func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` + MsigRemoveSigner func(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) `perm:"sign"` MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"` @@ -360,6 +365,35 @@ type WorkerStruct struct { } } +type GatewayStruct struct { + Internal struct { + // TODO: does the gateway need perms? + ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) + GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested func(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) + StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) + } +} + +type WalletStruct struct { + Internal struct { + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` + } +} + // CommonStruct func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) { @@ -749,6 +783,10 @@ func (c *FullNodeStruct) SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error return c.Internal.SyncUnmarkBad(ctx, bcid) } +func (c *FullNodeStruct) SyncUnmarkAllBad(ctx context.Context) error { + return c.Internal.SyncUnmarkAllBad(ctx) +} + func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) { return c.Internal.SyncCheckBad(ctx, bcid) } @@ -853,6 +891,10 @@ func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confide return c.Internal.StateWaitMsg(ctx, msgc, confidence) } +func (c *FullNodeStruct) StateWaitMsgLimited(ctx context.Context, msgc cid.Cid, confidence uint64, limit abi.ChainEpoch) (*api.MsgLookup, error) { + return c.Internal.StateWaitMsgLimited(ctx, msgc, confidence, limit) +} + func (c *FullNodeStruct) StateSearchMsg(ctx context.Context, msgc cid.Cid) (*api.MsgLookup, error) { return c.Internal.StateSearchMsg(ctx, msgc) } @@ -933,6 +975,10 @@ func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address. return c.Internal.MsigGetAvailableBalance(ctx, a, tsk) } +func (c *FullNodeStruct) MsigGetVestingSchedule(ctx context.Context, a address.Address, tsk types.TipSetKey) (api.MsigVesting, error) { + return c.Internal.MsigGetVestingSchedule(ctx, a, tsk) +} + func (c *FullNodeStruct) MsigGetVested(ctx context.Context, a address.Address, sTsk types.TipSetKey, eTsk types.TipSetKey) (types.BigInt, error) { return c.Internal.MsigGetVested(ctx, a, sTsk, eTsk) } @@ -945,8 +991,12 @@ func (c *FullNodeStruct) MsigPropose(ctx context.Context, msig address.Address, return c.Internal.MsigPropose(ctx, msig, to, amt, src, method, params) } -func (c *FullNodeStruct) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return c.Internal.MsigApprove(ctx, msig, txID, proposer, to, amt, src, method, params) +func (c *FullNodeStruct) MsigApprove(ctx context.Context, msig address.Address, txID uint64, signer address.Address) (cid.Cid, error) { + return c.Internal.MsigApprove(ctx, msig, txID, signer) +} + +func (c *FullNodeStruct) MsigApproveTxnHash(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { + return c.Internal.MsigApproveTxnHash(ctx, msig, txID, proposer, to, amt, src, method, params) } func (c *FullNodeStruct) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { @@ -977,6 +1027,10 @@ func (c *FullNodeStruct) MsigSwapCancel(ctx context.Context, msig address.Addres return c.Internal.MsigSwapCancel(ctx, msig, src, txID, oldAdd, newAdd) } +func (c *FullNodeStruct) MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) { + return c.Internal.MsigRemoveSigner(ctx, msig, proposer, toRemove, decrease) +} + func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) { return c.Internal.MarketEnsureAvailable(ctx, addr, wallet, amt) } @@ -1367,7 +1421,81 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } +func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { + return g.Internal.ChainHead(ctx) +} + +func (g GatewayStruct) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + return g.Internal.ChainGetTipSet(ctx, tsk) +} + +func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) +} + +func (g GatewayStruct) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + return g.Internal.GasEstimateMessageGas(ctx, msg, spec, tsk) +} + +func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + return g.Internal.MpoolPush(ctx, sm) +} + +func (g GatewayStruct) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + return g.Internal.MsigGetAvailableBalance(ctx, addr, tsk) +} + +func (g GatewayStruct) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + return g.Internal.MsigGetVested(ctx, addr, start, end) +} + +func (g GatewayStruct) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + return g.Internal.StateAccountKey(ctx, addr, tsk) +} + +func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + return g.Internal.StateGetActor(ctx, actor, ts) +} + +func (g GatewayStruct) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + return g.Internal.StateLookupID(ctx, addr, tsk) +} + +func (g GatewayStruct) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return g.Internal.StateWaitMsg(ctx, msg, confidence) +} + +func (c *WalletStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { + return c.Internal.WalletNew(ctx, typ) +} + +func (c *WalletStruct) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + return c.Internal.WalletHas(ctx, addr) +} + +func (c *WalletStruct) WalletList(ctx context.Context) ([]address.Address, error) { + return c.Internal.WalletList(ctx) +} + +func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { + return c.Internal.WalletSign(ctx, k, msg, meta) +} + +func (c *WalletStruct) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { + return c.Internal.WalletExport(ctx, a) +} + +func (c *WalletStruct) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { + return c.Internal.WalletImport(ctx, ki) +} + +func (c *WalletStruct) WalletDelete(ctx context.Context, addr address.Address) error { + return c.Internal.WalletDelete(ctx, addr) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} var _ api.WorkerAPI = &WorkerStruct{} +var _ api.GatewayAPI = &GatewayStruct{} +var _ api.WalletAPI = &WalletStruct{} diff --git a/api/client/client.go b/api/client/client.go index cd915acf0..7d8a466d3 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -82,3 +82,29 @@ func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) ( return &res, closer, err } + +// NewGatewayRPC creates a new http jsonrpc client for a gateway node. +func NewGatewayRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.GatewayAPI, jsonrpc.ClientCloser, error) { + var res apistruct.GatewayStruct + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", + []interface{}{ + &res.Internal, + }, + requestHeader, + opts..., + ) + + return &res, closer, err +} + +func NewWalletRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WalletAPI, jsonrpc.ClientCloser, error) { + var res apistruct.WalletStruct + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", + []interface{}{ + &res.Internal, + }, + requestHeader, + ) + + return &res, closer, err +} diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 97fb665ed..4a860c661 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -12,10 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" ) @@ -37,11 +34,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) { ctx := context.Background() - n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: build.ActorUpgradeNetworkVersion, - Height: upgradeHeight, - Migration: stmgr.UpgradeActorsV2, - }})) + n, sn := b(t, []FullNodeOpts{FullNodeWithUpgradeAt(upgradeHeight)}, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/api/test/deals.go b/api/test/deals.go index aa5bfa716..8b4a7fe8b 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -48,7 +48,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -85,7 +85,7 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -149,7 +149,7 @@ func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Durati _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -204,7 +204,7 @@ func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/api/test/mining.go b/api/test/mining.go index e19774a76..8147c224b 100644 --- a/api/test/mining.go +++ b/api/test/mining.go @@ -25,7 +25,7 @@ var log = logging.Logger("apitest") func (ts *testSuite) testMining(t *testing.T) { ctx := context.Background() - apis, sn := ts.makeNodes(t, 1, OneMiner) + apis, sn := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] newHeads, err := api.ChainNotify(ctx) @@ -54,7 +54,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) { }() ctx := context.Background() - apis, sn := ts.makeNodes(t, 1, OneMiner) + apis, sn := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] newHeads, err := api.ChainNotify(ctx) @@ -93,7 +93,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo // test making a deal with a fresh miner, and see if it starts to mine ctx := context.Background() - n, sn := b(t, 1, []StorageMiner{ + n, sn := b(t, OneFull, []StorageMiner{ {Full: 0, Preseal: PresealGenesis}, {Full: 0, Preseal: 0}, // TODO: Add support for miners on non-first full node }) diff --git a/api/test/paych.go b/api/test/paych.go index e95773b6a..a8ccebdde 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -33,7 +33,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 2, OneMiner) + n, sn := b(t, TwoFull, OneMiner) paymentCreator := n[0] paymentReceiver := n[1] diff --git a/api/test/test.go b/api/test/test.go index 853267eff..35b397740 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -4,11 +4,14 @@ import ( "context" "testing" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/miner" @@ -35,17 +38,27 @@ var PresealGenesis = -1 const GenesisPreseals = 2 +// Options for setting up a mock storage miner type StorageMiner struct { Full int Preseal int } +type OptionGenerator func([]TestNode) node.Option + +// Options for setting up a mock full node +type FullNodeOpts struct { + Lite bool // run node in "lite" mode + Opts OptionGenerator // generate dependency injection options +} + // APIBuilder is a function which is invoked in test suite to provide // test nodes and networks // +// fullOpts array defines options for each full node // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } @@ -63,13 +76,39 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testMiningReal", ts.testMiningReal) } +func DefaultFullOpts(nFull int) []FullNodeOpts { + full := make([]FullNodeOpts, nFull) + for i := range full { + full[i] = FullNodeOpts{ + Opts: func(nodes []TestNode) node.Option { + return node.Options() + }, + } + } + return full +} + var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}} +var OneFull = DefaultFullOpts(1) +var TwoFull = DefaultFullOpts(2) + +var FullNodeWithUpgradeAt = func(upgradeHeight abi.ChainEpoch) FullNodeOpts { + return FullNodeOpts{ + Opts: func(nodes []TestNode) node.Option { + return node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + Network: build.ActorUpgradeNetworkVersion, + Height: upgradeHeight, + Migration: stmgr.UpgradeActorsV2, + }}) + }, + } +} func (ts *testSuite) testVersion(t *testing.T) { build.RunningNodeType = build.NodeFull ctx := context.Background() - apis, _ := ts.makeNodes(t, 1, OneMiner) + apis, _ := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] v, err := api.Version(ctx) @@ -81,7 +120,7 @@ func (ts *testSuite) testVersion(t *testing.T) { func (ts *testSuite) testID(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 1, OneMiner) + apis, _ := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] id, err := api.ID(ctx) @@ -93,7 +132,7 @@ func (ts *testSuite) testID(t *testing.T) { func (ts *testSuite) testConnectTwo(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 2, OneMiner) + apis, _ := ts.makeNodes(t, TwoFull, OneMiner) p, err := apis[0].NetPeers(ctx) if err != nil { diff --git a/api/test/window_post.go b/api/test/window_post.go index eadcdbb05..28639cda8 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -3,6 +3,7 @@ package test import ( "context" "fmt" + "sync/atomic" "os" "strings" @@ -15,11 +16,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/mock" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" bminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl" @@ -33,8 +32,10 @@ func init() { } func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { - ctx := context.Background() - n, sn := b(t, 1, OneMiner) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -48,11 +49,11 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect } build.Clock.Sleep(time.Second) - mine := true + mine := int64(1) done := make(chan struct{}) go func() { defer close(done) - for mine { + for atomic.LoadInt64(&mine) != 0 { build.Clock.Sleep(blocktime) if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, abi.ChainEpoch, error) { @@ -64,7 +65,7 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect pledgeSectors(t, ctx, miner, nSectors, 0, nil) - mine = false + atomic.StoreInt64(&mine, 0) <-done } @@ -133,11 +134,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: build.ActorUpgradeNetworkVersion, - Height: upgradeHeight, - Migration: stmgr.UpgradeActorsV2, - }})) + n, sn := b(t, []FullNodeOpts{FullNodeWithUpgradeAt(upgradeHeight)}, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/build/params_2k.go b/build/params_2k.go index f4a17f724..c6538dc08 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -3,6 +3,9 @@ package build import ( + "math" + "os" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -13,8 +16,10 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 -const UpgradeLiftoffHeight = -3 -const UpgradeActorsV2Height = 10 +const UpgradeRefuelHeight = -3 + +var UpgradeActorsV2Height = abi.ChainEpoch(10) +var UpgradeLiftoffHeight = abi.ChainEpoch(-4) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, @@ -25,6 +30,11 @@ func init() { policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) + if os.Getenv("LOTUS_DISABLE_V2_ACTOR_MIGRATION") == "1" { + UpgradeActorsV2Height = math.MaxInt64 + UpgradeLiftoffHeight = 11 + } + BuildType |= Build2k } diff --git a/build/params_testground.go b/build/params_testground.go index bd064b3d6..6109cbc04 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -82,8 +82,9 @@ var ( UpgradeSmokeHeight abi.ChainEpoch = -1 UpgradeIgnitionHeight abi.ChainEpoch = -2 - UpgradeLiftoffHeight abi.ChainEpoch = -3 + UpgradeRefuelHeight abi.ChainEpoch = -3 UpgradeActorsV2Height abi.ChainEpoch = 10 + UpgradeLiftoffHeight abi.ChainEpoch = -4 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/version.go b/build/version.go index 0b317aa17..5baed1fb7 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.8.1" +const BuildVersion = "0.9.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 7def78dcf..cb24a2c33 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -22,6 +22,7 @@ import ( var SystemActorAddr = builtin0.SystemActorAddr var BurntFundsActorAddr = builtin0.BurntFundsActorAddr var ReserveAddress = makeAddress("t090") +var RootVerifierAddress = makeAddress("t080") // TODO: Why does actors have 2 different versions of this? type SectorInfo = proof0.SectorInfo diff --git a/chain/actors/builtin/miner/diff_deadlines.go b/chain/actors/builtin/miner/diff_deadlines.go new file mode 100644 index 000000000..e1e839960 --- /dev/null +++ b/chain/actors/builtin/miner/diff_deadlines.go @@ -0,0 +1,180 @@ +package miner + +import ( + "errors" + + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/exitcode" +) + +type DeadlinesDiff map[uint64]*DeadlineDiff + +func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { + changed, err := pre.DeadlinesChanged(cur) + if err != nil { + return nil, err + } + if !changed { + return nil, nil + } + + numDl, err := pre.NumDeadlines() + if err != nil { + return nil, err + } + dlDiff := make(DeadlinesDiff, numDl) + if err := pre.ForEachDeadline(func(idx uint64, preDl Deadline) error { + curDl, err := cur.LoadDeadline(idx) + if err != nil { + return err + } + + diff, err := DiffDeadline(preDl, curDl) + if err != nil { + return err + } + + dlDiff[idx] = diff + return nil + }); err != nil { + return nil, err + } + return &dlDiff, nil +} + +type DeadlineDiff map[uint64]*PartitionDiff + +func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { + changed, err := pre.PartitionsChanged(cur) + if err != nil { + return nil, err + } + if !changed { + return nil, nil + } + + partDiff := make(DeadlineDiff) + if err := pre.ForEachPartition(func(idx uint64, prePart Partition) error { + // try loading current partition at this index + curPart, err := cur.LoadPartition(idx) + if err != nil { + if errors.Is(err, exitcode.ErrNotFound) { + // TODO correctness? + return nil // the partition was removed. + } + return err + } + + // compare it with the previous partition + diff, err := DiffPartition(prePart, curPart) + if err != nil { + return err + } + + partDiff[idx] = diff + return nil + }); err != nil { + return nil, err + } + + // all previous partitions have been walked. + // all partitions in cur and not in prev are new... can they be faulty already? + // TODO is this correct? + if err := cur.ForEachPartition(func(idx uint64, curPart Partition) error { + if _, found := partDiff[idx]; found { + return nil + } + faults, err := curPart.FaultySectors() + if err != nil { + return err + } + recovering, err := curPart.RecoveringSectors() + if err != nil { + return err + } + partDiff[idx] = &PartitionDiff{ + Removed: bitfield.New(), + Recovered: bitfield.New(), + Faulted: faults, + Recovering: recovering, + } + + return nil + }); err != nil { + return nil, err + } + + return &partDiff, nil +} + +type PartitionDiff struct { + Removed bitfield.BitField + Recovered bitfield.BitField + Faulted bitfield.BitField + Recovering bitfield.BitField +} + +func DiffPartition(pre, cur Partition) (*PartitionDiff, error) { + prevLiveSectors, err := pre.LiveSectors() + if err != nil { + return nil, err + } + curLiveSectors, err := cur.LiveSectors() + if err != nil { + return nil, err + } + + removed, err := bitfield.SubtractBitField(prevLiveSectors, curLiveSectors) + if err != nil { + return nil, err + } + + prevRecoveries, err := pre.RecoveringSectors() + if err != nil { + return nil, err + } + + curRecoveries, err := cur.RecoveringSectors() + if err != nil { + return nil, err + } + + recovering, err := bitfield.SubtractBitField(curRecoveries, prevRecoveries) + if err != nil { + return nil, err + } + + prevFaults, err := pre.FaultySectors() + if err != nil { + return nil, err + } + + curFaults, err := cur.FaultySectors() + if err != nil { + return nil, err + } + + faulted, err := bitfield.SubtractBitField(curFaults, prevFaults) + if err != nil { + return nil, err + } + + // all current good sectors + curActiveSectors, err := cur.ActiveSectors() + if err != nil { + return nil, err + } + + // sectors that were previously fault and are now currently active are considered recovered. + recovered, err := bitfield.IntersectBitField(prevFaults, curActiveSectors) + if err != nil { + return nil, err + } + + return &PartitionDiff{ + Removed: removed, + Recovered: recovered, + Faulted: faulted, + Recovering: recovering, + }, nil +} diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index e683cfd96..bafb14de0 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -51,6 +51,7 @@ type State interface { MinerPower(address.Address) (Claim, bool, error) MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error) ListAllMiners() ([]address.Address, error) + ForEachClaim(func(miner address.Address, claim Claim) error) error } type Claim struct { diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index e2a9cf382..3f9a65777 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -96,3 +96,22 @@ func (s *state0) ListAllMiners() ([]address.Address, error) { return miners, nil } + +func (s *state0) ForEachClaim(cb func(miner address.Address, claim Claim) error) error { + claims, err := adt0.AsMap(s.store, s.Claims) + if err != nil { + return err + } + + var claim power0.Claim + return claims.ForEach(&claim, func(k string) error { + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + return cb(a, Claim{ + RawBytePower: claim.RawBytePower, + QualityAdjPower: claim.QualityAdjPower, + }) + }) +} diff --git a/chain/actors/builtin/power/v2.go b/chain/actors/builtin/power/v2.go index 6346a09b6..0c15f0669 100644 --- a/chain/actors/builtin/power/v2.go +++ b/chain/actors/builtin/power/v2.go @@ -96,3 +96,22 @@ func (s *state2) ListAllMiners() ([]address.Address, error) { return miners, nil } + +func (s *state2) ForEachClaim(cb func(miner address.Address, claim Claim) error) error { + claims, err := adt2.AsMap(s.store, s.Claims) + if err != nil { + return err + } + + var claim power2.Claim + return claims.ForEach(&claim, func(k string) error { + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + return cb(a, Claim{ + RawBytePower: claim.RawBytePower, + QualityAdjPower: claim.QualityAdjPower, + }) + }) +} diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 0efd0b482..6a6e6d12e 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -57,7 +57,7 @@ func (s *state0) CumsumBaseline() (abi.StoragePower, error) { } func (s *state0) CumsumRealized() (abi.StoragePower, error) { - return s.State.CumsumBaseline, nil + return s.State.CumsumRealized, nil } func (s *state0) InitialPledgeForPower(sectorWeight abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { diff --git a/chain/actors/builtin/reward/v2.go b/chain/actors/builtin/reward/v2.go index ec0709c39..b7cb49102 100644 --- a/chain/actors/builtin/reward/v2.go +++ b/chain/actors/builtin/reward/v2.go @@ -60,7 +60,7 @@ func (s *state2) CumsumBaseline() (abi.StoragePower, error) { } func (s *state2) CumsumRealized() (abi.StoragePower, error) { - return s.State.CumsumBaseline, nil + return s.State.CumsumRealized, nil } func (s *state2) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { diff --git a/chain/actors/version.go b/chain/actors/version.go index 385ff592f..17af8b08b 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -9,8 +9,8 @@ import ( type Version int const ( - Version0 = 0 - Version2 = 2 + Version0 Version = 0 + Version2 Version = 2 ) // Converts a network version into an actors adt version. diff --git a/chain/badtscache.go b/chain/badtscache.go index 3c5bf05ef..42564128e 100644 --- a/chain/badtscache.go +++ b/chain/badtscache.go @@ -60,6 +60,10 @@ func (bts *BadBlockCache) Remove(c cid.Cid) { bts.badBlocks.Remove(c) } +func (bts *BadBlockCache) Purge() { + bts.badBlocks.Purge() +} + func (bts *BadBlockCache) Has(c cid.Cid) (BadBlockReason, bool) { rval, ok := bts.badBlocks.Get(c) if !ok { diff --git a/chain/events/tscache.go b/chain/events/tscache.go index d47c71480..44699684e 100644 --- a/chain/events/tscache.go +++ b/chain/events/tscache.go @@ -2,6 +2,7 @@ package events import ( "context" + "sync" "github.com/filecoin-project/go-state-types/abi" "golang.org/x/xerrors" @@ -17,6 +18,8 @@ type tsCacheAPI interface { // tipSetCache implements a simple ring-buffer cache to keep track of recent // tipsets type tipSetCache struct { + mu sync.RWMutex + cache []*types.TipSet start int len int @@ -35,6 +38,9 @@ func newTSCache(cap abi.ChainEpoch, storage tsCacheAPI) *tipSetCache { } func (tsc *tipSetCache) add(ts *types.TipSet) error { + tsc.mu.Lock() + defer tsc.mu.Unlock() + if tsc.len > 0 { if tsc.cache[tsc.start].Height() >= ts.Height() { return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.cache[tsc.start].Height()+1, ts.Height()) @@ -65,6 +71,13 @@ func (tsc *tipSetCache) add(ts *types.TipSet) error { } func (tsc *tipSetCache) revert(ts *types.TipSet) error { + tsc.mu.Lock() + defer tsc.mu.Unlock() + + return tsc.revertUnlocked(ts) +} + +func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error { if tsc.len == 0 { return nil // this can happen, and it's fine } @@ -77,7 +90,7 @@ func (tsc *tipSetCache) revert(ts *types.TipSet) error { tsc.start = normalModulo(tsc.start-1, len(tsc.cache)) tsc.len-- - _ = tsc.revert(nil) // revert null block gap + _ = tsc.revertUnlocked(nil) // revert null block gap return nil } @@ -95,7 +108,10 @@ func (tsc *tipSetCache) getNonNull(height abi.ChainEpoch) (*types.TipSet, error) } func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { + tsc.mu.RLock() + if tsc.len == 0 { + tsc.mu.RUnlock() log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height) return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, types.EmptyTSK) } @@ -103,6 +119,7 @@ func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { headH := tsc.cache[tsc.start].Height() if height > headH { + tsc.mu.RUnlock() return nil, xerrors.Errorf("tipSetCache.get: requested tipset not in cache (req: %d, cache head: %d)", height, headH) } @@ -116,15 +133,20 @@ func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { } if height < tail.Height() { + tsc.mu.RUnlock() log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tail.Height()) return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, tail.Key()) } - return tsc.cache[normalModulo(tsc.start-int(headH-height), clen)], nil + ts := tsc.cache[normalModulo(tsc.start-int(headH-height), clen)] + tsc.mu.RUnlock() + return ts, nil } func (tsc *tipSetCache) best() (*types.TipSet, error) { + tsc.mu.RLock() best := tsc.cache[tsc.start] + tsc.mu.RUnlock() if best == nil { return tsc.storage.ChainHead(context.TODO()) } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 3ebd127f3..9133f14b4 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -39,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/repo" @@ -71,7 +72,7 @@ type ChainGen struct { GetMessages func(*ChainGen) ([]*types.SignedMessage, error) - w *wallet.Wallet + w *wallet.LocalWallet eppProvs map[address.Address]WinningPoStProver Miners []address.Address @@ -122,6 +123,7 @@ var DefaultRemainderAccountActor = genesis.Actor{ } func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { + j := journal.NilJournal() // TODO: we really shouldn't modify a global variable here. policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) @@ -153,14 +155,14 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("creating memrepo wallet failed: %w", err) } - banker, err := w.GenerateKey(crypto.SigTypeSecp256k1) + banker, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { return nil, xerrors.Errorf("failed to generate banker key: %w", err) } receievers := make([]address.Address, msgsPerBlock) for r := range receievers { - receievers[r], err = w.GenerateKey(crypto.SigTypeBLS) + receievers[r], err = w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { return nil, xerrors.Errorf("failed to generate receiver key: %w", err) } @@ -190,11 +192,11 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, err } - mk1, err := w.Import(k1) + mk1, err := w.WalletImport(context.Background(), k1) if err != nil { return nil, err } - mk2, err := w.Import(k2) + mk2, err := w.WalletImport(context.Background(), k2) if err != nil { return nil, err } @@ -229,12 +231,12 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { Timestamp: uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()), } - genb, err := genesis2.MakeGenesisBlock(context.TODO(), bs, sys, tpl) + genb, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, sys, tpl) if err != nil { return nil, xerrors.Errorf("make genesis block failed: %w", err) } - cs := store.NewChainStore(bs, ds, sys) + cs := store.NewChainStore(bs, ds, sys, j) genfb := &types.FullBlock{Header: genb.Genesis} gents := store.NewFullTipSet([]*types.FullBlock{genfb}) @@ -374,7 +376,13 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add return nil, nil, nil, xerrors.Errorf("get miner worker: %w", err) } - vrfout, err := ComputeVRF(ctx, cg.w.Sign, worker, ticketRand) + sf := func(ctx context.Context, a address.Address, i []byte) (*crypto.Signature, error) { + return cg.w.WalletSign(ctx, a, i, api.MsgMeta{ + Type: api.MTUnknown, + }) + } + + vrfout, err := ComputeVRF(ctx, sf, worker, ticketRand) if err != nil { return nil, nil, nil, xerrors.Errorf("compute VRF: %w", err) } @@ -506,7 +514,7 @@ func (cg *ChainGen) Banker() address.Address { return cg.banker } -func (cg *ChainGen) Wallet() *wallet.Wallet { +func (cg *ChainGen) Wallet() *wallet.LocalWallet { return cg.w } @@ -528,7 +536,9 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) { GasPremium: types.NewInt(0), } - sig, err := cg.w.Sign(context.TODO(), cg.banker, msg.Cid().Bytes()) + sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes(), api.MsgMeta{ + Type: api.MTUnknown, // testing + }) if err != nil { return nil, err } @@ -559,7 +569,7 @@ type MiningCheckAPI interface { } type mca struct { - w *wallet.Wallet + w *wallet.LocalWallet sm *stmgr.StateManager pv ffiwrapper.Verifier bcn beacon.Schedule @@ -588,7 +598,9 @@ func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoc } func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*crypto.Signature, error) { - return mca.w.Sign(ctx, a, v) + return mca.w.WalletSign(ctx, a, v, api.MsgMeta{ + Type: api.MTUnknown, + }) } type WinningPoStProver interface { diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index f532b9f5e..6a1090784 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/journal" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -26,7 +27,6 @@ import ( adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -117,7 +117,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("putting empty object: %w", err) } - state, err := state.NewStateTree(cst, actors.Version0) + state, err := state.NewStateTree(cst, types.StateTreeVersion0) if err != nil { return nil, nil, xerrors.Errorf("making new state tree: %w", err) } @@ -467,7 +467,10 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci return st, nil } -func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallBuilder, template genesis.Template) (*GenesisBootstrap, error) { +func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blockstore, sys vm.SyscallBuilder, template genesis.Template) (*GenesisBootstrap, error) { + if j == nil { + j = journal.NilJournal() + } st, keyIDs, err := MakeInitialStateTree(ctx, bs, template) if err != nil { return nil, xerrors.Errorf("make initial state tree failed: %w", err) @@ -479,7 +482,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB } // temp chainstore - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys, j) // Verify PreSealed Data stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs) diff --git a/chain/gen/mining.go b/chain/gen/mining.go index dd867da48..45a089452 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -15,11 +15,10 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/sigs/bls" ) -func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wallet, bt *api.BlockTemplate) (*types.FullBlock, error) { +func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletAPI, bt *api.BlockTemplate) (*types.FullBlock, error) { pts, err := sm.ChainStore().LoadTipSet(bt.Parents) if err != nil { @@ -131,7 +130,9 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal return nil, xerrors.Errorf("failed to get signing bytes for block: %w", err) } - sig, err := w.Sign(ctx, waddr, nosigbytes) + sig, err := w.WalletSign(ctx, waddr, nosigbytes, api.MsgMeta{ + Type: api.MTBlock, + }) if err != nil { return nil, xerrors.Errorf("failed to sign new block: %w", err) } diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go index aef3b98eb..8b5f85a12 100644 --- a/chain/market/fundmgr.go +++ b/chain/market/fundmgr.go @@ -120,9 +120,10 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add return cid.Undef, err } fm.lk.Lock() + defer fm.lk.Unlock() + bal, err := fm.api.StateMarketBalance(ctx, addr, types.EmptyTSK) if err != nil { - fm.lk.Unlock() return cid.Undef, err } @@ -138,7 +139,6 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add toAdd = types.NewInt(0) } fm.available[idAddr] = big.Add(avail, toAdd) - fm.lk.Unlock() log.Infof("Funds operation w/ Expected Balance: %s, In State: %s, Requested: %s, Adding: %s", avail.String(), stateAvail.String(), amt.String(), toAdd.String()) @@ -148,6 +148,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add params, err := actors.SerializeParams(&addr) if err != nil { + fm.available[idAddr] = avail return cid.Undef, err } @@ -159,6 +160,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add Params: params, }, nil) if err != nil { + fm.available[idAddr] = avail return cid.Undef, err } diff --git a/chain/market/fundmgr_test.go b/chain/market/fundmgr_test.go index b05db55d8..f5936f73d 100644 --- a/chain/market/fundmgr_test.go +++ b/chain/market/fundmgr_test.go @@ -57,9 +57,10 @@ func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Add } type expectedResult struct { - addAmt abi.TokenAmount - shouldAdd bool - err error + addAmt abi.TokenAmount + shouldAdd bool + err error + cachedAvailable abi.TokenAmount } func TestAddFunds(t *testing.T) { @@ -88,8 +89,9 @@ func TestAddFunds(t *testing.T) { addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)}, expectedResults: []expectedResult{ { - shouldAdd: false, - err: nil, + shouldAdd: false, + err: nil, + cachedAvailable: abi.NewTokenAmount(100), }, }, }, @@ -102,18 +104,21 @@ func TestAddFunds(t *testing.T) { err: nil, }, { - addAmt: abi.NewTokenAmount(100), - shouldAdd: true, - err: nil, + addAmt: abi.NewTokenAmount(100), + shouldAdd: true, + err: nil, + cachedAvailable: abi.NewTokenAmount(200), }, { - addAmt: abi.NewTokenAmount(50), - shouldAdd: true, - err: nil, + addAmt: abi.NewTokenAmount(50), + shouldAdd: true, + err: nil, + cachedAvailable: abi.NewTokenAmount(250), }, { - shouldAdd: false, - err: nil, + shouldAdd: false, + err: nil, + cachedAvailable: abi.NewTokenAmount(250), }, }, }, @@ -132,7 +137,8 @@ func TestAddFunds(t *testing.T) { addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)}, expectedResults: []expectedResult{ { - err: errors.New("something went wrong"), + err: errors.New("something went wrong"), + cachedAvailable: abi.NewTokenAmount(0), }, }, }, @@ -183,6 +189,10 @@ func TestAddFunds(t *testing.T) { } else { require.EqualError(t, err, expected.err.Error()) } + + if !expected.cachedAvailable.Nil() { + require.Equal(t, expected.cachedAvailable, fundMgr.available[addr]) + } } }) } diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 83aa5c6b7..d3c638b22 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -159,6 +159,7 @@ type MessagePool struct { sigValCache *lru.TwoQueueCache evtTypes [3]journal.EventType + journal journal.Journal } type msgSet struct { @@ -316,7 +317,7 @@ func (ms *msgSet) getRequiredFunds(nonce uint64) types.BigInt { return types.BigInt{Int: requiredFunds} } -func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*MessagePool, error) { +func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) { cache, _ := lru.New2Q(build.BlsSignatureCacheSize) verifcache, _ := lru.New2Q(build.VerifSigCacheSize) @@ -325,6 +326,10 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa return nil, xerrors.Errorf("error loading mpool config: %w", err) } + if j == nil { + j = journal.NilJournal() + } + mp := &MessagePool{ ds: ds, addSema: make(chan struct{}, 1), @@ -344,10 +349,11 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa netName: netName, cfg: cfg, evtTypes: [...]journal.EventType{ - evtTypeMpoolAdd: journal.J.RegisterEventType("mpool", "add"), - evtTypeMpoolRemove: journal.J.RegisterEventType("mpool", "remove"), - evtTypeMpoolRepub: journal.J.RegisterEventType("mpool", "repub"), + evtTypeMpoolAdd: j.RegisterEventType("mpool", "add"), + evtTypeMpoolRemove: j.RegisterEventType("mpool", "remove"), + evtTypeMpoolRepub: j.RegisterEventType("mpool", "repub"), }, + journal: j, } // enable initial prunes @@ -744,7 +750,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage, strict, untrusted bool) Message: m, }, localUpdates) - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolAdd], func() interface{} { + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolAdd], func() interface{} { return MessagePoolEvt{ Action: "add", Messages: []MessagePoolEvtMessage{{Message: m.Message, CID: m.Cid()}}, @@ -865,7 +871,7 @@ func (mp *MessagePool) remove(from address.Address, nonce uint64, applied bool) Message: m, }, localUpdates) - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRemove], func() interface{} { + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolRemove], func() interface{} { return MessagePoolEvt{ Action: "remove", Messages: []MessagePoolEvtMessage{{Message: m.Message, CID: m.Cid()}}} diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index a4aa059ca..063e0bbab 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -225,14 +225,14 @@ func TestMessagePool(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } a := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -266,14 +266,14 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } a := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -315,7 +315,7 @@ func TestRevertMessages(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -323,7 +323,7 @@ func TestRevertMessages(t *testing.T) { a := tma.nextBlock() b := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -378,7 +378,7 @@ func TestPruningSimple(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -386,7 +386,7 @@ func TestPruningSimple(t *testing.T) { a := tma.nextBlock() tma.applyBlock(t, a) - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -422,7 +422,7 @@ func TestLoadLocal(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -433,7 +433,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -443,7 +443,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -465,7 +465,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - mp, err = New(tma, ds, "mptest") + mp, err = New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -494,7 +494,7 @@ func TestClearAll(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -505,7 +505,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -515,7 +515,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -548,7 +548,7 @@ func TestClearNonLocal(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -559,7 +559,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -569,7 +569,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -609,7 +609,7 @@ func TestUpdates(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -620,7 +620,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -630,7 +630,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index db31e18c2..cdd169e1d 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" "github.com/ipfs/go-cid" ) @@ -148,14 +147,14 @@ loop: } if len(msgs) > 0 { - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { - msgs := make([]MessagePoolEvtMessage, 0, len(msgs)) + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { + msgsEv := make([]MessagePoolEvtMessage, 0, len(msgs)) for _, m := range msgs { - msgs = append(msgs, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) + msgsEv = append(msgsEv, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) } return MessagePoolEvt{ Action: "repub", - Messages: msgs, + Messages: msgsEv, } }) } diff --git a/chain/messagepool/repub_test.go b/chain/messagepool/repub_test.go index 491f484f5..3e1252eec 100644 --- a/chain/messagepool/repub_test.go +++ b/chain/messagepool/repub_test.go @@ -1,6 +1,7 @@ package messagepool import ( + "context" "testing" "time" @@ -21,7 +22,7 @@ func TestRepubMessages(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -32,7 +33,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -42,7 +43,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index ea19dad9c..9e4fe39e5 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -13,6 +13,10 @@ import ( "sort" "testing" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + logging "github.com/ipfs/go-log" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/build" @@ -21,12 +25,10 @@ import ( "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" + "github.com/filecoin-project/lotus/api" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - logging "github.com/ipfs/go-log" ) func init() { @@ -34,7 +36,7 @@ func init() { MaxActorPendingMessages = 1000000 } -func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, gasLimit int64, gasPrice uint64) *types.SignedMessage { +func makeTestMessage(w *wallet.LocalWallet, from, to address.Address, nonce uint64, gasLimit int64, gasPrice uint64) *types.SignedMessage { msg := &types.Message{ From: from, To: to, @@ -45,7 +47,7 @@ func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, g GasFeeCap: types.NewInt(100 + gasPrice), GasPremium: types.NewInt(gasPrice), } - sig, err := w.Sign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{}) if err != nil { panic(err) } @@ -58,7 +60,7 @@ func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, g func makeTestMpool() (*MessagePool, *testMpoolAPI) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "test") + mp, err := New(tma, ds, "test", nil) if err != nil { panic(err) } @@ -75,7 +77,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -85,7 +87,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -313,7 +315,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -323,7 +325,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -389,7 +391,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -399,7 +401,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -533,7 +535,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -543,7 +545,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -596,7 +598,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -606,7 +608,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -675,7 +677,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -685,7 +687,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -744,7 +746,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -754,7 +756,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -841,7 +843,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -851,7 +853,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -908,7 +910,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -918,7 +920,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -984,7 +986,7 @@ func TestOptimalMessageSelection3(t *testing.T) { nActors := 10 // the actors var actors []address.Address - var wallets []*wallet.Wallet + var wallets []*wallet.LocalWallet for i := 0; i < nActors; i++ { w, err := wallet.NewWallet(wallet.NewMemKeyStore()) @@ -992,7 +994,7 @@ func TestOptimalMessageSelection3(t *testing.T) { t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -1064,7 +1066,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu nActors := 300 // the actors var actors []address.Address - var wallets []*wallet.Wallet + var wallets []*wallet.LocalWallet for i := 0; i < nActors; i++ { w, err := wallet.NewWallet(wallet.NewMemKeyStore()) @@ -1072,7 +1074,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -1330,7 +1332,7 @@ readLoop: } actorMap := make(map[address.Address]address.Address) - actorWallets := make(map[address.Address]*wallet.Wallet) + actorWallets := make(map[address.Address]api.WalletAPI) for _, m := range msgs { baseNonce := baseNonces[m.Message.From] @@ -1342,7 +1344,7 @@ readLoop: t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -1360,7 +1362,7 @@ readLoop: m.Message.From = localActor m.Message.Nonce -= baseNonce - sig, err := w.Sign(context.TODO(), localActor, m.Message.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), localActor, m.Message.Cid().Bytes(), api.MsgMeta{}) if err != nil { t.Fatal(err) } diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index ac94d6a3e..9b8f86b64 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -5,40 +5,37 @@ import ( "context" "sync" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) const dsKeyActorNonce = "ActorNextNonce" var log = logging.Logger("messagesigner") -type mpoolAPI interface { +type MpoolNonceAPI interface { GetNonce(address.Address) (uint64, error) } // MessageSigner keeps track of nonces per address, and increments the nonce // when signing a message type MessageSigner struct { - wallet *wallet.Wallet + wallet api.WalletAPI lk sync.Mutex - mpool mpoolAPI + mpool MpoolNonceAPI ds datastore.Batching } -func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { - return newMessageSigner(wallet, mpool, ds) -} - -func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { +func NewMessageSigner(wallet api.WalletAPI, mpool MpoolNonceAPI, ds dtypes.MetadataDS) *MessageSigner { ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/")) return &MessageSigner{ wallet: wallet, @@ -61,7 +58,16 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb // Sign the message with the nonce msg.Nonce = nonce - sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes()) + + mb, err := msg.ToStorageBlock() + if err != nil { + return nil, xerrors.Errorf("serializing message: %w", err) + } + + sig, err := ms.wallet.WalletSign(ctx, msg.From, mb.Cid().Bytes(), api.MsgMeta{ + Type: api.MTChainMsg, + Extra: mb.RawData(), + }) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index 04869ff6d..0dafce9a1 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -47,13 +47,13 @@ func TestMessageSignerSignMessage(t *testing.T) { ctx := context.Background() w, _ := wallet.NewWallet(wallet.NewMemKeyStore()) - from1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + from1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - from2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + from2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - to1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + to1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - to2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + to2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) type msgSpec struct { @@ -177,7 +177,7 @@ func TestMessageSignerSignMessage(t *testing.T) { t.Run(tt.name, func(t *testing.T) { mpool := newMockMpool() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - ms := newMessageSigner(w, mpool, ds) + ms := NewMessageSigner(w, mpool, ds) for _, m := range tt.msgs { if len(m.mpoolNonce) == 1 { diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 3f9597420..e9b76ea77 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" cbg "github.com/whyrusleeping/cbor-gen" @@ -26,7 +27,7 @@ var log = logging.Logger("statetree") // StateTree stores actors state by their ID. type StateTree struct { root adt.Map - version actors.Version // TODO + version types.StateTreeVersion info cid.Cid Store cbor.IpldStore @@ -120,21 +121,41 @@ func (ss *stateSnaps) deleteActor(addr address.Address) { ss.layers[len(ss.layers)-1].actors[addr] = streeOp{Delete: true} } -func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error) { +// VersionForNetwork returns the state tree version for the given network +// version. +func VersionForNetwork(ver network.Version) types.StateTreeVersion { + if actors.VersionForNetwork(ver) == actors.Version0 { + return types.StateTreeVersion0 + } + return types.StateTreeVersion1 +} + +func adtForSTVersion(ver types.StateTreeVersion) actors.Version { + switch ver { + case types.StateTreeVersion0: + return actors.Version0 + case types.StateTreeVersion1: + return actors.Version2 + default: + panic("unhandled state tree version") + } +} + +func NewStateTree(cst cbor.IpldStore, ver types.StateTreeVersion) (*StateTree, error) { var info cid.Cid - switch version { - case actors.Version0: + switch ver { + case types.StateTreeVersion0: // info is undefined - case actors.Version2: + case types.StateTreeVersion1: var err error - info, err = cst.Put(context.TODO(), new(types.StateInfo)) + info, err = cst.Put(context.TODO(), new(types.StateInfo0)) if err != nil { return nil, err } default: - return nil, xerrors.Errorf("unsupported state tree version: %d", version) + return nil, xerrors.Errorf("unsupported state tree version: %d", ver) } - root, err := adt.NewMap(adt.WrapStore(context.TODO(), cst), version) + root, err := adt.NewMap(adt.WrapStore(context.TODO(), cst), adtForSTVersion(ver)) if err != nil { return nil, err } @@ -142,7 +163,7 @@ func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error return &StateTree{ root: root, info: info, - version: version, + version: ver, Store: cst, snaps: newStateSnaps(), }, nil @@ -154,13 +175,16 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { if err := cst.Get(context.TODO(), c, &root); err != nil { // We failed to decode as the new version, must be an old version. root.Actors = c - root.Version = actors.Version0 + root.Version = types.StateTreeVersion0 } switch root.Version { - case actors.Version0, actors.Version2: + case types.StateTreeVersion0, types.StateTreeVersion1: // Load the actual state-tree HAMT. - nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), root.Actors, actors.Version(root.Version)) + nd, err := adt.AsMap( + adt.WrapStore(context.TODO(), cst), root.Actors, + adtForSTVersion(root.Version), + ) if err != nil { log.Errorf("loading hamt node %s failed: %s", c, err) return nil, err @@ -169,7 +193,7 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { return &StateTree{ root: nd, info: root.Info, - version: actors.Version(root.Version), + version: root.Version, Store: cst, snaps: newStateSnaps(), }, nil @@ -309,11 +333,11 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) { return cid.Undef, xerrors.Errorf("failed to flush state-tree hamt: %w", err) } // If we're version 0, return a raw tree. - if st.version == actors.Version0 { + if st.version == types.StateTreeVersion0 { return root, nil } // Otherwise, return a versioned tree. - return st.Store.Put(ctx, &types.StateRoot{Version: uint64(st.version), Actors: root, Info: st.info}) + return st.Store.Put(ctx, &types.StateRoot{Version: st.version, Actors: root, Info: st.info}) } func (st *StateTree) Snapshot(ctx context.Context) error { @@ -400,7 +424,7 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error } // Version returns the version of the StateTree data structure in use. -func (st *StateTree) Version() actors.Version { +func (st *StateTree) Version() types.StateTreeVersion { return st.version } diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 3b08a4b53..ed1fb1889 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -13,13 +13,12 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) func BenchmarkStateTreeSet(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -46,7 +45,7 @@ func BenchmarkStateTreeSet(b *testing.B) { func BenchmarkStateTreeSetFlush(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -76,7 +75,7 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { func BenchmarkStateTree10kGetActor(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -118,7 +117,7 @@ func BenchmarkStateTree10kGetActor(b *testing.B) { func TestSetCache(t *testing.T) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -155,7 +154,7 @@ func TestSetCache(t *testing.T) { func TestSnapshots(t *testing.T) { ctx := context.Background() cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -239,7 +238,7 @@ func assertNotHas(t *testing.T, st *StateTree, addr address.Address) { func TestStateTreeConsistency(t *testing.T) { cst := cbor.NewMemCborStore() // TODO: ActorUpgrade: this test tests pre actors v2 - st, err := NewStateTree(cst, actors.VersionForNetwork(network.Version3)) + st, err := NewStateTree(cst, VersionForNetwork(network.Version3)) if err != nil { t.Fatal(err) } diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 15cbac53e..df3bfa357 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -2,6 +2,7 @@ package stmgr import ( "context" + "errors" "fmt" "github.com/filecoin-project/go-address" @@ -17,17 +18,38 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) +var ErrExpensiveFork = errors.New("refusing explicit call due to state fork at epoch") + func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { ctx, span := trace.StartSpan(ctx, "statemanager.Call") defer span.End() + // If no tipset is provided, try to find one without a fork. if ts == nil { ts = sm.cs.GetHeaviestTipSet() + + // Search back till we find a height with no fork, or we reach the beginning. + for ts.Height() > 0 && sm.hasExpensiveFork(ctx, ts.Height()-1) { + var err error + ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err) + } + } } bstate := ts.ParentState() bheight := ts.Height() + // If we have to run an expensive migration, and we're not at genesis, + // return an error because the migration will take too long. + // + // We allow this at height 0 for at-genesis migrations (for testing). + if bheight-1 > 0 && sm.hasExpensiveFork(ctx, bheight-1) { + return nil, ErrExpensiveFork + } + + // Run the (not expensive) migration. bstate, err := sm.handleStateForks(ctx, bstate, bheight-1, nil, ts) if err != nil { return nil, fmt.Errorf("failed to handle fork: %w", err) @@ -44,7 +66,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. BaseFee: types.NewInt(0), } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } @@ -106,6 +128,24 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri if ts == nil { ts = sm.cs.GetHeaviestTipSet() + + // Search back till we find a height with no fork, or we reach the beginning. + // We need the _previous_ height to have no fork, because we'll + // run the fork logic in `sm.TipSetState`. We need the _current_ + // height to have no fork, because we'll run it inside this + // function before executing the given message. + for ts.Height() > 0 && (sm.hasExpensiveFork(ctx, ts.Height()) || sm.hasExpensiveFork(ctx, ts.Height()-1)) { + var err error + ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err) + } + } + } + + // When we're not at the genesis block, make sure we don't have an expensive migration. + if ts.Height() > 0 && (sm.hasExpensiveFork(ctx, ts.Height()) || sm.hasExpensiveFork(ctx, ts.Height()-1)) { + return nil, ErrExpensiveFork } state, _, err := sm.TipSetState(ctx, ts) @@ -138,7 +178,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index d5d0dbf7e..fba92ee3f 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -6,6 +6,8 @@ import ( "encoding/binary" "math" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" @@ -22,23 +24,32 @@ import ( "github.com/filecoin-project/specs-actors/actors/migration/nv3" m2 "github.com/filecoin-project/specs-actors/v2/actors/migration" - states2 "github.com/filecoin-project/specs-actors/v2/actors/states" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/state" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + bstore "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" ) -type UpgradeFunc func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error) +// UpgradeFunc is a migration function run at every upgrade. +// +// - The oldState is the state produced by the upgrade epoch. +// - The returned newState is the new state that will be used by the next epoch. +// - The height is the upgrade epoch height (already executed). +// - The tipset is the tipset for the last non-null block before the upgrade. Do +// not assume that ts.Height() is the upgrade height. +type UpgradeFunc func(ctx context.Context, sm *StateManager, cb ExecCallback, oldState cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (newState cid.Cid, err error) type Upgrade struct { Height abi.ChainEpoch Network network.Version + Expensive bool Migration UpgradeFunc } @@ -47,7 +58,7 @@ type UpgradeSchedule []Upgrade func DefaultUpgradeSchedule() UpgradeSchedule { var us UpgradeSchedule - for _, u := range []Upgrade{{ + updates := []Upgrade{{ Height: build.UpgradeBreezeHeight, Network: network.Version1, Migration: UpgradeFaucetBurnRecovery, @@ -59,15 +70,46 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Height: build.UpgradeIgnitionHeight, Network: network.Version3, Migration: UpgradeIgnition, + }, { + Height: build.UpgradeRefuelHeight, + Network: network.Version3, + Migration: UpgradeRefuel, }, { Height: build.UpgradeActorsV2Height, Network: network.Version4, + Expensive: true, Migration: UpgradeActorsV2, }, { Height: build.UpgradeLiftoffHeight, Network: network.Version4, Migration: UpgradeLiftoff, - }} { + }} + + if build.UpgradeActorsV2Height == math.MaxInt64 { // disable actors upgrade + updates = []Upgrade{{ + Height: build.UpgradeBreezeHeight, + Network: network.Version1, + Migration: UpgradeFaucetBurnRecovery, + }, { + Height: build.UpgradeSmokeHeight, + Network: network.Version2, + Migration: nil, + }, { + Height: build.UpgradeIgnitionHeight, + Network: network.Version3, + Migration: UpgradeIgnition, + }, { + Height: build.UpgradeRefuelHeight, + Network: network.Version3, + Migration: UpgradeRefuel, + }, { + Height: build.UpgradeLiftoffHeight, + Network: network.Version3, + Migration: UpgradeLiftoff, + }} + } + + for _, u := range updates { if u.Height < 0 { // upgrade disabled continue @@ -109,7 +151,7 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig var err error f, ok := sm.stateMigrations[height] if ok { - retCid, err = f(ctx, sm, cb, root, ts) + retCid, err = f(ctx, sm, cb, root, height, ts) if err != nil { return cid.Undef, err } @@ -118,6 +160,11 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig return retCid, nil } +func (sm *StateManager) hasExpensiveFork(ctx context.Context, height abi.ChainEpoch) bool { + _, ok := sm.expensiveUpgrades[height] + return ok +} + func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { fromAct, err := tree.GetActor(from) if err != nil { @@ -180,7 +227,7 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, return nil } -func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { // Some initial parameters FundsForMiners := types.FromFil(1_000_000) LookbackEpoch := abi.ChainEpoch(32000) @@ -432,11 +479,9 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal return tree.Flush(ctx) } -func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) - epoch := ts.Height() - 1 - if build.UpgradeLiftoffHeight <= epoch { return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height") } @@ -489,12 +534,42 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return tree.Flush(ctx) } -func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeRefuel(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + store := sm.cs.Store(ctx) + tree, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } - epoch := ts.Height() - 1 + addr, err := address.NewFromString("t0122") + if err != nil { + return cid.Undef, xerrors.Errorf("getting address: %w", err) + } - info, err := store.Put(ctx, new(types.StateInfo)) + err = resetMultisigVesting(ctx, store, tree, addr, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + err = resetMultisigVesting(ctx, store, tree, builtin.ReserveAddress, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + err = resetMultisigVesting(ctx, store, tree, builtin.RootVerifierAddress, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + return tree.Flush(ctx) +} + +func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync()) + store := store.ActorStore(ctx, buf) + + info, err := store.Put(ctx, new(types.StateInfo0)) if err != nil { return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err) } @@ -504,22 +579,8 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("upgrading to actors v2: %w", err) } - newStateTree, err := states2.LoadTree(store, newHamtRoot) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to load new state tree: %w", err) - } - - // Check all state-tree invariants. - if msgs, err := states2.CheckStateInvariants(newStateTree, types.TotalFilecoinInt); err != nil { - return cid.Undef, xerrors.Errorf("failed to check new state tree: %w", err) - } else if !msgs.IsEmpty() { - // This error is going to be really nasty. - return cid.Undef, xerrors.Errorf("network upgrade failed: %v", msgs.Messages()) - } - newRoot, err := store.Put(ctx, &types.StateRoot{ - // TODO: ActorUpgrade: should be state-tree specific, not just the actors version. - Version: actors.Version2, + Version: types.StateTreeVersion1, Actors: newHamtRoot, Info: info, }) @@ -538,10 +599,19 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("failed to load init actor after upgrade: %w", err) } + { + from := buf + to := buf.Read() + + if err := vm.Copy(ctx, from, to, newRoot); err != nil { + return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) + } + } + return newRoot, nil } -func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { tree, err := sm.StateTree(root) if err != nil { return cid.Undef, xerrors.Errorf("getting state tree: %w", err) @@ -699,6 +769,7 @@ func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, erro return addr, nil } +// TODO: After the Liftoff epoch, refactor this to use resetMultisigVesting func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, startEpoch abi.ChainEpoch) error { gb, err := sm.cs.GetGenesis() if err != nil { @@ -748,3 +819,34 @@ func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, return nil } + +func resetMultisigVesting(ctx context.Context, store adt0.Store, tree *state.StateTree, addr address.Address, startEpoch abi.ChainEpoch, duration abi.ChainEpoch, balance abi.TokenAmount) error { + act, err := tree.GetActor(addr) + if err != nil { + return xerrors.Errorf("getting actor: %w", err) + } + + if !builtin.IsMultisigActor(act.Code) { + return xerrors.Errorf("actor wasn't msig: %w", err) + } + + var msigState multisig0.State + if err := store.Get(ctx, act.Head, &msigState); err != nil { + return xerrors.Errorf("reading multisig state: %w", err) + } + + msigState.StartEpoch = startEpoch + msigState.UnlockDuration = duration + msigState.InitialBalance = balance + + act.Head, err = store.Put(ctx, &msigState) + if err != nil { + return xerrors.Errorf("writing new multisig state: %w", err) + } + + if err := tree.SetActor(addr, act); err != nil { + return xerrors.Errorf("setting multisig actor: %w", err) + } + + return nil +} diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index bb03f13b9..0388af6ad 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -6,14 +6,21 @@ import ( "io" "testing" + "github.com/ipfs/go-cid" + ipldcbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" + "github.com/stretchr/testify/require" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/runtime" - "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -24,11 +31,6 @@ import ( "github.com/filecoin-project/lotus/chain/vm" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - - "github.com/ipfs/go-cid" - ipldcbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" - cbg "github.com/whyrusleeping/cbor-gen" ) func init() { @@ -76,7 +78,7 @@ func (ta testActor) Exports() []interface{} { func (ta *testActor) Constructor(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() rt.StateCreate(&testActorState{11}) - fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Receiver()) + //fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Receiver()) return abi.Empty } @@ -120,7 +122,7 @@ func TestForkHeightTriggers(t *testing.T) { Network: 1, Height: testForkHeight, Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, - root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore()) st, err := sm.StateTree(root) @@ -173,7 +175,7 @@ func TestForkHeightTriggers(t *testing.T) { var msgs []*types.SignedMessage - enc, err := actors.SerializeParams(&init_.ExecParams{CodeCID: (testActor{}).Code()}) + enc, err := actors.SerializeParams(&init0.ExecParams{CodeCID: (testActor{}).Code()}) if err != nil { t.Fatal(err) } @@ -185,7 +187,7 @@ func TestForkHeightTriggers(t *testing.T) { Params: enc, GasLimit: types.TestGasLimit, } - sig, err := cg.Wallet().Sign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes(), api.MsgMeta{}) if err != nil { t.Fatal(err) } @@ -213,7 +215,7 @@ func TestForkHeightTriggers(t *testing.T) { } nonce++ - sig, err := cg.Wallet().Sign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes(), api.MsgMeta{}) if err != nil { return nil, err } @@ -233,3 +235,84 @@ func TestForkHeightTriggers(t *testing.T) { } } } + +func TestForkRefuseCall(t *testing.T) { + logging.SetAllLoggers(logging.LevelInfo) + + ctx := context.TODO() + + cg, err := gen.NewGenerator() + if err != nil { + t.Fatal(err) + } + + sm, err := NewStateManagerWithUpgradeSchedule( + cg.ChainStore(), UpgradeSchedule{{ + Network: 1, + Expensive: true, + Height: testForkHeight, + Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, + root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + return root, nil + }}}) + if err != nil { + t.Fatal(err) + } + + inv := vm.NewActorRegistry() + inv.Register(nil, testActor{}) + + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) { + nvm, err := vm.NewVM(ctx, vmopt) + if err != nil { + return nil, err + } + nvm.SetInvoker(inv) + return nvm, nil + }) + + cg.SetStateManager(sm) + + enc, err := actors.SerializeParams(&init0.ExecParams{CodeCID: (testActor{}).Code()}) + if err != nil { + t.Fatal(err) + } + + m := &types.Message{ + From: cg.Banker(), + To: lotusinit.Address, + Method: builtin.MethodsInit.Exec, + Params: enc, + GasLimit: types.TestGasLimit, + Value: types.NewInt(0), + GasPremium: types.NewInt(0), + GasFeeCap: types.NewInt(0), + } + + for i := 0; i < 50; i++ { + ts, err := cg.NextTipSet() + if err != nil { + t.Fatal(err) + } + + ret, err := sm.CallWithGas(ctx, m, nil, ts.TipSet.TipSet()) + switch ts.TipSet.TipSet().Height() { + case testForkHeight, testForkHeight + 1: + // If I had a fork, or I _will_ have a fork, it should fail. + require.Equal(t, ErrExpensiveFork, err) + default: + require.NoError(t, err) + require.True(t, ret.MsgRct.ExitCode.IsSuccess()) + } + // Call just runs on the parent state for a tipset, so we only + // expect an error at the fork height. + ret, err = sm.Call(ctx, m, ts.TipSet.TipSet()) + switch ts.TipSet.TipSet().Height() { + case testForkHeight + 1: + require.Equal(t, ErrExpensiveFork, err) + default: + require.NoError(t, err) + require.True(t, ret.MsgRct.ExitCode.IsSuccess()) + } + } +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ac01ebb61..d6b6f4360 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -2,6 +2,7 @@ package stmgr import ( "context" + "errors" "fmt" "sync" @@ -37,8 +38,16 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) +const LookbackNoLimit = abi.ChainEpoch(-1) + var log = logging.Logger("statemgr") +type StateManagerAPI interface { + LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) + LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) + ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) +} + type versionSpec struct { networkVersion network.Version atOrBelow abi.ChainEpoch @@ -53,6 +62,10 @@ type StateManager struct { // Maps chain epochs to upgrade functions. stateMigrations map[abi.ChainEpoch]UpgradeFunc + // A set of potentially expensive/time consuming upgrades. Explicit + // calls for, e.g., gas estimation fail against this epoch with + // ErrExpensiveFork. + expensiveUpgrades map[abi.ChainEpoch]struct{} stCache map[string][]cid.Cid compWait map[string]chan struct{} @@ -78,6 +91,7 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule } stateMigrations := make(map[abi.ChainEpoch]UpgradeFunc, len(us)) + expensiveUpgrades := make(map[abi.ChainEpoch]struct{}, len(us)) var networkVersions []versionSpec lastVersion := network.Version0 if len(us) > 0 { @@ -87,6 +101,9 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule if upgrade.Migration != nil { stateMigrations[upgrade.Height] = upgrade.Migration } + if upgrade.Expensive { + expensiveUpgrades[upgrade.Height] = struct{}{} + } networkVersions = append(networkVersions, versionSpec{ networkVersion: lastVersion, atOrBelow: upgrade.Height, @@ -99,13 +116,14 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule } return &StateManager{ - networkVersions: networkVersions, - latestVersion: lastVersion, - stateMigrations: stateMigrations, - newVM: vm.NewVM, - cs: cs, - stCache: make(map[string][]cid.Cid), - compWait: make(map[string]chan struct{}), + networkVersions: networkVersions, + latestVersion: lastVersion, + stateMigrations: stateMigrations, + expensiveUpgrades: expensiveUpgrades, + newVM: vm.NewVM, + cs: cs, + stCache: make(map[string][]cid.Cid), + compWait: make(map[string]chan struct{}), }, nil } @@ -498,16 +516,7 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T return nil, fmt.Errorf("failed to load message: %w", err) } - r, _, err := sm.tipsetExecutedMessage(ts, msg, m.VMMessage()) - if err != nil { - return nil, err - } - - if r != nil { - return r, nil - } - - _, r, _, err = sm.searchBackForMsg(ctx, ts, m) + _, r, _, err := sm.searchBackForMsg(ctx, ts, m, LookbackNoLimit) if err != nil { return nil, fmt.Errorf("failed to look back through chain for message: %w", err) } @@ -516,9 +525,9 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T } // WaitForMessage blocks until a message appears on chain. It looks backwards in the chain to see if this has already -// happened. It guarantees that the message has been on chain for at least confidence epochs without being reverted -// before returning. -func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { +// happened, with an optional limit to how many epochs it will search. It guarantees that the message has been on +// chain for at least confidence epochs without being reverted before returning. +func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -556,7 +565,7 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confid var backFm cid.Cid backSearchWait := make(chan struct{}) go func() { - fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head[0].Val, msg) + fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head[0].Val, msg, lookbackLimit) if err != nil { log.Warnf("failed to look back through chain for message: %w", err) return @@ -648,7 +657,7 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty return head, r, foundMsg, nil } - fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg) + fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg, LookbackNoLimit) if err != nil { log.Warnf("failed to look back through chain for message %s", mcid) @@ -662,11 +671,33 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty return fts, r, foundMsg, nil } -func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet, m types.ChainMsg) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { +// searchBackForMsg searches up to limit tipsets backwards from the given +// tipset for a message receipt. +// If limit is +// - 0 then no tipsets are searched +// - 5 then five tipset are searched +// - LookbackNoLimit then there is no limit +func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet, m types.ChainMsg, limit abi.ChainEpoch) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { + limitHeight := from.Height() - limit + noLimit := limit == LookbackNoLimit cur := from + curActor, err := sm.LoadActor(ctx, m.VMMessage().From, cur) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("failed to load initital tipset") + } + + mFromId, err := sm.LookupID(ctx, m.VMMessage().From, from) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("looking up From id address: %w", err) + } + + mNonce := m.VMMessage().Nonce + for { - if cur.Height() == 0 { + // If we've reached the genesis block, or we've reached the limit of + // how far back to look + if cur.Height() == 0 || !noLimit && cur.Height() <= limitHeight { // it ain't here! return nil, nil, cid.Undef, nil } @@ -677,32 +708,37 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet default: } - act, err := sm.LoadActor(ctx, m.VMMessage().From, cur) - if err != nil { - return nil, nil, cid.Cid{}, err - } - // we either have no messages from the sender, or the latest message we found has a lower nonce than the one being searched for, // either way, no reason to lookback, it ain't there - if act.Nonce == 0 || act.Nonce < m.VMMessage().Nonce { + if curActor == nil || curActor.Nonce == 0 || curActor.Nonce < mNonce { return nil, nil, cid.Undef, nil } - ts, err := sm.cs.LoadTipSet(cur.Parents()) + pts, err := sm.cs.LoadTipSet(cur.Parents()) if err != nil { - return nil, nil, cid.Undef, fmt.Errorf("failed to load tipset during msg wait searchback: %w", err) + return nil, nil, cid.Undef, xerrors.Errorf("failed to load tipset during msg wait searchback: %w", err) } - r, foundMsg, err := sm.tipsetExecutedMessage(ts, m.Cid(), m.VMMessage()) - if err != nil { - return nil, nil, cid.Undef, fmt.Errorf("checking for message execution during lookback: %w", err) + act, err := sm.LoadActor(ctx, mFromId, pts) + actorNoExist := errors.Is(err, types.ErrActorNotFound) + if err != nil && !actorNoExist { + return nil, nil, cid.Cid{}, xerrors.Errorf("failed to load the actor: %w", err) } - if r != nil { - return ts, r, foundMsg, nil + // check that between cur and parent tipset the nonce fell into range of our message + if actorNoExist || (curActor.Nonce > mNonce && act.Nonce <= mNonce) { + r, foundMsg, err := sm.tipsetExecutedMessage(cur, m.Cid(), m.VMMessage()) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("checking for message execution during lookback: %w", err) + } + + if r != nil { + return pts, r, foundMsg, nil + } } - cur = ts + cur = pts + curActor = act } } @@ -1375,3 +1411,5 @@ func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (m } return actState, nil } + +var _ StateManagerAPI = (*StateManager)(nil) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 2c9c5ad94..c0f0c4d2f 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -387,7 +387,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return cid.Undef, nil, err } diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 63e08070c..5283d10dc 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -31,7 +31,7 @@ func TestIndexSeeks(t *testing.T) { ctx := context.TODO() nbs := blockstore.NewTemporarySync() - cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) + cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil) _, err = cs.Import(bytes.NewReader(gencar)) if err != nil { diff --git a/chain/store/store.go b/chain/store/store.go index aac28e5d3..c6fc0cbef 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -8,6 +8,7 @@ import ( "io" "os" "strconv" + "strings" "sync" "golang.org/x/sync/errgroup" @@ -38,7 +39,9 @@ import ( lru "github.com/hashicorp/golang-lru" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" dstore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" car "github.com/ipld/go-car" @@ -102,7 +105,7 @@ type HeadChangeEvt struct { // 2. a block => messages references cache. type ChainStore struct { bs bstore.Blockstore - ds dstore.Datastore + ds dstore.Batching heaviestLk sync.Mutex heaviest *types.TipSet @@ -124,11 +127,15 @@ type ChainStore struct { vmcalls vm.SyscallBuilder evtTypes [1]journal.EventType + journal journal.Journal } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder) *ChainStore { +func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore { c, _ := lru.NewARC(DefaultMsgMetaCacheSize) tsc, _ := lru.NewARC(DefaultTipSetCacheSize) + if j == nil { + j = journal.NilJournal() + } cs := &ChainStore{ bs: bs, ds: ds, @@ -137,10 +144,11 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallB mmCache: c, tsCache: tsc, vmcalls: vmcalls, + journal: j, } cs.evtTypes = [1]journal.EventType{ - evtTypeHeadChange: journal.J.RegisterEventType("sync", "head_change"), + evtTypeHeadChange: j.RegisterEventType("sync", "head_change"), } ci := NewChainIndex(cs.LoadTipSet) @@ -379,7 +387,7 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo continue } - journal.J.RecordEvent(cs.evtTypes[evtTypeHeadChange], func() interface{} { + cs.journal.RecordEvent(cs.evtTypes[evtTypeHeadChange], func() interface{} { return HeadChangeEvt{ From: r.old.Key(), FromHeight: r.old.Height(), @@ -441,6 +449,53 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet) return nil } +// FlushValidationCache removes all results of block validation from the +// chain metadata store. Usually the first step after a new chain import. +func (cs *ChainStore) FlushValidationCache() error { + log.Infof("clearing block validation cache...") + + dsWalk, err := cs.ds.Query(query.Query{ + // Potential TODO: the validation cache is not a namespace on its own + // but is rather constructed as prefixed-key `foo:bar` via .Instance(), which + // in turn does not work with the filter, which can match only on `foo/bar` + // + // If this is addressed (blockcache goes into its own sub-namespace) then + // strings.HasPrefix(...) below can be skipped + // + //Prefix: blockValidationCacheKeyPrefix.String() + KeysOnly: true, + }) + if err != nil { + return xerrors.Errorf("failed to initialize key listing query: %w", err) + } + + allKeys, err := dsWalk.Rest() + if err != nil { + return xerrors.Errorf("failed to run key listing query: %w", err) + } + + batch, err := cs.ds.Batch() + if err != nil { + return xerrors.Errorf("failed to open a DS batch: %w", err) + } + + delCnt := 0 + for _, k := range allKeys { + if strings.HasPrefix(k.Key, blockValidationCacheKeyPrefix.String()) { + delCnt++ + batch.Delete(datastore.RawKey(k.Key)) // nolint:errcheck + } + } + + if err := batch.Commit(); err != nil { + return xerrors.Errorf("failed to commit the DS batch: %w", err) + } + + log.Infof("%d block validation entries cleared.", delCnt) + + return nil +} + // SetHead sets the chainstores current 'best' head node. // This should only be called if something is broken and needs fixing func (cs *ChainStore) SetHead(ts *types.TipSet) error { diff --git a/chain/store/store_test.go b/chain/store/store_test.go index b7adfb595..326899f90 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -62,7 +62,7 @@ func BenchmarkGetRandomness(b *testing.B) { bs := blockstore.NewBlockstore(bds) - cs := store.NewChainStore(bs, mds, nil) + cs := store.NewChainStore(bs, mds, nil, nil) b.ResetTimer() @@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) { } nbs := blockstore.NewTemporary() - cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil) + cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, nil) root, err := cs.Import(buf) if err != nil { diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 07b3343d2..68ee5e20c 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -120,12 +120,6 @@ func FetchMessagesByCids( return err } - // FIXME: We already sort in `fetchCids`, we are duplicating too much work, - // we don't need to pass the index. - if out[i] != nil { - return fmt.Errorf("received duplicate message") - } - out[i] = msg return nil }) @@ -149,10 +143,6 @@ func FetchSignedMessagesByCids( return err } - if out[i] != nil { - return fmt.Errorf("received duplicate message") - } - out[i] = smsg return nil }) @@ -172,37 +162,41 @@ func fetchCids( cids []cid.Cid, cb func(int, blocks.Block) error, ) error { - fetchedBlocks := bserv.GetBlocks(ctx, cids) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() cidIndex := make(map[cid.Cid]int) for i, c := range cids { cidIndex[c] = i } + if len(cids) != len(cidIndex) { + return fmt.Errorf("duplicate CIDs in fetchCids input") + } - for i := 0; i < len(cids); i++ { - select { - case block, ok := <-fetchedBlocks: - if !ok { - // Closed channel, no more blocks fetched, check if we have all - // of the CIDs requested. - // FIXME: Review this check. We don't call the callback on the - // last index? - if i == len(cids)-1 { - break - } - - return fmt.Errorf("failed to fetch all messages") - } - - ix, ok := cidIndex[block.Cid()] - if !ok { - return fmt.Errorf("received message we didnt ask for") - } - - if err := cb(ix, block); err != nil { - return err - } + for block := range bserv.GetBlocks(ctx, cids) { + ix, ok := cidIndex[block.Cid()] + if !ok { + // Ignore duplicate/unexpected blocks. This shouldn't + // happen, but we can be safe. + log.Errorw("received duplicate/unexpected block when syncing", "cid", block.Cid()) + continue } + + // Record that we've received the block. + delete(cidIndex, block.Cid()) + + if err := cb(ix, block); err != nil { + return err + } + } + + if len(cidIndex) > 0 { + err := ctx.Err() + if err == nil { + err = fmt.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex)) + } + return err } return nil diff --git a/chain/sub/incoming_test.go b/chain/sub/incoming_test.go new file mode 100644 index 000000000..215439209 --- /dev/null +++ b/chain/sub/incoming_test.go @@ -0,0 +1,63 @@ +package sub + +import ( + "context" + "testing" + + address "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" +) + +type getter struct { + msgs []*types.Message +} + +func (g *getter) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { panic("NYI") } + +func (g *getter) GetBlocks(ctx context.Context, ks []cid.Cid) <-chan blocks.Block { + ch := make(chan blocks.Block, len(g.msgs)) + for _, m := range g.msgs { + by, err := m.Serialize() + if err != nil { + panic(err) + } + b, err := blocks.NewBlockWithCid(by, m.Cid()) + if err != nil { + panic(err) + } + ch <- b + } + close(ch) + return ch +} + +func TestFetchCidsWithDedup(t *testing.T) { + msgs := []*types.Message{} + for i := 0; i < 10; i++ { + msgs = append(msgs, &types.Message{ + From: address.TestAddress, + To: address.TestAddress, + + Nonce: uint64(i), + }) + } + cids := []cid.Cid{} + for _, m := range msgs { + cids = append(cids, m.Cid()) + } + g := &getter{msgs} + + // the cids have a duplicate + res, err := FetchMessagesByCids(context.TODO(), g, append(cids, cids[0])) + + t.Logf("err: %+v", err) + t.Logf("res: %+v", res) + if err == nil { + t.Errorf("there should be an error") + } + if err == nil && (res[0] == nil || res[len(res)-1] == nil) { + t.Fatalf("there is a nil message: first %p, last %p", res[0], res[len(res)-1]) + } +} diff --git a/chain/sync.go b/chain/sync.go index 655bb4a1d..9040e3f05 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -217,6 +217,12 @@ func (syncer *Syncer) Stop() { // This should be called when connecting to new peers, and additionally // when receiving new blocks from the network func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) bool { + defer func() { + if err := recover(); err != nil { + log.Errorf("panic in InformNewHead: ", err) + } + }() + ctx := context.Background() if fts == nil { log.Errorf("got nil tipset in InformNewHead") @@ -731,7 +737,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, use lbst, _, err := syncer.sm.TipSetState(ctx, lbts) if err != nil { - return xerrors.Errorf("failed to compute lookback tipset state: %w", err) + return xerrors.Errorf("failed to compute lookback tipset state (epoch %d): %w", lbts.Height(), err) } prevBeacon, err := syncer.store.GetLatestBeaconEntry(baseTs) @@ -1281,9 +1287,11 @@ func (syncer *Syncer) collectHeaders(ctx context.Context, incoming *types.TipSet blockSet := []*types.TipSet{incoming} + // Parent of the new (possibly better) tipset that we need to fetch next. at := incoming.Parents() - // we want to sync all the blocks until the height above the block we have + // we want to sync all the blocks until the height above our + // best tipset so far untilHeight := known.Height() + 1 ss.SetHeight(blockSet[len(blockSet)-1].Height()) @@ -1377,13 +1385,17 @@ loop: } base := blockSet[len(blockSet)-1] - if base.Parents() == known.Parents() { - // common case: receiving a block thats potentially part of the same tipset as our best block + if base.IsChildOf(known) { + // common case: receiving blocks that are building on top of our best tipset return blockSet, nil } - if types.CidArrsEqual(base.Parents().Cids(), known.Cids()) { - // common case: receiving blocks that are building on top of our best tipset + knownParent, err := syncer.store.LoadTipSet(known.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to load next local tipset: %w", err) + } + if base.IsChildOf(knownParent) { + // common case: receiving a block thats potentially part of the same tipset as our best block return blockSet, nil } @@ -1514,7 +1526,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS ss.SetStage(api.StageMessages) if batchErr != nil { - return xerrors.Errorf("failed to fetch messages: %w", err) + return xerrors.Errorf("failed to fetch messages: %w", batchErr) } for bsi := 0; bsi < len(bstout); bsi++ { @@ -1714,7 +1726,7 @@ func VerifyElectionPoStVRF(ctx context.Context, worker address.Address, rand []b return gen.VerifyVRF(ctx, worker, rand, evrf) } -func (syncer *Syncer) State() []SyncerState { +func (syncer *Syncer) State() []SyncerStateSnapshot { return syncer.syncmgr.State() } @@ -1728,6 +1740,10 @@ func (syncer *Syncer) UnmarkBad(blk cid.Cid) { syncer.bad.Remove(blk) } +func (syncer *Syncer) UnmarkAllBad() { + syncer.bad.Purge() +} + func (syncer *Syncer) CheckBadBlockCache(blk cid.Cid) (string, bool) { bbr, ok := syncer.bad.Has(blk) return bbr.String(), ok diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 811092bc7..c7fdea726 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -38,7 +38,7 @@ type SyncManager interface { SetPeerHead(ctx context.Context, p peer.ID, ts *types.TipSet) // State retrieves the state of the sync workers. - State() []SyncerState + State() []SyncerStateSnapshot } type syncManager struct { @@ -79,7 +79,7 @@ type syncResult struct { const syncWorkerCount = 3 func NewSyncManager(sync SyncFunc) SyncManager { - return &syncManager{ + sm := &syncManager{ bspThresh: 1, peerHeads: make(map[peer.ID]*types.TipSet), syncTargets: make(chan *types.TipSet), @@ -90,6 +90,10 @@ func NewSyncManager(sync SyncFunc) SyncManager { doSync: sync, stop: make(chan struct{}), } + for i := range sm.syncStates { + sm.syncStates[i] = new(SyncerState) + } + return sm } func (sm *syncManager) Start() { @@ -128,8 +132,8 @@ func (sm *syncManager) SetPeerHead(ctx context.Context, p peer.ID, ts *types.Tip sm.incomingTipSets <- ts } -func (sm *syncManager) State() []SyncerState { - ret := make([]SyncerState, 0, len(sm.syncStates)) +func (sm *syncManager) State() []SyncerStateSnapshot { + ret := make([]SyncerStateSnapshot, 0, len(sm.syncStates)) for _, s := range sm.syncStates { ret = append(ret, s.Snapshot()) } @@ -405,8 +409,7 @@ func (sm *syncManager) scheduleWorkSent() { } func (sm *syncManager) syncWorker(id int) { - ss := &SyncerState{} - sm.syncStates[id] = ss + ss := sm.syncStates[id] for { select { case ts, ok := <-sm.syncTargets: diff --git a/chain/sync_test.go b/chain/sync_test.go index 1b06f604b..0a8174c41 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -221,8 +221,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { sourceRepo, genesis, blocks := tu.repoWithChain(tu.t, gen) var out api.FullNode - // TODO: Don't ignore stop - _, err := node.New(tu.ctx, + stop, err := node.New(tu.ctx, node.FullAPI(&out), node.Online(), node.Repo(sourceRepo), @@ -232,6 +231,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { node.Override(new(modules.Genesis), modules.LoadGenesis(genesis)), ) require.NoError(tu.t, err) + tu.t.Cleanup(func() { _ = stop(context.Background()) }) lastTs := blocks[len(blocks)-1].Blocks for _, lastB := range lastTs { @@ -253,8 +253,7 @@ func (tu *syncTestUtil) addClientNode() int { var out api.FullNode - // TODO: Don't ignore stop - _, err := node.New(tu.ctx, + stop, err := node.New(tu.ctx, node.FullAPI(&out), node.Online(), node.Repo(repo.NewMemory(nil)), @@ -264,6 +263,7 @@ func (tu *syncTestUtil) addClientNode() int { node.Override(new(modules.Genesis), modules.LoadGenesis(tu.genesis)), ) require.NoError(tu.t, err) + tu.t.Cleanup(func() { _ = stop(context.Background()) }) tu.nds = append(tu.nds, out) return len(tu.nds) - 1 @@ -593,7 +593,7 @@ func TestDuplicateNonce(t *testing.T) { GasPremium: types.NewInt(0), } - sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes(), api.MsgMeta{}) require.NoError(t, err) return &types.SignedMessage{ @@ -685,7 +685,7 @@ func TestBadNonce(t *testing.T) { GasPremium: types.NewInt(0), } - sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes(), api.MsgMeta{}) require.NoError(t, err) return &types.SignedMessage{ diff --git a/chain/syncstate.go b/chain/syncstate.go index 06cd5d91e..26f9f1c39 100644 --- a/chain/syncstate.go +++ b/chain/syncstate.go @@ -11,8 +11,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -type SyncerState struct { - lk sync.Mutex +type SyncerStateSnapshot struct { Target *types.TipSet Base *types.TipSet Stage api.SyncStateStage @@ -22,6 +21,11 @@ type SyncerState struct { End time.Time } +type SyncerState struct { + lk sync.Mutex + data SyncerStateSnapshot +} + func (ss *SyncerState) SetStage(v api.SyncStateStage) { if ss == nil { return @@ -29,9 +33,9 @@ func (ss *SyncerState) SetStage(v api.SyncStateStage) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Stage = v + ss.data.Stage = v if v == api.StageSyncComplete { - ss.End = build.Clock.Now() + ss.data.End = build.Clock.Now() } } @@ -42,13 +46,13 @@ func (ss *SyncerState) Init(base, target *types.TipSet) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Target = target - ss.Base = base - ss.Stage = api.StageHeaders - ss.Height = 0 - ss.Message = "" - ss.Start = build.Clock.Now() - ss.End = time.Time{} + ss.data.Target = target + ss.data.Base = base + ss.data.Stage = api.StageHeaders + ss.data.Height = 0 + ss.data.Message = "" + ss.data.Start = build.Clock.Now() + ss.data.End = time.Time{} } func (ss *SyncerState) SetHeight(h abi.ChainEpoch) { @@ -58,7 +62,7 @@ func (ss *SyncerState) SetHeight(h abi.ChainEpoch) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Height = h + ss.data.Height = h } func (ss *SyncerState) Error(err error) { @@ -68,21 +72,13 @@ func (ss *SyncerState) Error(err error) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Message = err.Error() - ss.Stage = api.StageSyncErrored - ss.End = build.Clock.Now() + ss.data.Message = err.Error() + ss.data.Stage = api.StageSyncErrored + ss.data.End = build.Clock.Now() } -func (ss *SyncerState) Snapshot() SyncerState { +func (ss *SyncerState) Snapshot() SyncerStateSnapshot { ss.lk.Lock() defer ss.lk.Unlock() - return SyncerState{ - Base: ss.Base, - Target: ss.Target, - Stage: ss.Stage, - Height: ss.Height, - Message: ss.Message, - Start: ss.Start, - End: ss.End, - } + return ss.data } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index f95df33bc..d063ce8c9 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -1648,7 +1648,7 @@ func (t *StateRoot) MarshalCBOR(w io.Writer) error { scratch := make([]byte, 9) - // t.Version (uint64) (uint64) + // t.Version (types.StateTreeVersion) (uint64) if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Version)); err != nil { return err @@ -1687,7 +1687,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Version (uint64) (uint64) + // t.Version (types.StateTreeVersion) (uint64) { @@ -1698,7 +1698,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Version = uint64(extra) + t.Version = StateTreeVersion(extra) } // t.Actors (cid.Cid) (struct) @@ -1728,22 +1728,22 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { return nil } -var lengthBufStateInfo = []byte{128} +var lengthBufStateInfo0 = []byte{128} -func (t *StateInfo) MarshalCBOR(w io.Writer) error { +func (t *StateInfo0) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufStateInfo); err != nil { + if _, err := w.Write(lengthBufStateInfo0); err != nil { return err } return nil } -func (t *StateInfo) UnmarshalCBOR(r io.Reader) error { - *t = StateInfo{} +func (t *StateInfo0) UnmarshalCBOR(r io.Reader) error { + *t = StateInfo0{} br := cbg.GetPeeker(r) scratch := make([]byte, 8) diff --git a/chain/types/fil.go b/chain/types/fil.go index 99a896e38..7eac8ce93 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -12,11 +12,15 @@ import ( type FIL BigInt func (f FIL) String() string { + return f.Unitless() + " FIL" +} + +func (f FIL) Unitless() string { r := new(big.Rat).SetFrac(f.Int, big.NewInt(int64(build.FilecoinPrecision))) if r.Sign() == 0 { - return "0 FIL" + return "0" } - return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") + " FIL" + return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") } func (f FIL) Format(s fmt.State, ch rune) { diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 559630619..e4bb2fcee 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -22,7 +23,7 @@ func Address(i uint64) address.Address { return a } -func MkMessage(from, to address.Address, nonce uint64, w *wallet.Wallet) *types.SignedMessage { +func MkMessage(from, to address.Address, nonce uint64, w *wallet.LocalWallet) *types.SignedMessage { msg := &types.Message{ To: to, From: from, @@ -33,7 +34,7 @@ func MkMessage(from, to address.Address, nonce uint64, w *wallet.Wallet) *types. GasPremium: types.NewInt(1), } - sig, err := w.Sign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{}) if err != nil { panic(err) } @@ -59,9 +60,11 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types var pcids []cid.Cid var height abi.ChainEpoch weight := types.NewInt(weightInc) + var timestamp uint64 if parents != nil { pcids = parents.Cids() height = parents.Height() + 1 + timestamp = parents.MinTimestamp() + build.BlockDelaySecs weight = types.BigAdd(parents.Blocks()[0].ParentWeight, weight) } @@ -79,6 +82,7 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types ParentWeight: weight, Messages: c, Height: height, + Timestamp: timestamp, ParentStateRoot: pstateRoot, BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS, Data: []byte("boo! im a signature")}, ParentBaseFee: types.NewInt(uint64(build.MinimumBaseFee)), diff --git a/chain/types/state.go b/chain/types/state.go index b99eb19c2..a96883604 100644 --- a/chain/types/state.go +++ b/chain/types/state.go @@ -2,9 +2,20 @@ package types import "github.com/ipfs/go-cid" +// StateTreeVersion is the version of the state tree itself, independent of the +// network version or the actors version. +type StateTreeVersion uint64 + +const ( + // StateTreeVersion0 corresponds to actors < v2. + StateTreeVersion0 StateTreeVersion = iota + // StateTreeVersion1 corresponds to actors >= v2. + StateTreeVersion1 +) + type StateRoot struct { - // State root version. Versioned along with actors (for now). - Version uint64 + // State tree version. + Version StateTreeVersion // Actors tree. The structure depends on the state root version. Actors cid.Cid // Info. The structure depends on the state root version. @@ -12,4 +23,4 @@ type StateRoot struct { } // TODO: version this. -type StateInfo struct{} +type StateInfo0 struct{} diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index 51d907648..096548e04 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "math/rand" @@ -61,11 +62,11 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { panic(err) } - blsk, err := w.GenerateKey(crypto.SigTypeBLS) + blsk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { panic(err) } - bki, err := w.Export(blsk) + bki, err := w.WalletExport(context.Background(), blsk) if err != nil { panic(err) } @@ -85,11 +86,11 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { Signature: &bmsg.Signature, } - secpk, err := w.GenerateKey(crypto.SigTypeBLS) + secpk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { panic(err) } - ski, err := w.Export(secpk) + ski, err := w.WalletExport(context.Background(), secpk) if err != nil { panic(err) } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 1e9f04081..661e31178 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -70,12 +70,12 @@ func (ar *ActorRegistry) Invoke(codeCid cid.Cid, rt vmr.Runtime, method abi.Meth log.Errorf("no code for actor %s (Addr: %s)", codeCid, rt.Receiver()) return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "no code for actor %s(%d)(%s)", codeCid, method, hex.EncodeToString(params)) } + if err := act.predicate(rt, act.vmActor); err != nil { + return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "unsupported actor: %s", err) + } if method >= abi.MethodNum(len(act.methods)) || act.methods[method] == nil { return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "no method %d on actor", method) } - if err := act.predicate(rt, act.vmActor); err != nil { - return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "unsupported actor: %s", err) - } return act.methods[method](rt, params) } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 8d3e1ab4d..8f124247c 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -49,6 +49,9 @@ func (m *Message) ValueReceived() abi.TokenAmount { return m.msg.Value } +// EnableGasTracing, if true, outputs gas tracing in execution traces. +var EnableGasTracing = false + type Runtime struct { rt0.Message rt0.Syscalls @@ -477,7 +480,7 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { } func (rt *Runtime) finilizeGasTracing() { - if enableTracing { + if EnableGasTracing { if rt.lastGasCharge != nil { rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) } @@ -509,11 +512,9 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { } -var enableTracing = false - func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() - if enableTracing { + if EnableGasTracing { var callers [10]uintptr cout := 0 //gruntime.Callers(2+skip, callers[:]) diff --git a/chain/vm/runtime_test.go b/chain/vm/runtime_test.go index c22a8b615..9fc87f7c5 100644 --- a/chain/vm/runtime_test.go +++ b/chain/vm/runtime_test.go @@ -45,3 +45,23 @@ func TestRuntimePutErrors(t *testing.T) { rt.StorePut(&NotAVeryGoodMarshaler{}) t.Error("expected panic") } + +func BenchmarkRuntime_CreateRuntimeChargeGas_TracingDisabled(b *testing.B) { + var ( + cst = cbor.NewCborStore(nil) + gch = newGasCharge("foo", 1000, 1000) + ) + + b.ResetTimer() + + EnableGasTracing = false + noop := func() bool { return EnableGasTracing } + for n := 0; n < b.N; n++ { + // flip the value and access it to make sure + // the compiler doesn't optimize away + EnableGasTracing = true + _ = noop() + EnableGasTracing = false + _ = (&Runtime{cst: cst}).chargeGasInternal(gch, 0) + } +} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index bb20c14e9..a4efccb29 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -236,7 +236,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, } rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) - if enableTracing { + if EnableGasTracing { rt.lastGasChargeTime = start if parent != nil { rt.lastGasChargeTime = parent.lastGasChargeTime diff --git a/chain/wallet/key.go b/chain/wallet/key.go new file mode 100644 index 000000000..4b746a17a --- /dev/null +++ b/chain/wallet/key.go @@ -0,0 +1,81 @@ +package wallet + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" +) + +func GenerateKey(typ crypto.SigType) (*Key, error) { + pk, err := sigs.Generate(typ) + if err != nil { + return nil, err + } + ki := types.KeyInfo{ + Type: kstoreSigType(typ), + PrivateKey: pk, + } + return NewKey(ki) +} + +type Key struct { + types.KeyInfo + + PublicKey []byte + Address address.Address +} + +func NewKey(keyinfo types.KeyInfo) (*Key, error) { + k := &Key{ + KeyInfo: keyinfo, + } + + var err error + k.PublicKey, err = sigs.ToPublic(ActSigType(k.Type), k.PrivateKey) + if err != nil { + return nil, err + } + + switch k.Type { + case KTSecp256k1: + k.Address, err = address.NewSecp256k1Address(k.PublicKey) + if err != nil { + return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) + } + case KTBLS: + k.Address, err = address.NewBLSAddress(k.PublicKey) + if err != nil { + return nil, xerrors.Errorf("converting BLS to address: %w", err) + } + default: + return nil, xerrors.Errorf("unknown key type") + } + return k, nil + +} + +func kstoreSigType(typ crypto.SigType) string { + switch typ { + case crypto.SigTypeBLS: + return KTBLS + case crypto.SigTypeSecp256k1: + return KTSecp256k1 + default: + return "" + } +} + +func ActSigType(typ string) crypto.SigType { + switch typ { + case KTBLS: + return crypto.SigTypeBLS + case KTSecp256k1: + return crypto.SigTypeSecp256k1 + default: + return 0 + } +} diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go new file mode 100644 index 000000000..c7192f496 --- /dev/null +++ b/chain/wallet/remotewallet/remote.go @@ -0,0 +1,42 @@ +package remotewallet + +import ( + "context" + + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/node/modules/helpers" +) + +type RemoteWallet struct { + api.WalletAPI +} + +func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { + ai := cliutil.ParseApiInfo(info) + + url, err := ai.DialArgs() + if err != nil { + return nil, err + } + + wapi, closer, err := client.NewWalletRPC(mctx, url, ai.AuthHeader()) + if err != nil { + return nil, xerrors.Errorf("creating jsonrpc client: %w", err) + } + + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + closer() + return nil + }, + }) + + return &RemoteWallet{wapi}, nil + } +} diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 9438d2051..57e63625b 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures @@ -29,15 +30,20 @@ const ( KTSecp256k1 = "secp256k1" ) -type Wallet struct { +type LocalWallet struct { keys map[address.Address]*Key keystore types.KeyStore lk sync.Mutex } -func NewWallet(keystore types.KeyStore) (*Wallet, error) { - w := &Wallet{ +type Default interface { + GetDefault() (address.Address, error) + SetDefault(a address.Address) error +} + +func NewWallet(keystore types.KeyStore) (*LocalWallet, error) { + w := &LocalWallet{ keys: make(map[address.Address]*Key), keystore: keystore, } @@ -45,18 +51,18 @@ func NewWallet(keystore types.KeyStore) (*Wallet, error) { return w, nil } -func KeyWallet(keys ...*Key) *Wallet { +func KeyWallet(keys ...*Key) *LocalWallet { m := make(map[address.Address]*Key) for _, key := range keys { m[key.Address] = key } - return &Wallet{ + return &LocalWallet{ keys: m, } } -func (w *Wallet) Sign(ctx context.Context, addr address.Address, msg []byte) (*crypto.Signature, error) { +func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { ki, err := w.findKey(addr) if err != nil { return nil, err @@ -68,7 +74,7 @@ func (w *Wallet) Sign(ctx context.Context, addr address.Address, msg []byte) (*c return sigs.Sign(ActSigType(ki.Type), ki.PrivateKey, msg) } -func (w *Wallet) findKey(addr address.Address) (*Key, error) { +func (w *LocalWallet) findKey(addr address.Address) (*Key, error) { w.lk.Lock() defer w.lk.Unlock() @@ -96,7 +102,7 @@ func (w *Wallet) findKey(addr address.Address) (*Key, error) { return k, nil } -func (w *Wallet) tryFind(addr address.Address) (types.KeyInfo, error) { +func (w *LocalWallet) tryFind(addr address.Address) (types.KeyInfo, error) { ki, err := w.keystore.Get(KNamePrefix + addr.String()) if err == nil { @@ -110,14 +116,12 @@ func (w *Wallet) tryFind(addr address.Address) (types.KeyInfo, error) { // We got an ErrKeyInfoNotFound error // Try again, this time with the testnet prefix - aChars := []rune(addr.String()) - prefixRunes := []rune(address.TestnetPrefix) - if len(prefixRunes) != 1 { - return types.KeyInfo{}, xerrors.Errorf("unexpected prefix length: %d", len(prefixRunes)) + tAddress, err := swapMainnetForTestnetPrefix(addr.String()) + if err != nil { + return types.KeyInfo{}, err } - aChars[0] = prefixRunes[0] - ki, err = w.keystore.Get(KNamePrefix + string(aChars)) + ki, err = w.keystore.Get(KNamePrefix + tAddress) if err != nil { return types.KeyInfo{}, err } @@ -132,16 +136,19 @@ func (w *Wallet) tryFind(addr address.Address) (types.KeyInfo, error) { return ki, nil } -func (w *Wallet) Export(addr address.Address) (*types.KeyInfo, error) { +func (w *LocalWallet) WalletExport(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { k, err := w.findKey(addr) if err != nil { return nil, xerrors.Errorf("failed to find key to export: %w", err) } + if k == nil { + return nil, xerrors.Errorf("key not found") + } return &k.KeyInfo, nil } -func (w *Wallet) Import(ki *types.KeyInfo) (address.Address, error) { +func (w *LocalWallet) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -157,7 +164,7 @@ func (w *Wallet) Import(ki *types.KeyInfo) (address.Address, error) { return k.Address, nil } -func (w *Wallet) ListAddrs() ([]address.Address, error) { +func (w *LocalWallet) WalletList(ctx context.Context) ([]address.Address, error) { all, err := w.keystore.List() if err != nil { return nil, xerrors.Errorf("listing keystore: %w", err) @@ -190,7 +197,7 @@ func (w *Wallet) ListAddrs() ([]address.Address, error) { return out, nil } -func (w *Wallet) GetDefault() (address.Address, error) { +func (w *LocalWallet) GetDefault() (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -207,7 +214,7 @@ func (w *Wallet) GetDefault() (address.Address, error) { return k.Address, nil } -func (w *Wallet) SetDefault(a address.Address) error { +func (w *LocalWallet) SetDefault(a address.Address) error { w.lk.Lock() defer w.lk.Unlock() @@ -229,19 +236,7 @@ func (w *Wallet) SetDefault(a address.Address) error { return nil } -func GenerateKey(typ crypto.SigType) (*Key, error) { - pk, err := sigs.Generate(typ) - if err != nil { - return nil, err - } - ki := types.KeyInfo{ - Type: kstoreSigType(typ), - PrivateKey: pk, - } - return NewKey(ki) -} - -func (w *Wallet) GenerateKey(typ crypto.SigType) (address.Address, error) { +func (w *LocalWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -269,7 +264,7 @@ func (w *Wallet) GenerateKey(typ crypto.SigType) (address.Address, error) { return k.Address, nil } -func (w *Wallet) HasKey(addr address.Address) (bool, error) { +func (w *LocalWallet) WalletHas(ctx context.Context, addr address.Address) (bool, error) { k, err := w.findKey(addr) if err != nil { return false, err @@ -277,11 +272,14 @@ func (w *Wallet) HasKey(addr address.Address) (bool, error) { return k != nil, nil } -func (w *Wallet) DeleteKey(addr address.Address) error { +func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) error { k, err := w.findKey(addr) if err != nil { return xerrors.Errorf("failed to delete key %s : %w", addr, err) } + if k == nil { + return nil // already not there + } if err := w.keystore.Put(KTrashPrefix+k.Address.String(), k.KeyInfo); err != nil { return xerrors.Errorf("failed to mark key %s as trashed: %w", addr, err) @@ -291,63 +289,26 @@ func (w *Wallet) DeleteKey(addr address.Address) error { return xerrors.Errorf("failed to delete key %s: %w", addr, err) } + tAddr, err := swapMainnetForTestnetPrefix(addr.String()) + if err != nil { + return xerrors.Errorf("failed to swap prefixes: %w", err) + } + + // TODO: Does this always error in the not-found case? Just ignoring an error return for now. + _ = w.keystore.Delete(KNamePrefix + tAddr) + return nil } -type Key struct { - types.KeyInfo +var _ api.WalletAPI = &LocalWallet{} - PublicKey []byte - Address address.Address -} - -func NewKey(keyinfo types.KeyInfo) (*Key, error) { - k := &Key{ - KeyInfo: keyinfo, - } - - var err error - k.PublicKey, err = sigs.ToPublic(ActSigType(k.Type), k.PrivateKey) - if err != nil { - return nil, err - } - - switch k.Type { - case KTSecp256k1: - k.Address, err = address.NewSecp256k1Address(k.PublicKey) - if err != nil { - return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) - } - case KTBLS: - k.Address, err = address.NewBLSAddress(k.PublicKey) - if err != nil { - return nil, xerrors.Errorf("converting BLS to address: %w", err) - } - default: - return nil, xerrors.Errorf("unknown key type") - } - return k, nil - -} - -func kstoreSigType(typ crypto.SigType) string { - switch typ { - case crypto.SigTypeBLS: - return KTBLS - case crypto.SigTypeSecp256k1: - return KTSecp256k1 - default: - return "" - } -} - -func ActSigType(typ string) crypto.SigType { - switch typ { - case KTBLS: - return crypto.SigTypeBLS - case KTSecp256k1: - return crypto.SigTypeSecp256k1 - default: - return 0 - } +func swapMainnetForTestnetPrefix(addr string) (string, error) { + aChars := []rune(addr) + prefixRunes := []rune(address.TestnetPrefix) + if len(prefixRunes) != 1 { + return "", xerrors.Errorf("unexpected prefix length: %d", len(prefixRunes)) + } + + aChars[0] = prefixRunes[0] + return string(aChars), nil } diff --git a/cli/chain.go b/cli/chain.go index 20a8b7797..bc0d7b508 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -449,11 +449,16 @@ var chainInspectUsage = &cli.Command{ bySender := make(map[string]int64) byDest := make(map[string]int64) byMethod := make(map[string]int64) + bySenderC := make(map[string]int64) + byDestC := make(map[string]int64) + byMethodC := make(map[string]int64) var sum int64 for _, m := range msgs { bySender[m.Message.From.String()] += m.Message.GasLimit + bySenderC[m.Message.From.String()]++ byDest[m.Message.To.String()] += m.Message.GasLimit + byDestC[m.Message.To.String()]++ sum += m.Message.GasLimit code, err := lookupActorCode(m.Message.To) @@ -464,7 +469,7 @@ var chainInspectUsage = &cli.Command{ mm := stmgr.MethodsMap[code][m.Message.Method] byMethod[mm.Name] += m.Message.GasLimit - + byMethodC[mm.Name]++ } type keyGasPair struct { @@ -496,19 +501,19 @@ var chainInspectUsage = &cli.Command{ fmt.Printf("By Sender:\n") for i := 0; i < numRes && i < len(senderVals); i++ { sv := senderVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key]) } fmt.Println() fmt.Printf("By Receiver:\n") for i := 0; i < numRes && i < len(destVals); i++ { sv := destVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key]) } fmt.Println() fmt.Printf("By Method:\n") for i := 0; i < numRes && i < len(methodVals); i++ { sv := methodVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key]) } return nil @@ -516,8 +521,9 @@ var chainInspectUsage = &cli.Command{ } var chainListCmd = &cli.Command{ - Name: "list", - Usage: "View a segment of the chain", + Name: "list", + Aliases: []string{"love"}, + Usage: "View a segment of the chain", Flags: []cli.Flag{ &cli.Uint64Flag{Name: "height"}, &cli.IntFlag{Name: "count", Value: 30}, diff --git a/cli/client.go b/cli/client.go index 7494815bf..34d151ace 100644 --- a/cli/client.go +++ b/cli/client.go @@ -536,7 +536,7 @@ func interactiveDeal(cctx *cli.Context) error { state = "miner" case "miner": - fmt.Print("Miner Address (t0..): ") + fmt.Print("Miner Address (f0..): ") var maddrStr string _, err := fmt.Scan(&maddrStr) @@ -1039,6 +1039,10 @@ var clientListDeals = &cli.Command{ Usage: "use color in display output", Value: true, }, + &cli.BoolFlag{ + Name: "show-failed", + Usage: "show failed/failing deals", + }, &cli.BoolFlag{ Name: "watch", Usage: "watch deal updates in real-time, rather than a one time list", @@ -1055,6 +1059,7 @@ var clientListDeals = &cli.Command{ verbose := cctx.Bool("verbose") color := cctx.Bool("color") watch := cctx.Bool("watch") + showFailed := cctx.Bool("show-failed") localDeals, err := api.ClientListDeals(ctx) if err != nil { @@ -1071,7 +1076,7 @@ var clientListDeals = &cli.Command{ tm.Clear() tm.MoveCursor(1, 1) - err = outputStorageDeals(ctx, tm.Screen, api, localDeals, verbose, color) + err = outputStorageDeals(ctx, tm.Screen, api, localDeals, verbose, color, showFailed) if err != nil { return err } @@ -1097,7 +1102,7 @@ var clientListDeals = &cli.Command{ } } - return outputStorageDeals(ctx, os.Stdout, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color")) + return outputStorageDeals(ctx, os.Stdout, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color"), showFailed) }, } @@ -1120,7 +1125,7 @@ func dealFromDealInfo(ctx context.Context, full api.FullNode, head *types.TipSet } } -func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, localDeals []api.DealInfo, verbose bool, color bool) error { +func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, localDeals []lapi.DealInfo, verbose bool, color bool, showFailed bool) error { sort.Slice(localDeals, func(i, j int) bool { return localDeals[i].CreationTime.Before(localDeals[j].CreationTime) }) @@ -1132,12 +1137,14 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, l var deals []deal for _, localDeal := range localDeals { - deals = append(deals, dealFromDealInfo(ctx, full, head, localDeal)) + if showFailed || localDeal.State != storagemarket.StorageDealError { + deals = append(deals, dealFromDealInfo(ctx, full, head, localDeal)) + } } if verbose { w := tabwriter.NewWriter(out, 2, 4, 2, ' ', 0) - fmt.Fprintf(w, "Created\tDealCid\tDealId\tProvider\tState\tOn Chain?\tSlashed?\tPieceCID\tSize\tPrice\tDuration\tMessage\n") + fmt.Fprintf(w, "Created\tDealCid\tDealId\tProvider\tState\tOn Chain?\tSlashed?\tPieceCID\tSize\tPrice\tDuration\tVerified\tMessage\n") for _, d := range deals { onChain := "N" if d.OnChainDealState.SectorStartEpoch != -1 { @@ -1150,7 +1157,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, l } price := types.FIL(types.BigMul(d.LocalDeal.PricePerEpoch, types.NewInt(d.LocalDeal.Duration))) - fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", d.LocalDeal.CreationTime.Format(time.Stamp), d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, dealStateString(color, d.LocalDeal.State), onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), price, d.LocalDeal.Duration, d.LocalDeal.Message) + fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%v\t%s\n", d.LocalDeal.CreationTime.Format(time.Stamp), d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, dealStateString(color, d.LocalDeal.State), onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), price, d.LocalDeal.Duration, d.LocalDeal.Verified, d.LocalDeal.Message) } return w.Flush() } @@ -1165,6 +1172,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, l tablewriter.Col("Size"), tablewriter.Col("Price"), tablewriter.Col("Duration"), + tablewriter.Col("Verified"), tablewriter.NewLineCol("Message")) for _, d := range deals { @@ -1194,6 +1202,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, l "PieceCID": piece, "Size": types.SizeStr(types.NewInt(d.LocalDeal.Size)), "Price": price, + "Verified": d.LocalDeal.Verified, "Duration": d.LocalDeal.Duration, "Message": d.LocalDeal.Message, }) diff --git a/cli/cmd.go b/cli/cmd.go index 92a366eb5..eef73b241 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -11,8 +11,6 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -20,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/node/repo" ) @@ -46,37 +45,16 @@ func NewCliError(s string) error { // ApiConnector returns API instance type ApiConnector func() api.FullNode -type APIInfo struct { - Addr multiaddr.Multiaddr - Token []byte -} - -func (a APIInfo) DialArgs() (string, error) { - _, addr, err := manet.DialArgs(a.Addr) - - return "ws://" + addr + "/rpc/v0", err -} - -func (a APIInfo) AuthHeader() http.Header { - if len(a.Token) != 0 { - headers := http.Header{} - headers.Add("Authorization", "Bearer "+string(a.Token)) - return headers - } - log.Warn("API Token not set and requested, capabilities might be limited.") - return nil -} - // The flag passed on the command line with the listen address of the API // server (only used by the tests) func flagForAPI(t repo.RepoType) string { switch t { case repo.FullNode: - return "api" + return "api-url" case repo.StorageMiner: - return "miner-api" + return "miner-api-url" case repo.Worker: - return "worker-api" + return "worker-api-url" default: panic(fmt.Sprintf("Unknown repo type: %v", t)) } @@ -122,7 +100,7 @@ func envForRepoDeprecation(t repo.RepoType) string { } } -func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { +func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) { // Check if there was a flag passed with the listen address of the API // server (only used by the tests) apiFlag := flagForAPI(t) @@ -130,11 +108,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { strma := ctx.String(apiFlag) strma = strings.TrimSpace(strma) - apima, err := multiaddr.NewMultiaddr(strma) - if err != nil { - return APIInfo{}, err - } - return APIInfo{Addr: apima}, nil + return cliutil.APIInfo{Addr: strma}, nil } envKey := envForRepo(t) @@ -148,36 +122,24 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } } if ok { - sp := strings.SplitN(env, ":", 2) - if len(sp) != 2 { - log.Warnf("invalid env(%s) value, missing token or address", envKey) - } else { - ma, err := multiaddr.NewMultiaddr(sp[1]) - if err != nil { - return APIInfo{}, xerrors.Errorf("could not parse multiaddr from env(%s): %w", envKey, err) - } - return APIInfo{ - Addr: ma, - Token: []byte(sp[0]), - }, nil - } + return cliutil.ParseApiInfo(env), nil } repoFlag := flagForRepo(t) p, err := homedir.Expand(ctx.String(repoFlag)) if err != nil { - return APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err) + return cliutil.APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err) } r, err := repo.NewFS(p) if err != nil { - return APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err) + return cliutil.APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err) } ma, err := r.APIEndpoint() if err != nil { - return APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err) + return cliutil.APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err) } token, err := r.APIToken() @@ -185,8 +147,8 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err) } - return APIInfo{ - Addr: ma, + return cliutil.APIInfo{ + Addr: ma.String(), Token: token, }, nil } @@ -266,6 +228,15 @@ func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) return client.NewWorkerRPC(ctx.Context, addr, headers) } +func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) { + addr, headers, err := GetRawAPI(ctx, repo.FullNode) + if err != nil { + return nil, nil, err + } + + return client.NewGatewayRPC(ctx.Context, addr, headers) +} + func DaemonContext(cctx *cli.Context) context.Context { if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok { return mtCtx.(context.Context) diff --git a/cli/helper.go b/cli/helper.go index 70a168145..9398ead71 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -35,7 +35,7 @@ func RunApp(app *cli.App) { if os.Getenv("LOTUS_DEV") != "" { log.Warnf("%+v", err) } else { - fmt.Printf("ERROR: %s\n\n", err) + fmt.Fprintf(os.Stderr, "ERROR: %s\n\n", err) // nolint:errcheck } var phe *PrintHelpErr if xerrors.As(err, &phe) { diff --git a/cli/multisig.go b/cli/multisig.go index 6b3867cb3..c8f181a62 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -3,8 +3,10 @@ package cli import ( "bytes" "encoding/hex" + "encoding/json" "fmt" "os" + "reflect" "sort" "strconv" "text/tabwriter" @@ -14,6 +16,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/stmgr" + cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/go-state-types/big" @@ -174,6 +178,10 @@ var msigInspectCmd = &cli.Command{ Name: "vesting", Usage: "Include vesting details", }, + &cli.BoolFlag{ + Name: "decode-params", + Usage: "Decode parameters of transaction proposals", + }, }, Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { @@ -204,6 +212,11 @@ var msigInspectCmd = &cli.Command{ return err } + ownId, err := api.StateLookupID(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + mstate, err := multisig.Load(store, act) if err != nil { return err @@ -256,6 +269,7 @@ var msigInspectCmd = &cli.Command{ return xerrors.Errorf("reading pending transactions: %w", err) } + decParams := cctx.Bool("decode-params") fmt.Println("Transactions: ", len(pending)) if len(pending) > 0 { var txids []int64 @@ -266,11 +280,36 @@ var msigInspectCmd = &cli.Command{ return txids[i] < txids[j] }) - w := tabwriter.NewWriter(os.Stdout, 8, 4, 0, ' ', 0) + w := tabwriter.NewWriter(os.Stdout, 8, 4, 2, ' ', 0) fmt.Fprintf(w, "ID\tState\tApprovals\tTo\tValue\tMethod\tParams\n") for _, txid := range txids { tx := pending[txid] - fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%d\t%x\n", txid, "pending", len(tx.Approved), tx.To, types.FIL(tx.Value), tx.Method, tx.Params) + target := tx.To.String() + if tx.To == ownId { + target += " (self)" + } + targAct, err := api.StateGetActor(ctx, tx.To, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("failed to resolve 'To' address of multisig transaction %d: %w", txid, err) + } + method := stmgr.MethodsMap[targAct.Code][tx.Method] + + paramStr := fmt.Sprintf("%x", tx.Params) + if decParams && tx.Method != 0 { + ptyp := reflect.New(method.Params.Elem()).Interface().(cbg.CBORUnmarshaler) + if err := ptyp.UnmarshalCBOR(bytes.NewReader(tx.Params)); err != nil { + return xerrors.Errorf("failed to decode parameters of transaction %d: %w", txid, err) + } + + b, err := json.Marshal(ptyp) + if err != nil { + return xerrors.Errorf("could not json marshal parameter type: %w", err) + } + + paramStr = string(b) + } + + fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%s(%d)\t%s\n", txid, "pending", len(tx.Approved), target, types.FIL(tx.Value), method.Name, tx.Method, paramStr) } if err := w.Flush(); err != nil { return xerrors.Errorf("flushing output: %+v", err) @@ -398,7 +437,7 @@ var msigProposeCmd = &cli.Command{ var msigApproveCmd = &cli.Command{ Name: "approve", Usage: "Approve a multisig message", - ArgsUsage: "[multisigAddress messageId proposerAddress destination value (optional)]", + ArgsUsage: " [proposerAddress destination value [methodId methodParams]]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "from", @@ -484,7 +523,7 @@ var msigApproveCmd = &cli.Command{ from = defaddr } - msgCid, err := api.MsigApprove(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) + msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) if err != nil { return err } @@ -1129,7 +1168,7 @@ var msigLockApproveCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigApprove(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) if err != nil { return err } diff --git a/cli/paych_test.go b/cli/paych_test.go index 35f56d90e..862ca2e74 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -121,13 +121,13 @@ func TestPaymentChannelStatus(t *testing.T) { create := make(chan string) go func() { // creator: paych add-funds - cmd = []string{creatorAddr.String(), receiverAddr.String(), fmt.Sprintf("%d", channelAmt)} + cmd := []string{creatorAddr.String(), receiverAddr.String(), fmt.Sprintf("%d", channelAmt)} create <- creatorCLI.runCmd(paychAddFundsCmd, cmd) }() // Wait for the output to stop being "Channel does not exist" for regexp.MustCompile(noChannelState).MatchString(out) { - cmd = []string{creatorAddr.String(), receiverAddr.String()} + cmd := []string{creatorAddr.String(), receiverAddr.String()} out = creatorCLI.runCmd(paychStatusByFromToCmd, cmd) } fmt.Println(out) @@ -390,7 +390,7 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) { } func startTwoNodesOneMiner(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) { - n, sn := builder.RPCMockSbBuilder(t, 2, test.OneMiner) + n, sn := builder.RPCMockSbBuilder(t, test.TwoFull, test.OneMiner) paymentCreator := n[0] paymentReceiver := n[1] @@ -439,12 +439,12 @@ type mockCLI struct { } func newMockCLI(t *testing.T) *mockCLI { - // Create a CLI App with an --api flag so that we can specify which node + // Create a CLI App with an --api-url flag so that we can specify which node // the command should be executed against app := cli.NewApp() app.Flags = []cli.Flag{ &cli.StringFlag{ - Name: "api", + Name: "api-url", Hidden: true, }, } @@ -476,8 +476,8 @@ func (c *mockCLIClient) runCmd(cmd *cli.Command, input []string) string { } func (c *mockCLIClient) runCmdRaw(cmd *cli.Command, input []string) (string, error) { - // prepend --api= - apiFlag := "--api=" + c.addr.String() + // prepend --api-url= + apiFlag := "--api-url=" + c.addr.String() input = append([]string{apiFlag}, input...) fs := c.flagSet(cmd) @@ -493,7 +493,7 @@ func (c *mockCLIClient) runCmdRaw(cmd *cli.Command, input []string) (string, err } func (c *mockCLIClient) flagSet(cmd *cli.Command) *flag.FlagSet { - // Apply app level flags (so we can process --api flag) + // Apply app level flags (so we can process --api-url flag) fs := &flag.FlagSet{} for _, f := range c.cctx.App.Flags { err := f.Apply(fs) diff --git a/cli/pprof.go b/cli/pprof.go index 6819b362a..dccb97f9a 100644 --- a/cli/pprof.go +++ b/cli/pprof.go @@ -9,7 +9,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/node/repo" - manet "github.com/multiformats/go-multiaddr/net" ) var pprofCmd = &cli.Command{ @@ -37,7 +36,7 @@ var PprofGoroutines = &cli.Command{ if err != nil { return xerrors.Errorf("could not get API info: %w", err) } - _, addr, err := manet.DialArgs(ainfo.Addr) + addr, err := ainfo.Host() if err != nil { return err } diff --git a/cli/state.go b/cli/state.go index cb1c974a7..453cde77f 100644 --- a/cli/state.go +++ b/cli/state.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "io" + "io/ioutil" "os" "reflect" "sort" @@ -32,6 +33,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/lotus/api" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" @@ -70,6 +72,7 @@ var stateCmd = &cli.Command{ stateMsgCostCmd, stateMinerInfo, stateMarketCmd, + stateExecTraceCmd, }, } @@ -313,6 +316,74 @@ var stateActiveSectorsCmd = &cli.Command{ }, } +var stateExecTraceCmd = &cli.Command{ + Name: "exec-trace", + Usage: "Get the execution trace of a given message", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return ShowHelp(cctx, fmt.Errorf("must pass message cid")) + } + + mcid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("message cid was invalid: %s", err) + } + + capi, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + msg, err := capi.ChainGetMessage(ctx, mcid) + if err != nil { + return err + } + + lookup, err := capi.StateSearchMsg(ctx, mcid) + if err != nil { + return err + } + + ts, err := capi.ChainGetTipSet(ctx, lookup.TipSet) + if err != nil { + return err + } + + pts, err := capi.ChainGetTipSet(ctx, ts.Parents()) + if err != nil { + return err + } + + cso, err := capi.StateCompute(ctx, pts.Height(), nil, pts.Key()) + if err != nil { + return err + } + + var trace *api.InvocResult + for _, t := range cso.Trace { + if t.Msg.From == msg.From && t.Msg.Nonce == msg.Nonce { + trace = t + break + } + } + if trace == nil { + return fmt.Errorf("failed to find message in tipset trace output") + } + + out, err := json.MarshalIndent(trace, "", " ") + if err != nil { + return err + } + + fmt.Println(string(out)) + return nil + }, +} + var stateReplaySetCmd = &cli.Command{ Name: "replay", Usage: "Replay a particular message within a tipset", @@ -823,6 +894,10 @@ var stateComputeStateCmd = &cli.Command{ Name: "json", Usage: "generate json output", }, + &cli.StringFlag{ + Name: "compute-state-output", + Usage: "a json file containing pre-existing compute-state output, to generate html reports without rerunning state changes", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -862,9 +937,26 @@ var stateComputeStateCmd = &cli.Command{ } } - stout, err := api.StateCompute(ctx, h, msgs, ts.Key()) - if err != nil { - return err + var stout *lapi.ComputeStateOutput + if csofile := cctx.String("compute-state-output"); csofile != "" { + data, err := ioutil.ReadFile(csofile) + if err != nil { + return err + } + + var o lapi.ComputeStateOutput + if err := json.Unmarshal(data, &o); err != nil { + return err + } + + stout = &o + } else { + o, err := api.StateCompute(ctx, h, msgs, ts.Key()) + if err != nil { + return err + } + + stout = o } if cctx.Bool("json") { diff --git a/cli/sync.go b/cli/sync.go index bee87cf70..ea066cbda 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -122,8 +122,14 @@ var syncMarkBadCmd = &cli.Command{ } var syncUnmarkBadCmd = &cli.Command{ - Name: "unmark-bad", - Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it", + Name: "unmark-bad", + Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "all", + Usage: "drop the entire bad block cache", + }, + }, ArgsUsage: "[blockCid]", Action: func(cctx *cli.Context) error { napi, closer, err := GetFullNodeAPI(cctx) @@ -133,6 +139,10 @@ var syncUnmarkBadCmd = &cli.Command{ defer closer() ctx := ReqContext(cctx) + if cctx.Bool("all") { + return napi.SyncUnmarkAllBad(ctx) + } + if !cctx.Args().Present() { return fmt.Errorf("must specify block cid to unmark") } @@ -233,7 +243,13 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { samples := 8 i := 0 - var app, lastApp uint64 + var firstApp, app, lastApp uint64 + + state, err := napi.SyncState(ctx) + if err != nil { + return err + } + firstApp = state.VMApplied for { state, err := napi.SyncState(ctx) @@ -286,10 +302,10 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { if i%samples == 0 { lastApp = app - app = state.VMApplied + app = state.VMApplied - firstApp } if i > 0 { - fmt.Printf("Validated %d messages (%d per second)\n", state.VMApplied, (app-lastApp)*uint64(time.Second/tick)/uint64(samples)) + fmt.Printf("Validated %d messages (%d per second)\n", state.VMApplied-firstApp, (app-lastApp)*uint64(time.Second/tick)/uint64(samples)) lastLines++ } diff --git a/cli/util/apiinfo.go b/cli/util/apiinfo.go new file mode 100644 index 000000000..1f9a83769 --- /dev/null +++ b/cli/util/apiinfo.go @@ -0,0 +1,83 @@ +package cliutil + +import ( + "net/http" + "net/url" + "regexp" + "strings" + + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +var log = logging.Logger("cliutil") + +var ( + infoWithToken = regexp.MustCompile("^[a-zA-Z0-9\\-_]+?\\.[a-zA-Z0-9\\-_]+?\\.([a-zA-Z0-9\\-_]+)?:.+$") +) + +type APIInfo struct { + Addr string + Token []byte +} + +func ParseApiInfo(s string) APIInfo { + var tok []byte + if infoWithToken.Match([]byte(s)) { + sp := strings.SplitN(s, ":", 2) + tok = []byte(sp[0]) + s = sp[1] + } + + return APIInfo{ + Addr: s, + Token: tok, + } +} + +func (a APIInfo) DialArgs() (string, error) { + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } + + return "ws://" + addr + "/rpc/v0", nil + } + + _, err = url.Parse(a.Addr) + if err != nil { + return "", err + } + return a.Addr + "/rpc/v0", nil +} + +func (a APIInfo) Host() (string, error) { + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } + + return addr, nil + } + + spec, err := url.Parse(a.Addr) + if err != nil { + return "", err + } + return spec.Host, nil +} + +func (a APIInfo) AuthHeader() http.Header { + if len(a.Token) != 0 { + headers := http.Header{} + headers.Add("Authorization", "Bearer "+string(a.Token)) + return headers + } + log.Warn("API Token not set and requested, capabilities might be limited.") + return nil +} diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 3d93b0e5e..3f99d0453 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -183,7 +183,7 @@ var importBenchCmd = &cli.Command{ return nil } - cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) + cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier), nil) stm := stmgr.NewStateManager(cs) if cctx.Bool("global-profile") { diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 0a6365dbd..d5fac0a06 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -6,85 +6,174 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" - - "go.opencensus.io/trace" ) -const LookbackCap = time.Hour +const ( + LookbackCap = time.Hour + stateWaitLookbackLimit = abi.ChainEpoch(20) +) var ( ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap) ) -type GatewayAPI struct { - api api.FullNode +// gatewayDepsAPI defines the API methods that the GatewayAPI depends on +// (to make it easy to mock for tests) +type gatewayDepsAPI interface { + ChainHead(ctx context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) } -func (a *GatewayAPI) getTipsetTimestamp(ctx context.Context, tsk types.TipSetKey) (time.Time, error) { +type GatewayAPI struct { + api gatewayDepsAPI +} + +func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { if tsk.IsEmpty() { - return time.Now(), nil + return nil } ts, err := a.api.ChainGetTipSet(ctx, tsk) - if err != nil { - return time.Time{}, err - } - - return time.Unix(int64(ts.Blocks()[0].Timestamp), 0), nil -} - -func (a *GatewayAPI) checkTipset(ctx context.Context, ts types.TipSetKey) error { - when, err := a.getTipsetTimestamp(ctx, ts) if err != nil { return err } - if time.Since(when) > time.Hour { + return a.checkTipset(ts) +} + +func (a *GatewayAPI) checkTipset(ts *types.TipSet) error { + at := time.Unix(int64(ts.Blocks()[0].Timestamp), 0) + if err := a.checkTimestamp(at); err != nil { + return fmt.Errorf("bad tipset: %w", err) + } + return nil +} + +func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error { + tsBlock := ts.Blocks()[0] + heightDelta := time.Duration(uint64(tsBlock.Height-h)*build.BlockDelaySecs) * time.Second + timeAtHeight := time.Unix(int64(tsBlock.Timestamp), 0).Add(-heightDelta) + + if err := a.checkTimestamp(timeAtHeight); err != nil { + return fmt.Errorf("bad tipset height: %w", err) + } + return nil +} + +func (a *GatewayAPI) checkTimestamp(at time.Time) error { + if time.Since(at) > LookbackCap { return ErrLookbackTooLong } return nil } -func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - ctx, span := trace.StartSpan(ctx, "StateGetActor") - defer span.End() - - if err := a.checkTipset(ctx, ts); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) - } - - return a.api.StateGetActor(ctx, actor, ts) -} - func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { - ctx, span := trace.StartSpan(ctx, "ChainHead") - defer span.End() // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) return a.api.ChainHead(ctx) } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - ctx, span := trace.StartSpan(ctx, "ChainGetTipSet") - defer span.End() - - if err := a.checkTipset(ctx, tsk); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) - } - - // TODO: since we're limiting lookbacks, should just cache this (could really even cache the json response bytes) return a.api.ChainGetTipSet(ctx, tsk) } +func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + ts, err := a.api.ChainGetTipSet(ctx, tsk) + if err != nil { + return nil, err + } + + // Check if the tipset key refers to a tipset that's too far in the past + if err := a.checkTipset(ts); err != nil { + return nil, err + } + + // Check if the height is too far in the past + if err := a.checkTipsetHeight(ts, h); err != nil { + return nil, err + } + + return a.api.ChainGetTipSetByHeight(ctx, h, tsk) +} + +func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + + return a.api.GasEstimateMessageGas(ctx, msg, spec, tsk) +} + func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { - ctx, span := trace.StartSpan(ctx, "MpoolPush") - defer span.End() - // TODO: additional anti-spam checks - return a.api.MpoolPushUntrusted(ctx, sm) } + +func (a *GatewayAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.NewInt(0), err + } + + return a.api.MsigGetAvailableBalance(ctx, addr, tsk) +} + +func (a *GatewayAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, start); err != nil { + return types.NewInt(0), err + } + if err := a.checkTipsetKey(ctx, end); err != nil { + return types.NewInt(0), err + } + + return a.api.MsigGetVested(ctx, addr, start, end) +} + +func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return address.Undef, err + } + + return a.api.StateAccountKey(ctx, addr, tsk) +} + +func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + + return a.api.StateGetActor(ctx, actor, tsk) +} + +func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return address.Undef, err + } + + return a.api.StateLookupID(ctx, addr, tsk) +} + +func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) +} + +var _ api.GatewayAPI = (*GatewayAPI)(nil) +var _ full.ChainModuleAPI = (*GatewayAPI)(nil) +var _ full.GasModuleAPI = (*GatewayAPI)(nil) +var _ full.MpoolModuleAPI = (*GatewayAPI)(nil) +var _ full.StateModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go new file mode 100644 index 000000000..f34f887f5 --- /dev/null +++ b/cmd/lotus-gateway/api_test.go @@ -0,0 +1,191 @@ +package main + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/filecoin-project/lotus/build" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/lotus/chain/types/mock" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { + ctx := context.Background() + + lookbackTimestamp := uint64(time.Now().Unix()) - uint64(LookbackCap.Seconds()) + type args struct { + h abi.ChainEpoch + tskh abi.ChainEpoch + genesisTS uint64 + } + tests := []struct { + name string + args args + expErr bool + }{{ + name: "basic", + args: args{ + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "genesis", + args: args{ + h: abi.ChainEpoch(0), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "same epoch as tipset", + args: args{ + h: abi.ChainEpoch(5), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "tipset too old", + args: args{ + // Tipset height is 5, genesis is at LookbackCap - 10 epochs. + // So resulting tipset height will be 5 epochs earlier than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp - build.BlockDelaySecs*10, + }, + expErr: true, + }, { + name: "lookup height too old", + args: args{ + // Tipset height is 5, lookup height is 1, genesis is at LookbackCap - 3 epochs. + // So + // - lookup height will be 2 epochs earlier than LookbackCap. + // - tipset height will be 2 epochs later than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp - build.BlockDelaySecs*3, + }, + expErr: true, + }, { + name: "tipset and lookup height within acceptable range", + args: args{ + // Tipset height is 5, lookup height is 1, genesis is at LookbackCap. + // So + // - lookup height will be 1 epoch later than LookbackCap. + // - tipset height will be 5 epochs later than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp, + }, + }} + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + mock := &mockGatewayDepsAPI{} + a := &GatewayAPI{api: mock} + + // Create tipsets from genesis up to tskh and return the highest + ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS) + + got, err := a.ChainGetTipSetByHeight(ctx, tt.args.h, ts.Key()) + if tt.expErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.args.h, got.Height()) + } + }) + } +} + +type mockGatewayDepsAPI struct { + lk sync.RWMutex + tipsets []*types.TipSet +} + +func (m *mockGatewayDepsAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { + m.lk.RLock() + defer m.lk.RUnlock() + + return m.tipsets[len(m.tipsets)-1], nil +} + +func (m *mockGatewayDepsAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + m.lk.RLock() + defer m.lk.RUnlock() + + for _, ts := range m.tipsets { + if ts.Key() == tsk { + return ts, nil + } + } + + return nil, nil +} + +// createTipSets creates tipsets from genesis up to tskh and returns the highest +func (m *mockGatewayDepsAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *types.TipSet { + m.lk.Lock() + defer m.lk.Unlock() + + targeth := h + 1 // add one for genesis block + if genesisTimestamp == 0 { + genesisTimestamp = uint64(time.Now().Unix()) - build.BlockDelaySecs*uint64(targeth) + } + var currts *types.TipSet + for currh := abi.ChainEpoch(0); currh < targeth; currh++ { + blks := mock.MkBlock(currts, 1, 1) + if currh == 0 { + blks.Timestamp = genesisTimestamp + } + currts = mock.TipSet(blks) + m.tipsets = append(m.tipsets, currts) + } + + return m.tipsets[len(m.tipsets)-1] +} + +func (m *mockGatewayDepsAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + m.lk.Lock() + defer m.lk.Unlock() + + return m.tipsets[h], nil +} + +func (m *mockGatewayDepsAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { + panic("implement me") +} diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go new file mode 100644 index 000000000..206034968 --- /dev/null +++ b/cmd/lotus-gateway/endtoend_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "testing" + "time" + + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node" + builder "github.com/filecoin-project/lotus/node/test" +) + +func init() { + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) +} + +// TestEndToEnd tests that API calls can be made on a lite node that is +// connected through a gateway to a full API node +func TestEndToEnd(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + + blocktime := 5 * time.Millisecond + ctx := context.Background() + full, lite, closer := startNodes(ctx, t, blocktime) + defer closer() + + // The full node starts with a wallet + fullWalletAddr, err := full.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // Check the full node's wallet balance from the lite node + balance, err := lite.WalletBalance(ctx, fullWalletAddr) + require.NoError(t, err) + fmt.Println(balance) + + // Create a wallet on the lite node + liteWalletAddr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + require.NoError(t, err) + + // Send some funds from the full node to the lite node + err = sendFunds(ctx, t, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + require.NoError(t, err) + + // Send some funds from the lite node back to the full node + err = sendFunds(ctx, t, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) + require.NoError(t, err) + + // Sign some data with the lite node wallet address + data := []byte("hello") + sig, err := lite.WalletSign(ctx, liteWalletAddr, data) + require.NoError(t, err) + + // Verify the signature + ok, err := lite.WalletVerify(ctx, liteWalletAddr, data, sig) + require.NoError(t, err) + require.True(t, ok) + + // Create some wallets on the lite node to use for testing multisig + var walletAddrs []address.Address + for i := 0; i < 4; i++ { + addr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + require.NoError(t, err) + + walletAddrs = append(walletAddrs, addr) + + err = sendFunds(ctx, t, lite, liteWalletAddr, addr, types.NewInt(1e15)) + require.NoError(t, err) + } + + // Create an msig with three of the addresses and threshold of two sigs + msigAddrs := walletAddrs[:3] + amt := types.NewInt(1000) + addProposal, err := lite.MsigCreate(ctx, 2, msigAddrs, abi.ChainEpoch(50), amt, liteWalletAddr, types.NewInt(0)) + require.NoError(t, err) + + res, err := lite.StateWaitMsg(ctx, addProposal, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var execReturn init0.ExecReturn + err = execReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + + // Get available balance of msig: should be greater than zero and less + // than initial amount + msig := execReturn.IDAddress + msigBalance, err := lite.MsigGetAvailableBalance(ctx, msig, types.EmptyTSK) + require.NoError(t, err) + require.Greater(t, msigBalance.Int64(), int64(0)) + require.Less(t, msigBalance.Int64(), amt.Int64()) + + // Propose to add a new address to the msig + addProposal, err = lite.MsigAddPropose(ctx, msig, walletAddrs[0], walletAddrs[3], false) + require.NoError(t, err) + + res, err = lite.StateWaitMsg(ctx, addProposal, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var proposeReturn multisig.ProposeReturn + err = proposeReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + + // Approve proposal (proposer is first (implicit) signer, approver is + // second signer + txnID := uint64(proposeReturn.TxnID) + approval1, err := lite.MsigAddApprove(ctx, msig, walletAddrs[1], txnID, walletAddrs[0], walletAddrs[3], false) + require.NoError(t, err) + + res, err = lite.StateWaitMsg(ctx, approval1, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var approveReturn multisig.ApproveReturn + err = approveReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + require.True(t, approveReturn.Applied) +} + +func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { + msg := &types.Message{ + From: fromAddr, + To: toAddr, + Value: amt, + } + + sm, err := fromNode.MpoolPushMessage(ctx, msg, nil) + if err != nil { + return err + } + + res, err := fromNode.StateWaitMsg(ctx, sm.Cid(), 1) + if err != nil { + return err + } + if res.Receipt.ExitCode != 0 { + return xerrors.Errorf("send funds failed with exit code %d", res.Receipt.ExitCode) + } + + return nil +} + +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { + var closer jsonrpc.ClientCloser + + // Create one miner and two full nodes. + // - Put a gateway server in front of full node 1 + // - Start full node 2 in lite mode + // - Connect lite node -> gateway server -> full node + opts := append( + // Full node + test.OneFull, + // Lite node + test.FullNodeOpts{ + Lite: true, + Opts: func(nodes []test.TestNode) node.Option { + fullNode := nodes[0] + + // Create a gateway server in front of the full node + _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) + require.NoError(t, err) + + // Create a gateway client API that connects to the gateway server + var gapi api.GatewayAPI + gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) + require.NoError(t, err) + + // Provide the gateway API to dependency injection + return node.Override(new(api.GatewayAPI), gapi) + }, + }, + ) + n, sn := builder.RPCMockSbBuilder(t, opts, test.OneMiner) + + full := n[0] + lite := n[1] + miner := sn[0] + + // Get the listener address for the full node + fullAddr, err := full.NetAddrsListen(ctx) + require.NoError(t, err) + + // Connect the miner and the full node + err = miner.NetConnect(ctx, fullAddr) + require.NoError(t, err) + + // Start mining blocks + bm := test.NewBlockMiner(ctx, t, miner, blocktime) + bm.MineBlocks() + + return full, lite, closer +} diff --git a/cmd/lotus-keygen/main.go b/cmd/lotus-keygen/main.go index 8e2881ae9..4b971cf48 100644 --- a/cmd/lotus-keygen/main.go +++ b/cmd/lotus-keygen/main.go @@ -40,12 +40,12 @@ func main() { return fmt.Errorf("unrecognized key type: %q", cctx.String("type")) } - kaddr, err := w.GenerateKey(kt) + kaddr, err := w.WalletNew(cctx.Context, kt) if err != nil { return err } - ki, err := w.Export(kaddr) + ki, err := w.WalletExport(cctx.Context, kaddr) if err != nil { return err } diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 769fc4bb2..5491e4af2 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -15,6 +15,7 @@ import ( "time" "github.com/filecoin-project/lotus/chain/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -37,6 +38,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/tools/stats" @@ -342,6 +344,18 @@ var runCmd = &cli.Command{ Usage: "process ProveCommitSector messages", Value: true, }, + &cli.BoolFlag{ + Name: "windowed-post", + EnvVars: []string{"LOTUS_PCR_WINDOWED_POST"}, + Usage: "process SubmitWindowedPoSt messages and refund gas fees", + Value: false, + }, + &cli.BoolFlag{ + Name: "storage-deals", + EnvVars: []string{"LOTUS_PCR_STORAGE_DEALS"}, + Usage: "process PublishStorageDeals messages and refund gas fees", + Value: false, + }, &cli.IntFlag{ Name: "head-delay", EnvVars: []string{"LOTUS_PCR_HEAD_DELAY"}, @@ -378,6 +392,18 @@ var runCmd = &cli.Command{ Usage: "percent of refund to issue", Value: 110, }, + &cli.StringFlag{ + Name: "pre-fee-cap-max", + EnvVars: []string{"LOTUS_PCR_PRE_FEE_CAP_MAX"}, + Usage: "messages with a fee cap larger than this will be skipped when processing pre commit messages", + Value: "0.000000001", + }, + &cli.StringFlag{ + Name: "prove-fee-cap-max", + EnvVars: []string{"LOTUS_PCR_PROVE_FEE_CAP_MAX"}, + Usage: "messages with a prove cap larger than this will be skipped when processing pre commit messages", + Value: "0.000000001", + }, }, Action: func(cctx *cli.Context) error { go func() { @@ -421,6 +447,8 @@ var runCmd = &cli.Command{ dryRun := cctx.Bool("dry-run") preCommitEnabled := cctx.Bool("pre-commit") proveCommitEnabled := cctx.Bool("prove-commit") + windowedPoStEnabled := cctx.Bool("windowed-post") + publishStorageDealsEnabled := cctx.Bool("storage-deals") aggregateTipsets := cctx.Int("aggregate-tipsets") minerRecoveryEnabled := cctx.Bool("miner-recovery") minerRecoveryPeriod := abi.ChainEpoch(int64(cctx.Int("miner-recovery-period"))) @@ -428,6 +456,16 @@ var runCmd = &cli.Command{ minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff")) minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus")) + preFeeCapMax, err := types.ParseFIL(cctx.String("pre-fee-cap-max")) + if err != nil { + return err + } + + proveFeeCapMax, err := types.ParseFIL(cctx.String("prove-fee-cap-max")) + if err != nil { + return err + } + rf := &refunder{ api: api, wallet: from, @@ -438,6 +476,10 @@ var runCmd = &cli.Command{ dryRun: dryRun, preCommitEnabled: preCommitEnabled, proveCommitEnabled: proveCommitEnabled, + windowedPoStEnabled: windowedPoStEnabled, + publishStorageDealsEnabled: publishStorageDealsEnabled, + preFeeCapMax: types.BigInt(preFeeCapMax), + proveFeeCapMax: types.BigInt(proveFeeCapMax), } var refunds *MinersRefund = NewMinersRefund() @@ -589,7 +631,12 @@ type refunder struct { dryRun bool preCommitEnabled bool proveCommitEnabled bool + windowedPoStEnabled bool + publishStorageDealsEnabled bool threshold big.Int + + preFeeCapMax big.Int + proveFeeCapMax big.Int } func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) { @@ -817,6 +864,147 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet return refunds, nil } +func (r *refunder) processTipsetStorageMarketActor(ctx context.Context, tipset *types.TipSet, msg api.Message, recp *types.MessageReceipt) (bool, string, types.BigInt, error) { + + m := msg.Message + refundValue := types.NewInt(0) + var messageMethod string + + switch m.Method { + case builtin0.MethodsMarket.PublishStorageDeals: + if !r.publishStorageDealsEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "PublishStorageDeals" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) + } + + return true, messageMethod, refundValue, nil +} + +func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *types.TipSet, msg api.Message, recp *types.MessageReceipt) (bool, string, types.BigInt, error) { + + m := msg.Message + refundValue := types.NewInt(0) + var messageMethod string + + switch m.Method { + case builtin0.MethodsMiner.SubmitWindowedPoSt: + if !r.windowedPoStEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "SubmitWindowedPoSt" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) + case builtin0.MethodsMiner.ProveCommitSector: + if !r.proveCommitEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "ProveCommitSector" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + if m.GasFeeCap.GreaterThan(r.proveFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.proveFeeCapMax) + return false, messageMethod, types.NewInt(0), nil + } + + var sn abi.SectorNumber + + var proveCommitSector miner0.ProveCommitSectorParams + if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { + log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) + return false, messageMethod, types.NewInt(0), nil + } + + sn = proveCommitSector.SectorNumber + + // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed + precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) + if err != nil { + log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key()) + if err != nil { + log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key()) + if err != nil { + log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit) + if collateral.LessThan(big.Zero()) { + log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = collateral + if r.refundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + } + case builtin0.MethodsMiner.PreCommitSector: + if !r.preCommitEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "PreCommitSector" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + if m.GasFeeCap.GreaterThan(r.preFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.preFeeCapMax) + return false, messageMethod, types.NewInt(0), nil + } + + var precommitInfo miner.SectorPreCommitInfo + if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { + log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) + return false, messageMethod, types.NewInt(0), nil + } + + collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitInfo, tipset.Key()) + if err != nil { + log.Warnw("failed to calculate initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", precommitInfo.SectorNumber) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = collateral + if r.refundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + } + default: + return false, messageMethod, types.NewInt(0), nil + } + + return true, messageMethod, refundValue, nil +} + func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { cids := tipset.Cids() if len(cids) == 0 { @@ -841,9 +1029,9 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu return nil, nil } - refundValue := types.NewInt(0) tipsetRefunds := NewMinersRefund() for i, msg := range msgs { + refundValue := types.NewInt(0) m := msg.Message a, err := r.api.StateGetActor(ctx, m.To, tipset.Key()) @@ -852,91 +1040,23 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } - if !builtin.IsStorageMinerActor(a.Code) { - continue - } - var messageMethod string + var processed bool - switch m.Method { - case builtin0.MethodsMiner.ProveCommitSector: - if !r.proveCommitEnabled { - continue - } - - messageMethod = "ProveCommitSector" - - if recps[i].ExitCode != exitcode.Ok { - log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode) - continue - } - - var sn abi.SectorNumber - - var proveCommitSector miner0.ProveCommitSectorParams - if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { - log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) - continue - } - - sn = proveCommitSector.SectorNumber - - // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed - precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) - if err != nil { - log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) - continue - } - - precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key()) - if err != nil { - log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) - continue - } - - collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key()) - if err != nil { - log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) - } - - collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit) - if collateral.LessThan(big.Zero()) { - log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) - continue - } - - refundValue = collateral - case builtin0.MethodsMiner.PreCommitSector: - if !r.preCommitEnabled { - continue - } - - messageMethod = "PreCommitSector" - - if recps[i].ExitCode != exitcode.Ok { - log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode) - continue - } - - var precommitInfo miner.SectorPreCommitInfo - if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { - log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) - continue - } - - collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitInfo, tipset.Key()) - if err != nil { - log.Warnw("failed to calculate initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", precommitInfo.SectorNumber) - continue - } - - refundValue = collateral - default: - continue + if m.To == market.Address { + processed, messageMethod, refundValue, err = r.processTipsetStorageMarketActor(ctx, tipset, msg, recps[i]) } - if r.refundPercent > 0 { - refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + if builtin.IsStorageMinerActor(a.Code) { + processed, messageMethod, refundValue, err = r.processTipsetStorageMinerActor(ctx, tipset, msg, recps[i]) + } + + if err != nil { + log.Errorw("error while processing message", "cid", msg.Cid) + continue + } + if !processed { + continue } log.Debugw( diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index fba6d7724..c2099bb2d 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -7,6 +7,7 @@ import ( "github.com/docker/go-units" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" @@ -33,16 +34,19 @@ import ( ) type accountInfo struct { - Address address.Address - Balance types.FIL - Type string - Power abi.StoragePower - Worker address.Address - Owner address.Address - InitialPledge types.FIL - PreCommits types.FIL - LockedFunds types.FIL - Sectors uint64 + Address address.Address + Balance types.FIL + Type string + Power abi.StoragePower + Worker address.Address + Owner address.Address + InitialPledge types.FIL + PreCommits types.FIL + LockedFunds types.FIL + Sectors uint64 + VestingStart abi.ChainEpoch + VestingDuration abi.ChainEpoch + VestingAmount types.FIL } var auditsCmd = &cli.Command{ @@ -115,10 +119,8 @@ var chainBalanceCmd = &cli.Command{ infos = append(infos, ai) } - fmt.Printf("Address,Balance,Type,Power,Worker,Owner\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance, acc.Type, acc.Power, acc.Worker, acc.Owner) - } + printAccountInfos(infos, false) + return nil }, } @@ -171,7 +173,7 @@ var chainBalanceStateCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) @@ -196,6 +198,7 @@ var chainBalanceStateCmd = &cli.Command{ LockedFunds: types.FIL(big.NewInt(0)), InitialPledge: types.FIL(big.NewInt(0)), PreCommits: types.FIL(big.NewInt(0)), + VestingAmount: types.FIL(big.NewInt(0)), } if minerInfo && builtin.IsStorageMinerActor(act.Code) { @@ -234,6 +237,32 @@ var chainBalanceStateCmd = &cli.Command{ ai.Worker = minfo.Worker ai.Owner = minfo.Owner } + + if builtin.IsMultisigActor(act.Code) { + mst, err := multisig.Load(store, act) + if err != nil { + return err + } + + ai.VestingStart, err = mst.StartEpoch() + if err != nil { + return err + } + + ib, err := mst.InitialBalance() + if err != nil { + return err + } + + ai.VestingAmount = types.FIL(ib) + + ai.VestingDuration, err = mst.UnlockDuration() + if err != nil { + return err + } + + } + infos = append(infos, ai) return nil }) @@ -241,22 +270,27 @@ var chainBalanceStateCmd = &cli.Command{ return xerrors.Errorf("failed to loop over actors: %w", err) } - if minerInfo { - fmt.Printf("Address,Balance,Type,Sectors,Worker,Owner,InitialPledge,Locked,PreCommits\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance, acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge, acc.LockedFunds, acc.PreCommits) - } - } else { - fmt.Printf("Address,Balance,Type\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance, acc.Type) - } - } + printAccountInfos(infos, minerInfo) return nil }, } +func printAccountInfos(infos []accountInfo, minerInfo bool) { + if minerInfo { + fmt.Printf("Address,Balance,Type,Sectors,Worker,Owner,InitialPledge,Locked,PreCommits,VestingStart,VestingDuration,VestingAmount\n") + for _, acc := range infos { + fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s,%d,%d,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge.Unitless(), acc.LockedFunds.Unitless(), acc.PreCommits.Unitless(), acc.VestingStart, acc.VestingDuration, acc.VestingAmount.Unitless()) + } + } else { + fmt.Printf("Address,Balance,Type\n") + for _, acc := range infos { + fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type) + } + } + +} + var chainPledgeCmd = &cli.Command{ Name: "stateroot-pledge", Description: "Calculate sector pledge numbers", @@ -309,7 +343,7 @@ var chainPledgeCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) diff --git a/cmd/lotus-shed/consensus.go b/cmd/lotus-shed/consensus.go index 59d9555df..1fe7756c1 100644 --- a/cmd/lotus-shed/consensus.go +++ b/cmd/lotus-shed/consensus.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" @@ -111,7 +112,7 @@ var consensusCheckCmd = &cli.Command{ if err != nil { return err } - ainfo := lcli.APIInfo{Addr: apima} + ainfo := cliutil.APIInfo{Addr: apima.String()} addr, err := ainfo.DialArgs() if err != nil { return err diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index c12cbd82d..3be49f0e0 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -83,7 +83,7 @@ var exportChainCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, nil) + cs := store.NewChainStore(bs, mds, nil, nil) if err := cs.Load(); err != nil { return err } diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index 9a47d6561..4b197c58f 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -52,7 +52,7 @@ var genesisVerifyCmd = &cli.Command{ } bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil, nil) cf := cctx.Args().Get(0) f, err := os.Open(cf) diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index 6090b7a0f..fdd1fcb49 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -105,7 +105,7 @@ var keyinfoVerifyCmd = &cli.Command{ return err } - if _, err := w.Import(&keyInfo); err != nil { + if _, err := w.WalletImport(cctx.Context, &keyInfo); err != nil { return err } @@ -220,7 +220,7 @@ var keyinfoImportCmd = &cli.Command{ return err } - addr, err := w.Import(&keyInfo) + addr, err := w.WalletImport(cctx.Context, &keyInfo) if err != nil { return err } diff --git a/cmd/lotus-shed/nonce-fix.go b/cmd/lotus-shed/nonce-fix.go index 3cd9726f4..8102fd8a9 100644 --- a/cmd/lotus-shed/nonce-fix.go +++ b/cmd/lotus-shed/nonce-fix.go @@ -5,6 +5,8 @@ import ( "math" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/types" @@ -32,6 +34,10 @@ var noncefix = &cli.Command{ &cli.BoolFlag{ Name: "auto", }, + &cli.Int64Flag{ + Name: "gas-fee-cap", + Usage: "specify gas fee cap for nonce filling messages", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := lcli.GetFullNodeAPI(cctx) @@ -84,15 +90,32 @@ var noncefix = &cli.Command{ } fmt.Printf("Creating %d filler messages (%d ~ %d)\n", end-start, start, end) + ts, err := api.ChainHead(ctx) + if err != nil { + return err + } + + feeCap := big.Mul(ts.Blocks()[0].ParentBaseFee, big.NewInt(2)) // default fee cap to 2 * parent base fee + if fcf := cctx.Int64("gas-fee-cap"); fcf != 0 { + feeCap = abi.NewTokenAmount(fcf) + } + for i := start; i < end; i++ { msg := &types.Message{ - From: addr, - To: addr, - Value: types.NewInt(1), - Nonce: i, + From: addr, + To: addr, + Value: types.NewInt(0), + Nonce: i, + GasLimit: 1000000, + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(5), + } + smsg, err := api.WalletSignMessage(ctx, addr, msg) + if err != nil { + return err } - _, err = api.MpoolPushMessage(ctx, msg, nil) + _, err = api.MpoolPush(ctx, smsg) if err != nil { return err } diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index 79158c3a3..6cf4f8c6f 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -162,7 +162,7 @@ var stateTreePruneCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) if err := cs.Load(); err != nil { return fmt.Errorf("loading chainstore: %w", err) } diff --git a/cmd/lotus-storage-miner/allinfo_test.go b/cmd/lotus-storage-miner/allinfo_test.go index 8f744c4b3..a458c024b 100644 --- a/cmd/lotus-storage-miner/allinfo_test.go +++ b/cmd/lotus-storage-miner/allinfo_test.go @@ -5,8 +5,6 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/node" - logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" @@ -64,8 +62,8 @@ func TestMinerAllInfo(t *testing.T) { require.NoError(t, infoAllCmd.Action(cctx)) } - bp := func(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - n, sn = builder.Builder(t, nFull, storage, opts...) + bp := func(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + n, sn = builder.Builder(t, fullOpts, storage) t.Run("pre-info-all", run) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 9218faa77..0f830023f 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -464,13 +464,12 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - if jrnl, err := journal.OpenFSJournal(lr, journal.DefaultDisabledEvents); err == nil { - journal.J = jrnl - } else { + j, err := journal.OpenFSJournal(lr, journal.EnvDisabledEvents()) + if err != nil { return fmt.Errorf("failed to open filesystem journal: %w", err) } - m := miner.NewMiner(api, epp, a, slashfilter.New(mds)) + m := miner.NewMiner(api, epp, a, slashfilter.New(mds), j) { if err := m.Start(ctx); err != nil { return xerrors.Errorf("failed to start up genesis miner: %w", err) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 6e3743143..bb1ebd9ec 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -154,14 +154,14 @@ var setAskCmd = &cli.Command{ Name: "set-ask", Usage: "Configure the miner's ask", Flags: []cli.Flag{ - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "price", - Usage: "Set the price of the ask for unverified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", + Usage: "Set the price of the ask for unverified deals (specified as FIL / GiB / Epoch) to `PRICE`.", Required: true, }, - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "verified-price", - Usage: "Set the price of the ask for verified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", + Usage: "Set the price of the ask for verified deals (specified as FIL / GiB / Epoch) to `PRICE`", Required: true, }, &cli.StringFlag{ @@ -185,8 +185,15 @@ var setAskCmd = &cli.Command{ } defer closer() - pri := types.NewInt(cctx.Uint64("price")) - vpri := types.NewInt(cctx.Uint64("verified-price")) + pri, err := types.ParseFIL(cctx.String("price")) + if err != nil { + return err + } + + vpri, err := types.ParseFIL(cctx.String("verified-price")) + if err != nil { + return err + } dur, err := time.ParseDuration("720h0m0s") if err != nil { @@ -229,7 +236,7 @@ var setAskCmd = &cli.Command{ return xerrors.Errorf("max piece size (w/bit-padding) %s cannot exceed miner sector size %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) } - return api.MarketSetAsk(ctx, pri, vpri, abi.ChainEpoch(qty), abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) + return api.MarketSetAsk(ctx, types.BigInt(pri), types.BigInt(vpri), abi.ChainEpoch(qty), abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) }, } @@ -281,7 +288,7 @@ var getAskCmd = &cli.Command{ rem = (time.Second * time.Duration(int64(dlt)*int64(build.BlockDelaySecs))).String() } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%d\n", ask.Price, ask.VerifiedPrice, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%d\n", types.FIL(ask.Price), types.FIL(ask.VerifiedPrice), types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo) return w.Flush() }, diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index a5d996f78..98a9cfaba 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -33,7 +33,7 @@ var runCmd = &cli.Command{ Usage: "Start a lotus miner process", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "api", + Name: "miner-api", Usage: "2345", }, &cli.BoolFlag{ @@ -61,7 +61,7 @@ var runCmd = &cli.Command{ nodeApi, ncloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { - return err + return xerrors.Errorf("getting full node api: %w", err) } defer ncloser() ctx := lcli.DaemonContext(cctx) @@ -112,29 +112,29 @@ var runCmd = &cli.Command{ node.Online(), node.Repo(r), - node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, + node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("miner-api") }, node.Override(new(dtypes.APIEndpoint), func() (dtypes.APIEndpoint, error) { - return multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + cctx.String("api")) + return multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + cctx.String("miner-api")) })), node.Override(new(api.FullNode), nodeApi), ) if err != nil { - return err + return xerrors.Errorf("creating node: %w", err) } endpoint, err := r.APIEndpoint() if err != nil { - return err + return xerrors.Errorf("getting API endpoint: %w", err) } // Bootstrap with full node remoteAddrs, err := nodeApi.NetAddrsListen(ctx) if err != nil { - return err + return xerrors.Errorf("getting full node libp2p address: %w", err) } if err := minerapi.NetConnect(ctx, remoteAddrs); err != nil { - return err + return xerrors.Errorf("connecting to full node (libp2p): %w", err) } log.Infof("Remote version %s", v) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 8a3687877..77792f32a 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -31,6 +31,10 @@ const metaFile = "sectorstore.json" var storageCmd = &cli.Command{ Name: "storage", Usage: "manage sector storage", + Description: `Sectors can be stored across many filesystem paths. These +commands provide ways to manage the storage the miner will used to store sectors +long term for proving (references as 'store') as well as how sectors will be +stored while moving through the sealing pipeline (references as 'seal').`, Subcommands: []*cli.Command{ storageAttachCmd, storageListCmd, @@ -41,6 +45,25 @@ var storageCmd = &cli.Command{ var storageAttachCmd = &cli.Command{ Name: "attach", Usage: "attach local storage path", + Description: `Storage can be attached to the miner using this command. The storage volume +list is stored local to the miner in $LOTUS_MINER_PATH/storage.json. We do not +recommend manually modifying this value without further understanding of the +storage system. + +Each storage volume contains a configuration file which describes the +capabilities of the volume. When the '--init' flag is provided, this file will +be created using the additional flags. + +Weight +A high weight value means data will be more likely to be stored in this path + +Seal +Data for the sealing process will be stored here + +Store +Finalized sectors that will be moved here for long term storage and be proven +over time + `, Flags: []cli.Flag{ &cli.BoolFlag{ Name: "init", diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go new file mode 100644 index 000000000..3bcb3f867 --- /dev/null +++ b/cmd/lotus-wallet/logged.go @@ -0,0 +1,94 @@ +package main + +import ( + "bytes" + "context" + "encoding/hex" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" +) + +type LoggedWallet struct { + under api.WalletAPI +} + +func (c *LoggedWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { + n, err := typ.Name() + if err != nil { + return address.Address{}, err + } + + log.Infow("WalletNew", "type", n) + + return c.under.WalletNew(ctx, typ) +} + +func (c *LoggedWallet) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + log.Infow("WalletHas", "address", addr) + + return c.under.WalletHas(ctx, addr) +} + +func (c *LoggedWallet) WalletList(ctx context.Context) ([]address.Address, error) { + log.Infow("WalletList") + + return c.under.WalletList(ctx) +} + +func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { + switch meta.Type { + case api.MTChainMsg: + var cmsg types.Message + if err := cmsg.UnmarshalCBOR(bytes.NewReader(meta.Extra)); err != nil { + return nil, xerrors.Errorf("unmarshalling message: %w", err) + } + + _, bc, err := cid.CidFromBytes(msg) + if err != nil { + return nil, xerrors.Errorf("getting cid from signing bytes: %w", err) + } + + if !cmsg.Cid().Equals(bc) { + return nil, xerrors.Errorf("cid(meta.Extra).bytes() != msg") + } + + log.Infow("WalletSign", + "address", k, + "type", meta.Type, + "from", cmsg.From, + "to", cmsg.To, + "value", types.FIL(cmsg.Value), + "feecap", types.FIL(cmsg.RequiredFunds()), + "method", cmsg.Method, + "params", hex.EncodeToString(cmsg.Params)) + default: + log.Infow("WalletSign", "address", k, "type", meta.Type) + } + + return c.under.WalletSign(ctx, k, msg, meta) +} + +func (c *LoggedWallet) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { + log.Infow("WalletExport", "address", a) + + return c.under.WalletExport(ctx, a) +} + +func (c *LoggedWallet) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { + log.Infow("WalletImport", "type", ki.Type) + + return c.under.WalletImport(ctx, ki) +} + +func (c *LoggedWallet) WalletDelete(ctx context.Context, addr address.Address) error { + log.Infow("WalletDelete", "address", addr) + + return c.under.WalletDelete(ctx, addr) +} diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go new file mode 100644 index 000000000..ae3580a59 --- /dev/null +++ b/cmd/lotus-wallet/main.go @@ -0,0 +1,141 @@ +package main + +import ( + "context" + "net" + "net/http" + "os" + + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/wallet" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("main") + +const FlagWalletRepo = "wallet-repo" + +func main() { + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + runCmd, + } + + app := &cli.App{ + Name: "lotus-wallet", + Usage: "Basic external wallet", + Version: build.UserVersion(), + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagWalletRepo, + EnvVars: []string{"WALLET_PATH"}, + Value: "~/.lotuswallet", // TODO: Consider XDG_DATA_HOME + }, + }, + + Commands: local, + } + app.Setup() + + if err := app.Run(os.Args); err != nil { + log.Warnf("%+v", err) + return + } +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start lotus wallet", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "host address and port the wallet api will listen on", + Value: "0.0.0.0:1777", + }, + }, + Action: func(cctx *cli.Context) error { + log.Info("Starting lotus wallet") + + ctx := lcli.ReqContext(cctx) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + repoPath := cctx.String(FlagWalletRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + if err := r.Init(repo.Worker); err != nil { + return err + } + } + + lr, err := r.Lock(repo.Wallet) + if err != nil { + return err + } + + ks, err := lr.KeyStore() + if err != nil { + return err + } + + w, err := wallet.NewWallet(ks) + if err != nil { + return err + } + + address := cctx.String("listen") + mux := mux.NewRouter() + + log.Info("Setting up API endpoint at " + address) + + rpcServer := jsonrpc.NewServer() + rpcServer.Register("Filecoin", &LoggedWallet{under: w}) + + mux.Handle("/rpc/v0", rpcServer) + mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof + + /*ah := &auth.Handler{ + Verify: nodeApi.AuthVerify, + Next: mux.ServeHTTP, + }*/ + + srv := &http.Server{ + Handler: mux, + BaseContext: func(listener net.Listener) context.Context { + return ctx + }, + } + + go func() { + <-ctx.Done() + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + nl, err := net.Listen("tcp", address) + if err != nil { + return err + } + + return srv.Serve(nl) + }, +} diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index a0f754a60..4ff63be11 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,8 +15,6 @@ import ( "runtime/pprof" "strings" - "github.com/filecoin-project/lotus/chain/types" - paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" @@ -32,9 +30,11 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/peermgr" "github.com/filecoin-project/lotus/lib/ulimit" @@ -114,6 +114,11 @@ var DaemonCmd = &cli.Command{ Name: "halt-after-import", Usage: "halt the process after importing chain from file", }, + &cli.BoolFlag{ + Name: "lite", + Usage: "start lotus in lite mode", + Hidden: true, + }, &cli.StringFlag{ Name: "pprof", Usage: "specify name of file for writing cpu profile to", @@ -133,6 +138,8 @@ var DaemonCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + isLite := cctx.Bool("lite") + err := runmetrics.Enable(runmetrics.RunMetricOptions{ EnableCPU: true, EnableMemory: true, @@ -192,8 +199,10 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil { - return xerrors.Errorf("fetching proof parameters: %w", err) + if !isLite { + if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil { + return xerrors.Errorf("fetching proof parameters: %w", err) + } } var genBytes []byte @@ -240,10 +249,23 @@ var DaemonCmd = &cli.Command{ shutdownChan := make(chan struct{}) + // If the daemon is started in "lite mode", provide a GatewayAPI + // for RPC calls + liteModeDeps := node.Options() + if isLite { + gapi, closer, err := lcli.GetGatewayAPI(cctx) + if err != nil { + return err + } + + defer closer() + liteModeDeps = node.Override(new(api.GatewayAPI), gapi) + } + var api api.FullNode stop, err := node.New(ctx, - node.FullAPI(&api), + node.FullAPI(&api, node.Lite(isLite)), node.Override(new(dtypes.Bootstrapper), isBootstrapper), node.Override(new(dtypes.ShutdownChan), shutdownChan), @@ -251,6 +273,7 @@ var DaemonCmd = &cli.Command{ node.Repo(r), genesis, + liteModeDeps, node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, node.Override(node.SetApiEndpointKey, func(lr repo.LockedRepo) error { @@ -388,7 +411,11 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { bs := blockstore.NewBlockstore(ds) - cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + j, err := journal.OpenFSJournal(lr, journal.EnvDisabledEvents()) + if err != nil { + return xerrors.Errorf("failed to open journal: %w", err) + } + cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), j) log.Infof("importing chain from %s...", fname) @@ -409,6 +436,10 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { return xerrors.Errorf("importing chain failed: %w", err) } + if err := cst.FlushValidationCache(); err != nil { + return xerrors.Errorf("flushing validation cache failed: %w", err) + } + gb, err := cst.GetTipsetByHeight(context.TODO(), 0, ts, true) if err != nil { return err @@ -428,7 +459,7 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { } } - log.Info("accepting %s as new head", ts.Cids()) + log.Infof("accepting %s as new head", ts.Cids()) if err := cst.SetHead(ts); err != nil { return err } diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index fef245858..b0ed574df 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -198,8 +198,10 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: root, Epoch: execTs.Height(), Message: m, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, + // recorded randomness will be discarded. + Rand: conformance.NewRecordingRand(new(conformance.LogReporter), fapi), }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) @@ -212,6 +214,9 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { applyret *vm.ApplyRet carWriter func(w io.Writer) error retention = opts.retain + + // recordingRand will record randomness so we can embed it in the test vector. + recordingRand = conformance.NewRecordingRand(new(conformance.LogReporter), fapi) ) log.Printf("using state retention strategy: %s", retention) @@ -229,8 +234,9 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: preroot, Epoch: execTs.Height(), Message: msg, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, + Rand: recordingRand, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -260,8 +266,9 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: preroot, Epoch: execTs.Height(), Message: msg, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, + Rand: recordingRand, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -356,7 +363,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, - CAR: out.Bytes(), + Randomness: recordingRand.Recorded(), + CAR: out.Bytes(), Pre: &schema.Preconditions{ Epoch: int64(execTs.Height()), CircSupply: circSupply.Int, diff --git a/conformance/driver.go b/conformance/driver.go index 5648084ce..f49022b9c 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -2,6 +2,7 @@ package conformance import ( "context" + gobig "math/big" "os" "github.com/filecoin-project/lotus/chain/state" @@ -13,7 +14,11 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/lib/blockstore" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures + _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/test-vectors/schema" @@ -79,10 +84,10 @@ type ExecuteTipsetResult struct { // and reward withdrawal per miner. func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) { var ( - syscalls = mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)) - vmRand = new(testRand) + syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) + vmRand = NewFixedRand() - cs = store.NewChainStore(bs, ds, syscalls) + cs = store.NewChainStore(bs, ds, syscalls, nil) sm = stmgr.NewStateManager(cs) ) @@ -143,8 +148,12 @@ type ExecuteMessageParams struct { Preroot cid.Cid Epoch abi.ChainEpoch Message *types.Message - CircSupply *abi.TokenAmount - BaseFee *abi.TokenAmount + CircSupply abi.TokenAmount + BaseFee abi.TokenAmount + + // Rand is an optional vm.Rand implementation to use. If nil, the driver + // will use a vm.Rand that returns a fixed value for all calls. + Rand vm.Rand } // ExecuteMessage executes a conformance test vector message in a temporary VM. @@ -155,14 +164,8 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") } - basefee := DefaultBaseFee - if params.BaseFee != nil { - basefee = *params.BaseFee - } - - circSupply := DefaultCirculatingSupply - if params.CircSupply != nil { - circSupply = *params.CircSupply + if params.Rand == nil { + params.Rand = NewFixedRand() } // dummy state manager; only to reference the GetNetworkVersion method, @@ -172,13 +175,13 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP vmOpts := &vm.VMOpts{ StateBase: params.Preroot, Epoch: params.Epoch, - Rand: &testRand{}, // TODO always succeeds; need more flexibility. Bstore: bs, - Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. + Syscalls: vm.Syscalls(ffiwrapper.ProofVerifier), CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { - return circSupply, nil + return params.CircSupply, nil }, - BaseFee: basefee, + Rand: params.Rand, + BaseFee: params.BaseFee, NtwkVersion: sm.GetNtwkVersion, } @@ -231,3 +234,22 @@ func toChainMsg(msg *types.Message) (ret types.ChainMsg) { } return ret } + +// BaseFeeOrDefault converts a basefee as passed in a test vector (go *big.Int +// type) to an abi.TokenAmount, or if nil it returns the DefaultBaseFee. +func BaseFeeOrDefault(basefee *gobig.Int) abi.TokenAmount { + if basefee == nil { + return DefaultBaseFee + } + return big.NewFromGo(basefee) +} + +// CircSupplyOrDefault converts a circulating supply as passed in a test vector +// (go *big.Int type) to an abi.TokenAmount, or if nil it returns the +// DefaultCirculatingSupply. +func CircSupplyOrDefault(circSupply *gobig.Int) abi.TokenAmount { + if circSupply == nil { + return DefaultBaseFee + } + return big.NewFromGo(circSupply) +} diff --git a/conformance/rand_fixed.go b/conformance/rand_fixed.go new file mode 100644 index 000000000..d356b53d0 --- /dev/null +++ b/conformance/rand_fixed.go @@ -0,0 +1,28 @@ +package conformance + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/vm" +) + +type fixedRand struct{} + +var _ vm.Rand = (*fixedRand)(nil) + +// NewFixedRand creates a test vm.Rand that always returns fixed bytes value +// of utf-8 string 'i_am_random_____i_am_random_____'. +func NewFixedRand() vm.Rand { + return &fixedRand{} +} + +func (r *fixedRand) GetChainRandomness(_ context.Context, _ crypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) ([]byte, error) { + return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. +} + +func (r *fixedRand) GetBeaconRandomness(_ context.Context, _ crypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) ([]byte, error) { + return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. +} diff --git a/conformance/rand_record.go b/conformance/rand_record.go new file mode 100644 index 000000000..6f6d064dc --- /dev/null +++ b/conformance/rand_record.go @@ -0,0 +1,103 @@ +package conformance + +import ( + "context" + "fmt" + "sync" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/test-vectors/schema" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" +) + +type RecordingRand struct { + reporter Reporter + api api.FullNode + + // once guards the loading of the head tipset. + // can be removed when https://github.com/filecoin-project/lotus/issues/4223 + // is fixed. + once sync.Once + head types.TipSetKey + lk sync.Mutex + recorded schema.Randomness +} + +var _ vm.Rand = (*RecordingRand)(nil) + +// NewRecordingRand returns a vm.Rand implementation that proxies calls to a +// full Lotus node via JSON-RPC, and records matching rules and responses so +// they can later be embedded in test vectors. +func NewRecordingRand(reporter Reporter, api api.FullNode) *RecordingRand { + return &RecordingRand{reporter: reporter, api: api} +} + +func (r *RecordingRand) loadHead() { + head, err := r.api.ChainHead(context.Background()) + if err != nil { + panic(fmt.Sprintf("could not fetch chain head while fetching randomness: %s", err)) + } + r.head = head.Key() +} + +func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + r.once.Do(r.loadHead) + ret, err := r.api.ChainGetRandomnessFromTickets(ctx, r.head, pers, round, entropy) + if err != nil { + return ret, err + } + + r.reporter.Logf("fetched and recorded chain randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + + match := schema.RandomnessMatch{ + On: schema.RandomnessRule{ + Kind: schema.RandomnessChain, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + }, + Return: []byte(ret), + } + r.lk.Lock() + r.recorded = append(r.recorded, match) + r.lk.Unlock() + + return ret, err +} + +func (r *RecordingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + r.once.Do(r.loadHead) + ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, r.head, pers, round, entropy) + if err != nil { + return ret, err + } + + r.reporter.Logf("fetched and recorded beacon randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + + match := schema.RandomnessMatch{ + On: schema.RandomnessRule{ + Kind: schema.RandomnessBeacon, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + }, + Return: []byte(ret), + } + r.lk.Lock() + r.recorded = append(r.recorded, match) + r.lk.Unlock() + + return ret, err +} + +func (r *RecordingRand) Recorded() schema.Randomness { + r.lk.Lock() + defer r.lk.Unlock() + + return r.recorded +} diff --git a/conformance/rand_replay.go b/conformance/rand_replay.go new file mode 100644 index 000000000..1b73e5a08 --- /dev/null +++ b/conformance/rand_replay.go @@ -0,0 +1,79 @@ +package conformance + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/test-vectors/schema" + + "github.com/filecoin-project/lotus/chain/vm" +) + +type ReplayingRand struct { + reporter Reporter + recorded schema.Randomness + fallback vm.Rand +} + +var _ vm.Rand = (*ReplayingRand)(nil) + +// NewReplayingRand replays recorded randomness when requested, falling back to +// fixed randomness if the value cannot be found; hence this is a safe +// backwards-compatible replacement for fixedRand. +func NewReplayingRand(reporter Reporter, recorded schema.Randomness) *ReplayingRand { + return &ReplayingRand{ + reporter: reporter, + recorded: recorded, + fallback: NewFixedRand(), + } +} + +func (r *ReplayingRand) match(requested schema.RandomnessRule) ([]byte, bool) { + for _, other := range r.recorded { + if other.On.Kind == requested.Kind && + other.On.Epoch == requested.Epoch && + other.On.DomainSeparationTag == requested.DomainSeparationTag && + bytes.Equal(other.On.Entropy, requested.Entropy) { + return other.Return, true + } + } + return nil, false +} + +func (r *ReplayingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + rule := schema.RandomnessRule{ + Kind: schema.RandomnessChain, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + } + + if ret, ok := r.match(rule); ok { + r.reporter.Logf("returning saved chain randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + return ret, nil + } + + r.reporter.Logf("returning fallback chain randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy) + return r.fallback.GetChainRandomness(ctx, pers, round, entropy) +} + +func (r *ReplayingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + rule := schema.RandomnessRule{ + Kind: schema.RandomnessBeacon, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + } + + if ret, ok := r.match(rule); ok { + r.reporter.Logf("returning saved beacon randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + return ret, nil + } + + r.reporter.Logf("returning fallback beacon randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy) + return r.fallback.GetBeaconRandomness(ctx, pers, round, entropy) + +} diff --git a/conformance/runner.go b/conformance/runner.go index 626545b2b..d489ac288 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -13,9 +13,7 @@ import ( "github.com/fatih/color" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/test-vectors/schema" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -24,6 +22,8 @@ import ( "github.com/ipfs/go-merkledag" "github.com/ipld/go-car" + "github.com/filecoin-project/test-vectors/schema" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/lib/blockstore" @@ -46,18 +46,6 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Create a new Driver. driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true}) - var circSupply *abi.TokenAmount - if cs := vector.Pre.CircSupply; cs != nil { - ta := big.NewFromGo(cs) - circSupply = &ta - } - - var basefee *abi.TokenAmount - if bf := vector.Pre.BaseFee; bf != nil { - ta := big.NewFromGo(bf) - basefee = &ta - } - // Apply every message. for i, m := range vector.ApplyMessages { msg, err := types.DecodeMessage(m.Bytes) @@ -76,8 +64,9 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { Preroot: root, Epoch: abi.ChainEpoch(epoch), Message: msg, - CircSupply: circSupply, - BaseFee: basefee, + BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), + CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), + Rand: NewReplayingRand(r, vector.Randomness), }) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) diff --git a/conformance/stubs.go b/conformance/stubs.go deleted file mode 100644 index a7100892f..000000000 --- a/conformance/stubs.go +++ /dev/null @@ -1,56 +0,0 @@ -package conformance - -import ( - "context" - - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/state" - "github.com/filecoin-project/lotus/chain/vm" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/runtime" - - cbor "github.com/ipfs/go-ipld-cbor" -) - -type testRand struct{} - -var _ vm.Rand = (*testRand)(nil) - -func (r *testRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. -} - -func (r *testRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. -} - -type testSyscalls struct { - runtime.Syscalls -} - -// TODO VerifySignature this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifySignature(_ crypto.Signature, _ address.Address, _ []byte) error { - return nil -} - -// TODO VerifySeal this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifySeal(_ proof.SealVerifyInfo) error { - return nil -} - -// TODO VerifyPoSt this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifyPoSt(_ proof.WindowPoStVerifyInfo) error { - return nil -} - -func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder { - return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls { - return &testSyscalls{ - base(ctx, cstate, cst), - } - } -} diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index a6ad1ecb2..ec8071b57 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -83,11 +83,14 @@ * [MsigAddCancel](#MsigAddCancel) * [MsigAddPropose](#MsigAddPropose) * [MsigApprove](#MsigApprove) + * [MsigApproveTxnHash](#MsigApproveTxnHash) * [MsigCancel](#MsigCancel) * [MsigCreate](#MsigCreate) * [MsigGetAvailableBalance](#MsigGetAvailableBalance) * [MsigGetVested](#MsigGetVested) + * [MsigGetVestingSchedule](#MsigGetVestingSchedule) * [MsigPropose](#MsigPropose) + * [MsigRemoveSigner](#MsigRemoveSigner) * [MsigSwapApprove](#MsigSwapApprove) * [MsigSwapCancel](#MsigSwapCancel) * [MsigSwapPropose](#MsigSwapPropose) @@ -166,6 +169,7 @@ * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) * [StateVerifierStatus](#StateVerifierStatus) * [StateWaitMsg](#StateWaitMsg) + * [StateWaitMsgLimited](#StateWaitMsgLimited) * [Sync](#Sync) * [SyncCheckBad](#SyncCheckBad) * [SyncCheckpoint](#SyncCheckpoint) @@ -173,6 +177,7 @@ * [SyncMarkBad](#SyncMarkBad) * [SyncState](#SyncState) * [SyncSubmitBlock](#SyncSubmitBlock) + * [SyncUnmarkAllBad](#SyncUnmarkAllBad) * [SyncUnmarkBad](#SyncUnmarkBad) * [SyncValidateTipset](#SyncValidateTipset) * [Wallet](#Wallet) @@ -2005,7 +2010,33 @@ Response: ``` ### MsigApprove -MsigApprove approves a previously-proposed multisig message +MsigApprove approves a previously-proposed multisig message by transaction ID +It takes the following params: , + + +Perms: sign + +Inputs: +```json +[ + "f01234", + 42, + "f01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigApproveTxnHash +MsigApproveTxnHash approves a previously-proposed multisig message, specified +using both transaction ID and a hash of the parameters used in the +proposal. This method of approval can be used to ensure you only approve +exactly the transaction you think you are. It takes the following params: , , , , , , , @@ -2035,7 +2066,7 @@ Response: ### MsigCancel MsigCancel cancels a previously-proposed multisig message -It takes the following params: , , , , +It takes the following params: , , , , , , @@ -2143,6 +2174,36 @@ Inputs: Response: `"0"` +### MsigGetVestingSchedule +MsigGetVestingSchedule returns the vesting details of a given multisig. + + +Perms: read + +Inputs: +```json +[ + "f01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "InitialBalance": "0", + "StartEpoch": 10101, + "UnlockDuration": 10101 +} +``` + ### MsigPropose MsigPropose proposes a multisig message It takes the following params: , , , @@ -2170,6 +2231,33 @@ Response: } ``` +### MsigRemoveSigner +MsigRemoveSigner proposes the removal of a signer from the multisig. +It accepts the multisig to make the change on, the proposer address to +send the message from, the address to be removed, and a boolean +indicating whether or not the signing threshold should be lowered by one +along with the address removal. + + +Perms: sign + +Inputs: +```json +[ + "f01234", + "f01234", + "f01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + ### MsigSwapApprove MsigSwapApprove approves a previously proposed SwapSigner It takes the following params: , , , @@ -4290,6 +4378,49 @@ Response: } ``` +### StateWaitMsgLimited +StateWaitMsgLimited looks back up to limit epochs in the chain for a message. +If not found, it blocks until the message arrives on chain, and gets to the +indicated confidence depth. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + 42, + 10101 +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ReturnDec": {}, + "TipSet": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Height": 10101 +} +``` + ## Sync The Sync method group contains methods for interacting with and observing the lotus sync service. @@ -4471,6 +4602,16 @@ Inputs: Response: `{}` +### SyncUnmarkAllBad +SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad + + +Perms: admin + +Inputs: `null` + +Response: `{}` + ### SyncUnmarkBad SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index d75501838..bae6cafb4 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -290,10 +290,6 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s defer opr.Close() // nolint padwriter := fr32.NewPadWriter(out) - if err != nil { - perr = xerrors.Errorf("creating new padded writer: %w", err) - return - } bsize := uint64(size.Padded()) if bsize > uint64(runtime.NumCPU())*fr32.MTTresh { @@ -302,7 +298,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s bw := bufio.NewWriterSize(padwriter, int(abi.PaddedPieceSize(bsize).Unpadded())) - _, err = io.CopyN(bw, opr, int64(size)) + _, err := io.CopyN(bw, opr, int64(size)) if err != nil { perr = xerrors.Errorf("copying data: %w", err) return diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index 75387acc8..50968e7bd 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -30,10 +30,15 @@ type StoragePath struct { // LocalStorageMeta [path]/sectorstore.json type LocalStorageMeta struct { - ID ID + ID ID + + // A high weight means data is more likely to be stored in this path Weight uint64 // 0 = readonly - CanSeal bool + // Intermediate data for the sealing process will be stored here + CanSeal bool + + // Finalized sectors that will be proved over time will be stored here CanStore bool } diff --git a/extern/test-vectors b/extern/test-vectors index 3a6e0b5e0..a8f968ade 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 3a6e0b5e069b1452ce1a032aa315354d645f3ec4 +Subproject commit a8f968adeba1995f161f7be0048188affc425079 diff --git a/gen/main.go b/gen/main.go index bcb43a8f0..d5874af2c 100644 --- a/gen/main.go +++ b/gen/main.go @@ -27,7 +27,7 @@ func main() { types.ExpTipSet{}, types.BeaconEntry{}, types.StateRoot{}, - types.StateInfo{}, + types.StateInfo0{}, ) if err != nil { fmt.Println(err) diff --git a/go.mod b/go.mod index 8a25b3232..eb79a496c 100644 --- a/go.mod +++ b/go.mod @@ -29,8 +29,8 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.7 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.7.0 - github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 + github.com/filecoin-project/go-fil-markets v0.7.1 + github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 @@ -39,9 +39,9 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.0.1 + github.com/filecoin-project/specs-actors/v2 v2.0.3 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.3 + github.com/filecoin-project/test-vectors/schema v0.0.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect @@ -94,10 +94,10 @@ require ( github.com/libp2p/go-libp2p-discovery v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.8.3 github.com/libp2p/go-libp2p-mplex v0.2.4 - github.com/libp2p/go-libp2p-noise v0.1.1 + github.com/libp2p/go-libp2p-noise v0.1.2 github.com/libp2p/go-libp2p-peerstore v0.2.6 - github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18 - github.com/libp2p/go-libp2p-quic-transport v0.8.0 + github.com/libp2p/go-libp2p-pubsub v0.3.6 + github.com/libp2p/go-libp2p-quic-transport v0.8.2 github.com/libp2p/go-libp2p-record v0.1.3 github.com/libp2p/go-libp2p-routing-helpers v0.2.3 github.com/libp2p/go-libp2p-swarm v0.2.8 diff --git a/go.sum b/go.sum index 6062ee4ee..e13359f6c 100644 --- a/go.sum +++ b/go.sum @@ -244,16 +244,16 @@ github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.7.0 h1:tcEZiUNIYQJ4PBzgVpLwfdJ4ZdC4WCv9LsgvsoCXIls= -github.com/filecoin-project/go-fil-markets v0.7.0/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= +github.com/filecoin-project/go-fil-markets v0.7.1 h1:e0NlpSnaeGyDUhCOzevjcxkSA54kt9BzlXpLRgduUFI= +github.com/filecoin-project/go-fil-markets v0.7.1/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= -github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= -github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= +github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 h1:FSY245KeXFCUgyfFEu+bhrZNk8BGGJyfpSmQl2aiPU8= +github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 h1:+/4aUeUoKr6AKfPE3mBhXA5spIV6UcKdTYDPNU2Tdmg= @@ -276,12 +276,12 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.1 h1:bf08x6tqCDfClzrv2q/rmt/A/UbBOy1KgaoogQEcLhU= -github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= +github.com/filecoin-project/specs-actors/v2 v2.0.3 h1:Niy6xncgi8bI8aBCt1McdZfATBfG4Uxytt8KW4s3bAc= +github.com/filecoin-project/specs-actors/v2 v2.0.3/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= -github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= +github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= @@ -863,6 +863,8 @@ github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLK github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-noise v0.1.1 h1:vqYQWvnIcHpIoWJKC7Al4D6Hgj0H012TuXRhPwSMGpQ= github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM= +github.com/libp2p/go-libp2p-noise v0.1.2 h1:IH9GRihQJTx56obm+GnpdPX4KeVIlvpXrP6xnJ0wxWk= +github.com/libp2p/go-libp2p-noise v0.1.2/go.mod h1:9B10b7ueo7TIxZHHcjcDCo5Hd6kfKT2m77by82SFRfE= github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= @@ -884,12 +886,12 @@ github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= -github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18 h1:+ae7vHSv/PJ4xGXwLV6LKGj32zjyB8ttJHtyV4TXal0= -github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI= +github.com/libp2p/go-libp2p-pubsub v0.3.6 h1:9oO8W7qIWCYQYyz5z8nUsPcb3rrFehBlkbqvbSVjBxY= +github.com/libp2p/go-libp2p-pubsub v0.3.6/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M= -github.com/libp2p/go-libp2p-quic-transport v0.8.0 h1:mHA94K2+TD0e9XtjWx/P5jGGZn0GdQ4OFYwNllagv4E= -github.com/libp2p/go-libp2p-quic-transport v0.8.0/go.mod h1:F2FG/6Bzz0U6essUVxDzE0s9CrY4XGLbl7QEmDNvU7A= +github.com/libp2p/go-libp2p-quic-transport v0.8.2 h1:FDaXBCBJ1e5hY6gnWEJ4NbYyLk8eezr4J6AY3q3KqwM= +github.com/libp2p/go-libp2p-quic-transport v0.8.2/go.mod h1:L+e0q15ZNaYm3seHgbsXjWP8kXLEqz+elLWKk9l8DhM= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= @@ -1022,8 +1024,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/lucas-clemente/quic-go v0.16.0/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= -github.com/lucas-clemente/quic-go v0.18.0 h1:JhQDdqxdwdmGdKsKgXi1+coHRoGhvU6z0rNzOJqZ/4o= -github.com/lucas-clemente/quic-go v0.18.0/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= +github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys= +github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= diff --git a/journal/env.go b/journal/env.go new file mode 100644 index 000000000..68380844e --- /dev/null +++ b/journal/env.go @@ -0,0 +1,19 @@ +package journal + +import ( + "os" +) + +// envJournalDisabledEvents is the environment variable through which disabled +// journal events can be customized. +const envDisabledEvents = "LOTUS_JOURNAL_DISABLED_EVENTS" + +func EnvDisabledEvents() DisabledEvents { + if env, ok := os.LookupEnv(envDisabledEvents); ok { + if ret, err := ParseDisabledEvents(env); err == nil { + return ret + } + } + // fallback if env variable is not set, or if it failed to parse. + return DefaultDisabledEvents +} diff --git a/journal/global.go b/journal/global.go deleted file mode 100644 index b4d0e0a1b..000000000 --- a/journal/global.go +++ /dev/null @@ -1,9 +0,0 @@ -package journal - -var ( - // J is a globally accessible Journal. It starts being NilJournal, and early - // during the Lotus initialization routine, it is reset to whichever Journal - // is configured (by default, the filesystem journal). Components can safely - // record in the journal by calling: journal.J.RecordEvent(...). - J Journal = NilJournal() // nolint -) diff --git a/lotuspond/front/src/chain/methods.json b/lotuspond/front/src/chain/methods.json index 5e15b053b..b271bfae5 100644 --- a/lotuspond/front/src/chain/methods.json +++ b/lotuspond/front/src/chain/methods.json @@ -176,7 +176,8 @@ "CompactPartitions", "CompactSectorNumbers", "ConfirmUpdateWorkerKey", - "RepayDebt" + "RepayDebt", + "ChangeOwnerAddress" ], "fil/2/storagepower": [ "Send", diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 8b2e8661d..ce01b115e 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -11,16 +11,16 @@ import ( "sync/atomic" "time" - "github.com/filecoin-project/lotus/chain/actors/policy" - "github.com/filecoin-project/lotus/chain/types" - + "github.com/google/uuid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" ) @@ -69,6 +69,8 @@ func (api *api) Spawn() (nodeInfo, error) { Meta: (&genesis.AccountMeta{Owner: genm.Owner}).ActorMeta(), }) template.VerifregRootKey = gen.DefaultVerifregRootkeyActor + template.RemainderAccount = gen.DefaultRemainderAccountActor + template.NetworkName = "pond-" + uuid.New().String() tb, err := json.Marshal(&template) if err != nil { @@ -188,7 +190,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { mux := newWsMux() - cmd = exec.Command("./lotus-miner", "run", "--api", fmt.Sprintf("%d", 2500+id), "--nosync") + cmd = exec.Command("./lotus-miner", "run", "--miner-api", fmt.Sprintf("%d", 2500+id), "--nosync") cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) cmd.Env = append(os.Environ(), "LOTUS_MINER_PATH="+dir, "LOTUS_PATH="+fullNodeRepo) @@ -248,7 +250,7 @@ func (api *api) RestartNode(id int32) (nodeInfo, error) { var cmd *exec.Cmd if nd.meta.Storage { - cmd = exec.Command("./lotus-miner", "run", "--api", fmt.Sprintf("%d", 2500+id), "--nosync") + cmd = exec.Command("./lotus-miner", "run", "--miner-api", fmt.Sprintf("%d", 2500+id), "--nosync") } else { cmd = exec.Command("./lotus", "daemon", "--api", fmt.Sprintf("%d", 2500+id)) } diff --git a/markets/journal.go b/markets/journal.go index a73d28011..9c9c5be9c 100644 --- a/markets/journal.go +++ b/markets/journal.go @@ -28,9 +28,9 @@ type RetrievalProviderEvt struct { } // StorageClientJournaler records journal events from the storage client. -func StorageClientJournaler(evtType journal.EventType) func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { +func StorageClientJournaler(j journal.Journal, evtType journal.EventType) func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { return func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return StorageClientEvt{ Event: storagemarket.ClientEvents[event], Deal: deal, @@ -40,9 +40,9 @@ func StorageClientJournaler(evtType journal.EventType) func(event storagemarket. } // StorageProviderJournaler records journal events from the storage provider. -func StorageProviderJournaler(evtType journal.EventType) func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { +func StorageProviderJournaler(j journal.Journal, evtType journal.EventType) func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { return func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return StorageProviderEvt{ Event: storagemarket.ProviderEvents[event], Deal: deal, @@ -52,9 +52,9 @@ func StorageProviderJournaler(evtType journal.EventType) func(event storagemarke } // RetrievalClientJournaler records journal events from the retrieval client. -func RetrievalClientJournaler(evtType journal.EventType) func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { +func RetrievalClientJournaler(j journal.Journal, evtType journal.EventType) func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { return func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return RetrievalClientEvt{ Event: retrievalmarket.ClientEvents[event], Deal: deal, @@ -64,9 +64,9 @@ func RetrievalClientJournaler(evtType journal.EventType) func(event retrievalmar } // RetrievalProviderJournaler records journal events from the retrieval provider. -func RetrievalProviderJournaler(evtType journal.EventType) func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { +func RetrievalProviderJournaler(j journal.Journal, evtType journal.EventType) func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { return func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return RetrievalProviderEvt{ Event: retrievalmarket.ProviderEvents[event], Deal: deal, diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 411c86ec9..6496fffad 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -6,19 +6,22 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-state-types/big" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - "golang.org/x/xerrors" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/events" @@ -30,7 +33,6 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" - "github.com/ipfs/go-cid" ) type ClientNodeAdapter struct { @@ -187,7 +189,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor } // TODO: timeout - _, ret, _, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence) + _, ret, _, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence, stmgr.LookbackNoLimit) if err != nil { return 0, xerrors.Errorf("waiting for deal publish message: %w", err) } @@ -422,7 +424,9 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add return nil, err } - sig, err := c.Wallet.Sign(ctx, signer, buf) + sig, err := c.Wallet.WalletSign(ctx, signer, buf, api.MsgMeta{ + Type: api.MTDealProposal, + }) if err != nil { return nil, err } @@ -434,7 +438,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add } func (c *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { - addr, err := c.Wallet.GetDefault() + addr, err := c.DefWallet.GetDefault() return addr, err } @@ -475,7 +479,9 @@ func (c *ClientNodeAdapter) SignBytes(ctx context.Context, signer address.Addres return nil, err } - localSignature, err := c.Wallet.Sign(ctx, signer, b) + localSignature, err := c.Wallet.WalletSign(ctx, signer, b, api.MsgMeta{ + Type: api.MTUnknown, // TODO: pass type here + }) if err != nil { return nil, err } diff --git a/miner/miner.go b/miner/miner.go index a256e8b3a..73985a649 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -49,7 +49,7 @@ func randTimeOffset(width time.Duration) time.Duration { return val - (width / 2) } -func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter) *Miner { +func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter, j journal.Journal) *Miner { arc, err := lru.NewARC(10000) if err != nil { panic(err) @@ -74,8 +74,9 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf: sf, minedBlockHeights: arc, evtTypes: [...]journal.EventType{ - evtTypeBlockMined: journal.J.RegisterEventType("miner", "block_mined"), + evtTypeBlockMined: j.RegisterEventType("miner", "block_mined"), }, + journal: j, } } @@ -97,6 +98,7 @@ type Miner struct { minedBlockHeights *lru.ARCCache evtTypes [1]journal.EventType + journal journal.Journal } func (m *Miner) Address() address.Address { @@ -239,7 +241,7 @@ minerLoop: onDone(b != nil, h, nil) if b != nil { - journal.J.RecordEvent(m.evtTypes[evtTypeBlockMined], func() interface{} { + m.journal.RecordEvent(m.evtTypes[evtTypeBlockMined], func() interface{} { return map[string]interface{}{ "parents": base.TipSet.Cids(), "nulls": base.NullRounds, @@ -327,6 +329,7 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error) } ltsw, err := m.api.ChainTipSetWeight(ctx, m.lastWork.TipSet.Key()) if err != nil { + m.lastWork = nil return nil, err } diff --git a/miner/testminer.go b/miner/testminer.go index 64e3b3a62..5f461d884 100644 --- a/miner/testminer.go +++ b/miner/testminer.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" + "github.com/filecoin-project/lotus/journal" ) type MineReq struct { @@ -32,6 +33,7 @@ func NewTestMiner(nextCh <-chan MineReq, addr address.Address) func(api.FullNode minedBlockHeights: arc, address: addr, sf: slashfilter.New(ds.NewMapDatastore()), + journal: journal.NilJournal(), } if err := m.Start(context.TODO()); err != nil { diff --git a/node/builder.go b/node/builder.go index 0d48ec130..b91172386 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,9 +3,15 @@ package node import ( "context" "errors" - "os" "time" + "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/exchange" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node/hello" + logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -29,9 +35,7 @@ import ( storage2 "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" - "github.com/filecoin-project/lotus/chain/exchange" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/market" @@ -39,10 +43,8 @@ import ( "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/remotewallet" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -56,9 +58,9 @@ import ( "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/impl/common" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -71,10 +73,6 @@ import ( "github.com/filecoin-project/lotus/storage/sectorblocks" ) -// EnvJournalDisabledEvents is the environment variable through which disabled -// journal events can be customized. -const EnvJournalDisabledEvents = "LOTUS_JOURNAL_DISABLED_EVENTS" - //nolint:deadcode,varcheck var log = logging.Logger("builder") @@ -158,25 +156,14 @@ type Settings struct { Online bool // Online option applied Config bool // Config option applied - + Lite bool // Start node in "lite" mode } func defaults() []Option { return []Option{ // global system journal. - Override(new(journal.DisabledEvents), func() journal.DisabledEvents { - if env, ok := os.LookupEnv(EnvJournalDisabledEvents); ok { - if ret, err := journal.ParseDisabledEvents(env); err == nil { - return ret - } - } - // fallback if env variable is not set, or if it failed to parse. - return journal.DefaultDisabledEvents - }), + Override(new(journal.DisabledEvents), journal.EnvDisabledEvents), Override(new(journal.Journal), modules.OpenFilesystemJournal), - Override(InitJournalKey, func(j journal.Journal) { - journal.J = j // eagerly sets the global journal through fx.Invoke. - }), Override(new(helpers.MetricsCtx), context.Background), Override(new(record.Validator), modules.RecordValidator), @@ -232,6 +219,10 @@ func isType(t repo.RepoType) func(s *Settings) bool { // Online sets up basic libp2p node func Online() Option { + isFullOrLiteNode := func(s *Settings) bool { return s.nodeType == repo.FullNode } + isFullNode := func(s *Settings) bool { return s.nodeType == repo.FullNode && !s.Lite } + isLiteNode := func(s *Settings) bool { return s.nodeType == repo.FullNode && s.Lite } + return Options( // make sure that online is applied before Config. // This is important because Config overrides some of Online units @@ -245,23 +236,23 @@ func Online() Option { // common Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - // Full node - - ApplyIf(isType(repo.FullNode), + // Full node or lite node + ApplyIf(isFullOrLiteNode, // TODO: Fix offline mode Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), - Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), Override(new(vm.SyscallBuilder), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), - Override(new(*wallet.Wallet), wallet.NewWallet), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), + Override(new(*wallet.LocalWallet), wallet.NewWallet), + Override(new(api.WalletAPI), From(new(*wallet.LocalWallet))), + Override(new(wallet.Default), From(new(*wallet.LocalWallet))), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), @@ -288,12 +279,6 @@ func Online() Option { Override(new(dtypes.Graphsync), modules.Graphsync), Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)), - - Override(RunHelloKey, modules.RunHello), - Override(RunChainExchangeKey, modules.RunChainExchange), - Override(RunPeerMgrKey, modules.RunPeerMgr), - Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), Override(new(discovery.PeerResolver), modules.RetrievalResolver), @@ -312,8 +297,34 @@ func Online() Option { Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels), ), + // Lite node + ApplyIf(isLiteNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), + Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))), + Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + ), + + // Full node + ApplyIf(isFullNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), + Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), + Override(new(full.GasModuleAPI), From(new(full.GasModule))), + Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), + + Override(RunHelloKey, modules.RunHello), + Override(RunChainExchangeKey, modules.RunChainExchange), + Override(RunPeerMgrKey, modules.RunPeerMgr), + Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), + Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), + ), + // miner - ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, + ApplyIf(isType(repo.StorageMiner), Override(new(api.Common), From(new(common.CommonAPI))), Override(new(sectorstorage.StorageAuth), modules.StorageAuth), @@ -448,6 +459,9 @@ func ConfigFullNode(c interface{}) Option { If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), ), + If(cfg.Wallet.RemoteBackend != "", + Override(new(api.WalletAPI), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + ), ) } @@ -507,12 +521,22 @@ func Repo(r repo.Repo) Option { } } -func FullAPI(out *api.FullNode) Option { +type FullOption = Option + +func Lite(enable bool) FullOption { + return func(s *Settings) error { + s.Lite = enable + return nil + } +} + +func FullAPI(out *api.FullNode, fopts ...FullOption) Option { return Options( func(s *Settings) error { s.nodeType = repo.FullNode return nil }, + Options(fopts...), func(s *Settings) error { resAPI := &impl.FullNodeAPI{} s.invokes[ExtractApiKey] = fx.Populate(resAPI) diff --git a/node/config/def.go b/node/config/def.go index b05891e47..63340cfd5 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -22,6 +22,7 @@ type FullNode struct { Common Client Client Metrics Metrics + Wallet Wallet } // // Common @@ -107,6 +108,10 @@ type Client struct { IpfsUseForRetrieval bool } +type Wallet struct { + RemoteBackend string +} + func defCommon() Common { return Common{ API: API{ diff --git a/node/impl/client/client.go b/node/impl/client/client.go index f146dcea3..fe83fda7b 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -462,13 +462,19 @@ type retrievalSubscribeEvent struct { state rm.ClientDealState } -func readSubscribeEvents(ctx context.Context, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { +func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { for { var subscribeEvent retrievalSubscribeEvent select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") case subscribeEvent = <-subscribeEvents: + if subscribeEvent.state.ID != dealID { + // we can't check the deal ID ahead of time because: + // 1. We need to subscribe before retrieving. + // 2. We won't know the deal ID until after retrieving. + continue + } } select { @@ -531,19 +537,6 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return err }*/ - var dealID retrievalmarket.DealID - subscribeEvents := make(chan retrievalSubscribeEvent, 1) - subscribeCtx, cancel := context.WithCancel(ctx) - defer cancel() - unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { - if state.PayloadCID.Equals(order.Root) && state.ID == dealID { - select { - case <-subscribeCtx.Done(): - case subscribeEvents <- retrievalSubscribeEvent{event, state}: - } - } - }) - ppb := types.BigDiv(order.Total, types.NewInt(order.Size)) params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, shared.AllSelector(), order.Piece, order.UnsealPrice) @@ -562,7 +555,21 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref _ = a.RetrievalStoreMgr.ReleaseStore(store) }() - dealID, err = a.Retrieval.Retrieve( + // Subscribe to events before retrieving to avoid losing events. + subscribeEvents := make(chan retrievalSubscribeEvent, 1) + subscribeCtx, cancel := context.WithCancel(ctx) + defer cancel() + unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { + // We'll check the deal IDs inside readSubscribeEvents. + if state.PayloadCID.Equals(order.Root) { + select { + case <-subscribeCtx.Done(): + case subscribeEvents <- retrievalSubscribeEvent{event, state}: + } + } + }) + + dealID, err := a.Retrieval.Retrieve( ctx, order.Root, params, @@ -573,11 +580,12 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref store.StoreID()) if err != nil { + unsubscribe() finish(xerrors.Errorf("Retrieve failed: %w", err)) return } - err = readSubscribeEvents(ctx, subscribeEvents, events) + err = readSubscribeEvents(ctx, dealID, subscribeEvents, events) unsubscribe() if err != nil { diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index b405fb3dc..5b4f41114 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -39,10 +39,28 @@ import ( var log = logging.Logger("fullnode") +type ChainModuleAPI interface { + ChainHead(context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) +} + +// ChainModule provides a default implementation of ChainModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type ChainModule struct { + fx.In + + Chain *store.ChainStore +} + +var _ ChainModuleAPI = (*ChainModule)(nil) + type ChainAPI struct { fx.In WalletAPI + ChainModuleAPI Chain *store.ChainStore } @@ -51,8 +69,8 @@ func (a *ChainAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, e return a.Chain.SubHeadChanges(ctx), nil } -func (a *ChainAPI) ChainHead(context.Context) (*types.TipSet, error) { - return a.Chain.GetHeaviestTipSet(), nil +func (m *ChainModule) ChainHead(context.Context) (*types.TipSet, error) { + return m.Chain.GetHeaviestTipSet(), nil } func (a *ChainAPI) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) { @@ -77,8 +95,8 @@ func (a *ChainAPI) ChainGetBlock(ctx context.Context, msg cid.Cid) (*types.Block return a.Chain.GetBlock(msg) } -func (a *ChainAPI) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { - return a.Chain.LoadTipSet(key) +func (m *ChainModule) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { + return m.Chain.LoadTipSet(key) } func (a *ChainAPI) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api.BlockMessages, error) { @@ -180,12 +198,12 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([] return out, nil } -func (a *ChainAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *ChainModule) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.Chain.GetTipsetByHeight(ctx, h, ts, true) + return m.Chain.GetTipsetByHeight(ctx, h, ts, true) } func (a *ChainAPI) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { @@ -254,11 +272,31 @@ func (a *ChainAPI) ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) } func (a *ChainAPI) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error { - ts, err := a.Chain.GetTipSetFromKey(tsk) + newHeadTs, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.Chain.SetHead(ts) + + currentTs, err := a.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("getting head: %w", err) + } + + for currentTs.Height() >= newHeadTs.Height() { + for _, blk := range currentTs.Key().Cids() { + err = a.Chain.UnmarkBlockAsValidated(ctx, blk) + if err != nil { + return xerrors.Errorf("unmarking block as validated %s: %w", blk, err) + } + } + + currentTs, err = a.ChainGetTipSet(ctx, currentTs.Parents()) + if err != nil { + return xerrors.Errorf("loading tipset: %w", err) + } + } + + return a.Chain.SetHead(newHeadTs) } func (a *ChainAPI) ChainGetGenesis(ctx context.Context) (*types.TipSet, error) { diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index c912c7a8c..0cb1eb084 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -26,8 +26,27 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +type GasModuleAPI interface { + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) +} + +// GasModule provides a default implementation of GasModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type GasModule struct { + fx.In + Stmgr *stmgr.StateManager + Chain *store.ChainStore + Mpool *messagepool.MessagePool +} + +var _ GasModuleAPI = (*GasModule)(nil) + type GasAPI struct { fx.In + + GasModuleAPI + Stmgr *stmgr.StateManager Chain *store.ChainStore Mpool *messagepool.MessagePool @@ -36,9 +55,24 @@ type GasAPI struct { const MinGasPremium = 100e3 const MaxSpendOnFeeDenom = 100 -func (a *GasAPI) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64, - tsk types.TipSetKey) (types.BigInt, error) { - ts := a.Chain.GetHeaviestTipSet() +func (a *GasAPI) GasEstimateFeeCap( + ctx context.Context, + msg *types.Message, + maxqueueblks int64, + tsk types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateFeeCap(a.Chain, msg, maxqueueblks) +} +func (m *GasModule) GasEstimateFeeCap( + ctx context.Context, + msg *types.Message, + maxqueueblks int64, + tsk types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateFeeCap(m.Chain, msg, maxqueueblks) +} +func gasEstimateFeeCap(cstore *store.ChainStore, msg *types.Message, maxqueueblks int64) (types.BigInt, error) { + ts := cstore.GetHeaviestTipSet() parentBaseFee := ts.Blocks()[0].ParentBaseFee increaseFactor := math.Pow(1.+1./float64(build.BaseFeeMaxChangeDenom), float64(maxqueueblks)) @@ -82,9 +116,25 @@ func medianGasPremium(prices []gasMeta, blocks int) abi.TokenAmount { return premium } -func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, - sender address.Address, gaslimit int64, _ types.TipSetKey) (types.BigInt, error) { - +func (a *GasAPI) GasEstimateGasPremium( + ctx context.Context, + nblocksincl uint64, + sender address.Address, + gaslimit int64, + _ types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateGasPremium(a.Chain, nblocksincl) +} +func (m *GasModule) GasEstimateGasPremium( + ctx context.Context, + nblocksincl uint64, + sender address.Address, + gaslimit int64, + _ types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateGasPremium(m.Chain, nblocksincl) +} +func gasEstimateGasPremium(cstore *store.ChainStore, nblocksincl uint64) (types.BigInt, error) { if nblocksincl == 0 { nblocksincl = 1 } @@ -92,20 +142,20 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, var prices []gasMeta var blocks int - ts := a.Chain.GetHeaviestTipSet() + ts := cstore.GetHeaviestTipSet() for i := uint64(0); i < nblocksincl*2; i++ { if ts.Height() == 0 { break // genesis } - pts, err := a.Chain.LoadTipSet(ts.Parents()) + pts, err := cstore.LoadTipSet(ts.Parents()) if err != nil { return types.BigInt{}, err } blocks += len(pts.Blocks()) - msgs, err := a.Chain.MessagesForTipset(pts) + msgs, err := cstore.MessagesForTipset(pts) if err != nil { return types.BigInt{}, xerrors.Errorf("loading messages: %w", err) } @@ -142,25 +192,47 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, } func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) { - + return gasEstimateGasLimit(ctx, a.Chain, a.Stmgr, a.Mpool, msgIn) +} +func (m *GasModule) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) { + return gasEstimateGasLimit(ctx, m.Chain, m.Stmgr, m.Mpool, msgIn) +} +func gasEstimateGasLimit( + ctx context.Context, + cstore *store.ChainStore, + smgr *stmgr.StateManager, + mpool *messagepool.MessagePool, + msgIn *types.Message, +) (int64, error) { msg := *msgIn msg.GasLimit = build.BlockGasLimit msg.GasFeeCap = types.NewInt(uint64(build.MinimumBaseFee) + 1) msg.GasPremium = types.NewInt(1) - currTs := a.Chain.GetHeaviestTipSet() - fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msgIn.From, currTs) + currTs := cstore.GetHeaviestTipSet() + fromA, err := smgr.ResolveToKeyAddress(ctx, msgIn.From, currTs) if err != nil { return -1, xerrors.Errorf("getting key address: %w", err) } - pending, ts := a.Mpool.PendingFor(fromA) + pending, ts := mpool.PendingFor(fromA) priorMsgs := make([]types.ChainMsg, 0, len(pending)) for _, m := range pending { priorMsgs = append(priorMsgs, m) } - res, err := a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) + // Try calling until we find a height with no migration. + var res *api.InvocResult + for { + res, err = smgr.CallWithGas(ctx, &msg, priorMsgs, ts) + if err != stmgr.ErrExpensiveFork { + break + } + ts, err = cstore.GetTipSetFromKey(ts.Parents()) + if err != nil { + return -1, xerrors.Errorf("getting parent tipset: %w", err) + } + } if err != nil { return -1, xerrors.Errorf("CallWithGas failed: %w", err) } @@ -169,7 +241,7 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, } // Special case for PaymentChannel collect, which is deleting actor - st, err := a.Stmgr.ParentState(ts) + st, err := smgr.ParentState(ts) if err != nil { _ = err // somewhat ignore it as it can happen and we just want to detect @@ -195,17 +267,17 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, return res.MsgRct.GasUsed + 76e3, nil } -func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, _ types.TipSetKey) (*types.Message, error) { +func (m *GasModule) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, _ types.TipSetKey) (*types.Message, error) { if msg.GasLimit == 0 { - gasLimit, err := a.GasEstimateGasLimit(ctx, msg, types.TipSetKey{}) + gasLimit, err := m.GasEstimateGasLimit(ctx, msg, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas used: %w", err) } - msg.GasLimit = int64(float64(gasLimit) * a.Mpool.GetConfig().GasLimitOverestimation) + msg.GasLimit = int64(float64(gasLimit) * m.Mpool.GetConfig().GasLimitOverestimation) } if msg.GasPremium == types.EmptyInt || types.BigCmp(msg.GasPremium, types.NewInt(0)) == 0 { - gasPremium, err := a.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) + gasPremium, err := m.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas price: %w", err) } @@ -213,7 +285,7 @@ func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, } if msg.GasFeeCap == types.EmptyInt || types.BigCmp(msg.GasFeeCap, types.NewInt(0)) == 0 { - feeCap, err := a.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK) + feeCap, err := m.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("estimating fee cap: %w", err) } diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 1f093606c..8ad209f3f 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -10,14 +10,32 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) +type MpoolModuleAPI interface { + MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) +} + +// MpoolModule provides a default implementation of MpoolModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type MpoolModule struct { + fx.In + + Mpool *messagepool.MessagePool +} + +var _ MpoolModuleAPI = (*MpoolModule)(nil) + type MpoolAPI struct { fx.In + MpoolModuleAPI + WalletAPI GasAPI @@ -106,8 +124,8 @@ func (a *MpoolAPI) MpoolClear(ctx context.Context, local bool) error { return nil } -func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { - return a.Mpool.Push(smsg) +func (m *MpoolModule) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return m.Mpool.Push(smsg) } func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { @@ -162,7 +180,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe // Sign and push the message return a.MessageSigner.SignMessage(ctx, msg, func(smsg *types.SignedMessage) error { - if _, err := a.Mpool.Push(smsg); err != nil { + if _, err := a.MpoolModuleAPI.MpoolPush(ctx, smsg); err != nil { return xerrors.Errorf("mpool push: failed to push message: %w", err) } return nil diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 715689edc..d840620eb 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -23,9 +23,8 @@ import ( type MsigAPI struct { fx.In - WalletAPI WalletAPI - StateAPI StateAPI - MpoolAPI MpoolAPI + StateAPI StateAPI + MpoolAPI MpoolAPI } func (a *MsigAPI) messageBuilder(ctx context.Context, from address.Address) (multisig.MessageBuilder, error) { @@ -95,7 +94,7 @@ func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -122,7 +121,7 @@ func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -134,15 +133,64 @@ func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } -func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return a.msigApproveOrCancel(ctx, api.MsigApprove, msig, txID, proposer, to, amt, src, method, params) +func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { + return a.msigApproveOrCancelSimple(ctx, api.MsigApprove, msig, txID, src) +} + +func (a *MsigAPI) MsigApproveTxnHash(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { + return a.msigApproveOrCancelTxnHash(ctx, api.MsigApprove, msig, txID, proposer, to, amt, src, method, params) } func (a *MsigAPI) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return a.msigApproveOrCancel(ctx, api.MsigCancel, msig, txID, src, to, amt, src, method, params) + return a.msigApproveOrCancelTxnHash(ctx, api.MsigCancel, msig, txID, src, to, amt, src, method, params) } -func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { +func (a *MsigAPI) MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) { + enc, actErr := serializeRemoveParams(toRemove, decrease) + if actErr != nil { + return cid.Undef, actErr + } + + return a.MsigPropose(ctx, msig, msig, types.NewInt(0), proposer, uint64(builtin0.MethodsMultisig.RemoveSigner), enc) +} + +func (a *MsigAPI) msigApproveOrCancelSimple(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { + if msig == address.Undef { + return cid.Undef, xerrors.Errorf("must provide multisig address") + } + + if src == address.Undef { + return cid.Undef, xerrors.Errorf("must provide source address") + } + + mb, err := a.messageBuilder(ctx, src) + if err != nil { + return cid.Undef, err + } + + var msg *types.Message + switch operation { + case api.MsigApprove: + msg, err = mb.Approve(msig, txID, nil) + case api.MsigCancel: + msg, err = mb.Cancel(msig, txID, nil) + default: + return cid.Undef, xerrors.Errorf("Invalid operation for msigApproveOrCancel") + } + if err != nil { + return cid.Undef, err + } + + smsg, err := a.MpoolAPI.MpoolPushMessage(ctx, msg, nil) + if err != nil { + return cid.Undef, err + } + + return smsg.Cid(), nil + +} + +func (a *MsigAPI) msigApproveOrCancelTxnHash(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { if msig == address.Undef { return cid.Undef, xerrors.Errorf("must provide multisig address") } @@ -216,3 +264,15 @@ func serializeSwapParams(old address.Address, new address.Address) ([]byte, erro return enc, nil } + +func serializeRemoveParams(rem address.Address, dec bool) ([]byte, error) { + enc, actErr := actors.SerializeParams(&multisig0.RemoveSignerParams{ + Signer: rem, + Decrease: dec, + }) + if actErr != nil { + return nil, actErr + } + + return enc, nil +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 3a4390d03..db91433aa 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -5,14 +5,6 @@ import ( "context" "strconv" - "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/lotus/chain/actors/policy" - - "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" - - "github.com/filecoin-project/go-state-types/dline" - "github.com/filecoin-project/go-state-types/network" - cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "go.uber.org/fx" @@ -22,14 +14,19 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/state" @@ -42,12 +39,36 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" ) +type StateModuleAPI interface { + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) +} + +// StateModule provides a default implementation of StateModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type StateModule struct { + fx.In + + StateManager *stmgr.StateManager + Chain *store.ChainStore +} + +var _ StateModuleAPI = (*StateModule)(nil) + type StateAPI struct { fx.In // TODO: the wallet here is only needed because we have the MinerCreateBlock // API attached to the state API. It probably should live somewhere better - Wallet *wallet.Wallet + Wallet api.WalletAPI + DefWallet wallet.Default + + StateModuleAPI ProofVerifier ffiwrapper.Verifier StateManager *stmgr.StateManager @@ -307,12 +328,22 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, ts }, nil } -func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { +func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (res *api.InvocResult, err error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.Call(ctx, msg, ts) + for { + res, err = a.StateManager.Call(ctx, msg, ts) + if err != stmgr.ErrExpensiveFork { + break + } + ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("getting parent tipset: %w", err) + } + } + return res, err } func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { @@ -339,27 +370,33 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. }, nil } -func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { +func stateForTs(ctx context.Context, ts *types.TipSet, cstore *store.ChainStore, smgr *stmgr.StateManager) (*state.StateTree, error) { if ts == nil { - ts = a.Chain.GetHeaviestTipSet() + ts = cstore.GetHeaviestTipSet() } - st, _, err := a.StateManager.TipSetState(ctx, ts) + st, _, err := smgr.TipSetState(ctx, ts) if err != nil { return nil, err } - buf := bufbstore.NewBufferedBstore(a.Chain.Blockstore()) + buf := bufbstore.NewBufferedBstore(cstore.Blockstore()) cst := cbor.NewCborStore(buf) return state.LoadStateTree(cst, st) } +func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { + return stateForTs(ctx, ts, a.Chain, a.StateManager) +} +func (m *StateModule) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { + return stateForTs(ctx, ts, m.Chain, m.StateManager) +} -func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - state, err := a.stateForTs(ctx, ts) + state, err := m.stateForTs(ctx, ts) if err != nil { return nil, xerrors.Errorf("computing tipset state failed: %w", err) } @@ -367,26 +404,22 @@ func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, tsk return state.GetActor(actor) } -func (a *StateAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - state, err := a.stateForTs(ctx, ts) - if err != nil { - return address.Undef, err - } - return state.LookupID(addr) + return m.StateManager.LookupID(ctx, addr, ts) } -func (a *StateAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.ResolveToKeyAddress(ctx, addr, ts) + return m.StateManager.ResolveToKeyAddress(ctx, addr, ts) } func (a *StateAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { @@ -443,22 +476,28 @@ func (a *StateAPI) MinerCreateBlock(ctx context.Context, bt *api.BlockTemplate) return &out, nil } -func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { - ts, recpt, found, err := a.StateManager.WaitForMessage(ctx, msg, confidence) +func (m *StateModule) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return stateWaitMsgLimited(ctx, m.StateManager, m.Chain, msg, confidence, stmgr.LookbackNoLimit) +} +func (a *StateAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error) { + return stateWaitMsgLimited(ctx, a.StateManager, a.Chain, msg, confidence, lookbackLimit) +} +func stateWaitMsgLimited(ctx context.Context, smgr *stmgr.StateManager, cstore *store.ChainStore, msg cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error) { + ts, recpt, found, err := smgr.WaitForMessage(ctx, msg, confidence, lookbackLimit) if err != nil { return nil, err } var returndec interface{} if recpt.ExitCode == 0 && len(recpt.Return) > 0 { - cmsg, err := a.Chain.GetCMessage(msg) + cmsg, err := cstore.GetCMessage(msg) if err != nil { return nil, xerrors.Errorf("failed to load message after successful receipt search: %w", err) } vmsg := cmsg.VMMessage() - t, err := stmgr.GetReturnType(ctx, a.StateManager, vmsg.To, vmsg.Method, ts) + t, err := stmgr.GetReturnType(ctx, smgr, vmsg.To, vmsg.Method, ts) if err != nil { return nil, xerrors.Errorf("failed to get return type: %w", err) } @@ -789,17 +828,17 @@ func (a *StateAPI) StateCompute(ctx context.Context, height abi.ChainEpoch, msgs }, nil } -func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - act, err := a.StateManager.LoadActor(ctx, addr, ts) + act, err := m.StateManager.LoadActor(ctx, addr, ts) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor: %w", err) } - msas, err := multisig.Load(a.Chain.Store(ctx), act) + msas, err := multisig.Load(m.Chain.Store(ctx), act) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err) } @@ -810,13 +849,51 @@ func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Add return types.BigSub(act.Balance, locked), nil } -func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { - startTs, err := a.Chain.GetTipSetFromKey(start) +func (a *StateAPI) MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MsigVesting, error) { + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + + act, err := a.StateManager.LoadActor(ctx, addr, ts) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig actor: %w", err) + } + + msas, err := multisig.Load(a.Chain.Store(ctx), act) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig actor state: %w", err) + } + + ib, err := msas.InitialBalance() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig initial balance: %w", err) + } + + se, err := msas.StartEpoch() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig start epoch: %w", err) + } + + ud, err := msas.UnlockDuration() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig unlock duration: %w", err) + } + + return api.MsigVesting{ + InitialBalance: ib, + StartEpoch: se, + UnlockDuration: ud, + }, nil +} + +func (m *StateModule) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + startTs, err := m.Chain.GetTipSetFromKey(start) if err != nil { return types.EmptyInt, xerrors.Errorf("loading start tipset %s: %w", start, err) } - endTs, err := a.Chain.GetTipSetFromKey(end) + endTs, err := m.Chain.GetTipSetFromKey(end) if err != nil { return types.EmptyInt, xerrors.Errorf("loading end tipset %s: %w", end, err) } @@ -827,12 +904,12 @@ func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, star return big.Zero(), nil } - act, err := a.StateManager.LoadActor(ctx, addr, endTs) + act, err := m.StateManager.LoadActor(ctx, addr, endTs) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor at end epoch: %w", err) } - msas, err := multisig.Load(a.Chain.Store(ctx), act) + msas, err := multisig.Load(m.Chain.Store(ctx), act) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err) } diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index 1bd3af415..05d4c9cb7 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -118,6 +118,12 @@ func (a *SyncAPI) SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error { return nil } +func (a *SyncAPI) SyncUnmarkAllBad(ctx context.Context) error { + log.Warnf("Dropping bad block cache") + a.Syncer.UnmarkAllBad() + return nil +} + func (a *SyncAPI) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) { reason, ok := a.Syncer.CheckBadBlockCache(bcid) if !ok { diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index b2ecdebbd..05a7a5768 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -3,13 +3,14 @@ package full import ( "context" - "github.com/filecoin-project/go-address" "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -19,24 +20,13 @@ import ( type WalletAPI struct { fx.In - StateManager *stmgr.StateManager - Wallet *wallet.Wallet -} - -func (a *WalletAPI) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { - return a.Wallet.GenerateKey(typ) -} - -func (a *WalletAPI) WalletHas(ctx context.Context, addr address.Address) (bool, error) { - return a.Wallet.HasKey(addr) -} - -func (a *WalletAPI) WalletList(ctx context.Context) ([]address.Address, error) { - return a.Wallet.ListAddrs() + StateManagerAPI stmgr.StateManagerAPI + Default wallet.Default + api.WalletAPI } func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) { - act, err := a.StateManager.LoadActorTsk(ctx, addr, types.EmptyTSK) + act, err := a.StateManagerAPI.LoadActorTsk(ctx, addr, types.EmptyTSK) if xerrors.Is(err, types.ErrActorNotFound) { return big.Zero(), nil } else if err != nil { @@ -46,17 +36,30 @@ func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (ty } func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { - keyAddr, err := a.StateManager.ResolveToKeyAddress(ctx, k, nil) + keyAddr, err := a.StateManagerAPI.ResolveToKeyAddress(ctx, k, nil) if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.Wallet.Sign(ctx, keyAddr, msg) + return a.WalletAPI.WalletSign(ctx, keyAddr, msg, api.MsgMeta{ + Type: api.MTUnknown, + }) } func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - mcid := msg.Cid() + keyAddr, err := a.StateManagerAPI.ResolveToKeyAddress(ctx, k, nil) + if err != nil { + return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) + } - sig, err := a.WalletSign(ctx, k, mcid.Bytes()) + mb, err := msg.ToStorageBlock() + if err != nil { + return nil, xerrors.Errorf("serializing message: %w", err) + } + + sig, err := a.WalletAPI.WalletSign(ctx, k, mb.Cid().Bytes(), api.MsgMeta{ + Type: api.MTChainMsg, + Extra: mb.RawData(), + }) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } @@ -72,23 +75,11 @@ func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []b } func (a *WalletAPI) WalletDefaultAddress(ctx context.Context) (address.Address, error) { - return a.Wallet.GetDefault() + return a.Default.GetDefault() } func (a *WalletAPI) WalletSetDefault(ctx context.Context, addr address.Address) error { - return a.Wallet.SetDefault(addr) -} - -func (a *WalletAPI) WalletExport(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { - return a.Wallet.Export(addr) -} - -func (a *WalletAPI) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { - return a.Wallet.Import(ki) -} - -func (a *WalletAPI) WalletDelete(ctx context.Context, addr address.Address) error { - return a.Wallet.DeleteKey(addr) + return a.Default.SetDefault(addr) } func (a *WalletAPI) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) { diff --git a/node/modules/chain.go b/node/modules/chain.go index 66f54a76a..f563b4cdd 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -17,6 +17,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" @@ -51,9 +52,9 @@ func ChainBitswap(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt r return exch } -func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName) (*messagepool.MessagePool, error) { +func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName, j journal.Journal) (*messagepool.MessagePool, error) { mpp := messagepool.NewProvider(sm, ps) - mp, err := messagepool.New(mpp, ds, nn) + mp, err := messagepool.New(mpp, ds, nn, j) if err != nil { return nil, xerrors.Errorf("constructing mpool: %w", err) } @@ -88,8 +89,8 @@ func ChainBlockService(bs dtypes.ChainBlockstore, rem dtypes.ChainBitswap) dtype return blockservice.New(bs, rem) } -func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder) *store.ChainStore { - chain := store.NewChainStore(bs, ds, syscalls) +func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore { + chain := store.NewChainStore(bs, ds, syscalls, j) if err := chain.Load(); err != nil { log.Warnf("loading chain state from disk: %s", err) diff --git a/node/modules/client.go b/node/modules/client.go index 6972ca36e..d012e4539 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -113,7 +113,7 @@ func NewClientDealFunds(ds dtypes.MetadataDS) (ClientDealFunds, error) { return funds.NewDealFunds(ds, datastore.NewKey("/marketfunds/client")) } -func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, dealFunds ClientDealFunds) (storagemarket.StorageClient, error) { +func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, dealFunds ClientDealFunds, j journal.Journal) (storagemarket.StorageClient, error) { net := smnet.NewFromLibp2pHost(h) c, err := storageimpl.NewClient(net, ibs, mds, dataTransfer, discovery, deals, scn, dealFunds, storageimpl.DealPollingInterval(time.Second)) if err != nil { @@ -124,8 +124,8 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md OnStart: func(ctx context.Context) error { c.SubscribeToEvents(marketevents.StorageClientLogger) - evtType := journal.J.RegisterEventType("markets/storage/client", "state_change") - c.SubscribeToEvents(markets.StorageClientJournaler(evtType)) + evtType := j.RegisterEventType("markets/storage/client", "state_change") + c.SubscribeToEvents(markets.StorageClientJournaler(j, evtType)) return c.Start(ctx) }, @@ -137,7 +137,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md } // RetrievalClient creates a new retrieval client attached to the client blockstore -func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver, ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI) (retrievalmarket.RetrievalClient, error) { +func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver, ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, j journal.Journal) (retrievalmarket.RetrievalClient, error) { adapter := retrievaladapter.NewRetrievalClientNode(payAPI, chainAPI, stateAPI) network := rmnet.NewFromLibp2pHost(h) sc := storedcounter.New(ds, datastore.NewKey("/retr")) @@ -150,8 +150,8 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, OnStart: func(ctx context.Context) error { client.SubscribeToEvents(marketevents.RetrievalClientLogger) - evtType := journal.J.RegisterEventType("markets/retrieval/client", "state_change") - client.SubscribeToEvents(markets.RetrievalClientJournaler(evtType)) + evtType := j.RegisterEventType("markets/retrieval/client", "state_change") + client.SubscribeToEvents(markets.RetrievalClientJournaler(j, evtType)) return client.Start(ctx) }, diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 7dca3608f..9724eb3b4 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -187,6 +187,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { build.MessagesTopic(in.Nn): 1, } + var drandTopics []string for _, d := range in.Dr { topic, err := getDrandTopic(d.Config.ChainInfoJSON) if err != nil { @@ -194,6 +195,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { } topicParams[topic] = drandTopicParams pgTopicWeights[topic] = 5 + drandTopics = append(drandTopics, topic) } options := []pubsub.Option{ @@ -309,6 +311,17 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { options = append(options, pubsub.WithPeerGater(pgParams)) + allowTopics := []string{ + build.BlocksTopic(in.Nn), + build.MessagesTopic(in.Nn), + } + allowTopics = append(allowTopics, drandTopics...) + options = append(options, + pubsub.WithSubscriptionFilter( + pubsub.WrapLimitSubscriptionFilter( + pubsub.NewAllowlistSubscriptionFilter(allowTopics...), + 100))) + // tracer if in.Cfg.RemoteTracer != "" { a, err := ma.NewMultiaddr(in.Cfg.RemoteTracer) @@ -359,14 +372,9 @@ type tracerWrapper struct { topics map[string]struct{} } -func (trw *tracerWrapper) traceMessage(topics []string) bool { - for _, topic := range topics { - _, ok := trw.topics[topic] - if ok { - return true - } - } - return false +func (trw *tracerWrapper) traceMessage(topic string) bool { + _, ok := trw.topics[topic] + return ok } func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) { @@ -379,12 +387,12 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) { switch evt.GetType() { case pubsub_pb.TraceEvent_PUBLISH_MESSAGE: stats.Record(context.TODO(), metrics.PubsubPublishMessage.M(1)) - if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().Topics) { + if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().GetTopic()) { trw.tr.Trace(evt) } case pubsub_pb.TraceEvent_DELIVER_MESSAGE: stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1)) - if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().Topics) { + if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().GetTopic()) { trw.tr.Trace(evt) } case pubsub_pb.TraceEvent_REJECT_MESSAGE: diff --git a/node/modules/mpoolnonceapi.go b/node/modules/mpoolnonceapi.go new file mode 100644 index 000000000..294f4d954 --- /dev/null +++ b/node/modules/mpoolnonceapi.go @@ -0,0 +1,33 @@ +package modules + +import ( + "context" + + "go.uber.org/fx" + + "github.com/filecoin-project/lotus/node/impl/full" + + "github.com/filecoin-project/lotus/chain/messagesigner" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/go-address" +) + +// MpoolNonceAPI substitutes the mpool nonce with an implementation that +// doesn't rely on the mpool - it just gets the nonce from actor state +type MpoolNonceAPI struct { + fx.In + + StateAPI full.StateAPI +} + +// GetNonce gets the nonce from actor state +func (a *MpoolNonceAPI) GetNonce(addr address.Address) (uint64, error) { + act, err := a.StateAPI.StateGetActor(context.Background(), addr, types.EmptyTSK) + if err != nil { + return 0, err + } + return act.Nonce, nil +} + +var _ messagesigner.MpoolNonceAPI = (*MpoolNonceAPI)(nil) diff --git a/node/modules/rpcstatemanager.go b/node/modules/rpcstatemanager.go new file mode 100644 index 000000000..0ed054d45 --- /dev/null +++ b/node/modules/rpcstatemanager.go @@ -0,0 +1,33 @@ +package modules + +import ( + "context" + + "github.com/filecoin-project/lotus/api" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" +) + +type RPCStateManager struct { + gapi api.GatewayAPI +} + +func NewRPCStateManager(api api.GatewayAPI) *RPCStateManager { + return &RPCStateManager{gapi: api} +} + +func (s *RPCStateManager) LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) { + return s.gapi.StateGetActor(ctx, addr, tsk) +} + +func (s *RPCStateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { + return s.gapi.StateLookupID(ctx, addr, ts.Key()) +} + +func (s *RPCStateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { + return s.gapi.StateAccountKey(ctx, addr, ts.Key()) +} + +var _ stmgr.StateManagerAPI = (*RPCStateManager)(nil) diff --git a/node/modules/services.go b/node/modules/services.go index 4ee0abacc..e0a7c2eda 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -42,11 +42,13 @@ func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello. return xerrors.Errorf("failed to subscribe to event bus: %w", err) } + ctx := helpers.LifecycleCtx(mctx, lc) + go func() { for evt := range sub.Out() { pic := evt.(event.EvtPeerIdentificationCompleted) go func() { - if err := svc.SayHello(helpers.LifecycleCtx(mctx, lc), pic.Peer); err != nil { + if err := svc.SayHello(ctx, pic.Peer); err != nil { protos, _ := h.Peerstore().GetProtocols(pic.Peer) agent, _ := h.Peerstore().Get(pic.Peer, "AgentVersion") if protosContains(protos, hello.ProtocolID) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 95bfb6c11..9011c4821 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -161,6 +161,7 @@ type StorageMinerParams struct { SectorIDCounter sealing.SectorIDCounter Verifier ffiwrapper.Verifier GetSealingConfigFn dtypes.GetSealingConfigFunc + Journal journal.Journal } func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*storage.Miner, error) { @@ -175,6 +176,7 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st sc = params.SectorIDCounter verif = params.Verifier gsd = params.GetSealingConfigFn + j = params.Journal ) maddr, err := minerAddrFromDS(ds) @@ -194,12 +196,12 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st return nil, err } - fps, err := storage.NewWindowedPoStScheduler(api, fc, sealer, sealer, maddr, worker) + fps, err := storage.NewWindowedPoStScheduler(api, fc, sealer, sealer, j, maddr, worker) if err != nil { return nil, err } - sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd, fc) + sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd, fc, j) if err != nil { return nil, err } @@ -216,15 +218,15 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st } } -func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider) { +func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider, j journal.Journal) { m.OnReady(marketevents.ReadyLogger("retrieval provider")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { m.SubscribeToEvents(marketevents.RetrievalProviderLogger) - evtType := journal.J.RegisterEventType("markets/retrieval/provider", "state_change") - m.SubscribeToEvents(markets.RetrievalProviderJournaler(evtType)) + evtType := j.RegisterEventType("markets/retrieval/provider", "state_change") + m.SubscribeToEvents(markets.RetrievalProviderJournaler(j, evtType)) return m.Start(ctx) }, @@ -234,15 +236,15 @@ func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.Retrieva }) } -func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider) { +func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider, j journal.Journal) { ctx := helpers.LifecycleCtx(mctx, lc) h.OnReady(marketevents.ReadyLogger("storage provider")) lc.Append(fx.Hook{ OnStart: func(context.Context) error { h.SubscribeToEvents(marketevents.StorageProviderLogger) - evtType := journal.J.RegisterEventType("markets/storage/provider", "state_change") - h.SubscribeToEvents(markets.StorageProviderJournaler(evtType)) + evtType := j.RegisterEventType("markets/storage/provider", "state_change") + h.SubscribeToEvents(markets.StorageProviderJournaler(j, evtType)) return h.Start(ctx) }, @@ -354,13 +356,13 @@ func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Stagi return gs } -func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter) (*miner.Miner, error) { +func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter, j journal.Journal) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - m := miner.NewMiner(api, epp, minerAddr, sf) + m := miner.NewMiner(api, epp, minerAddr, sf, j) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index 40df68bf3..fa9e0cff7 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -23,17 +23,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" ) var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { +func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") - b, err := genesis2.MakeGenesisBlock(context.TODO(), bs, syscalls, template) + b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template) if err != nil { return nil, xerrors.Errorf("make genesis block failed: %w", err) } @@ -50,8 +51,8 @@ func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.Cha } } -func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { +func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") genesisTemplate, err := homedir.Expand(genesisTemplate) @@ -73,7 +74,7 @@ func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore template.Timestamp = uint64(build.Clock.Now().Unix()) } - b, err := genesis2.MakeGenesisBlock(context.TODO(), bs, syscalls, template) + b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template) if err != nil { return nil, xerrors.Errorf("make genesis block: %w", err) } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index c4571c478..c1b6b5233 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -44,6 +44,7 @@ const ( FullNode RepoType = iota StorageMiner Worker + Wallet ) func defConfForType(t RepoType) interface{} { @@ -54,6 +55,8 @@ func defConfForType(t RepoType) interface{} { return config.DefaultStorageMiner() case Worker: return &struct{}{} + case Wallet: + return &struct{}{} default: panic(fmt.Sprintf("unknown RepoType(%d)", int(t))) } @@ -93,6 +96,12 @@ func (fsr *FsRepo) Exists() (bool, error) { notexist := os.IsNotExist(err) if notexist { err = nil + + _, err = os.Stat(filepath.Join(fsr.path, fsKeystore)) + notexist = os.IsNotExist(err) + if notexist { + err = nil + } } return !notexist, err } diff --git a/node/test/builder.go b/node/test/builder.go index a3455f376..4aa8a55ea 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -7,10 +7,13 @@ import ( "io/ioutil" "net" "net/http/httptest" + "strings" "sync" "testing" "time" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" @@ -101,8 +104,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr var minerapi api.StorageMiner mineBlock := make(chan miner2.MineReq) - // TODO: use stop - _, err = node.New(ctx, + stop, err := node.New(ctx, node.StorageMiner(&minerapi), node.Online(), node.Repo(r), @@ -119,6 +121,8 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr t.Fatalf("failed to construct node: %v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) + /*// Bootstrap with full node remoteAddrs, err := tnd.NetAddrsListen(ctx) require.NoError(t, err) @@ -137,11 +141,29 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - ctx := context.Background() +func Builder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, false) +} + +func MockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, false) +} + +func RPCBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, true) +} + +func RPCMockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, true) +} + +func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, rpc bool) ([]test.TestNode, []test.TestStorageNode) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mn := mocknet.New(ctx) - fulls := make([]test.TestNode, nFull) + fulls := make([]test.TestNode, len(fullOpts)) storers := make([]test.TestStorageNode, len(storage)) pk, _, err := crypto.GenerateEd25519Key(rand.Reader) @@ -206,7 +228,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. // END PRESEAL SECTION - for i := 0; i < nFull; i++ { + for i := 0; i < len(fullOpts); i++ { var genesis node.Option if i == 0 { genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ)) @@ -214,22 +236,26 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } - var err error - // TODO: Don't ignore stop - _, err = node.New(ctx, - node.FullAPI(&fulls[i].FullNode), + stop, err := node.New(ctx, + node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), node.Test(), genesis, - node.Options(opts...), + + fullOpts[i].Opts(fulls), ) if err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = stop(context.Background()) }) + + if rpc { + fulls[i] = fullRpc(t, fulls[i]) + } } for i, def := range storage { @@ -261,6 +287,9 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. psd := presealDirs[i] */ + if rpc { + storers[i] = storerRpc(t, storers[i]) + } } if err := mn.LinkAll(); err != nil { @@ -286,11 +315,13 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. return fulls, storers } -func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - ctx := context.Background() +func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, rpc bool) ([]test.TestNode, []test.TestStorageNode) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mn := mocknet.New(ctx) - fulls := make([]test.TestNode, nFull) + fulls := make([]test.TestNode, len(fullOpts)) storers := make([]test.TestStorageNode, len(storage)) var genbuf bytes.Buffer @@ -354,7 +385,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options // END PRESEAL SECTION - for i := 0; i < nFull; i++ { + for i := 0; i < len(fullOpts); i++ { var genesis node.Option if i == 0 { genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ)) @@ -362,10 +393,8 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } - var err error - // TODO: Don't ignore stop - _, err = node.New(ctx, - node.FullAPI(&fulls[i].FullNode), + stop, err := node.New(ctx, + node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), @@ -374,11 +403,18 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), genesis, - node.Options(options...), + + fullOpts[i].Opts(fulls), ) if err != nil { t.Fatalf("%+v", err) } + + t.Cleanup(func() { _ = stop(context.Background()) }) + + if rpc { + fulls[i] = fullRpc(t, fulls[i]) + } } for i, def := range storage { @@ -413,6 +449,10 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), node.Unset(new(*sectorstorage.Manager)), )) + + if rpc { + storers[i] = storerRpc(t, storers[i]) + } } if err := mn.LinkAll(); err != nil { @@ -437,69 +477,66 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options return fulls, storers } -func RPCBuilder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return rpcWithBuilder(t, Builder, nFull, storage, opts...) +func fullRpc(t *testing.T, nd test.TestNode) test.TestNode { + ma, listenAddr, err := CreateRPCServer(nd) + require.NoError(t, err) + + var full test.TestNode + full.FullNode, _, err = client.NewFullNodeRPC(context.Background(), listenAddr, nil) + require.NoError(t, err) + + full.ListenAddr = ma + return full } -func RPCMockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return rpcWithBuilder(t, MockSbBuilder, nFull, storage, opts...) +func storerRpc(t *testing.T, nd test.TestStorageNode) test.TestStorageNode { + ma, listenAddr, err := CreateRPCServer(nd) + require.NoError(t, err) + + var storer test.TestStorageNode + storer.StorageMiner, _, err = client.NewStorageMinerRPC(context.Background(), listenAddr, nil) + require.NoError(t, err) + + storer.ListenAddr = ma + storer.MineOne = nd.MineOne + return storer } -func rpcWithBuilder(t *testing.T, b test.APIBuilder, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - fullApis, storaApis := b(t, nFull, storage, opts...) - fulls := make([]test.TestNode, nFull) - storers := make([]test.TestStorageNode, len(storage)) +func CreateRPCServer(handler interface{}) (multiaddr.Multiaddr, string, error) { + rpcServer := jsonrpc.NewServer() + rpcServer.Register("Filecoin", handler) + testServ := httptest.NewServer(rpcServer) // todo: close - for i, a := range fullApis { - rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", a) - testServ := httptest.NewServer(rpcServer) // todo: close - - addr := testServ.Listener.Addr() - listenAddr := "ws://" + addr.String() - var err error - fulls[i].FullNode, _, err = client.NewFullNodeRPC(context.Background(), listenAddr, nil) - if err != nil { - t.Fatal(err) - } - ma, err := parseWSSMultiAddr(addr) - if err != nil { - t.Fatal(err) - } - fulls[i].ListenAddr = ma + addr := testServ.Listener.Addr() + listenAddr := "ws://" + addr.String() + ma, err := parseWSMultiAddr(addr) + if err != nil { + return nil, "", err } - - for i, a := range storaApis { - rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", a) - testServ := httptest.NewServer(rpcServer) // todo: close - - addr := testServ.Listener.Addr() - listenAddr := "ws://" + addr.String() - var err error - storers[i].StorageMiner, _, err = client.NewStorageMinerRPC(context.Background(), listenAddr, nil) - if err != nil { - t.Fatal(err) - } - ma, err := parseWSSMultiAddr(addr) - if err != nil { - t.Fatal(err) - } - storers[i].ListenAddr = ma - storers[i].MineOne = a.MineOne - } - - return fulls, storers + return ma, listenAddr, err } -func parseWSSMultiAddr(addr net.Addr) (multiaddr.Multiaddr, error) { +func parseWSMultiAddr(addr net.Addr) (multiaddr.Multiaddr, error) { host, port, err := net.SplitHostPort(addr.String()) if err != nil { return nil, err } - ma, err := multiaddr.NewMultiaddr("/ip4/" + host + "/" + addr.Network() + "/" + port + "/wss") + ma, err := multiaddr.NewMultiaddr("/ip4/" + host + "/" + addr.Network() + "/" + port + "/ws") if err != nil { return nil, err } return ma, nil } + +func WSMultiAddrToString(addr multiaddr.Multiaddr) (string, error) { + parts := strings.Split(addr.String(), "/") + if len(parts) != 6 || parts[0] != "" { + return "", xerrors.Errorf("Malformed ws multiaddr %s", addr) + } + + host := parts[2] + port := parts[4] + proto := parts[5] + + return proto + "://" + host + ":" + port + "/rpc/v0", nil +} diff --git a/storage/miner.go b/storage/miner.go index c1b50fe89..74a048c8e 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -57,6 +57,8 @@ type Miner struct { sealing *sealing.Sealing sealingEvtType journal.EventType + + journal journal.Journal } // SealingStateEvt is a journal event that records a sector state transition. @@ -110,7 +112,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingConfigFunc, feeCfg config.MinerFeeConfig) (*Miner, error) { +func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingConfigFunc, feeCfg config.MinerFeeConfig, journal journal.Journal) (*Miner, error) { m := &Miner{ api: api, feeCfg: feeCfg, @@ -123,7 +125,8 @@ func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, d maddr: maddr, worker: worker, getSealConfig: gsd, - sealingEvtType: journal.J.RegisterEventType("storage", "sealing_states"), + journal: journal, + sealingEvtType: journal.RegisterEventType("storage", "sealing_states"), } return m, nil @@ -156,7 +159,7 @@ func (m *Miner) Run(ctx context.Context) error { } func (m *Miner) handleSealingNotifications(before, after sealing.SectorInfo) { - journal.J.RecordEvent(m.sealingEvtType, func() interface{} { + m.journal.RecordEvent(m.sealingEvtType, func() interface{} { return SealingStateEvt{ SectorNumber: before.SectorNumber, SectorType: before.SectorType, diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 6aa3d5188..d4ed4d64c 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -26,11 +26,10 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" ) func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dline.Info) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { c := evtCommon{Error: err} if ts != nil { c.Deadline = deadline @@ -54,7 +53,7 @@ func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dl // recordProofsEvent records a successful proofs_processed event in the // journal, even if it was a noop (no partitions). func (s *WindowPoStScheduler) recordProofsEvent(partitions []miner.PoStPartition, mcid cid.Cid) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { return &WdPoStProofsProcessedEvt{ evtCommon: s.getEvtCommon(nil), Partitions: partitions, @@ -74,7 +73,7 @@ func (s *WindowPoStScheduler) startGeneratePoST( go func() { defer abort() - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), State: SchedulerStateStarted, @@ -125,7 +124,7 @@ func (s *WindowPoStScheduler) startSubmitPoST( err := s.runSubmitPoST(ctx, ts, deadline, posts) if err == nil { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), State: SchedulerStateSucceeded, @@ -439,7 +438,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Errorf("checking sector recoveries: %v", err) } - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStRecoveries], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStRecoveries], func() interface{} { j := WdPoStRecoveriesProcessedEvt{ evtCommon: s.getEvtCommon(err), Declarations: recoveries, @@ -458,7 +457,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Errorf("checking sector faults: %v", err) } - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStFaults], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStFaults], func() interface{} { return WdPoStFaultsProcessedEvt{ evtCommon: s.getEvtCommon(err), Declarations: faults, diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 09b9aee5c..dd7ac4c24 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" ) type mockStorageMinerAPI struct { @@ -168,6 +169,7 @@ func TestWDPostDoPost(t *testing.T) { proofType: proofType, actor: postAct, worker: workerAct, + journal: journal.NilJournal(), } di := &dline.Info{ diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index ee380fbaf..99bea2e66 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -35,12 +35,13 @@ type WindowPoStScheduler struct { worker address.Address evtTypes [4]journal.EventType + journal journal.Journal // failed abi.ChainEpoch // eps // failLk sync.Mutex } -func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb storage.Prover, ft sectorstorage.FaultTracker, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) { +func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb storage.Prover, ft sectorstorage.FaultTracker, j journal.Journal, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) { mi, err := api.StateMinerInfo(context.TODO(), actor, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("getting sector size: %w", err) @@ -62,11 +63,12 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb actor: actor, worker: worker, evtTypes: [...]journal.EventType{ - evtTypeWdPoStScheduler: journal.J.RegisterEventType("wdpost", "scheduler"), - evtTypeWdPoStProofs: journal.J.RegisterEventType("wdpost", "proofs_processed"), - evtTypeWdPoStRecoveries: journal.J.RegisterEventType("wdpost", "recoveries_processed"), - evtTypeWdPoStFaults: journal.J.RegisterEventType("wdpost", "faults_processed"), + evtTypeWdPoStScheduler: j.RegisterEventType("wdpost", "scheduler"), + evtTypeWdPoStProofs: j.RegisterEventType("wdpost", "proofs_processed"), + evtTypeWdPoStRecoveries: j.RegisterEventType("wdpost", "recoveries_processed"), + evtTypeWdPoStFaults: j.RegisterEventType("wdpost", "faults_processed"), }, + journal: j, }, nil } @@ -166,7 +168,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, revert, apply *types.T // onAbort is called when generating proofs or submitting proofs is aborted func (s *WindowPoStScheduler) onAbort(ts *types.TipSet, deadline *dline.Info) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { c := evtCommon{} if ts != nil { c.Deadline = deadline diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index aee61b2aa..795203c40 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -269,25 +270,27 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) pl.AddPoint(p) - miners, err := api.StateListMiners(ctx, tipset.Key()) + powerActor, err := api.StateGetActor(ctx, power.Address, tipset.Key()) if err != nil { return err } - for _, addr := range miners { - mp, err := api.StateMinerPower(ctx, addr, tipset.Key()) - if err != nil { - return err - } - - if !mp.MinerPower.QualityAdjPower.IsZero() { - p = NewPoint("chain.miner_power", mp.MinerPower.QualityAdjPower.Int64()) - p.AddTag("miner", addr.String()) - pl.AddPoint(p) - } + powerActorState, err := power.Load(&ApiIpldStore{ctx, api}, powerActor) + if err != nil { + return err } - return nil + return powerActorState.ForEachClaim(func(addr address.Address, claim power.Claim) error { + if claim.QualityAdjPower.Int64() == 0 { + return nil + } + + p = NewPoint("chain.miner_power", claim.QualityAdjPower.Int64()) + p.AddTag("miner", addr.String()) + pl.AddPoint(p) + + return nil + }) } type msgTag struct {