diff --git a/.codecov.yml b/.codecov.yml index 1551f2276..a53081be7 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,5 +1,9 @@ comment: off ignore: - - "cbor_gen.go" + - "**/cbor_gen.go" + - "api/test/**/*" + - "api/test/*" + - "gen/**/*" + - "gen/*" github_checks: annotations: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d8d15b8..701f221b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,46 @@ # Lotus changelog +# 0.10.2 / 2020-10-14 + +This is an optional release of Lotus that updates markets to 0.9.1, which fixes an issue affecting deals that were mid-transfer when the node was upgraded to 0.9.0. This release also includes some tweaks to default gas values and minor performance improvements. + +## Changes + +- Use updated stored ask API (https://github.com/filecoin-project/lotus/pull/4384) +- tvx: trace puts to blockstore for inclusion in CAR. (https://github.com/filecoin-project/lotus/pull/4278) +- Add propose remove (https://github.com/filecoin-project/lotus/pull/4311) +- Update to 0.9.1 bugfix release (https://github.com/filecoin-project/lotus/pull/4402) +- Update drand endpoints (https://github.com/filecoin-project/lotus/pull/4125) +- fix: return true when deadlines changed (https://github.com/filecoin-project/lotus/pull/4403) +- sync wait --watch (https://github.com/filecoin-project/lotus/pull/4396) +- reduce garbage in blockstore (https://github.com/filecoin-project/lotus/pull/4406) +- give the TimeCacheBS tests a bit more time (https://github.com/filecoin-project/lotus/pull/4407) +- Improve gas defaults (https://github.com/filecoin-project/lotus/pull/4408) +- Change default gas premium to for 10 block inclusion (https://github.com/filecoin-project/lotus/pull/4222) + +# 0.10.1 / 2020-10-14 + +This is an optional release of Lotus that updates markets to 0.9.0, which adds the ability to restart data transfers. This release also introduces Ledger support, and various UX improvements. + +## Changes + +- Test the tape upgrade (https://github.com/filecoin-project/lotus/pull/4328) +- Adding in Ledger support (https://github.com/filecoin-project/lotus/pull/4290) +- Improve the UX for lotus-miner sealing workers (https://github.com/filecoin-project/lotus/pull/4329) +- Add a CLI tool for miner's to repay debt (https://github.com/filecoin-project/lotus/pull/4319) +- Rename params_testnet to params_mainnet (https://github.com/filecoin-project/lotus/pull/4336) +- Use seal-duration in calculating the earliest StartEpoch (https://github.com/filecoin-project/lotus/pull/4337) +- Reject deals that are > 7 days in the future in the BasicDealFilter (https://github.com/filecoin-project/lotus/pull/4173) +- Add an API endpoint to calculate the exact circulating supply (https://github.com/filecoin-project/lotus/pull/4148) +- lotus-pcr: ignore all other market messages (https://github.com/filecoin-project/lotus/pull/4341) +- Add message CID to InvocResult (https://github.com/filecoin-project/lotus/pull/4382) +- types: Add CID fields to messages in json marshalers (https://github.com/filecoin-project/lotus/pull/4338) +- fix(sync state): set state height to actual tipset height (https://github.com/filecoin-project/lotus/pull/4347) +- Fix off by one tipset in searchBackForMsg (https://github.com/filecoin-project/lotus/pull/4367) +- fix a panic on startup when we fail to load the tipset (https://github.com/filecoin-project/lotus/pull/4376) +- Avoid having the same message CID show up in execution traces (https://github.com/filecoin-project/lotus/pull/4350) +- feat(markets): update markets 0.9.0 and add data transfer restart (https://github.com/filecoin-project/lotus/pull/4363) + # 0.10.0 / 2020-10-12 This is a consensus-breaking hotfix that addresses an issue in specs-actors v2.0.3 that made it impossible to pledge new 32GiB sectors. The change in Lotus is to update to actors v2.1.0, behind the new network version 5. diff --git a/SECURITY.md b/SECURITY.md index ecb600deb..592206bc5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,9 +2,7 @@ ## Reporting a Vulnerability -For *critical* bugs, please send an email to security@filecoin.org. - -The bug reporting process differs between bugs that are critical and may crash the network, and others that are unlikely to cause problems if malicious parties know about it. For non-critical bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). +For *critical* bugs, please consult our Security Policy and Responsible Disclosure Program information at https://github.com/filecoin-project/community/blob/master/SECURITY.md Please try to provide a clear description of any bugs reported, along with how to reproduce the bug if possible. More detailed bug reports (especially those with a PoC included) will help us move forward much faster. Additionally, please avoid reporting bugs that already have open issues. Take a moment to search the issue list of the related GitHub repositories before writing up a new report. @@ -20,10 +18,6 @@ Here are some examples of bugs we would consider 'critical': This is not an exhaustive list, but should provide some idea of what we consider 'critical'. -## Supported Versions +## Reporting a non security bug -* TODO: This should be defined and set up by Mainnet launch. - -| Version | Supported | -| ------- | ------------------ | -| Testnet | :white_check_mark: | +For non-critical bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). diff --git a/api/api_full.go b/api/api_full.go index 957529eaa..c9662c73f 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" @@ -298,6 +299,8 @@ type FullNode interface { // ClientListTransfers returns the status of all ongoing transfers of data ClientListDataTransfers(ctx context.Context) ([]DataTransferChannel, error) ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) + // ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer + ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error // ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel // which are stuck due to insufficient funds ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error @@ -312,19 +315,20 @@ type FullNode interface { // MethodGroup: State // The State methods are used to query, inspect, and interact with chain state. - // All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. + // Most methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. // A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. // StateCall runs the given message and returns its result without any persisted changes. StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) - // StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. + // StateReplay replays a given message, assuming it was included in a block in the specified tipset. + // If no tipset key is provided, the appropriate tipset is looked up. StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) // StateGetActor returns the indicated actor's nonce and balance. StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // StateReadState returns the indicated actor's state. StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) // StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height. - StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) + StateListMessages(ctx context.Context, match *MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) // StateNetworkName returns the name of the network the node is synced to StateNetworkName(context.Context) (dtypes.NetworkName, error) @@ -369,8 +373,6 @@ type FullNode interface { StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) // StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) - // StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip - StateMsgGasCost(context.Context, cid.Cid, types.TipSetKey) (*MsgGasCost, error) // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) @@ -736,8 +738,10 @@ type RetrievalOrder struct { } type InvocResult struct { + MsgCid cid.Cid Msg *types.Message MsgRct *types.MessageReceipt + GasCost MsgGasCost ExecutionTrace types.ExecutionTrace Error string Duration time.Duration @@ -916,3 +920,8 @@ type MsigVesting struct { StartEpoch abi.ChainEpoch UnlockDuration abi.ChainEpoch } + +type MessageMatch struct { + To address.Address + From address.Address +} diff --git a/api/api_gateway.go b/api/api_gateway.go index 95d28887d..c99ff8408 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -10,9 +10,11 @@ import ( ) type GatewayAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b664f594f..180570742 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" @@ -166,6 +167,7 @@ type FullNodeStruct struct { ClientDealSize func(ctx context.Context, root cid.Cid) (api.DataSize, error) `perm:"read"` ClientListDataTransfers func(ctx context.Context) ([]api.DataTransferChannel, error) `perm:"write"` ClientDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error) `perm:"write"` + ClientRestartDataTransfer func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error `perm:"write"` ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error `perm:"write"` StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` @@ -191,7 +193,6 @@ type FullNodeStruct struct { StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` - StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` StateWaitMsgLimited func(context.Context, cid.Cid, uint64, abi.ChainEpoch) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` @@ -206,7 +207,7 @@ type FullNodeStruct struct { StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"` StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` - StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` + StateListMessages func(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` StateVerifierStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` @@ -382,9 +383,11 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? + ChainHasObj func(context.Context, cid.Cid) (bool, error) ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainHead func(ctx context.Context) (*types.TipSet, error) + ChainReadObj func(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) @@ -569,6 +572,10 @@ func (c *FullNodeStruct) ClientDataTransferUpdates(ctx context.Context) (<-chan return c.Internal.ClientDataTransferUpdates(ctx) } +func (c *FullNodeStruct) ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error { + return c.Internal.ClientRestartDataTransfer(ctx, transferID, otherPeer, isInitiator) +} + func (c *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error { return c.Internal.ClientRetrieveTryRestartInsufficientFunds(ctx, paymentChannel) } @@ -905,10 +912,6 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres return c.Internal.StateReadState(ctx, addr, tsk) } -func (c *FullNodeStruct) StateMsgGasCost(ctx context.Context, msgc cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { - return c.Internal.StateMsgGasCost(ctx, msgc, tsk) -} - func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) { return c.Internal.StateWaitMsg(ctx, msgc, confidence) } @@ -961,7 +964,7 @@ func (c *FullNodeStruct) StateGetReceipt(ctx context.Context, msg cid.Cid, tsk t return c.Internal.StateGetReceipt(ctx, msg, tsk) } -func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) { +func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) { return c.Internal.StateListMessages(ctx, match, tsk, toht) } @@ -1491,6 +1494,10 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } +func (g GatewayStruct) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { + return g.Internal.ChainHasObj(ctx, c) +} + func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { return g.Internal.ChainHead(ctx) } @@ -1503,6 +1510,10 @@ func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEp return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) } +func (g GatewayStruct) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + return g.Internal.ChainReadObj(ctx, c) +} + func (g GatewayStruct) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { return g.Internal.GasEstimateMessageGas(ctx, msg, spec, tsk) } diff --git a/api/test/test.go b/api/test/test.go index 947f2bef4..4074ce4a6 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -3,8 +3,10 @@ package test import ( "context" "testing" + "time" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" "github.com/multiformats/go-multiaddr" @@ -12,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -75,6 +78,7 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testConnectTwo", ts.testConnectTwo) t.Run("testMining", ts.testMining) t.Run("testMiningReal", ts.testMiningReal) + t.Run("testSearchMsg", ts.testSearchMsg) } func DefaultFullOpts(nFull int) []FullNodeOpts { @@ -120,6 +124,49 @@ func (ts *testSuite) testVersion(t *testing.T) { require.Equal(t, v.Version, build.BuildVersion) } +func (ts *testSuite) testSearchMsg(t *testing.T) { + apis, miners := ts.makeNodes(t, OneFull, OneMiner) + + api := apis[0] + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + senderAddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + t.Fatal(err) + } + + msg := &types.Message{ + From: senderAddr, + To: senderAddr, + Value: big.Zero(), + } + bm := NewBlockMiner(ctx, t, miners[0], 100*time.Millisecond) + bm.MineBlocks() + defer bm.Stop() + + sm, err := api.MpoolPushMessage(ctx, msg, nil) + if err != nil { + t.Fatal(err) + } + res, err := api.StateWaitMsg(ctx, sm.Cid(), 1) + if err != nil { + t.Fatal(err) + } + if res.Receipt.ExitCode != 0 { + t.Fatal("did not successfully send message") + } + + searchRes, err := api.StateSearchMsg(ctx, sm.Cid()) + if err != nil { + t.Fatal(err) + } + + if searchRes.TipSet != res.TipSet { + t.Fatalf("search ts: %s, different from wait ts: %s", searchRes.TipSet, res.TipSet) + } + +} + func (ts *testSuite) testID(t *testing.T) { ctx := context.Background() apis, _ := ts.makeNodes(t, OneFull, OneMiner) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index f3f8b8a55..886ac8e99 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1,9 +1,12 @@ -/dns4/bootstrap-0.testnet.fildev.network/tcp/1347/p2p/12D3KooWJTUBUjtzWJGWU1XSiY21CwmHaCNLNYn2E7jqHEHyZaP7 -/dns4/bootstrap-1.testnet.fildev.network/tcp/1347/p2p/12D3KooW9yeKXha4hdrJKq74zEo99T8DhriQdWNoojWnnQbsgB3v -/dns4/bootstrap-2.testnet.fildev.network/tcp/1347/p2p/12D3KooWCrx8yVG9U9Kf7w8KLN3Edkj5ZKDhgCaeMqQbcQUoB6CT -/dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 -/dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T -/dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W +/dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj +/dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc +/dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4 +/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R +/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc +/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH +/dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ +/dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf +/dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR /dns4/lotus-bootstrap.forceup.cn/tcp/41778/p2p/12D3KooWFQsv3nRMUevZNWWsY1Wu6NUzUbawnWU5NcRhgKuJA37C /dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz /dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u diff --git a/build/drand.go b/build/drand.go index 73299249a..3b976ac92 100644 --- a/build/drand.go +++ b/build/drand.go @@ -35,6 +35,7 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{ "https://api.drand.sh", "https://api2.drand.sh", "https://api3.drand.sh", + "https://drand.cloudflare.com", }, Relays: []string{ "/dnsaddr/api.drand.sh/", @@ -68,16 +69,6 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{ ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`, }, DrandIncentinet: { - Servers: []string{ - "https://pl-eu.incentinet.drand.sh", - "https://pl-us.incentinet.drand.sh", - "https://pl-sin.incentinet.drand.sh", - }, - Relays: []string{ - "/dnsaddr/pl-eu.incentinet.drand.sh/", - "/dnsaddr/pl-us.incentinet.drand.sh/", - "/dnsaddr/pl-sin.incentinet.drand.sh/", - }, ChainInfoJSON: `{"public_key":"8cad0c72c606ab27d36ee06de1d5b2db1faf92e447025ca37575ab3a8aac2eaae83192f846fc9e158bc738423753d000","period":30,"genesis_time":1595873820,"hash":"80c8b872c714f4c00fdd3daa465d5514049f457f01f85a4caf68cdcd394ba039","groupHash":"d9406aaed487f7af71851b4399448e311f2328923d454e971536c05398ce2d9b"}`, }, } diff --git a/build/version.go b/build/version.go index 90cb1cdf2..cb61ff8fb 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.10.0" +const BuildVersion = "0.10.2" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit @@ -83,7 +83,7 @@ func VersionForType(nodeType NodeType) (Version, error) { // semver versions of the rpc api exposed var ( - FullAPIVersion = newVer(0, 16, 0) + FullAPIVersion = newVer(0, 17, 0) MinerAPIVersion = newVer(0, 16, 0) WorkerAPIVersion = newVer(0, 16, 0) ) diff --git a/chain/actors/builtin/miner/diff_deadlines.go b/chain/actors/builtin/miner/diff_deadlines.go index e1e839960..7d686ece5 100644 --- a/chain/actors/builtin/miner/diff_deadlines.go +++ b/chain/actors/builtin/miner/diff_deadlines.go @@ -7,9 +7,9 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" ) -type DeadlinesDiff map[uint64]*DeadlineDiff +type DeadlinesDiff map[uint64]DeadlineDiff -func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { +func DiffDeadlines(pre, cur State) (DeadlinesDiff, error) { changed, err := pre.DeadlinesChanged(cur) if err != nil { return nil, err @@ -18,11 +18,7 @@ func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { return nil, nil } - numDl, err := pre.NumDeadlines() - if err != nil { - return nil, err - } - dlDiff := make(DeadlinesDiff, numDl) + dlDiff := make(DeadlinesDiff) if err := pre.ForEachDeadline(func(idx uint64, preDl Deadline) error { curDl, err := cur.LoadDeadline(idx) if err != nil { @@ -39,12 +35,12 @@ func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { }); err != nil { return nil, err } - return &dlDiff, nil + return dlDiff, nil } type DeadlineDiff map[uint64]*PartitionDiff -func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { +func DiffDeadline(pre, cur Deadline) (DeadlineDiff, error) { changed, err := pre.PartitionsChanged(cur) if err != nil { return nil, err @@ -104,7 +100,7 @@ func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { return nil, err } - return &partDiff, nil + return partDiff, nil } type PartitionDiff struct { diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 14718d002..c79f4a2f7 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -274,7 +274,7 @@ func (s *state0) DeadlinesChanged(other State) (bool, error) { return true, nil } - return s.State.Deadlines.Equals(other0.Deadlines), nil + return !s.State.Deadlines.Equals(other0.Deadlines), nil } func (s *state0) Info() (MinerInfo, error) { @@ -370,7 +370,7 @@ func (d *deadline0) PartitionsChanged(other Deadline) (bool, error) { return true, nil } - return d.Deadline.Partitions.Equals(other0.Deadline.Partitions), nil + return !d.Deadline.Partitions.Equals(other0.Deadline.Partitions), nil } func (d *deadline0) PostSubmissions() (bitfield.BitField, error) { diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 9e599f891..c686fa483 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -273,7 +273,7 @@ func (s *state2) DeadlinesChanged(other State) (bool, error) { return true, nil } - return s.State.Deadlines.Equals(other2.Deadlines), nil + return !s.State.Deadlines.Equals(other2.Deadlines), nil } func (s *state2) Info() (MinerInfo, error) { @@ -369,7 +369,7 @@ func (d *deadline2) PartitionsChanged(other Deadline) (bool, error) { return true, nil } - return d.Deadline.Partitions.Equals(other2.Deadline.Partitions), nil + return !d.Deadline.Partitions.Equals(other2.Deadline.Partitions), nil } func (d *deadline2) PostSubmissions() (bitfield.BitField, error) { diff --git a/chain/actors/builtin/multisig/message.go b/chain/actors/builtin/multisig/message.go index b19287432..5beb5319d 100644 --- a/chain/actors/builtin/multisig/message.go +++ b/chain/actors/builtin/multisig/message.go @@ -45,6 +45,7 @@ type MessageBuilder interface { // this type is the same between v0 and v2 type ProposalHashData = multisig2.ProposalHashData +type ProposeReturn = multisig2.ProposeReturn func txnParams(id uint64, data *ProposalHashData) ([]byte, error) { params := multisig2.TxnIDParams{ID: multisig2.TxnID(id)} diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index 0cb9c2ba8..d66ee7b54 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -12,7 +12,7 @@ import ( ) func TestPrintGroupInfo(t *testing.T) { - server := build.DrandConfigs[build.DrandIncentinet].Servers[0] + server := build.DrandConfigs[build.DrandDevnet].Servers[0] c, err := hclient.New(server, nil, nil) assert.NoError(t, err) cg := c.(interface { diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index d3c638b22..cc26be0b5 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -59,6 +59,8 @@ var MaxUntrustedActorPendingMessages = 10 var MaxNonceGap = uint64(4) +var DefaultMaxFee = abi.TokenAmount(types.MustParseFIL("0.007")) + var ( ErrMessageTooBig = errors.New("message too big") @@ -183,7 +185,7 @@ func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount { func CapGasFee(msg *types.Message, maxFee abi.TokenAmount) { if maxFee.Equals(big.Zero()) { - maxFee = types.NewInt(build.FilecoinPrecision / 10) + maxFee = DefaultMaxFee } gl := types.NewInt(uint64(msg.GasLimit)) @@ -368,11 +370,23 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, j journ return err }) - if err := mp.loadLocal(); err != nil { - log.Errorf("loading local messages: %+v", err) - } + mp.curTsLk.Lock() + mp.lk.Lock() - go mp.runLoop() + go func() { + err := mp.loadLocal() + + mp.lk.Unlock() + mp.curTsLk.Unlock() + + if err != nil { + log.Errorf("loading local messages: %+v", err) + } + + log.Info("mpool ready") + + mp.runLoop() + }() return mp, nil } @@ -665,11 +679,12 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { return err } - mp.curTsLk.Lock() - defer mp.curTsLk.Unlock() - curTs := mp.curTs + if curTs == nil { + return xerrors.Errorf("current tipset not loaded") + } + snonce, err := mp.getStateNonce(m.Message.From, curTs) if err != nil { return xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure) @@ -679,9 +694,6 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { return xerrors.Errorf("minimum expected nonce is %d: %w", snonce, ErrNonceTooLow) } - mp.lk.Lock() - defer mp.lk.Unlock() - _, err = mp.verifyMsgBeforeAdd(m, curTs, true) if err != nil { return err diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 8d0f6f6c1..3d7d284bc 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -113,6 +113,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. } return &api.InvocResult{ + MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, ExecutionTrace: ret.ExecutionTrace, @@ -228,8 +229,10 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri } return &api.InvocResult{ + MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, + GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Error: errs, Duration: ret.Duration, diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 7f2b57546..488f84167 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -168,7 +168,7 @@ func (sm *StateManager) hasExpensiveFork(ctx context.Context, height abi.ChainEp return ok } -func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { +func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmount, cb func(trace types.ExecutionTrace)) error { fromAct, err := tree.GetActor(from) if err != nil { return xerrors.Errorf("failed to get 'from' actor for transfer: %w", err) @@ -201,7 +201,6 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, From: from, To: to, Value: amt, - Nonce: math.MaxUint64, } fakeRct := &types.MessageReceipt{ ExitCode: 0, @@ -209,22 +208,14 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, GasUsed: 0, } - if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ - MessageReceipt: *fakeRct, - ActorErr: nil, - ExecutionTrace: types.ExecutionTrace{ - Msg: fakeMsg, - MsgRct: fakeRct, - Error: "", - Duration: 0, - GasCharges: nil, - Subcalls: nil, - }, - Duration: 0, - GasCosts: vm.ZeroGasOutputs(), - }); err != nil { - return xerrors.Errorf("recording transfer: %w", err) - } + cb(types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: nil, + }) } return nil @@ -277,6 +268,10 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } var transfers []transfer + subcalls := make([]types.ExecutionTrace, 0) + transferCb := func(trace types.ExecutionTrace) { + subcalls = append(subcalls, trace) + } // Take all excess funds away, put them into the reserve account err = tree.ForEach(func(addr address.Address, act *types.Actor) error { @@ -312,11 +307,13 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal available = st.GetAvailableBalance(act.Balance) } - transfers = append(transfers, transfer{ - From: addr, - To: builtin.ReserveAddress, - Amt: available, - }) + if !available.IsZero() { + transfers = append(transfers, transfer{ + From: addr, + To: builtin.ReserveAddress, + Amt: available, + }) + } } return nil }) @@ -326,7 +323,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal // Execute transfers from previous step for _, t := range transfers { - if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + if err := doTransfer(tree, t.From, t.To, t.Amt, transferCb); err != nil { return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } @@ -429,7 +426,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } for _, t := range transfersBack { - if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + if err := doTransfer(tree, t.From, t.To, t.Amt, transferCb); err != nil { return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } @@ -439,7 +436,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if err != nil { return cid.Undef, xerrors.Errorf("failed to load burnt funds actor: %w", err) } - if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, builtin.ReserveAddress, burntAct.Balance); err != nil { + if err := doTransfer(tree, builtin0.BurntFundsActorAddr, builtin.ReserveAddress, burntAct.Balance, transferCb); err != nil { return cid.Undef, xerrors.Errorf("failed to unburn funds: %w", err) } @@ -455,7 +452,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } difference := types.BigSub(DesiredReimbursementBalance, reimb.Balance) - if err := doTransfer(cb, tree, builtin.ReserveAddress, reimbAddr, difference); err != nil { + if err := doTransfer(tree, builtin.ReserveAddress, reimbAddr, difference, transferCb); err != nil { return cid.Undef, xerrors.Errorf("failed to top up reimbursement account: %w", err) } @@ -474,6 +471,39 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal return cid.Undef, xerrors.Errorf("resultant state tree account balance was not correct: %s", total) } + if cb != nil { + // record the transfer in execution traces + + fakeMsg := &types.Message{ + From: builtin.SystemActorAddr, + To: builtin.SystemActorAddr, + Value: big.Zero(), + Nonce: uint64(epoch), + } + fakeRct := &types.MessageReceipt{ + ExitCode: 0, + Return: nil, + GasUsed: 0, + } + + if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ + MessageReceipt: *fakeRct, + ActorErr: nil, + ExecutionTrace: types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: subcalls, + }, + Duration: 0, + GasCosts: nil, + }); err != nil { + return cid.Undef, xerrors.Errorf("recording transfers: %w", err) + } + } + return tree.Flush(ctx) } @@ -514,12 +544,12 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err) } - err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50) + err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50, epoch) if err != nil { return cid.Undef, xerrors.Errorf("splitting first msig: %w", err) } - err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50) + err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50, epoch) if err != nil { return cid.Undef, xerrors.Errorf("splitting second msig: %w", err) } @@ -645,7 +675,7 @@ func setNetworkName(ctx context.Context, store adt.Store, tree *state.StateTree, return nil } -func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64) error { +func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64, epoch abi.ChainEpoch) error { if portions < 1 { return xerrors.Errorf("cannot split into 0 portions") } @@ -714,6 +744,11 @@ func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Add } i := uint64(0) + subcalls := make([]types.ExecutionTrace, 0, portions) + transferCb := func(trace types.ExecutionTrace) { + subcalls = append(subcalls, trace) + } + for i < portions { keyAddr, err := makeKeyAddr(addr, i) if err != nil { @@ -730,13 +765,46 @@ func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Add return xerrors.Errorf("setting new msig actor state: %w", err) } - if err := doTransfer(cb, tree, addr, idAddr, newIbal); err != nil { + if err := doTransfer(tree, addr, idAddr, newIbal, transferCb); err != nil { return xerrors.Errorf("transferring split msig balance: %w", err) } i++ } + if cb != nil { + // record the transfer in execution traces + + fakeMsg := &types.Message{ + From: builtin.SystemActorAddr, + To: addr, + Value: big.Zero(), + Nonce: uint64(epoch), + } + fakeRct := &types.MessageReceipt{ + ExitCode: 0, + Return: nil, + GasUsed: 0, + } + + if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ + MessageReceipt: *fakeRct, + ActorErr: nil, + ExecutionTrace: types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: subcalls, + }, + Duration: 0, + GasCosts: nil, + }); err != nil { + return xerrors.Errorf("recording transfers: %w", err) + } + } + return nil } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b470cfe83..be14797d9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -200,6 +200,7 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { return func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { ir := &api.InvocResult{ + MsgCid: mcid, Msg: msg, MsgRct: &ret.MessageReceipt, ExecutionTrace: ret.ExecutionTrace, @@ -208,6 +209,9 @@ func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, if ret.ActorErr != nil { ir.Error = ret.ActorErr.Error() } + if ret.GasCosts != nil { + ir.GasCost = MakeMsgGasCost(msg, ret) + } *trace = append(*trace, ir) return nil } @@ -247,17 +251,12 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err) } - runCron := func() error { - // TODO: this nonce-getting is a tiny bit ugly - ca, err := vmi.StateTree().GetActor(builtin0.SystemActorAddr) - if err != nil { - return err - } + runCron := func(epoch abi.ChainEpoch) error { cronMsg := &types.Message{ To: builtin0.CronActorAddr, From: builtin0.SystemActorAddr, - Nonce: ca.Nonce, + Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), @@ -284,7 +283,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp for i := parentEpoch; i < epoch; i++ { if i > parentEpoch { // run cron for null rounds if any - if err := runCron(); err != nil { + if err := runCron(i); err != nil { return cid.Undef, cid.Undef, err } @@ -313,7 +312,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } var receipts []cbg.CBORMarshaler - processedMsgs := map[cid.Cid]bool{} + processedMsgs := make(map[cid.Cid]struct{}) for _, b := range bms { penalty := types.NewInt(0) gasReward := big.Zero() @@ -337,7 +336,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, err } } - processedMsgs[m.Cid()] = true + processedMsgs[m.Cid()] = struct{}{} } params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{ @@ -350,15 +349,10 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) } - sysAct, actErr := vmi.StateTree().GetActor(builtin0.SystemActorAddr) - if actErr != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to get system actor: %w", actErr) - } - rwMsg := &types.Message{ From: builtin0.SystemActorAddr, To: reward.Address, - Nonce: sysAct.Nonce, + Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), @@ -381,7 +375,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } } - if err := runCron(); err != nil { + if err := runCron(epoch); err != nil { return cid.Cid{}, cid.Cid{}, err } @@ -435,12 +429,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, ts *types.TipSet parentEpoch = parent.Height } - cids := make([]cid.Cid, len(blks)) - for i, v := range blks { - cids[i] = v.Cid() - } - - r := store.NewChainRand(sm.cs, cids) + r := store.NewChainRand(sm.cs, ts.Cids()) blkmsgs, err := sm.cs.BlockMsgsForTipset(ts) if err != nil { @@ -738,7 +727,7 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet } if r != nil { - return pts, r, foundMsg, nil + return cur, r, foundMsg, nil } } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index cbc78ca5b..de4f947df 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -701,3 +701,16 @@ func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi return sum, nil } + +func MakeMsgGasCost(msg *types.Message, ret *vm.ApplyRet) api.MsgGasCost { + return api.MsgGasCost{ + Message: msg.Cid(), + GasUsed: big.NewInt(ret.GasUsed), + BaseFeeBurn: ret.GasCosts.BaseFeeBurn, + OverEstimationBurn: ret.GasCosts.OverEstimationBurn, + MinerPenalty: ret.GasCosts.MinerPenalty, + MinerTip: ret.GasCosts.MinerTip, + Refund: ret.GasCosts.Refund, + TotalCost: big.Sub(msg.RequiredFunds(), ret.GasCosts.Refund), + } +} diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 68ee5e20c..99ca2fc65 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -37,6 +37,7 @@ import ( "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/node/impl/client" ) var log = logging.Logger("sub") @@ -44,6 +45,13 @@ var log = logging.Logger("sub") var ErrSoftFailure = errors.New("soft validation failure") var ErrInsufficientPower = errors.New("incoming block's miner does not have minimum power") +var msgCidPrefix = cid.Prefix{ + Version: 1, + Codec: cid.DagCBOR, + MhType: client.DefaultHashFunction, + MhLength: 32, +} + func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bs bserv.BlockService, cmgr connmgr.ConnManager) { // Timeout after (block time + propagation delay). This is useless at // this point. @@ -168,6 +176,9 @@ func fetchCids( cidIndex := make(map[cid.Cid]int) for i, c := range cids { + if c.Prefix() != msgCidPrefix { + return fmt.Errorf("invalid msg CID: %s", c) + } cidIndex[c] = i } if len(cids) != len(cidIndex) { diff --git a/chain/sync.go b/chain/sync.go index dd83bf319..97915d8df 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -323,25 +323,35 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc) } - // Collect the CIDs of both types of messages separately: BLS and Secpk. - var bcids, scids []cid.Cid - for _, m := range fblk.BlsMessages { - bcids = append(bcids, m.Cid()) - } - - for _, m := range fblk.SecpkMessages { - scids = append(scids, m.Cid()) - } - // TODO: IMPORTANT(GARBAGE). These message puts and the msgmeta // computation need to go into the 'temporary' side of the blockstore when // we implement that - blockstore := syncer.store.Blockstore() - bs := cbor.NewCborStore(blockstore) + // We use a temporary bstore here to avoid writing intermediate pieces + // into the blockstore. + blockstore := bstore.NewTemporary() + cst := cbor.NewCborStore(blockstore) + + var bcids, scids []cid.Cid + + for _, m := range fblk.BlsMessages { + c, err := store.PutMessage(blockstore, m) + if err != nil { + return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) + } + bcids = append(bcids, c) + } + + for _, m := range fblk.SecpkMessages { + c, err := store.PutMessage(blockstore, m) + if err != nil { + return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) + } + scids = append(scids, c) + } // Compute the root CID of the combined message trie. - smroot, err := computeMsgMeta(bs, bcids, scids) + smroot, err := computeMsgMeta(cst, bcids, scids) if err != nil { return xerrors.Errorf("validating msgmeta, compute failed: %w", err) } @@ -351,21 +361,8 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { return xerrors.Errorf("messages in full block did not match msgmeta root in header (%s != %s)", fblk.Header.Messages, smroot) } - for _, m := range fblk.BlsMessages { - _, err := store.PutMessage(blockstore, m) - if err != nil { - return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) - } - } - - for _, m := range fblk.SecpkMessages { - _, err := store.PutMessage(blockstore, m) - if err != nil { - return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) - } - } - - return nil + // Finally, flush. + return vm.Copy(context.TODO(), blockstore, syncer.store.Blockstore(), smroot) } func (syncer *Syncer) LocalPeer() peer.ID { @@ -1064,8 +1061,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return err } - cst := cbor.NewCborStore(syncer.store.Blockstore()) - st, err := state.LoadStateTree(cst, stateroot) + st, err := state.LoadStateTree(syncer.store.Store(ctx), stateroot) if err != nil { return xerrors.Errorf("failed to load base state tree: %w", err) } @@ -1111,21 +1107,28 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return nil } - store := adt0.WrapStore(ctx, cst) + // Validate message arrays in a temporary blockstore. + tmpbs := bstore.NewTemporary() + tmpstore := adt0.WrapStore(ctx, cbor.NewCborStore(tmpbs)) - bmArr := adt0.MakeEmptyArray(store) + bmArr := adt0.MakeEmptyArray(tmpstore) for i, m := range b.BlsMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid bls message at index %d: %w", i, err) } - c := cbg.CborCid(m.Cid()) - if err := bmArr.Set(uint64(i), &c); err != nil { + c, err := store.PutMessage(tmpbs, m) + if err != nil { + return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err) + } + + k := cbg.CborCid(c) + if err := bmArr.Set(uint64(i), &k); err != nil { return xerrors.Errorf("failed to put bls message at index %d: %w", i, err) } } - smArr := adt0.MakeEmptyArray(store) + smArr := adt0.MakeEmptyArray(tmpstore) for i, m := range b.SecpkMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err) @@ -1142,8 +1145,12 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return xerrors.Errorf("secpk message %s has invalid signature: %w", m.Cid(), err) } - c := cbg.CborCid(m.Cid()) - if err := smArr.Set(uint64(i), &c); err != nil { + c, err := store.PutMessage(tmpbs, m) + if err != nil { + return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err) + } + k := cbg.CborCid(c) + if err := smArr.Set(uint64(i), &k); err != nil { return xerrors.Errorf("failed to put secpk message at index %d: %w", i, err) } } @@ -1158,7 +1165,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return err } - mrcid, err := cst.Put(ctx, &types.MsgMeta{ + mrcid, err := tmpstore.Put(ctx, &types.MsgMeta{ BlsMessages: bmroot, SecpkMessages: smroot, }) @@ -1170,7 +1177,8 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return fmt.Errorf("messages didnt match message root in header") } - return nil + // Finally, flush. + return vm.Copy(ctx, tmpbs, syncer.store.Blockstore(), mrcid) } func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signature, msgs []cid.Cid, pubks [][]byte) error { diff --git a/chain/types/fil.go b/chain/types/fil.go index 7eac8ce93..0ea77660c 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -81,5 +81,14 @@ func ParseFIL(s string) (FIL, error) { return FIL{r.Num()}, nil } +func MustParseFIL(s string) FIL { + n, err := ParseFIL(s) + if err != nil { + panic(err) + } + + return n +} + var _ encoding.TextMarshaler = (*FIL)(nil) var _ encoding.TextUnmarshaler = (*FIL)(nil) diff --git a/chain/types/message.go b/chain/types/message.go index cbc7c1dad..c53ecc7c1 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -195,7 +195,7 @@ func (m *Message) ValidForBlockInclusion(minGas int64) error { // since prices might vary with time, this is technically semantic validation if m.GasLimit < minGas { - return xerrors.New("'GasLimit' field cannot be less than the cost of storing a message on chain") + return xerrors.Errorf("'GasLimit' field cannot be less than the cost of storing a message on chain %d < %d", m.GasLimit, minGas) } return nil diff --git a/chain/types/signedmessage.go b/chain/types/signedmessage.go index 7532bea35..c539ac240 100644 --- a/chain/types/signedmessage.go +++ b/chain/types/signedmessage.go @@ -78,7 +78,14 @@ func (sm *SignedMessage) MarshalJSON() ([]byte, error) { } func (sm *SignedMessage) ChainLength() int { - ser, err := sm.Serialize() + var ser []byte + var err error + if sm.Signature.Type == crypto.SigTypeBLS { + // BLS chain message length doesn't include signature + ser, err = sm.Message.Serialize() + } else { + ser, err = sm.Serialize() + } if err != nil { panic(err) } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index a4efccb29..72ad731aa 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -214,7 +214,7 @@ type ApplyRet struct { ActorErr aerrors.ActorError ExecutionTrace types.ExecutionTrace Duration time.Duration - GasCosts GasOutputs + GasCosts *GasOutputs } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, @@ -361,7 +361,7 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - GasCosts: GasOutputs{}, + GasCosts: nil, Duration: time.Since(start), }, actorErr } @@ -397,7 +397,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -417,7 +417,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "actor not found: %s", msg.From), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -434,7 +434,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "send from not account actor: %s", fromActor.Code), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -450,7 +450,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor nonce invalid: msg:%d != state:%d", msg.Nonce, fromActor.Nonce), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -466,7 +466,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor balance less than needed: %s < %s", types.FIL(fromActor.Balance), types.FIL(gascost)), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -560,7 +560,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } diff --git a/cli/client.go b/cli/client.go index 34d151ace..7035fb64b 100644 --- a/cli/client.go +++ b/cli/client.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -82,6 +83,7 @@ var clientCmd = &cli.Command{ WithCategory("util", clientCarGenCmd), WithCategory("util", clientInfoCmd), WithCategory("util", clientListTransfers), + WithCategory("util", clientRestartTransfer), }, } @@ -1326,6 +1328,66 @@ var clientInfoCmd = &cli.Command{ }, } +var clientRestartTransfer = &cli.Command{ + Name: "restart-transfer", + Usage: "Force restart a stalled data transfer", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "peerid", + Usage: "narrow to transfer with specific peer", + }, + &cli.BoolFlag{ + Name: "initiator", + Usage: "specify only transfers where peer is/is not initiator", + Value: true, + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + transferUint, err := strconv.ParseUint(cctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("Error reading transfer ID: %w", err) + } + transferID := datatransfer.TransferID(transferUint) + initiator := cctx.Bool("initiator") + var other peer.ID + if pidstr := cctx.String("peerid"); pidstr != "" { + p, err := peer.Decode(pidstr) + if err != nil { + return err + } + other = p + } else { + channels, err := api.ClientListDataTransfers(ctx) + if err != nil { + return err + } + found := false + for _, channel := range channels { + if channel.IsInitiator == initiator && channel.TransferID == transferID { + other = channel.OtherPeer + found = true + break + } + } + if !found { + return errors.New("unable to find matching data transfer") + } + } + + return api.ClientRestartDataTransfer(ctx, transferID, other, initiator) + }, +} + var clientListTransfers = &cli.Command{ Name: "list-transfers", Usage: "List ongoing data transfers for deals", diff --git a/cli/multisig.go b/cli/multisig.go index c8f181a62..7cf7ed750 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "os" "reflect" "sort" "strconv" @@ -24,6 +23,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-address" + cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -41,10 +41,18 @@ import ( var multisigCmd = &cli.Command{ Name: "msig", Usage: "Interact with a multisig wallet", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "confidence", + Usage: "number of block confirmations to wait for", + Value: int(build.MessageConfidence), + }, + }, Subcommands: []*cli.Command{ msigCreateCmd, msigInspectCmd, msigProposeCmd, + msigRemoveProposeCmd, msigApproveCmd, msigAddProposeCmd, msigAddApproveCmd, @@ -56,6 +64,7 @@ var multisigCmd = &cli.Command{ msigLockApproveCmd, msigLockCancelCmd, msigVestedCmd, + msigProposeThresholdCmd, }, } @@ -145,14 +154,14 @@ var msigCreateCmd = &cli.Command{ } // wait for it to get mined into a block - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } // check it executed successfully if wait.Receipt.ExitCode != 0 { - fmt.Println("actor creation failed!") + fmt.Fprintln(cctx.App.Writer, "actor creation failed!") return err } @@ -162,7 +171,7 @@ var msigCreateCmd = &cli.Command{ if err := execreturn.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil { return err } - fmt.Println("Created new multisig: ", execreturn.IDAddress, execreturn.RobustAddress) + fmt.Fprintln(cctx.App.Writer, "Created new multisig: ", execreturn.IDAddress, execreturn.RobustAddress) // TODO: maybe register this somewhere return nil @@ -226,25 +235,25 @@ var msigInspectCmd = &cli.Command{ return err } - fmt.Printf("Balance: %s\n", types.FIL(act.Balance)) - fmt.Printf("Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked))) + fmt.Fprintf(cctx.App.Writer, "Balance: %s\n", types.FIL(act.Balance)) + fmt.Fprintf(cctx.App.Writer, "Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked))) if cctx.Bool("vesting") { ib, err := mstate.InitialBalance() if err != nil { return err } - fmt.Printf("InitialBalance: %s\n", types.FIL(ib)) + fmt.Fprintf(cctx.App.Writer, "InitialBalance: %s\n", types.FIL(ib)) se, err := mstate.StartEpoch() if err != nil { return err } - fmt.Printf("StartEpoch: %d\n", se) + fmt.Fprintf(cctx.App.Writer, "StartEpoch: %d\n", se) ud, err := mstate.UnlockDuration() if err != nil { return err } - fmt.Printf("UnlockDuration: %d\n", ud) + fmt.Fprintf(cctx.App.Writer, "UnlockDuration: %d\n", ud) } signers, err := mstate.Signers() @@ -255,10 +264,10 @@ var msigInspectCmd = &cli.Command{ if err != nil { return err } - fmt.Printf("Threshold: %d / %d\n", threshold, len(signers)) - fmt.Println("Signers:") + fmt.Fprintf(cctx.App.Writer, "Threshold: %d / %d\n", threshold, len(signers)) + fmt.Fprintln(cctx.App.Writer, "Signers:") for _, s := range signers { - fmt.Printf("\t%s\n", s) + fmt.Fprintf(cctx.App.Writer, "\t%s\n", s) } pending := make(map[int64]multisig.Transaction) @@ -270,7 +279,7 @@ var msigInspectCmd = &cli.Command{ } decParams := cctx.Bool("decode-params") - fmt.Println("Transactions: ", len(pending)) + fmt.Fprintln(cctx.App.Writer, "Transactions: ", len(pending)) if len(pending) > 0 { var txids []int64 for txid := range pending { @@ -280,7 +289,7 @@ var msigInspectCmd = &cli.Command{ return txids[i] < txids[j] }) - w := tabwriter.NewWriter(os.Stdout, 8, 4, 2, ' ', 0) + w := tabwriter.NewWriter(cctx.App.Writer, 8, 4, 2, ' ', 0) fmt.Fprintf(w, "ID\tState\tApprovals\tTo\tValue\tMethod\tParams\n") for _, txid := range txids { tx := pending[txid] @@ -409,7 +418,7 @@ var msigProposeCmd = &cli.Command{ fmt.Println("send proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -445,14 +454,18 @@ var msigApproveCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - if cctx.Args().Len() < 5 { - return ShowHelp(cctx, fmt.Errorf("must pass multisig address, message ID, proposer address, destination, and value")) + if cctx.Args().Len() < 2 { + return ShowHelp(cctx, fmt.Errorf("must pass at least multisig address and message ID")) } if cctx.Args().Len() > 5 && cctx.Args().Len() != 7 { return ShowHelp(cctx, fmt.Errorf("usage: msig approve [ ]")) } + if cctx.Args().Len() > 2 && cctx.Args().Len() != 5 { + return ShowHelp(cctx, fmt.Errorf("usage: msig approve ")) + } + api, closer, err := GetFullNodeAPI(cctx) if err != nil { return err @@ -470,42 +483,121 @@ var msigApproveCmd = &cli.Command{ return err } - proposer, err := address.NewFromString(cctx.Args().Get(2)) - if err != nil { - return err + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr } - if proposer.Protocol() != address.ID { - proposer, err = api.StateLookupID(ctx, proposer, types.EmptyTSK) + var msgCid cid.Cid + if cctx.Args().Len() == 2 { + msgCid, err = api.MsigApprove(ctx, msig, txid, from) + if err != nil { + return err + } + } else { + proposer, err := address.NewFromString(cctx.Args().Get(2)) + if err != nil { + return err + } + + if proposer.Protocol() != address.ID { + proposer, err = api.StateLookupID(ctx, proposer, types.EmptyTSK) + if err != nil { + return err + } + } + + dest, err := address.NewFromString(cctx.Args().Get(3)) + if err != nil { + return err + } + + value, err := types.ParseFIL(cctx.Args().Get(4)) + if err != nil { + return err + } + + var method uint64 + var params []byte + if cctx.Args().Len() == 7 { + m, err := strconv.ParseUint(cctx.Args().Get(5), 10, 64) + if err != nil { + return err + } + method = m + + p, err := hex.DecodeString(cctx.Args().Get(6)) + if err != nil { + return err + } + params = p + } + + msgCid, err = api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) if err != nil { return err } } - dest, err := address.NewFromString(cctx.Args().Get(3)) + fmt.Println("sent approval in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } - value, err := types.ParseFIL(cctx.Args().Get(4)) + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("approve returned exit %d", wait.Receipt.ExitCode) + } + + return nil + }, +} + +var msigRemoveProposeCmd = &cli.Command{ + Name: "propose-remove", + Usage: "Propose to remove a signer", + ArgsUsage: "[multisigAddress signer]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "decrease-threshold", + Usage: "whether the number of required signers should be decreased", + }, + &cli.StringFlag{ + Name: "from", + Usage: "account to send the propose message from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 2 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address and signer address")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) if err != nil { return err } - var method uint64 - var params []byte - if cctx.Args().Len() == 7 { - m, err := strconv.ParseUint(cctx.Args().Get(5), 10, 64) - if err != nil { - return err - } - method = m - - p, err := hex.DecodeString(cctx.Args().Get(6)) - if err != nil { - return err - } - params = p + addr, err := address.NewFromString(cctx.Args().Get(1)) + if err != nil { + return err } var from address.Address @@ -523,22 +615,29 @@ var msigApproveCmd = &cli.Command{ from = defaddr } - msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) + msgCid, err := api.MsigRemoveSigner(ctx, msig, from, addr, cctx.Bool("decrease-threshold")) if err != nil { return err } - fmt.Println("sent approval in message: ", msgCid) + fmt.Println("sent remove proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } if wait.Receipt.ExitCode != 0 { - return fmt.Errorf("approve returned exit %d", wait.Receipt.ExitCode) + return fmt.Errorf("add proposal returned exit %d", wait.Receipt.ExitCode) } + var ret multisig.ProposeReturn + err = ret.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)) + if err != nil { + return xerrors.Errorf("decoding proposal return: %w", err) + } + fmt.Printf("TxnID: %d", ret.TxnID) + return nil }, } @@ -599,9 +698,9 @@ var msigAddProposeCmd = &cli.Command{ return err } - fmt.Println("sent add proposal in message: ", msgCid) + fmt.Fprintln(cctx.App.Writer, "sent add proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -683,7 +782,7 @@ var msigAddApproveCmd = &cli.Command{ fmt.Println("sent add approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -760,7 +859,7 @@ var msigAddCancelCmd = &cli.Command{ fmt.Println("sent add cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -832,7 +931,7 @@ var msigSwapProposeCmd = &cli.Command{ fmt.Println("sent swap proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -914,7 +1013,7 @@ var msigSwapApproveCmd = &cli.Command{ fmt.Println("sent swap approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -991,7 +1090,7 @@ var msigSwapCancelCmd = &cli.Command{ fmt.Println("sent swap cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1078,7 +1177,7 @@ var msigLockProposeCmd = &cli.Command{ fmt.Println("sent lock proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1175,7 +1274,7 @@ var msigLockApproveCmd = &cli.Command{ fmt.Println("sent lock approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1267,7 +1366,7 @@ var msigLockCancelCmd = &cli.Command{ fmt.Println("sent lock cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1341,3 +1440,78 @@ var msigVestedCmd = &cli.Command{ return nil }, } + +var msigProposeThresholdCmd = &cli.Command{ + Name: "propose-threshold", + Usage: "Propose setting a different signing threshold on the account", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "account to send the proposal from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 2 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address and new threshold value")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + newM, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) + if err != nil { + return err + } + + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr + } + + params, actErr := actors.SerializeParams(&msig0.ChangeNumApprovalsThresholdParams{ + NewThreshold: newM, + }) + + if actErr != nil { + return actErr + } + + msgCid, err := api.MsigPropose(ctx, msig, msig, types.NewInt(0), from, uint64(builtin2.MethodsMultisig.ChangeNumApprovalsThreshold), params) + if err != nil { + return fmt.Errorf("failed to propose change of threshold: %w", err) + } + + fmt.Println("sent change threshold proposal in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("change threshold proposal returned exit %d", wait.Receipt.ExitCode) + } + + return nil + }, +} diff --git a/cli/multisig_test.go b/cli/multisig_test.go new file mode 100644 index 000000000..09cdbe056 --- /dev/null +++ b/cli/multisig_test.go @@ -0,0 +1,55 @@ +package cli + +import ( + "context" + "os" + "testing" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api/test" + clitest "github.com/filecoin-project/lotus/cli/test" + builder "github.com/filecoin-project/lotus/node/test" +) + +// TestMultisig does a basic test to exercise the multisig CLI +// commands +func TestMultisig(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + + blocktime := 5 * time.Millisecond + ctx := context.Background() + nodes, _ := startNodes(ctx, t, blocktime) + clientNode := nodes[0] + clitest.RunMultisigTest(t, Commands, clientNode) +} + +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) { + n, sn := builder.RPCMockSbBuilder(t, test.OneFull, test.OneMiner) + + full := n[0] + miner := sn[0] + + // Get everyone connected + addrs, err := full.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrs); err != nil { + t.Fatal(err) + } + + // Start mining blocks + bm := test.NewBlockMiner(ctx, t, miner, blocktime) + bm.MineBlocks() + + // Get the creator's address + creatorAddr, err := full.WalletDefaultAddress(ctx) + if err != nil { + t.Fatal(err) + } + + // Create mock CLI + return n, []address.Address{creatorAddr} +} diff --git a/cli/paych_test.go b/cli/paych_test.go index b94ba27e7..77a6e61eb 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -437,6 +437,7 @@ type mockCLI struct { out *bytes.Buffer } +// TODO: refactor to use the methods in cli/test/mockcli.go func newMockCLI(t *testing.T) *mockCLI { // Create a CLI App with an --api-url flag so that we can specify which node // the command should be executed against diff --git a/cli/state.go b/cli/state.go index 1e7e24ddb..d0fcc4f95 100644 --- a/cli/state.go +++ b/cli/state.go @@ -60,7 +60,7 @@ var stateCmd = &cli.Command{ stateSectorCmd, stateGetActorCmd, stateLookupIDCmd, - stateReplaySetCmd, + stateReplayCmd, stateSectorSizeCmd, stateReadStateCmd, stateListMessagesCmd, @@ -69,7 +69,6 @@ var stateCmd = &cli.Command{ stateGetDealSetCmd, stateWaitMsgCmd, stateSearchMsgCmd, - stateMsgCostCmd, stateMinerInfo, stateMarketCmd, stateExecTraceCmd, @@ -384,20 +383,27 @@ var stateExecTraceCmd = &cli.Command{ }, } -var stateReplaySetCmd = &cli.Command{ +var stateReplayCmd = &cli.Command{ Name: "replay", - Usage: "Replay a particular message within a tipset", - ArgsUsage: "[tipsetKey messageCid]", + Usage: "Replay a particular message", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-trace", + Usage: "print out full execution trace for given message", + }, + &cli.BoolFlag{ + Name: "detailed-gas", + Usage: "print out detailed gas costs for given message", + }, + }, Action: func(cctx *cli.Context) error { - if cctx.Args().Len() < 1 { - fmt.Println("usage: [tipset] ") - fmt.Println("The last cid passed will be used as the message CID") - fmt.Println("All preceding ones will be used as the tipset") + if cctx.Args().Len() != 1 { + fmt.Println("must provide cid of message to replay") return nil } - args := cctx.Args().Slice() - mcid, err := cid.Decode(args[len(args)-1]) + mcid, err := cid.Decode(cctx.Args().First()) if err != nil { return fmt.Errorf("message cid was invalid: %s", err) } @@ -410,52 +416,7 @@ var stateReplaySetCmd = &cli.Command{ ctx := ReqContext(cctx) - var ts *types.TipSet - { - var tscids []cid.Cid - for _, s := range args[:len(args)-1] { - c, err := cid.Decode(s) - if err != nil { - return fmt.Errorf("tipset cid was invalid: %s", err) - } - tscids = append(tscids, c) - } - - if len(tscids) > 0 { - var headers []*types.BlockHeader - for _, c := range tscids { - h, err := fapi.ChainGetBlock(ctx, c) - if err != nil { - return err - } - - headers = append(headers, h) - } - - ts, err = types.NewTipSet(headers) - if err != nil { - return err - } - } else { - var r *api.MsgLookup - r, err = fapi.StateWaitMsg(ctx, mcid, build.MessageConfidence) - if err != nil { - return xerrors.Errorf("finding message in chain: %w", err) - } - - childTs, err := fapi.ChainGetTipSet(ctx, r.TipSet) - if err != nil { - return xerrors.Errorf("loading tipset: %w", err) - } - ts, err = fapi.ChainGetTipSet(ctx, childTs.Parents()) - if err != nil { - return err - } - } - - } - - res, err := fapi.StateReplay(ctx, ts.Key(), mcid) + res, err := fapi.StateReplay(ctx, types.EmptyTSK, mcid) if err != nil { return xerrors.Errorf("replay call failed: %w", err) } @@ -464,10 +425,25 @@ var stateReplaySetCmd = &cli.Command{ fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) fmt.Printf("Return: %x\n", res.MsgRct.Return) fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) + + if cctx.Bool("detailed-gas") { + fmt.Printf("Base Fee Burn: %d\n", res.GasCost.BaseFeeBurn) + fmt.Printf("Overestimaton Burn: %d\n", res.GasCost.OverEstimationBurn) + fmt.Printf("Miner Penalty: %d\n", res.GasCost.MinerPenalty) + fmt.Printf("Miner Tip: %d\n", res.GasCost.MinerTip) + fmt.Printf("Refund: %d\n", res.GasCost.Refund) + } + fmt.Printf("Total Message Cost: %d\n", res.GasCost.TotalCost) + if res.MsgRct.ExitCode != 0 { fmt.Printf("Error message: %q\n", res.Error) } + if cctx.Bool("show-trace") { + fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", res.Msg.From, res.Msg.To, res.Msg.Value, res.Msg.Method, res.Msg.Params, res.MsgRct.ExitCode, res.MsgRct.Return) + printInternalExecutions("\t", res.ExecutionTrace.Subcalls) + } + return nil }, } @@ -837,33 +813,66 @@ var stateListMessagesCmd = &cli.Command{ froma = a } - toh := cctx.Uint64("toheight") + toh := abi.ChainEpoch(cctx.Uint64("toheight")) ts, err := LoadTipSet(ctx, cctx, api) if err != nil { return err } - msgs, err := api.StateListMessages(ctx, &types.Message{To: toa, From: froma}, ts.Key(), abi.ChainEpoch(toh)) - if err != nil { - return err + if ts == nil { + head, err := api.ChainHead(ctx) + if err != nil { + return err + } + ts = head } - for _, c := range msgs { - if cctx.Bool("cids") { - fmt.Println(c.String()) - continue + windowSize := abi.ChainEpoch(100) + + cur := ts + for cur.Height() > toh { + if ctx.Err() != nil { + return ctx.Err() } - m, err := api.ChainGetMessage(ctx, c) + end := toh + if cur.Height()-windowSize > end { + end = cur.Height() - windowSize + } + + msgs, err := api.StateListMessages(ctx, &lapi.MessageMatch{To: toa, From: froma}, cur.Key(), end) if err != nil { return err } - b, err := json.MarshalIndent(m, "", " ") + + for _, c := range msgs { + if cctx.Bool("cids") { + fmt.Println(c.String()) + continue + } + + m, err := api.ChainGetMessage(ctx, c) + if err != nil { + return err + } + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + } + + if end <= 0 { + break + } + + next, err := api.ChainGetTipSetByHeight(ctx, end-1, cur.Key()) if err != nil { return err } - fmt.Println(string(b)) + + cur = next } return nil @@ -1424,60 +1433,6 @@ var stateSearchMsgCmd = &cli.Command{ }, } -var stateMsgCostCmd = &cli.Command{ - Name: "msg-cost", - Usage: "Get the detailed gas costs of a message", - ArgsUsage: "[messageCid]", - Action: func(cctx *cli.Context) error { - if !cctx.Args().Present() { - return fmt.Errorf("must specify message cid to get gas costs for") - } - - api, closer, err := GetFullNodeAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := ReqContext(cctx) - - msg, err := cid.Decode(cctx.Args().First()) - if err != nil { - return err - } - - tsk := types.EmptyTSK - - ts, err := LoadTipSet(ctx, cctx, api) - if err != nil { - return err - } - - if ts != nil { - tsk = ts.Key() - } - - mgc, err := api.StateMsgGasCost(ctx, msg, tsk) - if err != nil { - return err - } - - if mgc != nil { - fmt.Printf("Message CID: %s", mgc.Message) - fmt.Printf("\nGas Used: %d", mgc.GasUsed) - fmt.Printf("\nBase Fee Burn: %d", mgc.BaseFeeBurn) - fmt.Printf("\nOverestimation Burn: %d", mgc.OverEstimationBurn) - fmt.Printf("\nMiner Tip: %d", mgc.MinerTip) - fmt.Printf("\nRefund: %d", mgc.Refund) - fmt.Printf("\nTotal Cost: %d", mgc.TotalCost) - fmt.Printf("\nMiner Penalty: %d", mgc.MinerPenalty) - } else { - fmt.Print("message was not found on chain") - } - return nil - }, -} - var stateCallCmd = &cli.Command{ Name: "call", Usage: "Invoke a method on an actor locally", diff --git a/cli/sync.go b/cli/sync.go index ea066cbda..c3f25eb1d 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -84,6 +84,12 @@ var syncStatusCmd = &cli.Command{ var syncWaitCmd = &cli.Command{ Name: "wait", Usage: "Wait for sync to be complete", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "watch", + Usage: "don't exit after node is synced", + }, + }, Action: func(cctx *cli.Context) error { napi, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -92,7 +98,7 @@ var syncWaitCmd = &cli.Command{ defer closer() ctx := ReqContext(cctx) - return SyncWait(ctx, napi) + return SyncWait(ctx, napi, cctx.Bool("watch")) }, } @@ -234,7 +240,7 @@ var syncCheckpointCmd = &cli.Command{ }, } -func SyncWait(ctx context.Context, napi api.FullNode) error { +func SyncWait(ctx context.Context, napi api.FullNode, watch bool) error { tick := time.Second / 4 lastLines := 0 @@ -311,7 +317,7 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { _ = target // todo: maybe print? (creates a bunch of line wrapping issues with most tipsets) - if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { + if !watch && time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { fmt.Println("\nDone!") return nil } diff --git a/cli/test/mockcli.go b/cli/test/mockcli.go new file mode 100644 index 000000000..f5d5cfcba --- /dev/null +++ b/cli/test/mockcli.go @@ -0,0 +1,124 @@ +package test + +import ( + "bytes" + "flag" + "strings" + "testing" + + "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/require" + lcli "github.com/urfave/cli/v2" +) + +type mockCLI struct { + t *testing.T + cmds []*lcli.Command + cctx *lcli.Context + out *bytes.Buffer +} + +func newMockCLI(t *testing.T, cmds []*lcli.Command) *mockCLI { + // Create a CLI App with an --api-url flag so that we can specify which node + // the command should be executed against + app := &lcli.App{ + Flags: []lcli.Flag{ + &lcli.StringFlag{ + Name: "api-url", + Hidden: true, + }, + }, + Commands: cmds, + } + + var out bytes.Buffer + app.Writer = &out + app.Setup() + + cctx := lcli.NewContext(app, &flag.FlagSet{}, nil) + return &mockCLI{t: t, cmds: cmds, cctx: cctx, out: &out} +} + +func (c *mockCLI) client(addr multiaddr.Multiaddr) *mockCLIClient { + return &mockCLIClient{t: c.t, cmds: c.cmds, addr: addr, cctx: c.cctx, out: c.out} +} + +// mockCLIClient runs commands against a particular node +type mockCLIClient struct { + t *testing.T + cmds []*lcli.Command + addr multiaddr.Multiaddr + cctx *lcli.Context + out *bytes.Buffer +} + +func (c *mockCLIClient) run(cmd []string, params []string, args []string) string { + // Add parameter --api-url= + apiFlag := "--api-url=" + c.addr.String() + params = append([]string{apiFlag}, params...) + + err := c.cctx.App.Run(append(append(cmd, params...), args...)) + require.NoError(c.t, err) + + // Get the output + str := strings.TrimSpace(c.out.String()) + c.out.Reset() + return str +} + +func (c *mockCLIClient) runCmd(input []string) string { + cmd := c.cmdByNameSub(input[0], input[1]) + out, err := c.runCmdRaw(cmd, input[2:]) + require.NoError(c.t, err) + + return out +} + +func (c *mockCLIClient) cmdByNameSub(name string, sub string) *lcli.Command { + for _, c := range c.cmds { + if c.Name == name { + for _, s := range c.Subcommands { + if s.Name == sub { + return s + } + } + } + } + return nil +} + +func (c *mockCLIClient) runCmdRaw(cmd *lcli.Command, input []string) (string, error) { + // prepend --api-url= + apiFlag := "--api-url=" + c.addr.String() + input = append([]string{apiFlag}, input...) + + fs := c.flagSet(cmd) + err := fs.Parse(input) + require.NoError(c.t, err) + + err = cmd.Action(lcli.NewContext(c.cctx.App, fs, c.cctx)) + + // Get the output + str := strings.TrimSpace(c.out.String()) + c.out.Reset() + return str, err +} + +func (c *mockCLIClient) flagSet(cmd *lcli.Command) *flag.FlagSet { + // Apply app level flags (so we can process --api-url flag) + fs := &flag.FlagSet{} + for _, f := range c.cctx.App.Flags { + err := f.Apply(fs) + if err != nil { + c.t.Fatal(err) + } + } + // Apply command level flags + for _, f := range cmd.Flags { + err := f.Apply(fs) + if err != nil { + c.t.Fatal(err) + } + } + return fs +} diff --git a/cli/test/multisig.go b/cli/test/multisig.go new file mode 100644 index 000000000..e77879346 --- /dev/null +++ b/cli/test/multisig.go @@ -0,0 +1,110 @@ +package test + +import ( + "context" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + lcli "github.com/urfave/cli/v2" +) + +func QuietMiningLogs() { + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") +} + +func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode) { + ctx := context.Background() + + // Create mock CLI + mockCLI := newMockCLI(t, cmds) + clientCLI := mockCLI.client(clientNode.ListenAddr) + + // Create some wallets on the node to use for testing multisig + var walletAddrs []address.Address + for i := 0; i < 4; i++ { + addr, err := clientNode.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + walletAddrs = append(walletAddrs, addr) + + test.SendFunds(ctx, t, clientNode, addr, types.NewInt(1e15)) + } + + // Create an msig with three of the addresses and threshold of two sigs + // msig create --required=2 --duration=50 --value=1000attofil + amtAtto := types.NewInt(1000) + threshold := 2 + paramDuration := "--duration=50" + paramRequired := fmt.Sprintf("--required=%d", threshold) + paramValue := fmt.Sprintf("--value=%dattofil", amtAtto) + cmd := []string{ + "msig", "create", + paramRequired, + paramDuration, + paramValue, + walletAddrs[0].String(), + walletAddrs[1].String(), + walletAddrs[2].String(), + } + out := clientCLI.runCmd(cmd) + fmt.Println(out) + + // Extract msig robust address from output + expCreateOutPrefix := "Created new multisig:" + require.Regexp(t, regexp.MustCompile(expCreateOutPrefix), out) + parts := strings.Split(strings.TrimSpace(strings.Replace(out, expCreateOutPrefix, "", -1)), " ") + require.Len(t, parts, 2) + msigRobustAddr := parts[1] + fmt.Println("msig robust address:", msigRobustAddr) + + // Propose to add a new address to the msig + // msig add-propose --from= + paramFrom := fmt.Sprintf("--from=%s", walletAddrs[0]) + cmd = []string{ + "msig", "add-propose", + paramFrom, + msigRobustAddr, + walletAddrs[3].String(), + } + out = clientCLI.runCmd(cmd) + fmt.Println(out) + + // msig inspect + cmd = []string{"msig", "inspect", "--vesting", "--decode-params", msigRobustAddr} + out = clientCLI.runCmd(cmd) + fmt.Println(out) + + // Expect correct balance + require.Regexp(t, regexp.MustCompile("Balance: 0.000000000000001 FIL"), out) + // Expect 1 transaction + require.Regexp(t, regexp.MustCompile(`Transactions:\s*1`), out) + // Expect transaction to be "AddSigner" + require.Regexp(t, regexp.MustCompile(`AddSigner`), out) + + // Approve adding the new address + // msig add-approve --from= 0 false + txnID := "0" + paramFrom = fmt.Sprintf("--from=%s", walletAddrs[1]) + cmd = []string{ + "msig", "add-approve", + paramFrom, + msigRobustAddr, + walletAddrs[0].String(), + txnID, + walletAddrs[3].String(), + "false", + } + out = clientCLI.runCmd(cmd) + fmt.Println(out) +} diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index d5fac0a06..105f8881e 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -6,16 +6,23 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" ) const ( - LookbackCap = time.Hour + LookbackCap = time.Hour * 12 stateWaitLookbackLimit = abi.ChainEpoch(20) ) @@ -26,9 +33,12 @@ var ( // gatewayDepsAPI defines the API methods that the GatewayAPI depends on // (to make it easy to mock for tests) type gatewayDepsAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) + ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) @@ -37,10 +47,31 @@ type gatewayDepsAPI interface { StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) + StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) + StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) + StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) + StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) + StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) + StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) + StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) } type GatewayAPI struct { - api gatewayDepsAPI + api gatewayDepsAPI + lookbackCap time.Duration +} + +// NewGatewayAPI creates a new GatewayAPI with the default lookback cap +func NewGatewayAPI(api gatewayDepsAPI) *GatewayAPI { + return newGatewayAPI(api, LookbackCap) +} + +// used by the tests +func newGatewayAPI(api gatewayDepsAPI, lookbackCap time.Duration) *GatewayAPI { + return &GatewayAPI{api: api, lookbackCap: lookbackCap} } func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { @@ -76,13 +107,17 @@ func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error } func (a *GatewayAPI) checkTimestamp(at time.Time) error { - if time.Since(at) > LookbackCap { + if time.Since(at) > a.lookbackCap { return ErrLookbackTooLong } return nil } +func (a *GatewayAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { + return a.api.ChainHasObj(ctx, c) +} + func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) @@ -94,9 +129,19 @@ func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (* } func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.api.ChainGetTipSet(ctx, tsk) - if err != nil { - return nil, err + var ts *types.TipSet + if tsk.IsEmpty() { + head, err := a.api.ChainHead(ctx) + if err != nil { + return nil, err + } + ts = head + } else { + gts, err := a.api.ChainGetTipSet(ctx, tsk) + if err != nil { + return nil, err + } + ts = gts } // Check if the tipset key refers to a tipset that's too far in the past @@ -112,6 +157,14 @@ func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoc return a.api.ChainGetTipSetByHeight(ctx, h, tsk) } +func (a *GatewayAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + return a.api.ChainReadObj(ctx, c) +} + +func (a *GatewayAPI) ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) { + return a.api.ChainGetNode(ctx, p) +} + func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -172,6 +225,80 @@ func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence u return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } +func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateReadState(ctx, actor, tsk) +} + +func (a *GatewayAPI) StateMinerPower(ctx context.Context, m address.Address, tsk types.TipSetKey) (*api.MinerPower, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerPower(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerFaults(ctx context.Context, m address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return bitfield.BitField{}, err + } + return a.api.StateMinerFaults(ctx, m, tsk) +} +func (a *GatewayAPI) StateMinerRecoveries(ctx context.Context, m address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return bitfield.BitField{}, err + } + return a.api.StateMinerRecoveries(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerInfo(ctx context.Context, m address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return miner.MinerInfo{}, err + } + return a.api.StateMinerInfo(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]api.Deadline, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerDeadlines(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerAvailableBalance(ctx context.Context, m address.Address, tsk types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.BigInt{}, err + } + return a.api.StateMinerAvailableBalance(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerProvingDeadline(ctx context.Context, m address.Address, tsk types.TipSetKey) (*dline.Info, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerProvingDeadline(ctx, m, tsk) +} + +func (a *GatewayAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.BigInt{}, err + } + return a.api.StateCirculatingSupply(ctx, tsk) + +} + +func (a *GatewayAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return api.CirculatingSupply{}, err + } + return a.api.StateVMCirculatingSupplyInternal(ctx, tsk) +} + +func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { + return sigs.Verify(sig, k, msg) == nil, nil +} + var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.GasModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index f34f887f5..ae161945b 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -88,7 +88,7 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { mock := &mockGatewayDepsAPI{} - a := &GatewayAPI{api: mock} + a := NewGatewayAPI(mock) // Create tipsets from genesis up to tskh and return the highest ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS) @@ -107,6 +107,13 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { type mockGatewayDepsAPI struct { lk sync.RWMutex tipsets []*types.TipSet + + gatewayDepsAPI // satisfies all interface requirements but will panic if + // methods are called. easier than filling out with panic stubs IMO +} + +func (m *mockGatewayDepsAPI) ChainHasObj(context.Context, cid.Cid) (bool, error) { + panic("implement me") } func (m *mockGatewayDepsAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { @@ -158,6 +165,10 @@ func (m *mockGatewayDepsAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.C return m.tipsets[h], nil } +func (m *mockGatewayDepsAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + panic("implement me") +} + func (m *mockGatewayDepsAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { panic("implement me") } @@ -189,3 +200,7 @@ func (m *mockGatewayDepsAPI) StateLookupID(ctx context.Context, addr address.Add func (m *mockGatewayDepsAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { panic("implement me") } + +func (m *mockGatewayDepsAPI) StateReadState(ctx context.Context, act address.Address, ts types.TipSetKey) (*api.ActorState, error) { + panic("implement me") +} diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index eebf56a9f..379c96053 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -4,10 +4,14 @@ import ( "bytes" "context" "fmt" + "math" "os" "testing" "time" + "github.com/filecoin-project/lotus/cli" + clitest "github.com/filecoin-project/lotus/cli/test" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -26,20 +30,22 @@ import ( builder "github.com/filecoin-project/lotus/node/test" ) +const maxLookbackCap = time.Duration(math.MaxInt64) + func init() { policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } -// TestEndToEnd tests that API calls can be made on a lite node that is -// connected through a gateway to a full API node -func TestEndToEnd(t *testing.T) { +// TestEndToEndWalletMsig tests that wallet and msig API calls can be made +// on a lite node that is connected through a gateway to a full API node +func TestEndToEndWalletMsig(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") blocktime := 5 * time.Millisecond ctx := context.Background() - full, lite, closer := startNodes(ctx, t, blocktime) + full, lite, closer := startNodes(ctx, t, blocktime, maxLookbackCap) defer closer() // The full node starts with a wallet @@ -56,11 +62,11 @@ func TestEndToEnd(t *testing.T) { require.NoError(t, err) // Send some funds from the full node to the lite node - err = sendFunds(ctx, t, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) require.NoError(t, err) // Send some funds from the lite node back to the full node - err = sendFunds(ctx, t, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) + err = sendFunds(ctx, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) require.NoError(t, err) // Sign some data with the lite node wallet address @@ -81,7 +87,7 @@ func TestEndToEnd(t *testing.T) { walletAddrs = append(walletAddrs, addr) - err = sendFunds(ctx, t, lite, liteWalletAddr, addr, types.NewInt(1e15)) + err = sendFunds(ctx, lite, liteWalletAddr, addr, types.NewInt(1e15)) require.NoError(t, err) } @@ -135,7 +141,33 @@ func TestEndToEnd(t *testing.T) { require.True(t, approveReturn.Applied) } -func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { +// TestEndToEndMsigCLI tests that msig CLI calls can be made +// on a lite node that is connected through a gateway to a full API node +func TestEndToEndMsigCLI(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + clitest.QuietMiningLogs() + + blocktime := 5 * time.Millisecond + ctx := context.Background() + full, lite, closer := startNodes(ctx, t, blocktime, maxLookbackCap) + defer closer() + + // The full node starts with a wallet + fullWalletAddr, err := full.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // Create a wallet on the lite node + liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + // Send some funds from the full node to the lite node + err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + require.NoError(t, err) + + clitest.RunMultisigTest(t, cli.Commands, lite) +} + +func sendFunds(ctx context.Context, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { msg := &types.Message{ From: fromAddr, To: toAddr, @@ -158,7 +190,7 @@ func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAd return nil } -func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration, lookbackCap time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { var closer jsonrpc.ClientCloser // Create one miner and two full nodes. @@ -175,7 +207,7 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (tes fullNode := nodes[0] // Create a gateway server in front of the full node - _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) + _, addr, err := builder.CreateRPCServer(newGatewayAPI(fullNode, lookbackCap)) require.NoError(t, err) // Create a gateway client API that connects to the gateway server diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index c19599084..34e068ffd 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -76,7 +76,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &GatewayAPI{api: api}) + rpcServer.Register("Filecoin", NewGatewayAPI(api)) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index dfdaf9fa9..b12c069f5 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -5,6 +5,10 @@ import ( "fmt" "strconv" + "github.com/filecoin-project/lotus/chain/gen/genesis" + + _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/docker/go-units" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" @@ -136,6 +140,9 @@ var chainBalanceStateCmd = &cli.Command{ &cli.BoolFlag{ Name: "miner-info", }, + &cli.BoolFlag{ + Name: "robust-addresses", + }, }, Action: func(cctx *cli.Context) error { ctx := context.TODO() @@ -187,6 +194,33 @@ var chainBalanceStateCmd = &cli.Command{ minerInfo := cctx.Bool("miner-info") + robustMap := make(map[address.Address]address.Address) + if cctx.Bool("robust-addresses") { + iact, err := tree.GetActor(_init.Address) + if err != nil { + return xerrors.Errorf("failed to load init actor: %w", err) + } + + ist, err := _init.Load(store, iact) + if err != nil { + return xerrors.Errorf("failed to load init actor state: %w", err) + } + + err = ist.ForEachActor(func(id abi.ActorID, addr address.Address) error { + idAddr, err := address.NewIDAddress(uint64(id)) + if err != nil { + return xerrors.Errorf("failed to write to addr map: %w", err) + } + + robustMap[idAddr] = addr + + return nil + }) + if err != nil { + return xerrors.Errorf("failed to invert init address map: %w", err) + } + } + var infos []accountInfo err = tree.ForEach(func(addr address.Address, act *types.Actor) error { @@ -201,6 +235,23 @@ var chainBalanceStateCmd = &cli.Command{ VestingAmount: types.FIL(big.NewInt(0)), } + if cctx.Bool("robust-addresses") { + robust, found := robustMap[addr] + if found { + ai.Address = robust + } else { + id, err := address.IDFromAddress(addr) + if err != nil { + return xerrors.Errorf("failed to get ID address: %w", err) + } + + // TODO: This is not the correctest way to determine whether a robust address should exist + if id >= genesis.MinerStart { + return xerrors.Errorf("address doesn't have a robust address: %s", addr) + } + } + } + if minerInfo && builtin.IsStorageMinerActor(act.Code) { pow, _, _, err := stmgr.GetPowerRaw(ctx, sm, sroot, addr) if err != nil { diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index f3874ed41..656dd6463 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -156,7 +156,7 @@ var initCmd = &cli.Command{ log.Info("Checking full node sync status") if !cctx.Bool("genesis-miner") && !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, api); err != nil { + if err := lcli.SyncWait(ctx, api, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index bdbb99fe0..83a9ad87c 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -72,7 +72,7 @@ var initRestoreCmd = &cli.Command{ } if !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, api); err != nil { + if err := lcli.SyncWait(ctx, api, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 98a9cfaba..c043bd903 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -84,7 +84,7 @@ var runCmd = &cli.Command{ log.Info("Checking full node sync status") if !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, nodeApi); err != nil { + if err := lcli.SyncWait(ctx, nodeApi, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } diff --git a/cmd/tvx/codenames.go b/cmd/tvx/codenames.go new file mode 100644 index 000000000..851e9d841 --- /dev/null +++ b/cmd/tvx/codenames.go @@ -0,0 +1,37 @@ +package main + +import ( + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/build" +) + +// ProtocolCodenames is a table that summarises the protocol codenames that +// will be set on extracted vectors, depending on the original execution height. +// +// Implementers rely on these names to filter the vectors they can run through +// their implementations, based on their support level +var ProtocolCodenames = []struct { + firstEpoch abi.ChainEpoch + name string +}{ + {0, "genesis"}, + {build.UpgradeBreezeHeight + 1, "breeze"}, + {build.UpgradeSmokeHeight + 1, "smoke"}, + {build.UpgradeIgnitionHeight + 1, "ignition"}, + {build.UpgradeRefuelHeight + 1, "refuel"}, + {build.UpgradeActorsV2Height + 1, "actorsv2"}, + {build.UpgradeTapeHeight + 1, "tape"}, + {build.UpgradeLiftoffHeight + 1, "liftoff"}, +} + +// GetProtocolCodename gets the protocol codename associated with a height. +func GetProtocolCodename(height abi.ChainEpoch) string { + for i, v := range ProtocolCodenames { + if height < v.firstEpoch { + // found the cutoff, return previous. + return ProtocolCodenames[i-1].name + } + } + return ProtocolCodenames[len(ProtocolCodenames)-1].name +} diff --git a/cmd/tvx/codenames_test.go b/cmd/tvx/codenames_test.go new file mode 100644 index 000000000..00d107707 --- /dev/null +++ b/cmd/tvx/codenames_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "math" + "testing" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/build" +) + +func TestProtocolCodenames(t *testing.T) { + if height := abi.ChainEpoch(100); GetProtocolCodename(height) != "genesis" { + t.Fatal("expected genesis codename") + } + + if height := abi.ChainEpoch(build.UpgradeBreezeHeight + 1); GetProtocolCodename(height) != "breeze" { + t.Fatal("expected breeze codename") + } + + if height := build.UpgradeActorsV2Height + 1; GetProtocolCodename(height) != "actorsv2" { + t.Fatal("expected actorsv2 codename") + } + + if height := abi.ChainEpoch(math.MaxInt64); GetProtocolCodename(height) != ProtocolCodenames[len(ProtocolCodenames)-1].name { + t.Fatal("expected last codename") + } +} diff --git a/cmd/tvx/exec.go b/cmd/tvx/exec.go index 9ec6f9e2b..89ad23913 100644 --- a/cmd/tvx/exec.go +++ b/cmd/tvx/exec.go @@ -72,20 +72,24 @@ func runExecLotus(_ *cli.Context) error { func executeTestVector(tv schema.TestVector) error { log.Println("executing test vector:", tv.Meta.ID) - r := new(conformance.LogReporter) - switch class := tv.Class; class { - case "message": - conformance.ExecuteMessageVector(r, &tv) - case "tipset": - conformance.ExecuteTipsetVector(r, &tv) - default: - return fmt.Errorf("test vector class %s not supported", class) - } - if r.Failed() { - log.Println(color.HiRedString("❌ test vector failed")) - } else { - log.Println(color.GreenString("✅ test vector succeeded")) + for _, v := range tv.Pre.Variants { + r := new(conformance.LogReporter) + + switch class, v := tv.Class, v; class { + case "message": + conformance.ExecuteMessageVector(r, &tv, &v) + case "tipset": + conformance.ExecuteTipsetVector(r, &tv, &v) + default: + return fmt.Errorf("test vector class %s not supported", class) + } + + if r.Failed() { + log.Println(color.HiRedString("❌ test vector failed for variant %s", v.ID)) + } else { + log.Println(color.GreenString("✅ test vector succeeded for variant %s", v.ID)) + } } return nil diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 4bb391af6..4a8dd162c 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -347,6 +347,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { return err } + nv, err := fapi.StateNetworkVersion(ctx, execTs.Key()) + if err != nil { + return err + } + + codename := GetProtocolCodename(execTs.Height()) + // Write out the test vector. vector := schema.TestVector{ Class: schema.ClassMessage, @@ -363,10 +370,15 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, + Selector: schema.Selector{ + schema.SelectorMinProtocolVersion: codename, + }, Randomness: recordingRand.Recorded(), CAR: out.Bytes(), Pre: &schema.Preconditions{ - Epoch: int64(execTs.Height()), + Variants: []schema.Variant{ + {ID: codename, Epoch: int64(execTs.Height()), NetworkVersion: uint(nv)}, + }, CircSupply: circSupply.Int, BaseFee: basefee.Int, StateTree: &schema.StateTree{ diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index 93e0d215f..4f574c175 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -87,7 +87,7 @@ type proxyingBlockstore struct { ctx context.Context api api.FullNode - lk sync.RWMutex + lk sync.Mutex tracing bool traced map[cid.Cid]struct{} @@ -113,11 +113,11 @@ func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} { } func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { - pb.lk.RLock() + pb.lk.Lock() if pb.tracing { pb.traced[cid] = struct{}{} } - pb.lk.RUnlock() + pb.lk.Unlock() if block, err := pb.Blockstore.Get(cid); err == nil { return block, err @@ -140,3 +140,12 @@ func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { return block, nil } + +func (pb *proxyingBlockstore) Put(block blocks.Block) error { + pb.lk.Lock() + if pb.tracing { + pb.traced[block.Cid()] = struct{}{} + } + pb.lk.Unlock() + return pb.Blockstore.Put(block) +} diff --git a/conformance/corpus_test.go b/conformance/corpus_test.go index 3d447570d..a09f9a8d3 100644 --- a/conformance/corpus_test.go +++ b/conformance/corpus_test.go @@ -11,6 +11,11 @@ import ( "github.com/filecoin-project/test-vectors/schema" ) +var invokees = map[schema.Class]func(Reporter, *schema.TestVector, *schema.Variant){ + schema.ClassMessage: ExecuteMessageVector, + schema.ClassTipset: ExecuteTipsetVector, +} + const ( // EnvSkipConformance, if 1, skips the conformance test suite. EnvSkipConformance = "SKIP_CONFORMANCE" @@ -120,13 +125,16 @@ func TestConformance(t *testing.T) { } // dispatch the execution depending on the vector class. - switch vector.Class { - case "message": - ExecuteMessageVector(t, &vector) - case "tipset": - ExecuteTipsetVector(t, &vector) - default: - t.Fatalf("test vector class not supported: %s", vector.Class) + invokee, ok := invokees[vector.Class] + if !ok { + t.Fatalf("unsupported test vector class: %s", vector.Class) + } + + for _, variant := range vector.Pre.Variants { + variant := variant + t.Run(variant.ID, func(t *testing.T) { + invokee(t, &vector, &variant) + }) } }) } diff --git a/conformance/driver.go b/conformance/driver.go index f49022b9c..fbb7dda6d 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -82,7 +82,7 @@ type ExecuteTipsetResult struct { // This method returns the the receipts root, the poststate root, and the VM // message results. The latter _include_ implicit messages, such as cron ticks // and reward withdrawal per miner. -func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) { +func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset, execEpoch abi.ChainEpoch) (*ExecuteTipsetResult, error) { var ( syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) vmRand = NewFixedRand() @@ -121,11 +121,10 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot messages []*types.Message results []*vm.ApplyRet - epoch = abi.ChainEpoch(tipset.Epoch) basefee = abi.NewTokenAmount(tipset.BaseFee.Int64()) ) - postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, epoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { + postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, execEpoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { messages = append(messages, msg) results = append(results, ret) return nil diff --git a/conformance/runner.go b/conformance/runner.go index d489ac288..6f9d73305 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -30,11 +30,11 @@ import ( ) // ExecuteMessageVector executes a message-class test vector. -func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { +func ExecuteMessageVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) { var ( - ctx = context.Background() - epoch = vector.Pre.Epoch - root = vector.Pre.StateTree.RootCID + ctx = context.Background() + baseEpoch = variant.Epoch + root = vector.Pre.StateTree.RootCID ) // Load the CAR into a new temporary Blockstore. @@ -53,16 +53,16 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { r.Fatalf("failed to deserialize message: %s", err) } - // add an epoch if one's set. - if m.Epoch != nil { - epoch = *m.Epoch + // add the epoch offset if one is set. + if m.EpochOffset != nil { + baseEpoch += *m.EpochOffset } // Execute the message. var ret *vm.ApplyRet ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{ Preroot: root, - Epoch: abi.ChainEpoch(epoch), + Epoch: abi.ChainEpoch(baseEpoch), Message: msg, BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), @@ -86,10 +86,10 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // ExecuteTipsetVector executes a tipset-class test vector. -func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { +func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) { var ( ctx = context.Background() - prevEpoch = vector.Pre.Epoch + baseEpoch = abi.ChainEpoch(variant.Epoch) root = vector.Pre.StateTree.RootCID tmpds = ds.NewMapDatastore() ) @@ -105,9 +105,11 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { // Apply every tipset. var receiptsIdx int + var prevEpoch = baseEpoch for i, ts := range vector.ApplyTipsets { ts := ts // capture - ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts) + execEpoch := baseEpoch + abi.ChainEpoch(ts.EpochOffset) + ret, err := driver.ExecuteTipset(bs, tmpds, root, prevEpoch, &ts, execEpoch) if err != nil { r.Fatalf("failed to apply tipset %d message: %s", i, err) } @@ -122,7 +124,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual) } - prevEpoch = ts.Epoch + prevEpoch = execEpoch root = ret.PostStateRoot } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index eefc53ab7..c5bc24b04 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -46,6 +46,7 @@ * [ClientMinerQueryOffer](#ClientMinerQueryOffer) * [ClientQueryAsk](#ClientQueryAsk) * [ClientRemoveImport](#ClientRemoveImport) + * [ClientRestartDataTransfer](#ClientRestartDataTransfer) * [ClientRetrieve](#ClientRetrieve) * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) @@ -155,7 +156,6 @@ * [StateMinerRecoveries](#StateMinerRecoveries) * [StateMinerSectorCount](#StateMinerSectorCount) * [StateMinerSectors](#StateMinerSectors) - * [StateMsgGasCost](#StateMsgGasCost) * [StateNetworkName](#StateNetworkName) * [StateNetworkVersion](#StateNetworkVersion) * [StateReadState](#StateReadState) @@ -227,7 +227,7 @@ Response: ```json { "Version": "string value", - "APIVersion": 4096, + "APIVersion": 4352, "BlockDelay": 42 } ``` @@ -1164,6 +1164,23 @@ Inputs: Response: `{}` +### ClientRestartDataTransfer +ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer + + +Perms: write + +Inputs: +```json +[ + 3, + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + true +] +``` + +Response: `{}` + ### ClientRetrieve ClientRetrieve initiates the retrieval of a file, as specified in the order. @@ -2971,7 +2988,7 @@ Response: ## State The State methods are used to query, inspect, and interact with chain state. -All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. +Most methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. @@ -3059,6 +3076,9 @@ Inputs: Response: ```json { + "MsgCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, "Msg": { "Version": 42, "To": "f01234", @@ -3079,6 +3099,18 @@ Response: "Return": "Ynl0ZSBhcnJheQ==", "GasUsed": 9 }, + "GasCost": { + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" + }, "ExecutionTrace": { "Msg": { "Version": 42, @@ -3331,19 +3363,8 @@ Inputs: ```json [ { - "Version": 42, "To": "f01234", - "From": "f01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==", - "CID": { - "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" - } + "From": "f01234" }, [ { @@ -3953,45 +3974,6 @@ Inputs: Response: `null` -### StateMsgGasCost -StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "GasUsed": "0", - "BaseFeeBurn": "0", - "OverEstimationBurn": "0", - "MinerPenalty": "0", - "MinerTip": "0", - "Refund": "0", - "TotalCost": "0" -} -``` - ### StateNetworkName StateNetworkName returns the name of the network the node is synced to @@ -4054,7 +4036,8 @@ Response: ``` ### StateReplay -StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. +StateReplay replays a given message, assuming it was included in a block in the specified tipset. +If no tipset key is provided, the appropriate tipset is looked up. Perms: read @@ -4079,6 +4062,9 @@ Inputs: Response: ```json { + "MsgCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, "Msg": { "Version": 42, "To": "f01234", @@ -4099,6 +4085,18 @@ Response: "Return": "Ynl0ZSBhcnJheQ==", "GasUsed": 9 }, + "GasCost": { + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" + }, "ExecutionTrace": { "Msg": { "Version": 42, diff --git a/extern/test-vectors b/extern/test-vectors index a8f968ade..d9a75a787 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit a8f968adeba1995f161f7be0048188affc425079 +Subproject commit d9a75a7873aee0db28b87e3970d2ea16a2f37c6a diff --git a/go.mod b/go.mod index adff07d32..a6a074e49 100644 --- a/go.mod +++ b/go.mod @@ -27,9 +27,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.1 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.7 + github.com/filecoin-project/go-data-transfer v0.9.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.7.1 + github.com/filecoin-project/go-fil-markets v0.9.1 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -41,7 +41,7 @@ require ( github.com/filecoin-project/specs-actors v0.9.12 github.com/filecoin-project/specs-actors/v2 v2.1.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.4 + github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect @@ -66,7 +66,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.2.1 + github.com/ipfs/go-graphsync v0.3.0 github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 576690c10..4cf7b3628 100644 --- a/go.sum +++ b/go.sum @@ -240,14 +240,14 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.7 h1:Kacr5qz2YWtd3sensU6aXFtES7joeapVDeXApeUD35I= -github.com/filecoin-project/go-data-transfer v0.6.7/go.mod h1:C++k1U6+jMQODOaen5OPDo9XQbth9Yq3ie94vNjBJbk= +github.com/filecoin-project/go-data-transfer v0.9.0 h1:nTT8j7Hu3TM0wRWrGy83/ctawG7sleJGdFWtIsUsKgY= +github.com/filecoin-project/go-data-transfer v0.9.0/go.mod h1:i2CqUy7TMQGKukj9BgqIxiP8nDHDXU2VLd771KVaCaQ= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.7.1 h1:e0NlpSnaeGyDUhCOzevjcxkSA54kt9BzlXpLRgduUFI= -github.com/filecoin-project/go-fil-markets v0.7.1/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= +github.com/filecoin-project/go-fil-markets v0.9.1 h1:MgO+UkpreD6x8DV2Zkw2xlBogixfpw9/wf4+nBii7bU= +github.com/filecoin-project/go-fil-markets v0.9.1/go.mod h1:h+bJ/IUnYjnW5HMKyt9JQSnhslqetkpuzwwugc3K8vM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -264,12 +264,10 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 h github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= -github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b h1:bMUfG6Sy6YSMbsjQAO1Q2vEZldbSdsbRy/FX3OlTck0= github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= @@ -277,15 +275,15 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= -github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= +github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT+dGVWJTr5yDevU00g= github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= -github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= +github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= @@ -540,8 +538,8 @@ github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPi github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= -github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= -github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.3.0 h1:I6Y20kSuCWkUvPoUWo4V3am704/9QjgDVVkf0zIV8+8= +github.com/ipfs/go-graphsync v0.3.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= diff --git a/lib/blockstore/blockstore.go b/lib/blockstore/blockstore.go index 9e74f4373..f274cd708 100644 --- a/lib/blockstore/blockstore.go +++ b/lib/blockstore/blockstore.go @@ -18,19 +18,18 @@ import ( "context" ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" ) // NewTemporary returns a temporary blockstore. -func NewTemporary() blockstore.Blockstore { - return NewBlockstore(ds.NewMapDatastore()) +func NewTemporary() MemStore { + return make(MemStore) } // NewTemporarySync returns a thread-safe temporary blockstore. -func NewTemporarySync() blockstore.Blockstore { - return NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) +func NewTemporarySync() *SyncStore { + return &SyncStore{bs: make(MemStore)} } // WrapIDStore wraps the underlying blockstore in an "identity" blockstore. diff --git a/lib/blockstore/memstore.go b/lib/blockstore/memstore.go new file mode 100644 index 000000000..9745d6f03 --- /dev/null +++ b/lib/blockstore/memstore.go @@ -0,0 +1,80 @@ +package blockstore + +import ( + "context" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" +) + +type MemStore map[cid.Cid]blocks.Block + +func (m MemStore) DeleteBlock(k cid.Cid) error { + delete(m, k) + return nil +} +func (m MemStore) Has(k cid.Cid) (bool, error) { + _, ok := m[k] + return ok, nil +} +func (m MemStore) Get(k cid.Cid) (blocks.Block, error) { + b, ok := m[k] + if !ok { + return nil, blockstore.ErrNotFound + } + return b, nil +} + +// GetSize returns the CIDs mapped BlockSize +func (m MemStore) GetSize(k cid.Cid) (int, error) { + b, ok := m[k] + if !ok { + return 0, blockstore.ErrNotFound + } + return len(b.RawData()), nil +} + +// Put puts a given block to the underlying datastore +func (m MemStore) Put(b blocks.Block) error { + // Convert to a basic block for safety, but try to reuse the existing + // block if it's already a basic block. + k := b.Cid() + if _, ok := b.(*blocks.BasicBlock); !ok { + // If we already have the block, abort. + if _, ok := m[k]; ok { + return nil + } + // the error is only for debugging. + b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid()) + } + m[b.Cid()] = b + return nil +} + +// PutMany puts a slice of blocks at the same time using batching +// capabilities of the underlying datastore whenever possible. +func (m MemStore) PutMany(bs []blocks.Block) error { + for _, b := range bs { + _ = m.Put(b) // can't fail + } + return nil +} + +// AllKeysChan returns a channel from which +// the CIDs in the Blockstore can be read. It should respect +// the given context, closing the channel if it becomes Done. +func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + ch := make(chan cid.Cid, len(m)) + for k := range m { + ch <- k + } + close(ch) + return ch, nil +} + +// HashOnRead specifies if every read block should be +// rehashed to make sure it matches its CID. +func (m MemStore) HashOnRead(enabled bool) { + // no-op +} diff --git a/lib/blockstore/syncstore.go b/lib/blockstore/syncstore.go new file mode 100644 index 000000000..be9f6b5c4 --- /dev/null +++ b/lib/blockstore/syncstore.go @@ -0,0 +1,68 @@ +package blockstore + +import ( + "context" + "sync" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" +) + +type SyncStore struct { + mu sync.RWMutex + bs MemStore // specifically use a memStore to save indirection overhead. +} + +func (m *SyncStore) DeleteBlock(k cid.Cid) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.DeleteBlock(k) +} +func (m *SyncStore) Has(k cid.Cid) (bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.Has(k) +} +func (m *SyncStore) Get(k cid.Cid) (blocks.Block, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.Get(k) +} + +// GetSize returns the CIDs mapped BlockSize +func (m *SyncStore) GetSize(k cid.Cid) (int, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.GetSize(k) +} + +// Put puts a given block to the underlying datastore +func (m *SyncStore) Put(b blocks.Block) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.Put(b) +} + +// PutMany puts a slice of blocks at the same time using batching +// capabilities of the underlying datastore whenever possible. +func (m *SyncStore) PutMany(bs []blocks.Block) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.PutMany(bs) +} + +// AllKeysChan returns a channel from which +// the CIDs in the Blockstore can be read. It should respect +// the given context, closing the channel if it becomes Done. +func (m *SyncStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + m.mu.RLock() + defer m.mu.RUnlock() + // this blockstore implementation doesn't do any async work. + return m.bs.AllKeysChan(ctx) +} + +// HashOnRead specifies if every read block should be +// rehashed to make sure it matches its CID. +func (m *SyncStore) HashOnRead(enabled bool) { + // noop +} diff --git a/lib/bufbstore/buf_bstore.go b/lib/bufbstore/buf_bstore.go index a766c2b52..4ea746444 100644 --- a/lib/bufbstore/buf_bstore.go +++ b/lib/bufbstore/buf_bstore.go @@ -19,10 +19,12 @@ type BufferedBS struct { } func NewBufferedBstore(base bstore.Blockstore) *BufferedBS { - buf := bstore.NewTemporary() + var buf bstore.Blockstore if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" { log.Warn("VM BLOCKSTORE BUFFERING IS DISABLED") buf = base + } else { + buf = bstore.NewTemporary() } return &BufferedBS{ diff --git a/lib/timedbs/timedbs.go b/lib/timedbs/timedbs.go new file mode 100644 index 000000000..bb03a59e9 --- /dev/null +++ b/lib/timedbs/timedbs.go @@ -0,0 +1,156 @@ +package timedbs + +import ( + "context" + "fmt" + "sync" + "time" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "go.uber.org/multierr" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/lib/blockstore" +) + +// TimedCacheBS is a blockstore that keeps blocks for at least the specified +// caching interval before discarding them. Garbage collection must be started +// and stopped by calling Start/Stop. +// +// Under the covers, it's implemented with an active and an inactive blockstore +// that are rotated every cache time interval. This means all blocks will be +// stored at most 2x the cache interval. +type TimedCacheBS struct { + mu sync.RWMutex + active, inactive blockstore.MemStore + + interval time.Duration + closeCh chan struct{} +} + +func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS { + return &TimedCacheBS{ + active: blockstore.NewTemporary(), + inactive: blockstore.NewTemporary(), + interval: cacheTime, + } +} + +func (t *TimedCacheBS) Start(ctx context.Context) error { + t.mu.Lock() + defer t.mu.Unlock() + if t.closeCh != nil { + return fmt.Errorf("already started") + } + t.closeCh = make(chan struct{}) + go func() { + ticker := build.Clock.Ticker(t.interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + t.rotate() + case <-t.closeCh: + return + } + } + }() + return nil +} + +func (t *TimedCacheBS) Stop(ctx context.Context) error { + t.mu.Lock() + defer t.mu.Unlock() + if t.closeCh == nil { + return fmt.Errorf("not started started") + } + select { + case <-t.closeCh: + // already closed + default: + close(t.closeCh) + } + return nil +} + +func (t *TimedCacheBS) rotate() { + newBs := blockstore.NewTemporary() + + t.mu.Lock() + t.inactive, t.active = t.active, newBs + t.mu.Unlock() +} + +func (t *TimedCacheBS) Put(b blocks.Block) error { + // Don't check the inactive set here. We want to keep this block for at + // least one interval. + t.mu.Lock() + defer t.mu.Unlock() + return t.active.Put(b) +} + +func (t *TimedCacheBS) PutMany(bs []blocks.Block) error { + t.mu.Lock() + defer t.mu.Unlock() + return t.active.PutMany(bs) +} + +func (t *TimedCacheBS) Get(k cid.Cid) (blocks.Block, error) { + t.mu.RLock() + defer t.mu.RUnlock() + b, err := t.active.Get(k) + if err == blockstore.ErrNotFound { + b, err = t.inactive.Get(k) + } + return b, err +} + +func (t *TimedCacheBS) GetSize(k cid.Cid) (int, error) { + t.mu.RLock() + defer t.mu.RUnlock() + size, err := t.active.GetSize(k) + if err == blockstore.ErrNotFound { + size, err = t.inactive.GetSize(k) + } + return size, err +} + +func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) { + t.mu.RLock() + defer t.mu.RUnlock() + if has, err := t.active.Has(k); err != nil { + return false, err + } else if has { + return true, nil + } + return t.inactive.Has(k) +} + +func (t *TimedCacheBS) HashOnRead(_ bool) { + // no-op +} + +func (t *TimedCacheBS) DeleteBlock(k cid.Cid) error { + t.mu.Lock() + defer t.mu.Unlock() + return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k)) +} + +func (t *TimedCacheBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + t.mu.RLock() + defer t.mu.RUnlock() + + ch := make(chan cid.Cid, len(t.active)+len(t.inactive)) + for c := range t.active { + ch <- c + } + for c := range t.inactive { + if _, ok := t.active[c]; ok { + continue + } + ch <- c + } + close(ch) + return ch, nil +} diff --git a/lib/timedbs/timedbs_test.go b/lib/timedbs/timedbs_test.go new file mode 100644 index 000000000..2126b9287 --- /dev/null +++ b/lib/timedbs/timedbs_test.go @@ -0,0 +1,78 @@ +package timedbs_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/lib/timedbs" +) + +func TestTimedBSSimple(t *testing.T) { + tc := timedbs.NewTimedCacheBS(10 * time.Millisecond) + _ = tc.Start(context.Background()) + defer func() { + _ = tc.Stop(context.Background()) + }() + + b1 := blocks.NewBlock([]byte("foo")) + require.NoError(t, tc.Put(b1)) + + b2 := blocks.NewBlock([]byte("bar")) + require.NoError(t, tc.Put(b2)) + + b3 := blocks.NewBlock([]byte("baz")) + + b1out, err := tc.Get(b1.Cid()) + require.NoError(t, err) + require.Equal(t, b1.RawData(), b1out.RawData()) + + has, err := tc.Has(b1.Cid()) + require.NoError(t, err) + require.True(t, has) + + time.Sleep(15 * time.Millisecond) + + // We should still have everything. + has, err = tc.Has(b1.Cid()) + require.NoError(t, err) + require.True(t, has) + + has, err = tc.Has(b2.Cid()) + require.NoError(t, err) + require.True(t, has) + + // extend b2, add b3. + require.NoError(t, tc.Put(b2)) + require.NoError(t, tc.Put(b3)) + + // all keys once. + allKeys, err := tc.AllKeysChan(context.Background()) + var ks []cid.Cid + for k := range allKeys { + ks = append(ks, k) + } + require.NoError(t, err) + require.ElementsMatch(t, ks, []cid.Cid{b1.Cid(), b2.Cid(), b3.Cid()}) + + time.Sleep(10 * time.Millisecond) + + // should still have b2, and b3, but not b1 + + has, err = tc.Has(b1.Cid()) + require.NoError(t, err) + require.False(t, has) + + has, err = tc.Has(b2.Cid()) + require.NoError(t, err) + require.True(t, has) + + has, err = tc.Has(b3.Cid()) + require.NoError(t, err) + require.True(t, has) +} diff --git a/markets/dealfilter/cli.go b/markets/dealfilter/cli.go index 2cb9d6c4f..af832bfa0 100644 --- a/markets/dealfilter/cli.go +++ b/markets/dealfilter/cli.go @@ -6,35 +6,57 @@ import ( "encoding/json" "os/exec" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func CliDealFilter(cmd string) dtypes.DealFilter { - // TODO: run some checks on the cmd string - +func CliStorageDealFilter(cmd string) dtypes.StorageDealFilter { return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { - j, err := json.MarshalIndent(deal, "", " ") - if err != nil { - return false, "", err + d := struct { + storagemarket.MinerDeal + DealType string + }{ + MinerDeal: deal, + DealType: "storage", } - - var out bytes.Buffer - - c := exec.Command("sh", "-c", cmd) - c.Stdin = bytes.NewReader(j) - c.Stdout = &out - c.Stderr = &out - - switch err := c.Run().(type) { - case nil: - return true, "", nil - case *exec.ExitError: - return false, out.String(), nil - default: - return false, "filter cmd run error", err - } - + return runDealFilter(ctx, cmd, d) + } +} + +func CliRetrievalDealFilter(cmd string) dtypes.RetrievalDealFilter { + return func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) { + d := struct { + retrievalmarket.ProviderDealState + DealType string + }{ + ProviderDealState: deal, + DealType: "retrieval", + } + return runDealFilter(ctx, cmd, d) + } +} + +func runDealFilter(ctx context.Context, cmd string, deal interface{}) (bool, string, error) { + j, err := json.MarshalIndent(deal, "", " ") + if err != nil { + return false, "", err + } + + var out bytes.Buffer + + c := exec.Command("sh", "-c", cmd) + c.Stdin = bytes.NewReader(j) + c.Stdout = &out + c.Stderr = &out + + switch err := c.Run().(type) { + case nil: + return true, "", nil + case *exec.ExitError: + return false, out.String(), nil + default: + return false, "filter cmd run error", err } } diff --git a/node/builder.go b/node/builder.go index de1f6d856..dcff54e2b 100644 --- a/node/builder.go +++ b/node/builder.go @@ -356,7 +356,8 @@ func Online() Option { Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), Override(new(*storedask.StoredAsk), modules.NewStorageAsk), - Override(new(dtypes.DealFilter), modules.BasicDealFilter(nil)), + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)), + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), Override(new(modules.ProviderDealFunds), modules.NewProviderDealFunds), Override(new(storagemarket.StorageProvider), modules.StorageProvider), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil)), @@ -486,7 +487,11 @@ func ConfigStorageMiner(c interface{}) Option { ConfigCommon(&cfg.Common), If(cfg.Dealmaking.Filter != "", - Override(new(dtypes.DealFilter), modules.BasicDealFilter(dealfilter.CliDealFilter(cfg.Dealmaking.Filter))), + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), + ), + + If(cfg.Dealmaking.RetrievalFilter != "", + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))), ), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees)), diff --git a/node/config/def.go b/node/config/def.go index 7eb7e19f0..1298ed45a 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -45,7 +45,8 @@ type DealmakingConfig struct { PieceCidBlocklist []cid.Cid ExpectedSealDuration Duration - Filter string + Filter string + RetrievalFilter string } type SealingConfig struct { @@ -182,11 +183,11 @@ func DefaultStorageMiner() *StorageMiner { }, Fees: MinerFeeConfig{ - MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05 - MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), - MaxWindowPoStGasFee: types.FIL(types.FromFil(50)), - MaxPublishDealsFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(33))), // 0.03ish - MaxMarketBalanceAddFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(100))), // 0.01 + MaxPreCommitGasFee: types.MustParseFIL("0.025"), + MaxCommitGasFee: types.MustParseFIL("0.05"), + MaxWindowPoStGasFee: types.MustParseFIL("5"), + MaxPublishDealsFee: types.MustParseFIL("0.05"), + MaxMarketBalanceAddFee: types.MustParseFIL("0.007"), }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/impl/client/client.go b/node/impl/client/client.go index fe83fda7b..f04d310ce 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -8,7 +8,6 @@ import ( "github.com/filecoin-project/go-state-types/dline" - datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/big" "golang.org/x/xerrors" @@ -33,6 +32,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/discovery" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -850,6 +850,14 @@ func (a *API) ClientDataTransferUpdates(ctx context.Context) (<-chan api.DataTra return channels, nil } +func (a *API) ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error { + selfPeer := a.Host.ID() + if isInitiator { + return a.DataTransfer.RestartDataTransferChannel(ctx, datatransfer.ChannelID{Initiator: selfPeer, Responder: otherPeer, ID: transferID}) + } + return a.DataTransfer.RestartDataTransferChannel(ctx, datatransfer.ChannelID{Initiator: otherPeer, Responder: selfPeer, ID: transferID}) +} + func newDealInfo(v storagemarket.ClientDeal) api.DealInfo { return api.DealInfo{ ProposalCid: v.ProposalCid, diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 5b4f41114..095b51ed8 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -40,9 +40,11 @@ import ( var log = logging.Logger("fullnode") type ChainModuleAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) } // ChainModule provides a default implementation of ChainModuleAPI. @@ -206,8 +208,8 @@ func (m *ChainModule) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpo return m.Chain.GetTipsetByHeight(ctx, h, ts, true) } -func (a *ChainAPI) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { - blk, err := a.Chain.Blockstore().Get(obj) +func (m *ChainModule) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { + blk, err := m.Chain.Blockstore().Get(obj) if err != nil { return nil, xerrors.Errorf("blockstore get: %w", err) } @@ -219,8 +221,8 @@ func (a *ChainAPI) ChainDeleteObj(ctx context.Context, obj cid.Cid) error { return a.Chain.Blockstore().DeleteBlock(obj) } -func (a *ChainAPI) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) { - return a.Chain.Blockstore().Has(obj) +func (m *ChainModule) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) { + return m.Chain.Blockstore().Has(obj) } func (a *ChainAPI) ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (api.ObjStat, error) { diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 0cb1eb084..2e5063ccb 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -277,7 +277,7 @@ func (m *GasModule) GasEstimateMessageGas(ctx context.Context, msg *types.Messag } if msg.GasPremium == types.EmptyInt || types.BigCmp(msg.GasPremium, types.NewInt(0)) == 0 { - gasPremium, err := m.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) + gasPremium, err := m.GasEstimateGasPremium(ctx, 10, msg.From, msg.GasLimit, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas price: %w", err) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 2cdbf00e4..43efd3d6a 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -347,11 +347,33 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. } func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) + msgToReplay := mc + var ts *types.TipSet + if tsk == types.EmptyTSK { + mlkp, err := a.StateSearchMsg(ctx, mc) + if err != nil { + return nil, xerrors.Errorf("searching for msg %s: %w", mc, err) + } + if mlkp == nil { + return nil, xerrors.Errorf("didn't find msg %s", mc) + } + + msgToReplay = mlkp.Message + + executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) + } + + ts, err = a.Chain.LoadTipSet(executionTs.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) + } + } else { + ts = a.Chain.GetHeaviestTipSet() } - m, r, err := a.StateManager.Replay(ctx, ts, mc) + + m, r, err := a.StateManager.Replay(ctx, ts, msgToReplay) if err != nil { return nil, err } @@ -362,8 +384,10 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. } return &api.InvocResult{ + MsgCid: msgToReplay, Msg: m, MsgRct: &r.MessageReceipt, + GasCost: stmgr.MakeMsgGasCost(m, r), ExecutionTrace: r.ExecutionTrace, Error: errstr, Duration: r.Duration, @@ -759,7 +783,7 @@ func (a *StateAPI) StateSectorPartition(ctx context.Context, maddr address.Addre return mas.FindSector(sectorNumber) } -func (a *StateAPI) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toheight abi.ChainEpoch) ([]cid.Cid, error) { +func (a *StateAPI) StateListMessages(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toheight abi.ChainEpoch) ([]cid.Cid, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -1284,52 +1308,3 @@ func (a *StateAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) return a.StateManager.GetNtwkVersion(ctx, ts.Height()), nil } - -func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { - var msg cid.Cid - var ts *types.TipSet - var err error - if tsk != types.EmptyTSK { - msg = inputMsg - ts, err = a.Chain.LoadTipSet(tsk) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) - } - } else { - mlkp, err := a.StateSearchMsg(ctx, inputMsg) - if err != nil { - return nil, xerrors.Errorf("searching for msg %s: %w", inputMsg, err) - } - if mlkp == nil { - return nil, xerrors.Errorf("didn't find msg %s", inputMsg) - } - - executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) - } - - ts, err = a.Chain.LoadTipSet(executionTs.Parents()) - if err != nil { - return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) - } - - msg = mlkp.Message - } - - m, r, err := a.StateManager.Replay(ctx, ts, msg) - if err != nil { - return nil, err - } - - return &api.MsgGasCost{ - Message: msg, - GasUsed: big.NewInt(r.GasUsed), - BaseFeeBurn: r.GasCosts.BaseFeeBurn, - OverEstimationBurn: r.GasCosts.OverEstimationBurn, - MinerPenalty: r.GasCosts.MinerPenalty, - MinerTip: r.GasCosts.MinerTip, - Refund: r.GasCosts.Refund, - TotalCost: big.Sub(m.RequiredFunds(), r.GasCosts.Refund), - }, nil -} diff --git a/node/modules/chain.go b/node/modules/chain.go index f563b4cdd..ce3e9f749 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "os" + "time" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" @@ -30,6 +31,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" + "github.com/filecoin-project/lotus/lib/timedbs" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" @@ -41,8 +44,15 @@ func ChainBitswap(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt r bitswapNetwork := network.NewFromIpfsHost(host, rt, network.Prefix("/chain")) bitswapOptions := []bitswap.Option{bitswap.ProvideEnabled(false)} + // Write all incoming bitswap blocks into a temporary blockstore for two + // block times. If they validate, they'll be persisted later. + cache := timedbs.NewTimedCacheBS(2 * time.Duration(build.BlockDelaySecs) * time.Second) + lc.Append(fx.Hook{OnStop: cache.Stop, OnStart: cache.Start}) + + bitswapBs := bufbstore.NewTieredBstore(bs, cache) + // Use just exch.Close(), closing the context is not needed - exch := bitswap.New(mctx, bitswapNetwork, bs, bitswapOptions...) + exch := bitswap.New(mctx, bitswapNetwork, bitswapBs, bitswapOptions...) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return exch.Close() diff --git a/node/modules/client.go b/node/modules/client.go index d012e4539..f1380bc97 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -91,6 +91,7 @@ func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Grap return nil, err } + dt.OnReady(marketevents.ReadyLogger("client data transfer")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return dt.Start(ctx) diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 5bb439b4d..1ef157b7e 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" @@ -71,4 +72,5 @@ type SetExpectedSealDurationFunc func(time.Duration) error // too determine how long sealing is expected to take type GetExpectedSealDurationFunc func() (time.Duration, error) -type DealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) +type StorageDealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) +type RetrievalDealFilter func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index e7a349e06..9040a91a4 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -268,6 +268,7 @@ func NewProviderDAGServiceDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.S return nil, err } + dt.OnReady(marketevents.ReadyLogger("provider data transfer")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return dt.Start(ctx) @@ -393,13 +394,13 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat if err != nil { return nil, err } - storedAsk, err := storedask.NewStoredAsk(namespace.Wrap(providerDs, datastore.NewKey("/storage-ask")), datastore.NewKey("latest"), spn, address.Address(minerAddress)) + storedAsk, err := storedask.NewStoredAsk(namespace.Wrap(providerDs, datastore.NewKey("/storage-ask")), datastore.NewKey("latest"), spn, address.Address(minerAddress), + storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) if err != nil { return nil, err } - // Hacky way to set max piece size to the sector size a := storedAsk.GetAsk().Ask - err = storedAsk.SetAsk(a.Price, a.VerifiedPrice, a.Expiry-a.Timestamp, storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) + err = storedAsk.SetAsk(a.Price, a.VerifiedPrice, a.Expiry-a.Timestamp) if err != nil { return storedAsk, err } @@ -412,16 +413,16 @@ func NewProviderDealFunds(ds dtypes.MetadataDS) (ProviderDealFunds, error) { return funds.NewDealFunds(ds, datastore.NewKey("/marketfunds/provider")) } -func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, +func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc, expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc, - spn storagemarket.StorageProviderNode) dtypes.DealFilter { + spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter { return func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc, expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc, - spn storagemarket.StorageProviderNode) dtypes.DealFilter { + spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter { return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { b, err := onlineOk() @@ -474,7 +475,7 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline // Reject if it's more than 7 days in the future // TODO: read from cfg - maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.EpochsInDay) + maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.SecondsInDay/build.BlockDelaySecs) if deal.Proposal.StartEpoch > maxStartEpoch { return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil } @@ -497,7 +498,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, - df dtypes.DealFilter, + df dtypes.StorageDealFilter, funds ProviderDealFunds, ) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) @@ -511,8 +512,52 @@ func StorageProvider(minerAddress dtypes.MinerAddress, return storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), store, mds, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk, funds, opt) } +func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter { + return func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter { + return func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) { + b, err := onlineOk() + if err != nil { + return false, "miner error", err + } + + if !b { + log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client") + return false, "miner is not accepting online retrieval deals", nil + } + + b, err = offlineOk() + if err != nil { + return false, "miner error", err + } + + if !b { + log.Info("offline retrieval has not been implemented yet") + } + + if userFilter != nil { + return userFilter(ctx, state) + } + + return true, "", nil + } + } +} + // RetrievalProvider creates a new retrieval provider attached to the provider blockstore -func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, mds dtypes.StagingMultiDstore, dt dtypes.ProviderDataTransfer, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) { +func RetrievalProvider(h host.Host, + miner *storage.Miner, + sealer sectorstorage.SectorManager, + full lapi.FullNode, + ds dtypes.MetadataDS, + pieceStore dtypes.ProviderPieceStore, + mds dtypes.StagingMultiDstore, + dt dtypes.ProviderDataTransfer, + onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc, + userFilter dtypes.RetrievalDealFilter, +) (retrievalmarket.RetrievalProvider, error) { adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full) maddr, err := minerAddrFromDS(ds) @@ -521,29 +566,7 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S } netwk := rmnet.NewFromLibp2pHost(h) - - opt := retrievalimpl.DealDeciderOpt(func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) { - b, err := onlineOk() - if err != nil { - return false, "miner error", err - } - - if !b { - log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client") - return false, "miner is not accepting online retrieval deals", nil - } - - b, err = offlineOk() - if err != nil { - return false, "miner error", err - } - - if !b { - log.Info("offline retrieval has not been implemented yet") - } - - return true, "", nil - }) + opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter)) return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt) }