From f90cfda2e63a90eb9ec6874b21c96b134177681f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 4 Sep 2020 22:17:28 +0200 Subject: [PATCH 001/125] wallet: Add interface layer --- chain/gen/gen.go | 20 ++--- chain/gen/mining.go | 4 +- chain/messagepool/messagepool_test.go | 24 +++--- chain/messagepool/repub_test.go | 5 +- chain/messagepool/selection_test.go | 44 +++++----- chain/stmgr/forks_test.go | 4 +- chain/sync_test.go | 2 +- chain/types/mock/chain.go | 4 +- chain/vectors/gen/main.go | 9 +- chain/wallet/interface.go | 27 ++++++ chain/wallet/key.go | 81 ++++++++++++++++++ chain/wallet/wallet.go | 114 +++++++------------------- cmd/lotus-keygen/main.go | 4 +- cmd/lotus-shed/keyinfo.go | 2 +- markets/storageadapter/client.go | 4 +- node/builder.go | 4 +- node/impl/full/state.go | 2 +- node/impl/full/wallet.go | 45 ++-------- 18 files changed, 214 insertions(+), 185 deletions(-) create mode 100644 chain/wallet/interface.go create mode 100644 chain/wallet/key.go diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 551c3703f..4ba127ce9 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -68,7 +68,7 @@ type ChainGen struct { GetMessages func(*ChainGen) ([]*types.SignedMessage, error) - w *wallet.Wallet + w *wallet.LocalWallet eppProvs map[address.Address]WinningPoStProver Miners []address.Address @@ -151,14 +151,14 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("creating memrepo wallet failed: %w", err) } - banker, err := w.GenerateKey(crypto.SigTypeSecp256k1) + banker, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { return nil, xerrors.Errorf("failed to generate banker key: %w", err) } receievers := make([]address.Address, msgsPerBlock) for r := range receievers { - receievers[r], err = w.GenerateKey(crypto.SigTypeBLS) + receievers[r], err = w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { return nil, xerrors.Errorf("failed to generate receiver key: %w", err) } @@ -188,11 +188,11 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, err } - mk1, err := w.Import(k1) + mk1, err := w.WalletImport(context.Background(), k1) if err != nil { return nil, err } - mk2, err := w.Import(k2) + mk2, err := w.WalletImport(context.Background(), k2) if err != nil { return nil, err } @@ -372,7 +372,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add return nil, nil, nil, xerrors.Errorf("get miner worker: %w", err) } - vrfout, err := ComputeVRF(ctx, cg.w.Sign, worker, ticketRand) + vrfout, err := ComputeVRF(ctx, cg.w.WalletSign, worker, ticketRand) if err != nil { return nil, nil, nil, xerrors.Errorf("compute VRF: %w", err) } @@ -501,7 +501,7 @@ func (cg *ChainGen) Banker() address.Address { return cg.banker } -func (cg *ChainGen) Wallet() *wallet.Wallet { +func (cg *ChainGen) Wallet() *wallet.LocalWallet { return cg.w } @@ -523,7 +523,7 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) { GasPremium: types.NewInt(0), } - sig, err := cg.w.Sign(context.TODO(), cg.banker, msg.Cid().Bytes()) + sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes()) if err != nil { return nil, err } @@ -554,7 +554,7 @@ type MiningCheckAPI interface { } type mca struct { - w *wallet.Wallet + w *wallet.LocalWallet sm *stmgr.StateManager pv ffiwrapper.Verifier bcn beacon.RandomBeacon @@ -583,7 +583,7 @@ func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoc } func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*crypto.Signature, error) { - return mca.w.Sign(ctx, a, v) + return mca.w.WalletSign(ctx, a, v) } type WinningPoStProver interface { diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 260c96808..a9ac47a60 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -19,7 +19,7 @@ import ( "github.com/filecoin-project/lotus/lib/sigs/bls" ) -func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wallet, bt *api.BlockTemplate) (*types.FullBlock, error) { +func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.LocalWallet, bt *api.BlockTemplate) (*types.FullBlock, error) { pts, err := sm.ChainStore().LoadTipSet(bt.Parents) if err != nil { @@ -131,7 +131,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal return nil, xerrors.Errorf("failed to get signing bytes for block: %w", err) } - sig, err := w.Sign(ctx, waddr, nosigbytes) + sig, err := w.WalletSign(ctx, waddr, nosigbytes) if err != nil { return nil, xerrors.Errorf("failed to sign new block: %w", err) } diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 484c72746..cf77f3fb0 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -221,7 +221,7 @@ func TestMessagePool(t *testing.T) { a := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -262,7 +262,7 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) { a := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -312,7 +312,7 @@ func TestRevertMessages(t *testing.T) { a := tma.nextBlock() b := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -375,7 +375,7 @@ func TestPruningSimple(t *testing.T) { a := tma.nextBlock() tma.applyBlock(t, a) - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -422,7 +422,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -432,7 +432,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -494,7 +494,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -504,7 +504,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -548,7 +548,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -558,7 +558,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -609,7 +609,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -619,7 +619,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/repub_test.go b/chain/messagepool/repub_test.go index c89367f0e..2e3fa123f 100644 --- a/chain/messagepool/repub_test.go +++ b/chain/messagepool/repub_test.go @@ -1,6 +1,7 @@ package messagepool import ( + "context" "testing" "time" @@ -32,7 +33,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -42,7 +43,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index a9ead3c01..ecbf08b13 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -29,7 +29,7 @@ func init() { MaxActorPendingMessages = 1000000 } -func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, gasLimit int64, gasPrice uint64) *types.SignedMessage { +func makeTestMessage(w *wallet.LocalWallet, from, to address.Address, nonce uint64, gasLimit int64, gasPrice uint64) *types.SignedMessage { msg := &types.Message{ From: from, To: to, @@ -40,7 +40,7 @@ func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, g GasFeeCap: types.NewInt(100 + gasPrice), GasPremium: types.NewInt(gasPrice), } - sig, err := w.Sign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes()) if err != nil { panic(err) } @@ -70,7 +70,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -308,7 +308,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -318,7 +318,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -384,7 +384,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -394,7 +394,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -528,7 +528,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -538,7 +538,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -591,7 +591,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -601,7 +601,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -670,7 +670,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -680,7 +680,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -740,7 +740,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -750,7 +750,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -807,7 +807,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -817,7 +817,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -883,7 +883,7 @@ func TestOptimalMessageSelection3(t *testing.T) { nActors := 10 // the actors var actors []address.Address - var wallets []*wallet.Wallet + var wallets []*wallet.LocalWallet for i := 0; i < nActors; i++ { w, err := wallet.NewWallet(wallet.NewMemKeyStore()) @@ -891,7 +891,7 @@ func TestOptimalMessageSelection3(t *testing.T) { t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -963,7 +963,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu nActors := 300 // the actors var actors []address.Address - var wallets []*wallet.Wallet + var wallets []*wallet.LocalWallet for i := 0; i < nActors; i++ { w, err := wallet.NewWallet(wallet.NewMemKeyStore()) @@ -971,7 +971,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index caa63c879..e5874d51d 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -174,7 +174,7 @@ func TestForkHeightTriggers(t *testing.T) { Params: enc, GasLimit: types.TestGasLimit, } - sig, err := cg.Wallet().Sign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes()) if err != nil { t.Fatal(err) } @@ -202,7 +202,7 @@ func TestForkHeightTriggers(t *testing.T) { } nonce++ - sig, err := cg.Wallet().Sign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes()) if err != nil { return nil, err } diff --git a/chain/sync_test.go b/chain/sync_test.go index cf1385baa..9a98b3b36 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -566,7 +566,7 @@ func TestDuplicateNonce(t *testing.T) { GasPremium: types.NewInt(0), } - sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) require.NoError(t, err) return &types.SignedMessage{ diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 33b13d408..19b1352ad 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -21,7 +21,7 @@ func Address(i uint64) address.Address { return a } -func MkMessage(from, to address.Address, nonce uint64, w *wallet.Wallet) *types.SignedMessage { +func MkMessage(from, to address.Address, nonce uint64, w *wallet.LocalWallet) *types.SignedMessage { msg := &types.Message{ To: to, From: from, @@ -32,7 +32,7 @@ func MkMessage(from, to address.Address, nonce uint64, w *wallet.Wallet) *types. GasPremium: types.NewInt(1), } - sig, err := w.Sign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes()) if err != nil { panic(err) } diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index ce8d137e8..631fb6ad1 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "math/rand" @@ -63,11 +64,11 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { panic(err) } - blsk, err := w.GenerateKey(crypto.SigTypeBLS) + blsk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { panic(err) } - bki, err := w.Export(blsk) + bki, err := w.WalletExport(context.Background(), blsk) if err != nil { panic(err) } @@ -87,11 +88,11 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { Signature: &bmsg.Signature, } - secpk, err := w.GenerateKey(crypto.SigTypeBLS) + secpk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { panic(err) } - ski, err := w.Export(secpk) + ski, err := w.WalletExport(context.Background(), secpk) if err != nil { panic(err) } diff --git a/chain/wallet/interface.go b/chain/wallet/interface.go new file mode 100644 index 000000000..9dc95a32c --- /dev/null +++ b/chain/wallet/interface.go @@ -0,0 +1,27 @@ +package wallet + +import ( + "context" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/crypto" +) + +type Wallet interface { + WalletNew(context.Context, crypto.SigType) (address.Address, error) + WalletHas(context.Context, address.Address) (bool, error) + WalletList(context.Context) ([]address.Address, error) + WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) + // nonce keeping done by wallet app + WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) + WalletExport(context.Context, address.Address) (*types.KeyInfo, error) + WalletImport(context.Context, *types.KeyInfo) (address.Address, error) + WalletDelete(context.Context, address.Address) error +} + +type Default interface { + GetDefault() (address.Address, error) + SetDefault(a address.Address) error +} diff --git a/chain/wallet/key.go b/chain/wallet/key.go new file mode 100644 index 000000000..2d4ecafd7 --- /dev/null +++ b/chain/wallet/key.go @@ -0,0 +1,81 @@ +package wallet + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/crypto" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" +) + +func GenerateKey(typ crypto.SigType) (*Key, error) { + pk, err := sigs.Generate(typ) + if err != nil { + return nil, err + } + ki := types.KeyInfo{ + Type: kstoreSigType(typ), + PrivateKey: pk, + } + return NewKey(ki) +} + +type Key struct { + types.KeyInfo + + PublicKey []byte + Address address.Address +} + +func NewKey(keyinfo types.KeyInfo) (*Key, error) { + k := &Key{ + KeyInfo: keyinfo, + } + + var err error + k.PublicKey, err = sigs.ToPublic(ActSigType(k.Type), k.PrivateKey) + if err != nil { + return nil, err + } + + switch k.Type { + case KTSecp256k1: + k.Address, err = address.NewSecp256k1Address(k.PublicKey) + if err != nil { + return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) + } + case KTBLS: + k.Address, err = address.NewBLSAddress(k.PublicKey) + if err != nil { + return nil, xerrors.Errorf("converting BLS to address: %w", err) + } + default: + return nil, xerrors.Errorf("unknown key type") + } + return k, nil + +} + +func kstoreSigType(typ crypto.SigType) string { + switch typ { + case crypto.SigTypeBLS: + return KTBLS + case crypto.SigTypeSecp256k1: + return KTSecp256k1 + default: + return "" + } +} + +func ActSigType(typ string) crypto.SigType { + switch typ { + case KTBLS: + return crypto.SigTypeBLS + case KTSecp256k1: + return crypto.SigTypeSecp256k1 + default: + return 0 + } +} diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 9c069d819..7fdea8963 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -29,15 +29,15 @@ const ( KTSecp256k1 = "secp256k1" ) -type Wallet struct { +type LocalWallet struct { keys map[address.Address]*Key keystore types.KeyStore lk sync.Mutex } -func NewWallet(keystore types.KeyStore) (*Wallet, error) { - w := &Wallet{ +func NewWallet(keystore types.KeyStore) (*LocalWallet, error) { + w := &LocalWallet{ keys: make(map[address.Address]*Key), keystore: keystore, } @@ -45,18 +45,18 @@ func NewWallet(keystore types.KeyStore) (*Wallet, error) { return w, nil } -func KeyWallet(keys ...*Key) *Wallet { +func KeyWallet(keys ...*Key) *LocalWallet { m := make(map[address.Address]*Key) for _, key := range keys { m[key.Address] = key } - return &Wallet{ + return &LocalWallet{ keys: m, } } -func (w *Wallet) Sign(ctx context.Context, addr address.Address, msg []byte) (*crypto.Signature, error) { +func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg []byte) (*crypto.Signature, error) { ki, err := w.findKey(addr) if err != nil { return nil, err @@ -68,7 +68,21 @@ func (w *Wallet) Sign(ctx context.Context, addr address.Address, msg []byte) (*c return sigs.Sign(ActSigType(ki.Type), ki.PrivateKey, msg) } -func (w *Wallet) findKey(addr address.Address) (*Key, error) { +func (w *LocalWallet) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { + mcid := msg.Cid() + + sig, err := w.WalletSign(ctx, k, mcid.Bytes()) + if err != nil { + return nil, xerrors.Errorf("failed to sign message: %w", err) + } + + return &types.SignedMessage{ + Message: *msg, + Signature: *sig, + }, nil +} + +func (w *LocalWallet) findKey(addr address.Address) (*Key, error) { w.lk.Lock() defer w.lk.Unlock() @@ -96,7 +110,7 @@ func (w *Wallet) findKey(addr address.Address) (*Key, error) { return k, nil } -func (w *Wallet) Export(addr address.Address) (*types.KeyInfo, error) { +func (w *LocalWallet) WalletExport(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { k, err := w.findKey(addr) if err != nil { return nil, xerrors.Errorf("failed to find key to export: %w", err) @@ -105,7 +119,7 @@ func (w *Wallet) Export(addr address.Address) (*types.KeyInfo, error) { return &k.KeyInfo, nil } -func (w *Wallet) Import(ki *types.KeyInfo) (address.Address, error) { +func (w *LocalWallet) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -121,7 +135,7 @@ func (w *Wallet) Import(ki *types.KeyInfo) (address.Address, error) { return k.Address, nil } -func (w *Wallet) ListAddrs() ([]address.Address, error) { +func (w *LocalWallet) WalletList(ctx context.Context) ([]address.Address, error) { all, err := w.keystore.List() if err != nil { return nil, xerrors.Errorf("listing keystore: %w", err) @@ -144,7 +158,7 @@ func (w *Wallet) ListAddrs() ([]address.Address, error) { return out, nil } -func (w *Wallet) GetDefault() (address.Address, error) { +func (w *LocalWallet) GetDefault() (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -161,7 +175,7 @@ func (w *Wallet) GetDefault() (address.Address, error) { return k.Address, nil } -func (w *Wallet) SetDefault(a address.Address) error { +func (w *LocalWallet) SetDefault(a address.Address) error { w.lk.Lock() defer w.lk.Unlock() @@ -183,19 +197,7 @@ func (w *Wallet) SetDefault(a address.Address) error { return nil } -func GenerateKey(typ crypto.SigType) (*Key, error) { - pk, err := sigs.Generate(typ) - if err != nil { - return nil, err - } - ki := types.KeyInfo{ - Type: kstoreSigType(typ), - PrivateKey: pk, - } - return NewKey(ki) -} - -func (w *Wallet) GenerateKey(typ crypto.SigType) (address.Address, error) { +func (w *LocalWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -223,7 +225,7 @@ func (w *Wallet) GenerateKey(typ crypto.SigType) (address.Address, error) { return k.Address, nil } -func (w *Wallet) HasKey(addr address.Address) (bool, error) { +func (w *LocalWallet) WalletHas(ctx context.Context, addr address.Address) (bool, error) { k, err := w.findKey(addr) if err != nil { return false, err @@ -231,7 +233,7 @@ func (w *Wallet) HasKey(addr address.Address) (bool, error) { return k != nil, nil } -func (w *Wallet) DeleteKey(addr address.Address) error { +func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) error { k, err := w.findKey(addr) if err != nil { return xerrors.Errorf("failed to delete key %s : %w", addr, err) @@ -248,60 +250,4 @@ func (w *Wallet) DeleteKey(addr address.Address) error { return nil } -type Key struct { - types.KeyInfo - - PublicKey []byte - Address address.Address -} - -func NewKey(keyinfo types.KeyInfo) (*Key, error) { - k := &Key{ - KeyInfo: keyinfo, - } - - var err error - k.PublicKey, err = sigs.ToPublic(ActSigType(k.Type), k.PrivateKey) - if err != nil { - return nil, err - } - - switch k.Type { - case KTSecp256k1: - k.Address, err = address.NewSecp256k1Address(k.PublicKey) - if err != nil { - return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) - } - case KTBLS: - k.Address, err = address.NewBLSAddress(k.PublicKey) - if err != nil { - return nil, xerrors.Errorf("converting BLS to address: %w", err) - } - default: - return nil, xerrors.Errorf("unknown key type") - } - return k, nil - -} - -func kstoreSigType(typ crypto.SigType) string { - switch typ { - case crypto.SigTypeBLS: - return KTBLS - case crypto.SigTypeSecp256k1: - return KTSecp256k1 - default: - return "" - } -} - -func ActSigType(typ string) crypto.SigType { - switch typ { - case KTBLS: - return crypto.SigTypeBLS - case KTSecp256k1: - return crypto.SigTypeSecp256k1 - default: - return 0 - } -} +var _ Wallet = &LocalWallet{} diff --git a/cmd/lotus-keygen/main.go b/cmd/lotus-keygen/main.go index 33124107e..8835a0928 100644 --- a/cmd/lotus-keygen/main.go +++ b/cmd/lotus-keygen/main.go @@ -40,12 +40,12 @@ func main() { return fmt.Errorf("unrecognized key type: %q", cctx.String("type")) } - kaddr, err := w.GenerateKey(kt) + kaddr, err := w.WalletNew(cctx.Context, kt) if err != nil { return err } - ki, err := w.Export(kaddr) + ki, err := w.WalletExport(cctx.Context, kaddr) if err != nil { return err } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index f397bd4c7..b2ff97e4e 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -130,7 +130,7 @@ var keyinfoImportCmd = &cli.Command{ return err } - addr, err := w.Import(&keyInfo) + addr, err := w.WalletImport(cctx.Context, &keyInfo) if err != nil { return err } diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 91fc6a054..03e5b5353 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -440,7 +440,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add return nil, err } - sig, err := c.Wallet.Sign(ctx, signer, buf) + sig, err := c.Wallet.WalletSign(ctx, signer, buf) if err != nil { return nil, err } @@ -524,7 +524,7 @@ func (c *ClientNodeAdapter) SignBytes(ctx context.Context, signer address.Addres return nil, err } - localSignature, err := c.Wallet.Sign(ctx, signer, b) + localSignature, err := c.Wallet.WalletSign(ctx, signer, b) if err != nil { return nil, err } diff --git a/node/builder.go b/node/builder.go index 5b6966cd4..7c3251df2 100644 --- a/node/builder.go +++ b/node/builder.go @@ -234,7 +234,9 @@ func Online() Option { Override(new(vm.SyscallBuilder), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), Override(new(*stmgr.StateManager), stmgr.NewStateManager), - Override(new(*wallet.Wallet), wallet.NewWallet), + Override(new(*wallet.LocalWallet), wallet.NewWallet), + Override(new(wallet.Wallet), From(new(*wallet.LocalWallet))), + Override(new(wallet.Default), From(new(*wallet.LocalWallet))), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 36721a93d..0e1aad27c 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -48,7 +48,7 @@ type StateAPI struct { // TODO: the wallet here is only needed because we have the MinerCreateBlock // API attached to the state API. It probably should live somewhere better - Wallet *wallet.Wallet + Wallet *wallet.LocalWallet ProofVerifier ffiwrapper.Verifier StateManager *stmgr.StateManager diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 440f9642f..22b674737 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -20,19 +20,8 @@ type WalletAPI struct { fx.In StateManager *stmgr.StateManager - Wallet *wallet.Wallet -} - -func (a *WalletAPI) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { - return a.Wallet.GenerateKey(typ) -} - -func (a *WalletAPI) WalletHas(ctx context.Context, addr address.Address) (bool, error) { - return a.Wallet.HasKey(addr) -} - -func (a *WalletAPI) WalletList(ctx context.Context) ([]address.Address, error) { - return a.Wallet.ListAddrs() + Default wallet.Default + wallet.Wallet } func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) { @@ -53,21 +42,15 @@ func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byt if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.Wallet.Sign(ctx, keyAddr, msg) + return a.WalletSign(ctx, keyAddr, msg) } func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - mcid := msg.Cid() - - sig, err := a.WalletSign(ctx, k, mcid.Bytes()) + keyAddr, err := a.StateManager.ResolveToKeyAddress(ctx, k, nil) if err != nil { - return nil, xerrors.Errorf("failed to sign message: %w", err) + return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - - return &types.SignedMessage{ - Message: *msg, - Signature: *sig, - }, nil + return a.Wallet.WalletSignMessage(ctx, keyAddr, msg) } func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool { @@ -75,21 +58,9 @@ func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []b } func (a *WalletAPI) WalletDefaultAddress(ctx context.Context) (address.Address, error) { - return a.Wallet.GetDefault() + return a.Default.GetDefault() } func (a *WalletAPI) WalletSetDefault(ctx context.Context, addr address.Address) error { - return a.Wallet.SetDefault(addr) -} - -func (a *WalletAPI) WalletExport(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { - return a.Wallet.Export(addr) -} - -func (a *WalletAPI) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { - return a.Wallet.Import(ki) -} - -func (a *WalletAPI) WalletDelete(ctx context.Context, addr address.Address) error { - return a.Wallet.DeleteKey(addr) + return a.Default.SetDefault(addr) } From 780f6dba34afbb65f178d208fad8a3e636cf56a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 5 Sep 2020 21:36:32 +0200 Subject: [PATCH 002/125] Remote wallet backends --- .gitignore | 2 + .../wallet/interface.go => api/api_wallet.go | 10 +- api/apistruct/permissioned.go | 6 + api/apistruct/struct.go | 46 ++++++ api/client/client.go | 12 ++ chain/wallet/remotewallet/remote.go | 51 +++++++ chain/wallet/wallet.go | 8 +- cmd/lotus-wallet/logged.go | 68 +++++++++ cmd/lotus-wallet/main.go | 141 ++++++++++++++++++ go.sum | 3 + node/builder.go | 6 +- node/config/def.go | 5 + node/impl/full/wallet.go | 5 +- node/repo/fsrepo.go | 9 ++ 14 files changed, 360 insertions(+), 12 deletions(-) rename chain/wallet/interface.go => api/api_wallet.go (84%) create mode 100644 chain/wallet/remotewallet/remote.go create mode 100644 cmd/lotus-wallet/logged.go create mode 100644 cmd/lotus-wallet/main.go diff --git a/.gitignore b/.gitignore index 0424c1f24..a1d152294 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ /lotus-fountain /lotus-stats /lotus-bench +/lotus-wallet +/lotus-pcr /bench.json /lotuspond/front/node_modules /lotuspond/front/build diff --git a/chain/wallet/interface.go b/api/api_wallet.go similarity index 84% rename from chain/wallet/interface.go rename to api/api_wallet.go index 9dc95a32c..06b00a75b 100644 --- a/chain/wallet/interface.go +++ b/api/api_wallet.go @@ -1,15 +1,14 @@ -package wallet +package api import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/crypto" ) -type Wallet interface { +type WalletAPI interface { WalletNew(context.Context, crypto.SigType) (address.Address, error) WalletHas(context.Context, address.Address) (bool, error) WalletList(context.Context) ([]address.Address, error) @@ -20,8 +19,3 @@ type Wallet interface { WalletImport(context.Context, *types.KeyInfo) (address.Address, error) WalletDelete(context.Context, address.Address) error } - -type Default interface { - GetDefault() (address.Address, error) - SetDefault(a address.Address) error -} diff --git a/api/apistruct/permissioned.go b/api/apistruct/permissioned.go index c93662733..86902d31b 100644 --- a/api/apistruct/permissioned.go +++ b/api/apistruct/permissioned.go @@ -36,3 +36,9 @@ func PermissionedWorkerAPI(a api.WorkerAPI) api.WorkerAPI { auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal) return &out } + +func PermissionedWalletAPI(a api.WalletAPI) api.WalletAPI { + var out WalletStruct + auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal) + return &out +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index dff614001..5eb757c7b 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -336,6 +336,19 @@ type WorkerStruct struct { } } +type WalletStruct struct { + Internal struct { + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` + WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` + } +} + // CommonStruct func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) { @@ -1271,7 +1284,40 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } +func (c *WalletStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { + return c.Internal.WalletNew(ctx, typ) +} + +func (c *WalletStruct) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + return c.Internal.WalletHas(ctx, addr) +} + +func (c *WalletStruct) WalletList(ctx context.Context) ([]address.Address, error) { + return c.Internal.WalletList(ctx) +} + +func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { + return c.Internal.WalletSign(ctx, k, msg) +} + +func (c *WalletStruct) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { + return c.Internal.WalletSignMessage(ctx, k, msg) +} + +func (c *WalletStruct) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { + return c.Internal.WalletExport(ctx, a) +} + +func (c *WalletStruct) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { + return c.Internal.WalletImport(ctx, ki) +} + +func (c *WalletStruct) WalletDelete(ctx context.Context, addr address.Address) error { + return c.Internal.WalletDelete(ctx, addr) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} var _ api.WorkerAPI = &WorkerStruct{} +var _ api.WalletAPI = &WalletStruct{} diff --git a/api/client/client.go b/api/client/client.go index cd915acf0..6e624bdd3 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -82,3 +82,15 @@ func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) ( return &res, closer, err } + +func NewWalletRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WalletAPI, jsonrpc.ClientCloser, error) { + var res apistruct.WalletStruct + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", + []interface{}{ + &res.Internal, + }, + requestHeader, + ) + + return &res, closer, err +} diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go new file mode 100644 index 000000000..712f270b4 --- /dev/null +++ b/chain/wallet/remotewallet/remote.go @@ -0,0 +1,51 @@ +package remotewallet + +import ( + "context" + "go.uber.org/fx" + "golang.org/x/xerrors" + "net/http" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/node/modules/helpers" +) + +type RemoteWallet struct { + api.WalletAPI +} + +func SetupRemoteWallet(url string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { + /*sp := strings.SplitN(env, ":", 2) + if len(sp) != 2 { + log.Warnf("invalid env(%s) value, missing token or address", envKey) + } else { + ma, err := multiaddr.NewMultiaddr(sp[1]) + if err != nil { + return APIInfo{}, xerrors.Errorf("could not parse multiaddr from env(%s): %w", envKey, err) + } + return APIInfo{ + Addr: ma, + Token: []byte(sp[0]), + }, nil + }*/ + + headers := http.Header{} + /*headers.Add("Authorization", "Bearer "+token)*/ + + wapi, closer, err := client.NewWalletRPC(mctx, url, headers) + if err != nil { + return nil, xerrors.Errorf("creating jsonrpc client: %w", err) + } + + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + closer() + return nil + }, + }) + + return &RemoteWallet{wapi}, nil + } +} \ No newline at end of file diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 7fdea8963..bcc8d07b1 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures @@ -36,6 +37,11 @@ type LocalWallet struct { lk sync.Mutex } +type Default interface { + GetDefault() (address.Address, error) + SetDefault(a address.Address) error +} + func NewWallet(keystore types.KeyStore) (*LocalWallet, error) { w := &LocalWallet{ keys: make(map[address.Address]*Key), @@ -250,4 +256,4 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er return nil } -var _ Wallet = &LocalWallet{} +var _ api.WalletAPI = &LocalWallet{} diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go new file mode 100644 index 000000000..e73daae90 --- /dev/null +++ b/cmd/lotus-wallet/logged.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/crypto" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" +) + +type LoggedWallet struct { + under api.WalletAPI +} + +func (c *LoggedWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { + n, err := typ.Name() + if err != nil { + return address.Address{}, err + } + + log.Infow("WalletNew", "type", n) + + return c.under.WalletNew(ctx, typ) +} + +func (c *LoggedWallet) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + log.Infow("WalletHas", "address", addr) + + return c.under.WalletHas(ctx, addr) +} + +func (c *LoggedWallet) WalletList(ctx context.Context) ([]address.Address, error) { + log.Infow("WalletList") + + return c.under.WalletList(ctx) +} + +func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { + log.Infow("WalletSign", "address", k) + + return c.under.WalletSign(ctx, k, msg) +} + +func (c *LoggedWallet) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { + log.Infow("WalletSignMessage", "address", k) + + return c.under.WalletSignMessage(ctx, k, msg) +} + +func (c *LoggedWallet) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { + log.Infow("WalletExport", "address", a) + + return c.under.WalletExport(ctx, a) +} + +func (c *LoggedWallet) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { + log.Infow("WalletImport", "type", ki.Type) + + return c.under.WalletImport(ctx, ki) +} + +func (c *LoggedWallet) WalletDelete(ctx context.Context, addr address.Address) error { + log.Infow("WalletDelete", "address", addr) + + return c.under.WalletDelete(ctx, addr) +} \ No newline at end of file diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go new file mode 100644 index 000000000..ae3580a59 --- /dev/null +++ b/cmd/lotus-wallet/main.go @@ -0,0 +1,141 @@ +package main + +import ( + "context" + "net" + "net/http" + "os" + + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/wallet" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("main") + +const FlagWalletRepo = "wallet-repo" + +func main() { + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + runCmd, + } + + app := &cli.App{ + Name: "lotus-wallet", + Usage: "Basic external wallet", + Version: build.UserVersion(), + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagWalletRepo, + EnvVars: []string{"WALLET_PATH"}, + Value: "~/.lotuswallet", // TODO: Consider XDG_DATA_HOME + }, + }, + + Commands: local, + } + app.Setup() + + if err := app.Run(os.Args); err != nil { + log.Warnf("%+v", err) + return + } +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start lotus wallet", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "host address and port the wallet api will listen on", + Value: "0.0.0.0:1777", + }, + }, + Action: func(cctx *cli.Context) error { + log.Info("Starting lotus wallet") + + ctx := lcli.ReqContext(cctx) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + repoPath := cctx.String(FlagWalletRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + if err := r.Init(repo.Worker); err != nil { + return err + } + } + + lr, err := r.Lock(repo.Wallet) + if err != nil { + return err + } + + ks, err := lr.KeyStore() + if err != nil { + return err + } + + w, err := wallet.NewWallet(ks) + if err != nil { + return err + } + + address := cctx.String("listen") + mux := mux.NewRouter() + + log.Info("Setting up API endpoint at " + address) + + rpcServer := jsonrpc.NewServer() + rpcServer.Register("Filecoin", &LoggedWallet{under: w}) + + mux.Handle("/rpc/v0", rpcServer) + mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof + + /*ah := &auth.Handler{ + Verify: nodeApi.AuthVerify, + Next: mux.ServeHTTP, + }*/ + + srv := &http.Server{ + Handler: mux, + BaseContext: func(listener net.Listener) context.Context { + return ctx + }, + } + + go func() { + <-ctx.Done() + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + nl, err := net.Listen("tcp", address) + if err != nil { + return err + } + + return srv.Serve(nl) + }, +} diff --git a/go.sum b/go.sum index 0c525579e..ea7b3711e 100644 --- a/go.sum +++ b/go.sum @@ -65,9 +65,11 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -1867,6 +1869,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/node/builder.go b/node/builder.go index 7c3251df2..b653fa429 100644 --- a/node/builder.go +++ b/node/builder.go @@ -40,6 +40,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/remotewallet" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -235,7 +236,7 @@ func Online() Option { Override(new(*store.ChainStore), modules.ChainStore), Override(new(*stmgr.StateManager), stmgr.NewStateManager), Override(new(*wallet.LocalWallet), wallet.NewWallet), - Override(new(wallet.Wallet), From(new(*wallet.LocalWallet))), + Override(new(api.WalletAPI), From(new(*wallet.LocalWallet))), Override(new(wallet.Default), From(new(*wallet.LocalWallet))), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), @@ -419,6 +420,9 @@ func ConfigFullNode(c interface{}) Option { If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), ), + If(cfg.Wallet.RemoteBackend != "", + Override(new(api.WalletAPI), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + ), ) } diff --git a/node/config/def.go b/node/config/def.go index 9fee8895a..f5071ada5 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -22,6 +22,7 @@ type FullNode struct { Common Client Client Metrics Metrics + Wallet Wallet } // // Common @@ -105,6 +106,10 @@ type Client struct { IpfsUseForRetrieval bool } +type Wallet struct { + RemoteBackend string +} + func defCommon() Common { return Common{ API: API{ diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 22b674737..263452cf3 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -21,7 +22,7 @@ type WalletAPI struct { StateManager *stmgr.StateManager Default wallet.Default - wallet.Wallet + api.WalletAPI } func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) { @@ -50,7 +51,7 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.Wallet.WalletSignMessage(ctx, keyAddr, msg) + return a.WalletAPI.WalletSignMessage(ctx, keyAddr, msg) } func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool { diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 14085d4ac..2b7fa9ed3 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -44,6 +44,7 @@ const ( FullNode RepoType = iota StorageMiner Worker + Wallet ) func defConfForType(t RepoType) interface{} { @@ -54,6 +55,8 @@ func defConfForType(t RepoType) interface{} { return config.DefaultStorageMiner() case Worker: return &struct{}{} + case Wallet: + return &struct{}{} default: panic(fmt.Sprintf("unknown RepoType(%d)", int(t))) } @@ -87,6 +90,12 @@ func (fsr *FsRepo) Exists() (bool, error) { notexist := os.IsNotExist(err) if notexist { err = nil + + _, err = os.Stat(filepath.Join(fsr.path, fsKeystore)) + notexist = os.IsNotExist(err) + if notexist { + err = nil + } } return !notexist, err } From c66f087f4c4f67396fa24da377f41fac777c781c Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 22 Sep 2020 18:15:42 +0000 Subject: [PATCH 003/125] lotus-miner: add more help text to storage / attach --- cmd/lotus-storage-miner/storage.go | 19 +++++++++++++++++++ extern/sector-storage/stores/local.go | 9 +++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 8a3687877..0c366cb01 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -31,6 +31,9 @@ const metaFile = "sectorstore.json" var storageCmd = &cli.Command{ Name: "storage", Usage: "manage sector storage", + Description: `Sectors can be stored across many filesystem paths. These commands provide ways to + manage the storage the miner will used to store sectors long term for proving (refernces as 'store') + as well as how sectors will be stored while moving through the sealing pipeline (references as 'seal').`, Subcommands: []*cli.Command{ storageAttachCmd, storageListCmd, @@ -41,6 +44,22 @@ var storageCmd = &cli.Command{ var storageAttachCmd = &cli.Command{ Name: "attach", Usage: "attach local storage path", + Description: `Storage can be attach to the miner using this command. The storage volume list is stored local + to the miner in $LOTUS_MINER_PATH/storage.json. We do not recommend modifying this value without further + understanding of the storage system. + + Each storage volume contains a configuration file which descripbes the capabilities of the volume. When the + '--init' flag is provided, this file will be created using the additional flags. + + Weight + A high weight value means data will be more likely to be stored + + Seal + Intermittment data for the sealing process will be stored here + + Store + Finalized sectors that will be proved over will be stored here + `, Flags: []cli.Flag{ &cli.BoolFlag{ Name: "init", diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index 75387acc8..70b37f10e 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -30,10 +30,15 @@ type StoragePath struct { // LocalStorageMeta [path]/sectorstore.json type LocalStorageMeta struct { - ID ID + ID ID + + // A height wait means data is more likely to be stored here Weight uint64 // 0 = readonly - CanSeal bool + // Intermittment data for the sealing process will be stored here + CanSeal bool + + // Finalized sectors that will be proved over will be stored here CanStore bool } From ef3987e2abbc38346974f9eba8b3d46a0c8b7dae Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 2 Oct 2020 14:51:32 -0700 Subject: [PATCH 004/125] decode parameters for multisig transactions in inspect --- cli/multisig.go | 45 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 5b0977382..ade938f3d 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -3,13 +3,17 @@ package cli import ( "bytes" "encoding/hex" + "encoding/json" "fmt" "os" + "reflect" "sort" "strconv" "text/tabwriter" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/stmgr" + cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/go-state-types/big" @@ -171,6 +175,10 @@ var msigInspectCmd = &cli.Command{ Name: "vesting", Usage: "Include vesting details", }, + &cli.BoolFlag{ + Name: "decode-params", + Usage: "Decode parameters of transaction proposals", + }, }, Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { @@ -201,6 +209,11 @@ var msigInspectCmd = &cli.Command{ return err } + ownId, err := api.StateLookupID(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + mstate, err := multisig.Load(store, act) if err != nil { return err @@ -253,6 +266,7 @@ var msigInspectCmd = &cli.Command{ return xerrors.Errorf("reading pending transactions: %w", err) } + decParams := cctx.Bool("decode-params") fmt.Println("Transactions: ", len(pending)) if len(pending) > 0 { var txids []int64 @@ -263,11 +277,36 @@ var msigInspectCmd = &cli.Command{ return txids[i] < txids[j] }) - w := tabwriter.NewWriter(os.Stdout, 8, 4, 0, ' ', 0) + w := tabwriter.NewWriter(os.Stdout, 8, 4, 2, ' ', 0) fmt.Fprintf(w, "ID\tState\tApprovals\tTo\tValue\tMethod\tParams\n") for _, txid := range txids { tx := pending[txid] - fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%d\t%x\n", txid, "pending", len(tx.Approved), tx.To, types.FIL(tx.Value), tx.Method, tx.Params) + target := tx.To.String() + if tx.To == ownId { + target += " (self)" + } + targAct, err := api.StateGetActor(ctx, tx.To, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("failed to resolve 'To' address of multisig transaction %d: %w", txid, err) + } + method := stmgr.MethodsMap[targAct.Code][tx.Method] + + paramStr := fmt.Sprintf("%x", tx.Params) + if decParams && tx.Method != 0 { + ptyp := reflect.New(method.Params.Elem()).Interface().(cbg.CBORUnmarshaler) + if err := ptyp.UnmarshalCBOR(bytes.NewReader(tx.Params)); err != nil { + return xerrors.Errorf("failed to decode parameters of transaction %d: %w", txid, err) + } + + b, err := json.Marshal(ptyp) + if err != nil { + return xerrors.Errorf("could not json marshal parameter type: %w", err) + } + + paramStr = string(b) + } + + fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%s(%d)\t%s\n", txid, "pending", len(tx.Approved), target, types.FIL(tx.Value), method.Name, tx.Method, paramStr) } if err := w.Flush(); err != nil { return xerrors.Errorf("flushing output: %+v", err) @@ -395,7 +434,7 @@ var msigProposeCmd = &cli.Command{ var msigApproveCmd = &cli.Command{ Name: "approve", Usage: "Approve a multisig message", - ArgsUsage: "[multisigAddress messageId proposerAddress destination value (optional)]", + ArgsUsage: " [proposerAddress destination value [methodId methodParams]]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "from", From 1fc23fb466e61327d953a909411f3c954194b912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 3 Oct 2020 00:34:09 +0200 Subject: [PATCH 005/125] lotus-miner: Cleanup storage attach helptext a bit --- cmd/lotus-storage-miner/storage.go | 32 +++++++++++++++------------ extern/sector-storage/stores/local.go | 6 ++--- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 0c366cb01..77792f32a 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -31,9 +31,10 @@ const metaFile = "sectorstore.json" var storageCmd = &cli.Command{ Name: "storage", Usage: "manage sector storage", - Description: `Sectors can be stored across many filesystem paths. These commands provide ways to - manage the storage the miner will used to store sectors long term for proving (refernces as 'store') - as well as how sectors will be stored while moving through the sealing pipeline (references as 'seal').`, + Description: `Sectors can be stored across many filesystem paths. These +commands provide ways to manage the storage the miner will used to store sectors +long term for proving (references as 'store') as well as how sectors will be +stored while moving through the sealing pipeline (references as 'seal').`, Subcommands: []*cli.Command{ storageAttachCmd, storageListCmd, @@ -44,21 +45,24 @@ var storageCmd = &cli.Command{ var storageAttachCmd = &cli.Command{ Name: "attach", Usage: "attach local storage path", - Description: `Storage can be attach to the miner using this command. The storage volume list is stored local - to the miner in $LOTUS_MINER_PATH/storage.json. We do not recommend modifying this value without further - understanding of the storage system. + Description: `Storage can be attached to the miner using this command. The storage volume +list is stored local to the miner in $LOTUS_MINER_PATH/storage.json. We do not +recommend manually modifying this value without further understanding of the +storage system. - Each storage volume contains a configuration file which descripbes the capabilities of the volume. When the - '--init' flag is provided, this file will be created using the additional flags. +Each storage volume contains a configuration file which describes the +capabilities of the volume. When the '--init' flag is provided, this file will +be created using the additional flags. - Weight - A high weight value means data will be more likely to be stored +Weight +A high weight value means data will be more likely to be stored in this path - Seal - Intermittment data for the sealing process will be stored here +Seal +Data for the sealing process will be stored here - Store - Finalized sectors that will be proved over will be stored here +Store +Finalized sectors that will be moved here for long term storage and be proven +over time `, Flags: []cli.Flag{ &cli.BoolFlag{ diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index 70b37f10e..50968e7bd 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -32,13 +32,13 @@ type StoragePath struct { type LocalStorageMeta struct { ID ID - // A height wait means data is more likely to be stored here + // A high weight means data is more likely to be stored in this path Weight uint64 // 0 = readonly - // Intermittment data for the sealing process will be stored here + // Intermediate data for the sealing process will be stored here CanSeal bool - // Finalized sectors that will be proved over will be stored here + // Finalized sectors that will be proved over time will be stored here CanStore bool } From d4cbd4fb55077ccfd96cb5619b4bca2a29fef069 Mon Sep 17 00:00:00 2001 From: frrist Date: Mon, 5 Oct 2020 16:48:29 -0700 Subject: [PATCH 006/125] feat(miner): add miner deadline diffing - logic gathering all removed, faulted, recovered, and recovering sectors for a miner. --- chain/actors/builtin/miner/diff_deadlines.go | 180 +++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 chain/actors/builtin/miner/diff_deadlines.go diff --git a/chain/actors/builtin/miner/diff_deadlines.go b/chain/actors/builtin/miner/diff_deadlines.go new file mode 100644 index 000000000..e1e839960 --- /dev/null +++ b/chain/actors/builtin/miner/diff_deadlines.go @@ -0,0 +1,180 @@ +package miner + +import ( + "errors" + + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/exitcode" +) + +type DeadlinesDiff map[uint64]*DeadlineDiff + +func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { + changed, err := pre.DeadlinesChanged(cur) + if err != nil { + return nil, err + } + if !changed { + return nil, nil + } + + numDl, err := pre.NumDeadlines() + if err != nil { + return nil, err + } + dlDiff := make(DeadlinesDiff, numDl) + if err := pre.ForEachDeadline(func(idx uint64, preDl Deadline) error { + curDl, err := cur.LoadDeadline(idx) + if err != nil { + return err + } + + diff, err := DiffDeadline(preDl, curDl) + if err != nil { + return err + } + + dlDiff[idx] = diff + return nil + }); err != nil { + return nil, err + } + return &dlDiff, nil +} + +type DeadlineDiff map[uint64]*PartitionDiff + +func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { + changed, err := pre.PartitionsChanged(cur) + if err != nil { + return nil, err + } + if !changed { + return nil, nil + } + + partDiff := make(DeadlineDiff) + if err := pre.ForEachPartition(func(idx uint64, prePart Partition) error { + // try loading current partition at this index + curPart, err := cur.LoadPartition(idx) + if err != nil { + if errors.Is(err, exitcode.ErrNotFound) { + // TODO correctness? + return nil // the partition was removed. + } + return err + } + + // compare it with the previous partition + diff, err := DiffPartition(prePart, curPart) + if err != nil { + return err + } + + partDiff[idx] = diff + return nil + }); err != nil { + return nil, err + } + + // all previous partitions have been walked. + // all partitions in cur and not in prev are new... can they be faulty already? + // TODO is this correct? + if err := cur.ForEachPartition(func(idx uint64, curPart Partition) error { + if _, found := partDiff[idx]; found { + return nil + } + faults, err := curPart.FaultySectors() + if err != nil { + return err + } + recovering, err := curPart.RecoveringSectors() + if err != nil { + return err + } + partDiff[idx] = &PartitionDiff{ + Removed: bitfield.New(), + Recovered: bitfield.New(), + Faulted: faults, + Recovering: recovering, + } + + return nil + }); err != nil { + return nil, err + } + + return &partDiff, nil +} + +type PartitionDiff struct { + Removed bitfield.BitField + Recovered bitfield.BitField + Faulted bitfield.BitField + Recovering bitfield.BitField +} + +func DiffPartition(pre, cur Partition) (*PartitionDiff, error) { + prevLiveSectors, err := pre.LiveSectors() + if err != nil { + return nil, err + } + curLiveSectors, err := cur.LiveSectors() + if err != nil { + return nil, err + } + + removed, err := bitfield.SubtractBitField(prevLiveSectors, curLiveSectors) + if err != nil { + return nil, err + } + + prevRecoveries, err := pre.RecoveringSectors() + if err != nil { + return nil, err + } + + curRecoveries, err := cur.RecoveringSectors() + if err != nil { + return nil, err + } + + recovering, err := bitfield.SubtractBitField(curRecoveries, prevRecoveries) + if err != nil { + return nil, err + } + + prevFaults, err := pre.FaultySectors() + if err != nil { + return nil, err + } + + curFaults, err := cur.FaultySectors() + if err != nil { + return nil, err + } + + faulted, err := bitfield.SubtractBitField(curFaults, prevFaults) + if err != nil { + return nil, err + } + + // all current good sectors + curActiveSectors, err := cur.ActiveSectors() + if err != nil { + return nil, err + } + + // sectors that were previously fault and are now currently active are considered recovered. + recovered, err := bitfield.IntersectBitField(prevFaults, curActiveSectors) + if err != nil { + return nil, err + } + + return &PartitionDiff{ + Removed: removed, + Recovered: recovered, + Faulted: faulted, + Recovering: recovering, + }, nil +} From 4a3081c77a99fc1d6ec989545ad4d55b532ab465 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 6 Oct 2020 20:13:48 +0000 Subject: [PATCH 007/125] lotus-pcr: limit refunds to properly priced messages --- cmd/lotus-pcr/main.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 4ce5bbb9f..f446b2c9d 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -376,6 +376,18 @@ var runCmd = &cli.Command{ Usage: "percent of refund to issue", Value: 110, }, + &cli.StringFlag{ + Name: "pre-fee-cap-max", + EnvVars: []string{"LOTUS_PCR_PRE_FEE_CAP_MAX"}, + Usage: "messages with a fee cap larger than this will be skipped when processing pre commit messages", + Value: "0.0000000001", + }, + &cli.StringFlag{ + Name: "prove-fee-cap-max", + EnvVars: []string{"LOTUS_PCR_PROVE_FEE_CAP_MAX"}, + Usage: "messages with a prove cap larger than this will be skipped when processing pre commit messages", + Value: "0.0000000001", + }, }, Action: func(cctx *cli.Context) error { go func() { @@ -426,6 +438,16 @@ var runCmd = &cli.Command{ minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff")) minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus")) + preFeeCapMax, err := types.ParseFIL(cctx.String("pre-fee-cap-max")) + if err != nil { + return err + } + + proveFeeCapMax, err := types.ParseFIL(cctx.String("prove-fee-cap-max")) + if err != nil { + return err + } + rf := &refunder{ api: api, wallet: from, @@ -436,6 +458,8 @@ var runCmd = &cli.Command{ dryRun: dryRun, preCommitEnabled: preCommitEnabled, proveCommitEnabled: proveCommitEnabled, + preFeeCapMax: types.BigInt(preFeeCapMax), + proveFeeCapMax: types.BigInt(proveFeeCapMax), } var refunds *MinersRefund = NewMinersRefund() @@ -588,6 +612,9 @@ type refunder struct { preCommitEnabled bool proveCommitEnabled bool threshold big.Int + + preFeeCapMax big.Int + proveFeeCapMax big.Int } func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) { @@ -869,6 +896,11 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } + if m.GasFeeCap.GreaterThan(r.proveFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.proveFeeCapMax) + continue + } + var sn abi.SectorNumber var proveCommitSector miner0.ProveCommitSectorParams @@ -916,6 +948,11 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } + if m.GasFeeCap.GreaterThan(r.preFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.preFeeCapMax) + continue + } + var precommitInfo miner.SectorPreCommitInfo if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) From 087030fe371476cfad576796a2060f81ec2b7487 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 6 Oct 2020 22:35:04 +0000 Subject: [PATCH 008/125] lotus-pcr: refund windowed post and storage deal gas fees --- cmd/lotus-pcr/main.go | 260 ++++++++++++++++++++++++++++-------------- 1 file changed, 174 insertions(+), 86 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index f446b2c9d..14f4778c2 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -340,6 +340,18 @@ var runCmd = &cli.Command{ Usage: "process ProveCommitSector messages", Value: true, }, + &cli.BoolFlag{ + Name: "windowed-post", + EnvVars: []string{"LOTUS_PCR_WINDOWED_POST"}, + Usage: "process SubmitWindowedPoSt messages and refund gas fees", + Value: false, + }, + &cli.BoolFlag{ + Name: "storage-deals", + EnvVars: []string{"LOTUS_PCR_STORAGE_DEALS"}, + Usage: "process PublishStorageDeals messages and refund gas fees", + Value: false, + }, &cli.IntFlag{ Name: "head-delay", EnvVars: []string{"LOTUS_PCR_HEAD_DELAY"}, @@ -431,6 +443,8 @@ var runCmd = &cli.Command{ dryRun := cctx.Bool("dry-run") preCommitEnabled := cctx.Bool("pre-commit") proveCommitEnabled := cctx.Bool("prove-commit") + windowedPoStEnabled := cctx.Bool("windowed-post") + publishStorageDealsEnabled := cctx.Bool("storage-deals") aggregateTipsets := cctx.Int("aggregate-tipsets") minerRecoveryEnabled := cctx.Bool("miner-recovery") minerRecoveryPeriod := abi.ChainEpoch(int64(cctx.Int("miner-recovery-period"))) @@ -458,6 +472,8 @@ var runCmd = &cli.Command{ dryRun: dryRun, preCommitEnabled: preCommitEnabled, proveCommitEnabled: proveCommitEnabled, + windowedPoStEnabled: windowedPoStEnabled, + publishStorageDealsEnabled: publishStorageDealsEnabled, preFeeCapMax: types.BigInt(preFeeCapMax), proveFeeCapMax: types.BigInt(proveFeeCapMax), } @@ -611,6 +627,8 @@ type refunder struct { dryRun bool preCommitEnabled bool proveCommitEnabled bool + windowedPoStEnabled bool + publishStorageDealsEnabled bool threshold big.Int preFeeCapMax big.Int @@ -842,6 +860,147 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet return refunds, nil } +func (r *refunder) processTipsetStorageMarketActor(ctx context.Context, tipset *types.TipSet, msg api.Message, recp *types.MessageReceipt) (bool, string, types.BigInt, error) { + + m := msg.Message + refundValue := types.NewInt(0) + var messageMethod string + + switch m.Method { + case builtin.MethodsMarket.PublishStorageDeals: + if !r.publishStorageDealsEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "PublishStorageDeals" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) + } + + return true, messageMethod, refundValue, nil +} + +func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *types.TipSet, msg api.Message, recp *types.MessageReceipt) (bool, string, types.BigInt, error) { + + m := msg.Message + refundValue := types.NewInt(0) + var messageMethod string + + switch m.Method { + case builtin.MethodsMiner.SubmitWindowedPoSt: + if !r.windowedPoStEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "SubmitWindowedPoSt" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) + case builtin.MethodsMiner.ProveCommitSector: + if !r.proveCommitEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "ProveCommitSector" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + if m.GasFeeCap.GreaterThan(r.proveFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.proveFeeCapMax) + return false, messageMethod, types.NewInt(0), nil + } + + var sn abi.SectorNumber + + var proveCommitSector miner0.ProveCommitSectorParams + if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { + log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) + return false, messageMethod, types.NewInt(0), nil + } + + sn = proveCommitSector.SectorNumber + + // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed + precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) + if err != nil { + log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key()) + if err != nil { + log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key()) + if err != nil { + log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit) + if collateral.LessThan(big.Zero()) { + log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = collateral + if r.refundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + } + case builtin.MethodsMiner.PreCommitSector: + if !r.preCommitEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "PreCommitSector" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + if m.GasFeeCap.GreaterThan(r.preFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.preFeeCapMax) + return false, messageMethod, types.NewInt(0), nil + } + + var precommitInfo miner.SectorPreCommitInfo + if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { + log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) + return false, messageMethod, types.NewInt(0), nil + } + + collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitInfo, tipset.Key()) + if err != nil { + log.Warnw("failed to calculate initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", precommitInfo.SectorNumber) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = collateral + if r.refundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + } + default: + return false, messageMethod, types.NewInt(0), nil + } + + return true, messageMethod, refundValue, nil +} + func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { cids := tipset.Cids() if len(cids) == 0 { @@ -877,101 +1036,30 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } - if !a.IsStorageMinerActor() { - continue - } - var messageMethod string - switch m.Method { - case builtin.MethodsMiner.ProveCommitSector: - if !r.proveCommitEnabled { - continue - } - - messageMethod = "ProveCommitSector" - - if recps[i].ExitCode != exitcode.Ok { - log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode) - continue - } - - if m.GasFeeCap.GreaterThan(r.proveFeeCapMax) { - log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.proveFeeCapMax) - continue - } - - var sn abi.SectorNumber - - var proveCommitSector miner0.ProveCommitSectorParams - if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { - log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) - continue - } - - sn = proveCommitSector.SectorNumber - - // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed - precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) + if m.To == builtin.StorageMarketActorAddr { + var err error + var processed bool + processed, messageMethod, refundValue, err = r.processTipsetStorageMarketActor(ctx, tipset, msg, recps[i]) if err != nil { - log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) continue } - - precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key()) - if err != nil { - log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + if !processed { continue } - - collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key()) - if err != nil { - log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) - } - - collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit) - if collateral.LessThan(big.Zero()) { - log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) - continue - } - - refundValue = collateral - case builtin.MethodsMiner.PreCommitSector: - if !r.preCommitEnabled { - continue - } - - messageMethod = "PreCommitSector" - - if recps[i].ExitCode != exitcode.Ok { - log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode) - continue - } - - if m.GasFeeCap.GreaterThan(r.preFeeCapMax) { - log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.preFeeCapMax) - continue - } - - var precommitInfo miner.SectorPreCommitInfo - if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { - log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) - continue - } - - collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitInfo, tipset.Key()) - if err != nil { - log.Warnw("failed to calculate initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", precommitInfo.SectorNumber) - continue - } - - refundValue = collateral - default: - continue } - if r.refundPercent > 0 { - refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + if a.IsStorageMinerActor() { + var err error + var processed bool + processed, messageMethod, refundValue, err = r.processTipsetStorageMinerActor(ctx, tipset, msg, recps[i]) + if err != nil { + continue + } + if !processed { + continue + } } log.Debugw( From bd474617ed6f51726f645119f1da4bf2ece092bb Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 6 Oct 2020 16:06:24 -0700 Subject: [PATCH 009/125] implement command to get execution traces of any message --- cli/state.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/cli/state.go b/cli/state.go index 7baf57df2..371ef4af0 100644 --- a/cli/state.go +++ b/cli/state.go @@ -72,6 +72,7 @@ var stateCmd = &cli.Command{ stateMsgCostCmd, stateMinerInfo, stateMarketCmd, + stateExecTraceCmd, }, } @@ -315,6 +316,74 @@ var stateActiveSectorsCmd = &cli.Command{ }, } +var stateExecTraceCmd = &cli.Command{ + Name: "exec-trace", + Usage: "Get the execution trace of a given message", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return ShowHelp(cctx, fmt.Errorf("must pass message cid")) + } + + mcid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("message cid was invalid: %s", err) + } + + capi, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + msg, err := capi.ChainGetMessage(ctx, mcid) + if err != nil { + return err + } + + lookup, err := capi.StateSearchMsg(ctx, mcid) + if err != nil { + return err + } + + ts, err := capi.ChainGetTipSet(ctx, lookup.TipSet) + if err != nil { + return err + } + + pts, err := capi.ChainGetTipSet(ctx, ts.Parents()) + if err != nil { + return err + } + + cso, err := capi.StateCompute(ctx, pts.Height(), nil, pts.Key()) + if err != nil { + return err + } + + var trace *api.InvocResult + for _, t := range cso.Trace { + if t.Msg.From == msg.From && t.Msg.Nonce == msg.Nonce { + trace = t + break + } + } + if trace == nil { + return fmt.Errorf("failed to find message in tipset trace output") + } + + out, err := json.MarshalIndent(trace, "", " ") + if err != nil { + return err + } + + fmt.Println(string(out)) + return nil + }, +} + var stateReplaySetCmd = &cli.Command{ Name: "replay", Usage: "Replay a particular message within a tipset", From 15670942c36dd6ffc185ae80152c2cd3f0651eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 16:29:16 +0200 Subject: [PATCH 010/125] Fix pond --- cmd/lotus-storage-miner/run.go | 16 ++++++++-------- lotuspond/spawn.go | 12 +++++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index a5d996f78..98a9cfaba 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -33,7 +33,7 @@ var runCmd = &cli.Command{ Usage: "Start a lotus miner process", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "api", + Name: "miner-api", Usage: "2345", }, &cli.BoolFlag{ @@ -61,7 +61,7 @@ var runCmd = &cli.Command{ nodeApi, ncloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { - return err + return xerrors.Errorf("getting full node api: %w", err) } defer ncloser() ctx := lcli.DaemonContext(cctx) @@ -112,29 +112,29 @@ var runCmd = &cli.Command{ node.Online(), node.Repo(r), - node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, + node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("miner-api") }, node.Override(new(dtypes.APIEndpoint), func() (dtypes.APIEndpoint, error) { - return multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + cctx.String("api")) + return multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + cctx.String("miner-api")) })), node.Override(new(api.FullNode), nodeApi), ) if err != nil { - return err + return xerrors.Errorf("creating node: %w", err) } endpoint, err := r.APIEndpoint() if err != nil { - return err + return xerrors.Errorf("getting API endpoint: %w", err) } // Bootstrap with full node remoteAddrs, err := nodeApi.NetAddrsListen(ctx) if err != nil { - return err + return xerrors.Errorf("getting full node libp2p address: %w", err) } if err := minerapi.NetConnect(ctx, remoteAddrs); err != nil { - return err + return xerrors.Errorf("connecting to full node (libp2p): %w", err) } log.Infof("Remote version %s", v) diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 8b2e8661d..ce01b115e 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -11,16 +11,16 @@ import ( "sync/atomic" "time" - "github.com/filecoin-project/lotus/chain/actors/policy" - "github.com/filecoin-project/lotus/chain/types" - + "github.com/google/uuid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" ) @@ -69,6 +69,8 @@ func (api *api) Spawn() (nodeInfo, error) { Meta: (&genesis.AccountMeta{Owner: genm.Owner}).ActorMeta(), }) template.VerifregRootKey = gen.DefaultVerifregRootkeyActor + template.RemainderAccount = gen.DefaultRemainderAccountActor + template.NetworkName = "pond-" + uuid.New().String() tb, err := json.Marshal(&template) if err != nil { @@ -188,7 +190,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { mux := newWsMux() - cmd = exec.Command("./lotus-miner", "run", "--api", fmt.Sprintf("%d", 2500+id), "--nosync") + cmd = exec.Command("./lotus-miner", "run", "--miner-api", fmt.Sprintf("%d", 2500+id), "--nosync") cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) cmd.Env = append(os.Environ(), "LOTUS_MINER_PATH="+dir, "LOTUS_PATH="+fullNodeRepo) @@ -248,7 +250,7 @@ func (api *api) RestartNode(id int32) (nodeInfo, error) { var cmd *exec.Cmd if nd.meta.Storage { - cmd = exec.Command("./lotus-miner", "run", "--api", fmt.Sprintf("%d", 2500+id), "--nosync") + cmd = exec.Command("./lotus-miner", "run", "--miner-api", fmt.Sprintf("%d", 2500+id), "--nosync") } else { cmd = exec.Command("./lotus", "daemon", "--api", fmt.Sprintf("%d", 2500+id)) } From be92dd9e63a840324a757d55bbdba7ff8f7c781e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 6 Oct 2020 18:07:34 -0700 Subject: [PATCH 011/125] allow manual setting of noncefix fee cap --- cmd/lotus-shed/nonce-fix.go | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-shed/nonce-fix.go b/cmd/lotus-shed/nonce-fix.go index 3cd9726f4..8102fd8a9 100644 --- a/cmd/lotus-shed/nonce-fix.go +++ b/cmd/lotus-shed/nonce-fix.go @@ -5,6 +5,8 @@ import ( "math" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/types" @@ -32,6 +34,10 @@ var noncefix = &cli.Command{ &cli.BoolFlag{ Name: "auto", }, + &cli.Int64Flag{ + Name: "gas-fee-cap", + Usage: "specify gas fee cap for nonce filling messages", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := lcli.GetFullNodeAPI(cctx) @@ -84,15 +90,32 @@ var noncefix = &cli.Command{ } fmt.Printf("Creating %d filler messages (%d ~ %d)\n", end-start, start, end) + ts, err := api.ChainHead(ctx) + if err != nil { + return err + } + + feeCap := big.Mul(ts.Blocks()[0].ParentBaseFee, big.NewInt(2)) // default fee cap to 2 * parent base fee + if fcf := cctx.Int64("gas-fee-cap"); fcf != 0 { + feeCap = abi.NewTokenAmount(fcf) + } + for i := start; i < end; i++ { msg := &types.Message{ - From: addr, - To: addr, - Value: types.NewInt(1), - Nonce: i, + From: addr, + To: addr, + Value: types.NewInt(0), + Nonce: i, + GasLimit: 1000000, + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(5), + } + smsg, err := api.WalletSignMessage(ctx, addr, msg) + if err != nil { + return err } - _, err = api.MpoolPushMessage(ctx, msg, nil) + _, err = api.MpoolPush(ctx, smsg) if err != nil { return err } From b7c8f65af5c474310d988b59da0d036b9d252a26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Oct 2020 01:15:19 +0200 Subject: [PATCH 012/125] stmgr: Make v2 upgrade faster --- chain/stmgr/forks.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index d5d0dbf7e..8143524c0 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -4,6 +4,9 @@ import ( "bytes" "context" "encoding/binary" + "github.com/filecoin-project/lotus/chain/store" + bstore "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" "math" "github.com/filecoin-project/go-address" @@ -490,7 +493,8 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo } func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { - store := sm.cs.Store(ctx) + buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync()) + store := store.ActorStore(ctx, buf) epoch := ts.Height() - 1 @@ -538,6 +542,15 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("failed to load init actor after upgrade: %w", err) } + { + from := buf + to := buf.Read() + + if err := vm.Copy(ctx, from, to, newRoot); err != nil { + return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) + } + } + return newRoot, nil } From 3b12a9f425c81715bf9fe0ad8087b6b880aee437 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 6 Oct 2020 16:38:31 -0700 Subject: [PATCH 013/125] Add flag to accept previous output of compute-state as input --- cli/state.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/cli/state.go b/cli/state.go index cb1c974a7..b2beb8ffb 100644 --- a/cli/state.go +++ b/cli/state.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "io" + "io/ioutil" "os" "reflect" "sort" @@ -32,6 +33,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/lotus/api" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" @@ -823,6 +825,10 @@ var stateComputeStateCmd = &cli.Command{ Name: "json", Usage: "generate json output", }, + &cli.StringFlag{ + Name: "compute-state-output", + Usage: "a json file containing pre-existing compute-state output, to generate html reports without rerunning state changes", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -862,9 +868,26 @@ var stateComputeStateCmd = &cli.Command{ } } - stout, err := api.StateCompute(ctx, h, msgs, ts.Key()) - if err != nil { - return err + var stout *lapi.ComputeStateOutput + if csofile := cctx.String("compute-state-output"); csofile != "" { + data, err := ioutil.ReadFile(csofile) + if err != nil { + return err + } + + var o lapi.ComputeStateOutput + if err := json.Unmarshal(data, &o); err != nil { + return err + } + + stout = &o + } else { + o, err := api.StateCompute(ctx, h, msgs, ts.Key()) + if err != nil { + return err + } + + stout = o } if cctx.Bool("json") { From d3dc560f53aee4eb843a3c081b44282e36c6dca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 12:45:11 +0100 Subject: [PATCH 014/125] conformance: minor refactors. --- cmd/tvx/extract.go | 12 +++++----- conformance/driver.go | 47 +++++++++++++++++++++++++++------------ conformance/rand_fixed.go | 28 +++++++++++++++++++++++ conformance/runner.go | 20 ++++------------- conformance/stubs.go | 14 +----------- 5 files changed, 72 insertions(+), 49 deletions(-) create mode 100644 conformance/rand_fixed.go diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index fef245858..afdeb9540 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -198,8 +198,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: root, Epoch: execTs.Height(), Message: m, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) @@ -229,8 +229,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: preroot, Epoch: execTs.Height(), Message: msg, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -260,8 +260,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: preroot, Epoch: execTs.Height(), Message: msg, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) diff --git a/conformance/driver.go b/conformance/driver.go index 9ced12d74..d51798a52 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -2,6 +2,7 @@ package conformance import ( "context" + gobig "math/big" "os" "github.com/filecoin-project/lotus/chain/state" @@ -14,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/test-vectors/schema" @@ -80,7 +82,7 @@ type ExecuteTipsetResult struct { func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) { var ( syscalls = mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)) - vmRand = new(testRand) + vmRand = NewFixedRand() cs = store.NewChainStore(bs, ds, syscalls) sm = stmgr.NewStateManager(cs) @@ -143,8 +145,12 @@ type ExecuteMessageParams struct { Preroot cid.Cid Epoch abi.ChainEpoch Message *types.Message - CircSupply *abi.TokenAmount - BaseFee *abi.TokenAmount + CircSupply abi.TokenAmount + BaseFee abi.TokenAmount + + // Rand is an optional vm.Rand implementation to use. If nil, the driver + // will use a vm.Rand that returns a fixed value for all calls. + Rand vm.Rand } // ExecuteMessage executes a conformance test vector message in a temporary VM. @@ -155,14 +161,8 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") } - basefee := DefaultBaseFee - if params.BaseFee != nil { - basefee = *params.BaseFee - } - - circSupply := DefaultCirculatingSupply - if params.CircSupply != nil { - circSupply = *params.CircSupply + if params.Rand == nil { + params.Rand = NewFixedRand() } // dummy state manager; only to reference the GetNetworkVersion method, @@ -172,13 +172,13 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP vmOpts := &vm.VMOpts{ StateBase: params.Preroot, Epoch: params.Epoch, - Rand: &testRand{}, // TODO always succeeds; need more flexibility. Bstore: bs, Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { - return circSupply, nil + return params.CircSupply, nil }, - BaseFee: basefee, + Rand: params.Rand, + BaseFee: params.BaseFee, NtwkVersion: sm.GetNtwkVersion, } @@ -231,3 +231,22 @@ func toChainMsg(msg *types.Message) (ret types.ChainMsg) { } return ret } + +// BaseFeeOrDefault converts a basefee as passed in a test vector (go *big.Int +// type) to an abi.TokenAmount, or if nil it returns the DefaultBaseFee. +func BaseFeeOrDefault(basefee *gobig.Int) abi.TokenAmount { + if basefee == nil { + return DefaultBaseFee + } + return big.NewFromGo(basefee) +} + +// CircSupplyOrDefault converts a circulating supply as passed in a test vector +// (go *big.Int type) to an abi.TokenAmount, or if nil it returns the +// DefaultCirculatingSupply. +func CircSupplyOrDefault(circSupply *gobig.Int) abi.TokenAmount { + if circSupply == nil { + return DefaultBaseFee + } + return big.NewFromGo(circSupply) +} diff --git a/conformance/rand_fixed.go b/conformance/rand_fixed.go new file mode 100644 index 000000000..d356b53d0 --- /dev/null +++ b/conformance/rand_fixed.go @@ -0,0 +1,28 @@ +package conformance + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/vm" +) + +type fixedRand struct{} + +var _ vm.Rand = (*fixedRand)(nil) + +// NewFixedRand creates a test vm.Rand that always returns fixed bytes value +// of utf-8 string 'i_am_random_____i_am_random_____'. +func NewFixedRand() vm.Rand { + return &fixedRand{} +} + +func (r *fixedRand) GetChainRandomness(_ context.Context, _ crypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) ([]byte, error) { + return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. +} + +func (r *fixedRand) GetBeaconRandomness(_ context.Context, _ crypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) ([]byte, error) { + return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. +} diff --git a/conformance/runner.go b/conformance/runner.go index 2db53b3e4..a240c9067 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -13,9 +13,7 @@ import ( "github.com/fatih/color" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/test-vectors/schema" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -24,6 +22,8 @@ import ( "github.com/ipfs/go-merkledag" "github.com/ipld/go-car" + "github.com/filecoin-project/test-vectors/schema" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/lib/blockstore" @@ -46,18 +46,6 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Create a new Driver. driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true}) - var circSupply *abi.TokenAmount - if cs := vector.Pre.CircSupply; cs != nil { - ta := big.NewFromGo(cs) - circSupply = &ta - } - - var basefee *abi.TokenAmount - if bf := vector.Pre.BaseFee; bf != nil { - ta := big.NewFromGo(bf) - basefee = &ta - } - // Apply every message. for i, m := range vector.ApplyMessages { msg, err := types.DecodeMessage(m.Bytes) @@ -76,8 +64,8 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { Preroot: root, Epoch: abi.ChainEpoch(epoch), Message: msg, - CircSupply: circSupply, - BaseFee: basefee, + BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), + CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), }) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) diff --git a/conformance/stubs.go b/conformance/stubs.go index a7100892f..9307fdd65 100644 --- a/conformance/stubs.go +++ b/conformance/stubs.go @@ -6,28 +6,16 @@ import ( "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/runtime" cbor "github.com/ipfs/go-ipld-cbor" ) -type testRand struct{} - -var _ vm.Rand = (*testRand)(nil) - -func (r *testRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. -} - -func (r *testRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. -} - type testSyscalls struct { runtime.Syscalls } From d6fa64497446d8f8337b16411e840bcdaa46ded4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Oct 2020 14:10:23 +0200 Subject: [PATCH 015/125] Fix lint --- chain/stmgr/forks.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 8143524c0..2c2304a1f 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -4,9 +4,6 @@ import ( "bytes" "context" "encoding/binary" - "github.com/filecoin-project/lotus/chain/store" - bstore "github.com/filecoin-project/lotus/lib/blockstore" - "github.com/filecoin-project/lotus/lib/bufbstore" "math" "github.com/filecoin-project/go-address" @@ -33,8 +30,11 @@ import ( init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/state" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + bstore "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" ) type UpgradeFunc func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error) @@ -547,7 +547,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo to := buf.Read() if err := vm.Copy(ctx, from, to, newRoot); err != nil { - return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) + return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) } } From 39bc816a79c76d22e401310f9cae3fc3d3725992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Oct 2020 15:01:30 +0200 Subject: [PATCH 016/125] build: Env var to keep test address output --- build/params_testnet.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/params_testnet.go b/build/params_testnet.go index 49abd30c0..1369b2c70 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -5,6 +5,8 @@ package build import ( + "os" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -36,7 +38,9 @@ func init() { abi.RegisteredSealProof_StackedDrg64GiBV1, ) - SetAddressNetwork(address.Mainnet) + if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" { + SetAddressNetwork(address.Mainnet) + } Devnet = false } From 7cd1330acd9802ee5709b324ba584f3882d8d2af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 15:52:03 +0100 Subject: [PATCH 017/125] make vm.EnableGasTracing public. --- chain/vm/runtime.go | 9 +++++---- chain/vm/runtime_test.go | 20 ++++++++++++++++++++ chain/vm/vm.go | 2 +- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 72dd413ed..c612e374f 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -28,6 +28,9 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +// EnableGasTracing, if true, outputs gas tracing in execution traces. +var EnableGasTracing = false + type Runtime struct { types.Message rt0.Syscalls @@ -459,7 +462,7 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { } func (rt *Runtime) finilizeGasTracing() { - if enableTracing { + if EnableGasTracing { if rt.lastGasCharge != nil { rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) } @@ -491,11 +494,9 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { } -var enableTracing = false - func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() - if enableTracing { + if EnableGasTracing { var callers [10]uintptr cout := 0 //gruntime.Callers(2+skip, callers[:]) diff --git a/chain/vm/runtime_test.go b/chain/vm/runtime_test.go index c22a8b615..9fc87f7c5 100644 --- a/chain/vm/runtime_test.go +++ b/chain/vm/runtime_test.go @@ -45,3 +45,23 @@ func TestRuntimePutErrors(t *testing.T) { rt.StorePut(&NotAVeryGoodMarshaler{}) t.Error("expected panic") } + +func BenchmarkRuntime_CreateRuntimeChargeGas_TracingDisabled(b *testing.B) { + var ( + cst = cbor.NewCborStore(nil) + gch = newGasCharge("foo", 1000, 1000) + ) + + b.ResetTimer() + + EnableGasTracing = false + noop := func() bool { return EnableGasTracing } + for n := 0; n < b.N; n++ { + // flip the value and access it to make sure + // the compiler doesn't optimize away + EnableGasTracing = true + _ = noop() + EnableGasTracing = false + _ = (&Runtime{cst: cst}).chargeGasInternal(gch, 0) + } +} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 44979454f..e2ccd33c7 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -227,7 +227,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, } rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) - if enableTracing { + if EnableGasTracing { rt.lastGasChargeTime = start if parent != nil { rt.lastGasChargeTime = parent.lastGasChargeTime From 0d34881a0ca3b3b823dfd3c16e84e66625b1c331 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 7 Oct 2020 18:50:50 +0000 Subject: [PATCH 018/125] lotus-pcr: ignore all other messages --- cmd/lotus-pcr/main.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 14f4778c2..5b59ff03d 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -1037,29 +1037,22 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu } var messageMethod string + var processed bool if m.To == builtin.StorageMarketActorAddr { - var err error - var processed bool processed, messageMethod, refundValue, err = r.processTipsetStorageMarketActor(ctx, tipset, msg, recps[i]) - if err != nil { - continue - } - if !processed { - continue - } } if a.IsStorageMinerActor() { - var err error - var processed bool processed, messageMethod, refundValue, err = r.processTipsetStorageMinerActor(ctx, tipset, msg, recps[i]) - if err != nil { - continue - } - if !processed { - continue - } + } + + if err != nil { + log.Errorw("error while processing message", "cid", msg.Cid) + continue + } + if !processed { + continue } log.Debugw( From 5bd6a3cdadd47fc5f26027b522fd0e94724350df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 19:52:37 +0100 Subject: [PATCH 019/125] conformance: record randomness in tvx; replay in driver. --- cmd/tvx/extract.go | 10 ++++- conformance/rand_record.go | 86 ++++++++++++++++++++++++++++++++++++++ conformance/rand_replay.go | 78 ++++++++++++++++++++++++++++++++++ conformance/runner.go | 1 + go.mod | 2 +- go.sum | 4 +- 6 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 conformance/rand_record.go create mode 100644 conformance/rand_replay.go diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index afdeb9540..b0ed574df 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -200,6 +200,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: m, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + // recorded randomness will be discarded. + Rand: conformance.NewRecordingRand(new(conformance.LogReporter), fapi), }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) @@ -212,6 +214,9 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { applyret *vm.ApplyRet carWriter func(w io.Writer) error retention = opts.retain + + // recordingRand will record randomness so we can embed it in the test vector. + recordingRand = conformance.NewRecordingRand(new(conformance.LogReporter), fapi) ) log.Printf("using state retention strategy: %s", retention) @@ -231,6 +236,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: msg, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + Rand: recordingRand, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -262,6 +268,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: msg, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + Rand: recordingRand, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -356,7 +363,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, - CAR: out.Bytes(), + Randomness: recordingRand.Recorded(), + CAR: out.Bytes(), Pre: &schema.Preconditions{ Epoch: int64(execTs.Height()), CircSupply: circSupply.Int, diff --git a/conformance/rand_record.go b/conformance/rand_record.go new file mode 100644 index 000000000..2d3477c94 --- /dev/null +++ b/conformance/rand_record.go @@ -0,0 +1,86 @@ +package conformance + +import ( + "context" + "sync" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/test-vectors/schema" +) + +type RecordingRand struct { + reporter Reporter + api api.FullNode + + lk sync.Mutex + recorded schema.Randomness +} + +var _ vm.Rand = (*RecordingRand)(nil) + +// NewRecordingRand returns a vm.Rand implementation that proxies calls to a +// full Lotus node via JSON-RPC, and records matching rules and responses so +// they can later be embedded in test vectors. +func NewRecordingRand(reporter Reporter, api api.FullNode) *RecordingRand { + return &RecordingRand{reporter: reporter, api: api} +} + +func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + ret, err := r.api.ChainGetRandomnessFromTickets(ctx, types.EmptyTSK, pers, round, entropy) + if err != nil { + return ret, err + } + + r.reporter.Logf("fetched and recorded chain randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + + match := schema.RandomnessMatch{ + On: schema.RandomnessRule{ + Kind: schema.RandomnessChain, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + }, + Return: []byte(ret), + } + r.lk.Lock() + r.recorded = append(r.recorded, match) + r.lk.Unlock() + + return ret, err +} + +func (r *RecordingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, types.EmptyTSK, pers, round, entropy) + if err != nil { + return ret, err + } + + r.reporter.Logf("fetched and recorded beacon randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + + match := schema.RandomnessMatch{ + On: schema.RandomnessRule{ + Kind: schema.RandomnessBeacon, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + }, + Return: []byte(ret), + } + r.lk.Lock() + r.recorded = append(r.recorded, match) + r.lk.Unlock() + + return ret, err +} + +func (r *RecordingRand) Recorded() schema.Randomness { + r.lk.Lock() + defer r.lk.Unlock() + + return r.recorded +} diff --git a/conformance/rand_replay.go b/conformance/rand_replay.go new file mode 100644 index 000000000..b820b8ced --- /dev/null +++ b/conformance/rand_replay.go @@ -0,0 +1,78 @@ +package conformance + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/test-vectors/schema" +) + +type ReplayingRand struct { + reporter Reporter + recorded schema.Randomness + fallback vm.Rand +} + +var _ vm.Rand = (*ReplayingRand)(nil) + +// NewReplayingRand replays recorded randomness when requested, falling back to +// fixed randomness if the value cannot be found; hence this is a safe +// backwards-compatible replacement for fixedRand. +func NewReplayingRand(reporter Reporter, recorded schema.Randomness) *ReplayingRand { + return &ReplayingRand{ + reporter: reporter, + recorded: recorded, + fallback: NewFixedRand(), + } +} + +func (r *ReplayingRand) match(requested schema.RandomnessRule) ([]byte, bool) { + for _, other := range r.recorded { + if other.On.Kind == requested.Kind && + other.On.Epoch == requested.Epoch && + other.On.DomainSeparationTag == requested.DomainSeparationTag && + bytes.Equal(other.On.Entropy, requested.Entropy) { + return other.Return, true + } + } + return nil, false +} + +func (r *ReplayingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + rule := schema.RandomnessRule{ + Kind: schema.RandomnessChain, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + } + + if ret, ok := r.match(rule); ok { + r.reporter.Logf("returning saved chain randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + return ret, nil + } + + r.reporter.Logf("returning fallback chain randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy) + return r.fallback.GetChainRandomness(ctx, pers, round, entropy) +} + +func (r *ReplayingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + rule := schema.RandomnessRule{ + Kind: schema.RandomnessBeacon, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + } + + if ret, ok := r.match(rule); ok { + r.reporter.Logf("returning saved beacon randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + return ret, nil + } + + r.reporter.Logf("returning fallback beacon randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy) + return r.fallback.GetBeaconRandomness(ctx, pers, round, entropy) + +} diff --git a/conformance/runner.go b/conformance/runner.go index a240c9067..3897ad853 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -66,6 +66,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { Message: msg, BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), + Rand: NewReplayingRand(r, vector.Randomness), }) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) diff --git a/go.mod b/go.mod index 170dc3003..e401af36d 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.3 + github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index e820ede3a..7d6c743ae 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= -github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a h1:2jRLaT3/sHyAinWR2asCAkvzcnOAIi13eTlWf3YEVvY= +github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From e803cf151f1dd6df0dd30681a44665a07e7b4ef9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 15:22:54 -0700 Subject: [PATCH 020/125] introduce separate state-tree versions Instead of versioning the state tree along with the actors, version it separately. This structure may not upgrade every time we update actors. --- chain/gen/genesis/genesis.go | 3 +- chain/state/statetree.go | 56 ++++++++++++++++++-------- chain/state/statetree_test.go | 13 +++--- chain/stmgr/forks.go | 2 +- chain/types/cbor_gen.go | 16 ++++---- chain/types/state.go | 17 ++++++-- gen/main.go | 2 +- lotuspond/front/src/chain/methods.json | 3 +- 8 files changed, 73 insertions(+), 39 deletions(-) diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index f532b9f5e..9f15ecaed 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -26,7 +26,6 @@ import ( adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -117,7 +116,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("putting empty object: %w", err) } - state, err := state.NewStateTree(cst, actors.Version0) + state, err := state.NewStateTree(cst, types.StateTreeVersion0) if err != nil { return nil, nil, xerrors.Errorf("making new state tree: %w", err) } diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 3f9597420..e9b76ea77 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" cbg "github.com/whyrusleeping/cbor-gen" @@ -26,7 +27,7 @@ var log = logging.Logger("statetree") // StateTree stores actors state by their ID. type StateTree struct { root adt.Map - version actors.Version // TODO + version types.StateTreeVersion info cid.Cid Store cbor.IpldStore @@ -120,21 +121,41 @@ func (ss *stateSnaps) deleteActor(addr address.Address) { ss.layers[len(ss.layers)-1].actors[addr] = streeOp{Delete: true} } -func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error) { +// VersionForNetwork returns the state tree version for the given network +// version. +func VersionForNetwork(ver network.Version) types.StateTreeVersion { + if actors.VersionForNetwork(ver) == actors.Version0 { + return types.StateTreeVersion0 + } + return types.StateTreeVersion1 +} + +func adtForSTVersion(ver types.StateTreeVersion) actors.Version { + switch ver { + case types.StateTreeVersion0: + return actors.Version0 + case types.StateTreeVersion1: + return actors.Version2 + default: + panic("unhandled state tree version") + } +} + +func NewStateTree(cst cbor.IpldStore, ver types.StateTreeVersion) (*StateTree, error) { var info cid.Cid - switch version { - case actors.Version0: + switch ver { + case types.StateTreeVersion0: // info is undefined - case actors.Version2: + case types.StateTreeVersion1: var err error - info, err = cst.Put(context.TODO(), new(types.StateInfo)) + info, err = cst.Put(context.TODO(), new(types.StateInfo0)) if err != nil { return nil, err } default: - return nil, xerrors.Errorf("unsupported state tree version: %d", version) + return nil, xerrors.Errorf("unsupported state tree version: %d", ver) } - root, err := adt.NewMap(adt.WrapStore(context.TODO(), cst), version) + root, err := adt.NewMap(adt.WrapStore(context.TODO(), cst), adtForSTVersion(ver)) if err != nil { return nil, err } @@ -142,7 +163,7 @@ func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error return &StateTree{ root: root, info: info, - version: version, + version: ver, Store: cst, snaps: newStateSnaps(), }, nil @@ -154,13 +175,16 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { if err := cst.Get(context.TODO(), c, &root); err != nil { // We failed to decode as the new version, must be an old version. root.Actors = c - root.Version = actors.Version0 + root.Version = types.StateTreeVersion0 } switch root.Version { - case actors.Version0, actors.Version2: + case types.StateTreeVersion0, types.StateTreeVersion1: // Load the actual state-tree HAMT. - nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), root.Actors, actors.Version(root.Version)) + nd, err := adt.AsMap( + adt.WrapStore(context.TODO(), cst), root.Actors, + adtForSTVersion(root.Version), + ) if err != nil { log.Errorf("loading hamt node %s failed: %s", c, err) return nil, err @@ -169,7 +193,7 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { return &StateTree{ root: nd, info: root.Info, - version: actors.Version(root.Version), + version: root.Version, Store: cst, snaps: newStateSnaps(), }, nil @@ -309,11 +333,11 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) { return cid.Undef, xerrors.Errorf("failed to flush state-tree hamt: %w", err) } // If we're version 0, return a raw tree. - if st.version == actors.Version0 { + if st.version == types.StateTreeVersion0 { return root, nil } // Otherwise, return a versioned tree. - return st.Store.Put(ctx, &types.StateRoot{Version: uint64(st.version), Actors: root, Info: st.info}) + return st.Store.Put(ctx, &types.StateRoot{Version: st.version, Actors: root, Info: st.info}) } func (st *StateTree) Snapshot(ctx context.Context) error { @@ -400,7 +424,7 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error } // Version returns the version of the StateTree data structure in use. -func (st *StateTree) Version() actors.Version { +func (st *StateTree) Version() types.StateTreeVersion { return st.version } diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 3b08a4b53..ed1fb1889 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -13,13 +13,12 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) func BenchmarkStateTreeSet(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -46,7 +45,7 @@ func BenchmarkStateTreeSet(b *testing.B) { func BenchmarkStateTreeSetFlush(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -76,7 +75,7 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { func BenchmarkStateTree10kGetActor(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -118,7 +117,7 @@ func BenchmarkStateTree10kGetActor(b *testing.B) { func TestSetCache(t *testing.T) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -155,7 +154,7 @@ func TestSetCache(t *testing.T) { func TestSnapshots(t *testing.T) { ctx := context.Background() cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -239,7 +238,7 @@ func assertNotHas(t *testing.T, st *StateTree, addr address.Address) { func TestStateTreeConsistency(t *testing.T) { cst := cbor.NewMemCborStore() // TODO: ActorUpgrade: this test tests pre actors v2 - st, err := NewStateTree(cst, actors.VersionForNetwork(network.Version3)) + st, err := NewStateTree(cst, VersionForNetwork(network.Version3)) if err != nil { t.Fatal(err) } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index d5d0dbf7e..fd6058127 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -494,7 +494,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo epoch := ts.Height() - 1 - info, err := store.Put(ctx, new(types.StateInfo)) + info, err := store.Put(ctx, new(types.StateInfo0)) if err != nil { return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err) } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index f95df33bc..d063ce8c9 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -1648,7 +1648,7 @@ func (t *StateRoot) MarshalCBOR(w io.Writer) error { scratch := make([]byte, 9) - // t.Version (uint64) (uint64) + // t.Version (types.StateTreeVersion) (uint64) if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Version)); err != nil { return err @@ -1687,7 +1687,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Version (uint64) (uint64) + // t.Version (types.StateTreeVersion) (uint64) { @@ -1698,7 +1698,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Version = uint64(extra) + t.Version = StateTreeVersion(extra) } // t.Actors (cid.Cid) (struct) @@ -1728,22 +1728,22 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { return nil } -var lengthBufStateInfo = []byte{128} +var lengthBufStateInfo0 = []byte{128} -func (t *StateInfo) MarshalCBOR(w io.Writer) error { +func (t *StateInfo0) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufStateInfo); err != nil { + if _, err := w.Write(lengthBufStateInfo0); err != nil { return err } return nil } -func (t *StateInfo) UnmarshalCBOR(r io.Reader) error { - *t = StateInfo{} +func (t *StateInfo0) UnmarshalCBOR(r io.Reader) error { + *t = StateInfo0{} br := cbg.GetPeeker(r) scratch := make([]byte, 8) diff --git a/chain/types/state.go b/chain/types/state.go index b99eb19c2..a96883604 100644 --- a/chain/types/state.go +++ b/chain/types/state.go @@ -2,9 +2,20 @@ package types import "github.com/ipfs/go-cid" +// StateTreeVersion is the version of the state tree itself, independent of the +// network version or the actors version. +type StateTreeVersion uint64 + +const ( + // StateTreeVersion0 corresponds to actors < v2. + StateTreeVersion0 StateTreeVersion = iota + // StateTreeVersion1 corresponds to actors >= v2. + StateTreeVersion1 +) + type StateRoot struct { - // State root version. Versioned along with actors (for now). - Version uint64 + // State tree version. + Version StateTreeVersion // Actors tree. The structure depends on the state root version. Actors cid.Cid // Info. The structure depends on the state root version. @@ -12,4 +23,4 @@ type StateRoot struct { } // TODO: version this. -type StateInfo struct{} +type StateInfo0 struct{} diff --git a/gen/main.go b/gen/main.go index bcb43a8f0..d5874af2c 100644 --- a/gen/main.go +++ b/gen/main.go @@ -27,7 +27,7 @@ func main() { types.ExpTipSet{}, types.BeaconEntry{}, types.StateRoot{}, - types.StateInfo{}, + types.StateInfo0{}, ) if err != nil { fmt.Println(err) diff --git a/lotuspond/front/src/chain/methods.json b/lotuspond/front/src/chain/methods.json index 5e15b053b..b271bfae5 100644 --- a/lotuspond/front/src/chain/methods.json +++ b/lotuspond/front/src/chain/methods.json @@ -176,7 +176,8 @@ "CompactPartitions", "CompactSectorNumbers", "ConfirmUpdateWorkerKey", - "RepayDebt" + "RepayDebt", + "ChangeOwnerAddress" ], "fil/2/storagepower": [ "Send", From c17fa4bc35d3a7847fcc50a967fce468b56993f9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 16:58:18 -0700 Subject: [PATCH 021/125] update test-vectors for StateManager constructor change --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 3a6e0b5e0..56f0676eb 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 3a6e0b5e069b1452ce1a032aa315354d645f3ec4 +Subproject commit 56f0676eb3be4b1e1dea892eea330614de755177 From cca17f607898615ded848d89a5de27c139413f90 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 17:51:43 -0700 Subject: [PATCH 022/125] fix state tree version in v2 upgrade --- chain/actors/version.go | 4 ++-- chain/stmgr/forks.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/chain/actors/version.go b/chain/actors/version.go index 385ff592f..17af8b08b 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -9,8 +9,8 @@ import ( type Version int const ( - Version0 = 0 - Version2 = 2 + Version0 Version = 0 + Version2 Version = 2 ) // Converts a network version into an actors adt version. diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index fd6058127..52c764ba2 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -25,7 +25,6 @@ import ( states2 "github.com/filecoin-project/specs-actors/v2/actors/states" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" @@ -518,8 +517,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo } newRoot, err := store.Put(ctx, &types.StateRoot{ - // TODO: ActorUpgrade: should be state-tree specific, not just the actors version. - Version: actors.Version2, + Version: types.StateTreeVersion1, Actors: newHamtRoot, Info: info, }) From c02daca9e973354a75660c18bf1cb921e4ff096e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 20:12:35 +0100 Subject: [PATCH 023/125] work around #4223. --- conformance/rand_record.go | 23 ++++++++++++++++++++--- conformance/rand_replay.go | 3 ++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/conformance/rand_record.go b/conformance/rand_record.go index 2d3477c94..6f6d064dc 100644 --- a/conformance/rand_record.go +++ b/conformance/rand_record.go @@ -2,21 +2,28 @@ package conformance import ( "context" + "fmt" "sync" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/test-vectors/schema" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/test-vectors/schema" ) type RecordingRand struct { reporter Reporter api api.FullNode + // once guards the loading of the head tipset. + // can be removed when https://github.com/filecoin-project/lotus/issues/4223 + // is fixed. + once sync.Once + head types.TipSetKey lk sync.Mutex recorded schema.Randomness } @@ -30,8 +37,17 @@ func NewRecordingRand(reporter Reporter, api api.FullNode) *RecordingRand { return &RecordingRand{reporter: reporter, api: api} } +func (r *RecordingRand) loadHead() { + head, err := r.api.ChainHead(context.Background()) + if err != nil { + panic(fmt.Sprintf("could not fetch chain head while fetching randomness: %s", err)) + } + r.head = head.Key() +} + func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - ret, err := r.api.ChainGetRandomnessFromTickets(ctx, types.EmptyTSK, pers, round, entropy) + r.once.Do(r.loadHead) + ret, err := r.api.ChainGetRandomnessFromTickets(ctx, r.head, pers, round, entropy) if err != nil { return ret, err } @@ -55,7 +71,8 @@ func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.Doma } func (r *RecordingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, types.EmptyTSK, pers, round, entropy) + r.once.Do(r.loadHead) + ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, r.head, pers, round, entropy) if err != nil { return ret, err } diff --git a/conformance/rand_replay.go b/conformance/rand_replay.go index b820b8ced..1b73e5a08 100644 --- a/conformance/rand_replay.go +++ b/conformance/rand_replay.go @@ -7,8 +7,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/test-vectors/schema" + + "github.com/filecoin-project/lotus/chain/vm" ) type ReplayingRand struct { From 6f4a76266441873da7d5f19dad57a9d610c83869 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 7 Oct 2020 21:13:04 +0200 Subject: [PATCH 024/125] Fix Messages field in MessagePoolEvtMessage journal entry Signed-off-by: Jakub Sztandera --- chain/messagepool/repub.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index db31e18c2..672119ba9 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -149,13 +149,13 @@ loop: if len(msgs) > 0 { journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { - msgs := make([]MessagePoolEvtMessage, 0, len(msgs)) + msgsEv := make([]MessagePoolEvtMessage, 0, len(msgs)) for _, m := range msgs { - msgs = append(msgs, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) + msgsEv = append(msgsEv, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) } return MessagePoolEvt{ Action: "repub", - Messages: msgs, + Messages: msgsEv, } }) } From 767c346cf82dd2a41d72926dd2b406707f049a0c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 12:17:32 -0700 Subject: [PATCH 025/125] update test-vectors --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 56f0676eb..7471e2805 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 56f0676eb3be4b1e1dea892eea330614de755177 +Subproject commit 7471e2805fc3e459e4ee325775633e8ec76cb7c6 From e55b00f3582215e974cf6534eb3e09fbe411ee9d Mon Sep 17 00:00:00 2001 From: Jennifer <42981373+jennijuju@users.noreply.github.com> Date: Wed, 7 Oct 2020 17:07:19 -0400 Subject: [PATCH 026/125] Increase the message fee cap upper bounds in PCR. --- cmd/lotus-pcr/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 5b59ff03d..df329d8d3 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -392,13 +392,13 @@ var runCmd = &cli.Command{ Name: "pre-fee-cap-max", EnvVars: []string{"LOTUS_PCR_PRE_FEE_CAP_MAX"}, Usage: "messages with a fee cap larger than this will be skipped when processing pre commit messages", - Value: "0.0000000001", + Value: "0.000000001", }, &cli.StringFlag{ Name: "prove-fee-cap-max", EnvVars: []string{"LOTUS_PCR_PROVE_FEE_CAP_MAX"}, Usage: "messages with a prove cap larger than this will be skipped when processing pre commit messages", - Value: "0.0000000001", + Value: "0.000000001", }, }, Action: func(cctx *cli.Context) error { From ae3691e0a2ee49858ba2d2f7f8dda26f728462c9 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 7 Oct 2020 21:53:56 +0000 Subject: [PATCH 027/125] lotus-pcr: zero refund --- cmd/lotus-pcr/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 5b59ff03d..a1406789e 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -1025,9 +1025,9 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu return nil, nil } - refundValue := types.NewInt(0) tipsetRefunds := NewMinersRefund() for i, msg := range msgs { + refundValue := types.NewInt(0) m := msg.Message a, err := r.api.StateGetActor(ctx, m.To, tipset.Key()) From 12cd478e11c5ee6e1eadab98392710c379f1f93a Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Oct 2020 15:14:46 -0700 Subject: [PATCH 028/125] Add message counts to inspect chain output --- cli/chain.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 20a8b7797..763752f23 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -449,11 +449,16 @@ var chainInspectUsage = &cli.Command{ bySender := make(map[string]int64) byDest := make(map[string]int64) byMethod := make(map[string]int64) + bySenderC := make(map[string]int64) + byDestC := make(map[string]int64) + byMethodC := make(map[string]int64) var sum int64 for _, m := range msgs { bySender[m.Message.From.String()] += m.Message.GasLimit + bySenderC[m.Message.From.String()]++ byDest[m.Message.To.String()] += m.Message.GasLimit + byDestC[m.Message.To.String()]++ sum += m.Message.GasLimit code, err := lookupActorCode(m.Message.To) @@ -464,7 +469,7 @@ var chainInspectUsage = &cli.Command{ mm := stmgr.MethodsMap[code][m.Message.Method] byMethod[mm.Name] += m.Message.GasLimit - + byMethodC[mm.Name]++ } type keyGasPair struct { @@ -496,19 +501,19 @@ var chainInspectUsage = &cli.Command{ fmt.Printf("By Sender:\n") for i := 0; i < numRes && i < len(senderVals); i++ { sv := senderVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key]) } fmt.Println() fmt.Printf("By Receiver:\n") for i := 0; i < numRes && i < len(destVals); i++ { sv := destVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key]) } fmt.Println() fmt.Printf("By Method:\n") for i := 0; i < numRes && i < len(methodVals); i++ { sv := methodVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key]) } return nil From fe912223bdfa000b008c143b21f2cca1c0ac7388 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 13:56:38 -0700 Subject: [PATCH 029/125] pass an explicit upgrade height to migrations The tipset height may not be the correct one, given null blocks. --- chain/stmgr/forks.go | 23 +++++++++++++---------- chain/stmgr/forks_test.go | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 52c764ba2..b897c86eb 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -33,7 +33,14 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -type UpgradeFunc func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error) +// UpgradeFunc is a migration function run ate very upgrade. +// +// - The oldState is the state produced by the upgrade epoch. +// - The returned newState is the new state that will be used by the next epoch. +// - The height is the upgrade epoch height (already executed). +// - The tipset is the tipset for the last non-null block before the upgrade. Do +// not assume that ts.Height() is the upgrade height. +type UpgradeFunc func(ctx context.Context, sm *StateManager, cb ExecCallback, oldState cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (newState cid.Cid, err error) type Upgrade struct { Height abi.ChainEpoch @@ -108,7 +115,7 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig var err error f, ok := sm.stateMigrations[height] if ok { - retCid, err = f(ctx, sm, cb, root, ts) + retCid, err = f(ctx, sm, cb, root, height, ts) if err != nil { return cid.Undef, err } @@ -179,7 +186,7 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, return nil } -func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { // Some initial parameters FundsForMiners := types.FromFil(1_000_000) LookbackEpoch := abi.ChainEpoch(32000) @@ -431,11 +438,9 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal return tree.Flush(ctx) } -func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) - epoch := ts.Height() - 1 - if build.UpgradeLiftoffHeight <= epoch { return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height") } @@ -488,11 +493,9 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return tree.Flush(ctx) } -func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) - epoch := ts.Height() - 1 - info, err := store.Put(ctx, new(types.StateInfo0)) if err != nil { return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err) @@ -539,7 +542,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return newRoot, nil } -func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { tree, err := sm.StateTree(root) if err != nil { return cid.Undef, xerrors.Errorf("getting state tree: %w", err) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index bb03f13b9..daa39a8d6 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -120,7 +120,7 @@ func TestForkHeightTriggers(t *testing.T) { Network: 1, Height: testForkHeight, Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, - root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore()) st, err := sm.StateTree(root) From dab1107f5b86f6f6dbef8e630b92fc9d0b4efac7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 14:35:07 -0700 Subject: [PATCH 030/125] avoid estimating gas and explicitly calling blocks on fork tipsets These tipsets can be slow. --- chain/stmgr/call.go | 39 +++++++++++++++++++++++++++++++++++++++ chain/stmgr/forks.go | 5 +++++ node/impl/full/gas.go | 13 ++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 15cbac53e..0497a20ed 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -2,6 +2,7 @@ package stmgr import ( "context" + "errors" "fmt" "github.com/filecoin-project/go-address" @@ -17,17 +18,37 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) +var ErrWouldFork = errors.New("refusing explicit call due to state fork at epoch") + func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { ctx, span := trace.StartSpan(ctx, "statemanager.Call") defer span.End() + // If no tipset is provided, try to find one without a fork. if ts == nil { ts = sm.cs.GetHeaviestTipSet() + + // Search back till we find a height with no fork, or we reach the beginning. + for ts.Height() > 0 && sm.hasStateFork(ctx, ts.Height()-1) { + var err error + ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err) + } + } } bstate := ts.ParentState() bheight := ts.Height() + // If we have to run a migration, and we're not at genesis, return an + // error because the migration will take too long. + // + // We allow this at height 0 for at-genesis migrations (for testing). + if bheight-1 > 0 && sm.hasStateFork(ctx, bheight-1) { + return nil, ErrWouldFork + } + bstate, err := sm.handleStateForks(ctx, bstate, bheight-1, nil, ts) if err != nil { return nil, fmt.Errorf("failed to handle fork: %w", err) @@ -106,6 +127,24 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri if ts == nil { ts = sm.cs.GetHeaviestTipSet() + + // Search back till we find a height with no fork, or we reach the beginning. + // We need the _previous_ height to have no fork, because we'll + // run the fork logic in `sm.TipSetState`. We need the _current_ + // height to have no fork, because we'll run it inside this + // function before executing the given message. + for ts.Height() > 0 && (sm.hasStateFork(ctx, ts.Height()) || sm.hasStateFork(ctx, ts.Height()-1)) { + var err error + ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err) + } + } + } + + // When we're not at the genesis block, make sure we're at a migration height. + if ts.Height() > 0 && (sm.hasStateFork(ctx, ts.Height()) || sm.hasStateFork(ctx, ts.Height()-1)) { + return nil, ErrWouldFork } state, _, err := sm.TipSetState(ctx, ts) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index b897c86eb..69159f0fe 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -124,6 +124,11 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig return retCid, nil } +func (sm *StateManager) hasStateFork(ctx context.Context, height abi.ChainEpoch) bool { + _, ok := sm.stateMigrations[height] + return ok +} + func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { fromAct, err := tree.GetActor(from) if err != nil { diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index c912c7a8c..1b07aa950 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -160,7 +160,18 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, priorMsgs = append(priorMsgs, m) } - res, err := a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) + // Try calling until we find a height with no migration. + var res *api.InvocResult + for { + res, err = a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) + if err != stmgr.ErrWouldFork { + break + } + ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) + if err != nil { + return -1, xerrors.Errorf("getting parent tipset: %w", err) + } + } if err != nil { return -1, xerrors.Errorf("CallWithGas failed: %w", err) } From 9b7b6146ebf6e3c18ecd421a0f148045d937325e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 15:26:08 -0700 Subject: [PATCH 031/125] construct the new vm with the state manager's vm constructor --- chain/stmgr/call.go | 4 ++-- chain/stmgr/utils.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 0497a20ed..14a4359c1 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -65,7 +65,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. BaseFee: types.NewInt(0), } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } @@ -177,7 +177,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 2c9c5ad94..c0f0c4d2f 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -387,7 +387,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return cid.Undef, nil, err } From 1fea550ce5028997cbe7423b0e9347251dc5878f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 15:28:16 -0700 Subject: [PATCH 032/125] test that we refuse explicit calls at the migration epochs --- chain/stmgr/forks_test.go | 87 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index daa39a8d6..bd3e960d0 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -10,8 +10,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/runtime" + "github.com/stretchr/testify/require" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors" @@ -76,7 +77,7 @@ func (ta testActor) Exports() []interface{} { func (ta *testActor) Constructor(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() rt.StateCreate(&testActorState{11}) - fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Receiver()) + //fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Receiver()) return abi.Empty } @@ -173,7 +174,7 @@ func TestForkHeightTriggers(t *testing.T) { var msgs []*types.SignedMessage - enc, err := actors.SerializeParams(&init_.ExecParams{CodeCID: (testActor{}).Code()}) + enc, err := actors.SerializeParams(&init0.ExecParams{CodeCID: (testActor{}).Code()}) if err != nil { t.Fatal(err) } @@ -233,3 +234,83 @@ func TestForkHeightTriggers(t *testing.T) { } } } + +func TestForkRefuseCall(t *testing.T) { + logging.SetAllLoggers(logging.LevelInfo) + + ctx := context.TODO() + + cg, err := gen.NewGenerator() + if err != nil { + t.Fatal(err) + } + + sm, err := NewStateManagerWithUpgradeSchedule( + cg.ChainStore(), UpgradeSchedule{{ + Network: 1, + Height: testForkHeight, + Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, + root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + return root, nil + }}}) + if err != nil { + t.Fatal(err) + } + + inv := vm.NewActorRegistry() + inv.Register(nil, testActor{}) + + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) { + nvm, err := vm.NewVM(ctx, vmopt) + if err != nil { + return nil, err + } + nvm.SetInvoker(inv) + return nvm, nil + }) + + cg.SetStateManager(sm) + + enc, err := actors.SerializeParams(&init0.ExecParams{CodeCID: (testActor{}).Code()}) + if err != nil { + t.Fatal(err) + } + + m := &types.Message{ + From: cg.Banker(), + To: lotusinit.Address, + Method: builtin.MethodsInit.Exec, + Params: enc, + GasLimit: types.TestGasLimit, + Value: types.NewInt(0), + GasPremium: types.NewInt(0), + GasFeeCap: types.NewInt(0), + } + + for i := 0; i < 50; i++ { + ts, err := cg.NextTipSet() + if err != nil { + t.Fatal(err) + } + + ret, err := sm.CallWithGas(ctx, m, nil, ts.TipSet.TipSet()) + switch ts.TipSet.TipSet().Height() { + case testForkHeight, testForkHeight + 1: + // If I had a fork, or I _will_ have a fork, it should fail. + require.Equal(t, ErrWouldFork, err) + default: + require.NoError(t, err) + require.True(t, ret.MsgRct.ExitCode.IsSuccess()) + } + // Call just runs on the parent state for a tipset, so we only + // expect an error at the fork height. + ret, err = sm.Call(ctx, m, ts.TipSet.TipSet()) + switch ts.TipSet.TipSet().Height() { + case testForkHeight + 1: + require.Equal(t, ErrWouldFork, err) + default: + require.NoError(t, err) + require.True(t, ret.MsgRct.ExitCode.IsSuccess()) + } + } +} From b0bea5f14558f49a676d471c5c612d97c925158b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 15:46:56 -0700 Subject: [PATCH 033/125] return an illegal actor error when we see an unsupported actor version As far as the chain is concerned, this actor does not exist. --- chain/vm/invoker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 1e9f04081..661e31178 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -70,12 +70,12 @@ func (ar *ActorRegistry) Invoke(codeCid cid.Cid, rt vmr.Runtime, method abi.Meth log.Errorf("no code for actor %s (Addr: %s)", codeCid, rt.Receiver()) return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "no code for actor %s(%d)(%s)", codeCid, method, hex.EncodeToString(params)) } + if err := act.predicate(rt, act.vmActor); err != nil { + return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "unsupported actor: %s", err) + } if method >= abi.MethodNum(len(act.methods)) || act.methods[method] == nil { return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "no method %d on actor", method) } - if err := act.predicate(rt, act.vmActor); err != nil { - return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "unsupported actor: %s", err) - } return act.methods[method](rt, params) } From 8f4911ac0d558a978acd3981d526c07792f8e65f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Oct 2020 16:00:52 -0700 Subject: [PATCH 034/125] dont print fil suffix in lotus shed audit balances --- chain/types/fil.go | 6 +++++- cmd/lotus-shed/balances.go | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/chain/types/fil.go b/chain/types/fil.go index 99a896e38..64f6a52b3 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -12,11 +12,15 @@ import ( type FIL BigInt func (f FIL) String() string { + return f.Unitless() + " FIL" +} + +func (f FIL) Unitless() string { r := new(big.Rat).SetFrac(f.Int, big.NewInt(int64(build.FilecoinPrecision))) if r.Sign() == 0 { return "0 FIL" } - return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") + " FIL" + return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") } func (f FIL) Format(s fmt.State, ch rune) { diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 1c89a00cf..3aeb0c879 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -117,7 +117,7 @@ var chainBalanceCmd = &cli.Command{ fmt.Printf("Address,Balance,Type,Power,Worker,Owner\n") for _, acc := range infos { - fmt.Printf("%s,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance, acc.Type, acc.Power, acc.Worker, acc.Owner) + fmt.Printf("%s,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Power, acc.Worker, acc.Owner) } return nil }, @@ -244,12 +244,12 @@ var chainBalanceStateCmd = &cli.Command{ if minerInfo { fmt.Printf("Address,Balance,Type,Sectors,Worker,Owner,InitialPledge,Locked,PreCommits\n") for _, acc := range infos { - fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance, acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge, acc.LockedFunds, acc.PreCommits) + fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge, acc.LockedFunds, acc.PreCommits) } } else { fmt.Printf("Address,Balance,Type\n") for _, acc := range infos { - fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance, acc.Type) + fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type) } } From a4e954197caf1c10ee67eb6487398640a96028b5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 16:03:42 -0700 Subject: [PATCH 035/125] retry StateCall at different height if we're at an expensive fork height --- node/impl/full/state.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 3a4390d03..0a5c856bd 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -307,12 +307,22 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, ts }, nil } -func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { +func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (res *api.InvocResult, err error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.Call(ctx, msg, ts) + for { + res, err = a.StateManager.Call(ctx, msg, ts) + if err != stmgr.ErrWouldFork { + break + } + ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("getting parent tipset: %w", err) + } + } + return res, err } func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { From e8253d22c6600d8b03733bf1d32fb1b508101fa3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 16:14:11 -0700 Subject: [PATCH 036/125] only forbid Call* at expensive forks --- chain/stmgr/call.go | 21 +++++++++++---------- chain/stmgr/forks.go | 6 ++++-- chain/stmgr/forks_test.go | 9 +++++---- chain/stmgr/stmgr.go | 23 ++++++++++++++++------- node/impl/full/gas.go | 2 +- node/impl/full/state.go | 2 +- 6 files changed, 38 insertions(+), 25 deletions(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 14a4359c1..df3bfa357 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -var ErrWouldFork = errors.New("refusing explicit call due to state fork at epoch") +var ErrExpensiveFork = errors.New("refusing explicit call due to state fork at epoch") func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { ctx, span := trace.StartSpan(ctx, "statemanager.Call") @@ -29,7 +29,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. ts = sm.cs.GetHeaviestTipSet() // Search back till we find a height with no fork, or we reach the beginning. - for ts.Height() > 0 && sm.hasStateFork(ctx, ts.Height()-1) { + for ts.Height() > 0 && sm.hasExpensiveFork(ctx, ts.Height()-1) { var err error ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) if err != nil { @@ -41,14 +41,15 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. bstate := ts.ParentState() bheight := ts.Height() - // If we have to run a migration, and we're not at genesis, return an - // error because the migration will take too long. + // If we have to run an expensive migration, and we're not at genesis, + // return an error because the migration will take too long. // // We allow this at height 0 for at-genesis migrations (for testing). - if bheight-1 > 0 && sm.hasStateFork(ctx, bheight-1) { - return nil, ErrWouldFork + if bheight-1 > 0 && sm.hasExpensiveFork(ctx, bheight-1) { + return nil, ErrExpensiveFork } + // Run the (not expensive) migration. bstate, err := sm.handleStateForks(ctx, bstate, bheight-1, nil, ts) if err != nil { return nil, fmt.Errorf("failed to handle fork: %w", err) @@ -133,7 +134,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri // run the fork logic in `sm.TipSetState`. We need the _current_ // height to have no fork, because we'll run it inside this // function before executing the given message. - for ts.Height() > 0 && (sm.hasStateFork(ctx, ts.Height()) || sm.hasStateFork(ctx, ts.Height()-1)) { + for ts.Height() > 0 && (sm.hasExpensiveFork(ctx, ts.Height()) || sm.hasExpensiveFork(ctx, ts.Height()-1)) { var err error ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) if err != nil { @@ -142,9 +143,9 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri } } - // When we're not at the genesis block, make sure we're at a migration height. - if ts.Height() > 0 && (sm.hasStateFork(ctx, ts.Height()) || sm.hasStateFork(ctx, ts.Height()-1)) { - return nil, ErrWouldFork + // When we're not at the genesis block, make sure we don't have an expensive migration. + if ts.Height() > 0 && (sm.hasExpensiveFork(ctx, ts.Height()) || sm.hasExpensiveFork(ctx, ts.Height()-1)) { + return nil, ErrExpensiveFork } state, _, err := sm.TipSetState(ctx, ts) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 69159f0fe..6aa5e5d87 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -45,6 +45,7 @@ type UpgradeFunc func(ctx context.Context, sm *StateManager, cb ExecCallback, ol type Upgrade struct { Height abi.ChainEpoch Network network.Version + Expensive bool Migration UpgradeFunc } @@ -68,6 +69,7 @@ func DefaultUpgradeSchedule() UpgradeSchedule { }, { Height: build.UpgradeActorsV2Height, Network: network.Version4, + Expensive: true, Migration: UpgradeActorsV2, }, { Height: build.UpgradeLiftoffHeight, @@ -124,8 +126,8 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig return retCid, nil } -func (sm *StateManager) hasStateFork(ctx context.Context, height abi.ChainEpoch) bool { - _, ok := sm.stateMigrations[height] +func (sm *StateManager) hasExpensiveFork(ctx context.Context, height abi.ChainEpoch) bool { + _, ok := sm.expensiveUpgrades[height] return ok } diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index bd3e960d0..bf1c711e4 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -247,8 +247,9 @@ func TestForkRefuseCall(t *testing.T) { sm, err := NewStateManagerWithUpgradeSchedule( cg.ChainStore(), UpgradeSchedule{{ - Network: 1, - Height: testForkHeight, + Network: 1, + Expensive: true, + Height: testForkHeight, Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { return root, nil @@ -297,7 +298,7 @@ func TestForkRefuseCall(t *testing.T) { switch ts.TipSet.TipSet().Height() { case testForkHeight, testForkHeight + 1: // If I had a fork, or I _will_ have a fork, it should fail. - require.Equal(t, ErrWouldFork, err) + require.Equal(t, ErrExpensiveFork, err) default: require.NoError(t, err) require.True(t, ret.MsgRct.ExitCode.IsSuccess()) @@ -307,7 +308,7 @@ func TestForkRefuseCall(t *testing.T) { ret, err = sm.Call(ctx, m, ts.TipSet.TipSet()) switch ts.TipSet.TipSet().Height() { case testForkHeight + 1: - require.Equal(t, ErrWouldFork, err) + require.Equal(t, ErrExpensiveFork, err) default: require.NoError(t, err) require.True(t, ret.MsgRct.ExitCode.IsSuccess()) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ac01ebb61..d81cf1c72 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -53,6 +53,10 @@ type StateManager struct { // Maps chain epochs to upgrade functions. stateMigrations map[abi.ChainEpoch]UpgradeFunc + // A set of potentially expensive/time consuming upgrades. Explicit + // calls for, e.g., gas estimation fail against this epoch with + // ErrExpensiveFork. + expensiveUpgrades map[abi.ChainEpoch]struct{} stCache map[string][]cid.Cid compWait map[string]chan struct{} @@ -78,6 +82,7 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule } stateMigrations := make(map[abi.ChainEpoch]UpgradeFunc, len(us)) + expensiveUpgrades := make(map[abi.ChainEpoch]struct{}, len(us)) var networkVersions []versionSpec lastVersion := network.Version0 if len(us) > 0 { @@ -87,6 +92,9 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule if upgrade.Migration != nil { stateMigrations[upgrade.Height] = upgrade.Migration } + if upgrade.Expensive { + expensiveUpgrades[upgrade.Height] = struct{}{} + } networkVersions = append(networkVersions, versionSpec{ networkVersion: lastVersion, atOrBelow: upgrade.Height, @@ -99,13 +107,14 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule } return &StateManager{ - networkVersions: networkVersions, - latestVersion: lastVersion, - stateMigrations: stateMigrations, - newVM: vm.NewVM, - cs: cs, - stCache: make(map[string][]cid.Cid), - compWait: make(map[string]chan struct{}), + networkVersions: networkVersions, + latestVersion: lastVersion, + stateMigrations: stateMigrations, + expensiveUpgrades: expensiveUpgrades, + newVM: vm.NewVM, + cs: cs, + stCache: make(map[string][]cid.Cid), + compWait: make(map[string]chan struct{}), }, nil } diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 1b07aa950..3580ca26d 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -164,7 +164,7 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, var res *api.InvocResult for { res, err = a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) - if err != stmgr.ErrWouldFork { + if err != stmgr.ErrExpensiveFork { break } ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 0a5c856bd..7d654985a 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -314,7 +314,7 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. } for { res, err = a.StateManager.Call(ctx, msg, ts) - if err != stmgr.ErrWouldFork { + if err != stmgr.ErrExpensiveFork { break } ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) From d97eb10349398e8fae800bf11b2bfc046d96d57e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 16:32:54 -0700 Subject: [PATCH 037/125] fix spelling nit --- chain/stmgr/forks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 6aa5e5d87..1888d1ee9 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -33,7 +33,7 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -// UpgradeFunc is a migration function run ate very upgrade. +// UpgradeFunc is a migration function run at every upgrade. // // - The oldState is the state produced by the upgrade epoch. // - The returned newState is the new state that will be used by the next epoch. From d1555106a474bf75c2e5329df02579165ffcbff8 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 17:47:03 -0400 Subject: [PATCH 038/125] Set actorsv2 upgrade epoch --- build/params_2k.go | 3 +- build/params_testground.go | 3 +- build/params_testnet.go | 5 +-- chain/actors/builtin/builtin.go | 1 + chain/stmgr/forks.go | 68 +++++++++++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 5 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index f4a17f724..a9277831d 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -13,8 +13,9 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 -const UpgradeLiftoffHeight = -3 +const UpgradeRefuelHeight = -3 const UpgradeActorsV2Height = 10 +const UpgradeLiftoffHeight = -4 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_testground.go b/build/params_testground.go index bd064b3d6..6109cbc04 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -82,8 +82,9 @@ var ( UpgradeSmokeHeight abi.ChainEpoch = -1 UpgradeIgnitionHeight abi.ChainEpoch = -2 - UpgradeLiftoffHeight abi.ChainEpoch = -3 + UpgradeRefuelHeight abi.ChainEpoch = -3 UpgradeActorsV2Height abi.ChainEpoch = 10 + UpgradeLiftoffHeight abi.ChainEpoch = -4 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_testnet.go b/build/params_testnet.go index 16de8ff75..aef3ff450 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -25,9 +25,8 @@ const BreezeGasTampingDuration = 120 const UpgradeSmokeHeight = 51000 const UpgradeIgnitionHeight = 94000 - -// TODO: Actual epoch needs to be filled in -const UpgradeActorsV2Height = 128888 +const UpgradeRefuelHeight = 130800 +const UpgradeActorsV2Height = 138720 // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 7def78dcf..cb24a2c33 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -22,6 +22,7 @@ import ( var SystemActorAddr = builtin0.SystemActorAddr var BurntFundsActorAddr = builtin0.BurntFundsActorAddr var ReserveAddress = makeAddress("t090") +var RootVerifierAddress = makeAddress("t080") // TODO: Why does actors have 2 different versions of this? type SectorInfo = proof0.SectorInfo diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 1888d1ee9..989a94870 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -6,6 +6,8 @@ import ( "encoding/binary" "math" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" @@ -66,6 +68,10 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Height: build.UpgradeIgnitionHeight, Network: network.Version3, Migration: UpgradeIgnition, + }, { + Height: build.UpgradeRefuelHeight, + Network: network.Version3, + Migration: UpgradeRefuel, }, { Height: build.UpgradeActorsV2Height, Network: network.Version4, @@ -500,6 +506,36 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return tree.Flush(ctx) } +func UpgradeRefuel(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + store := sm.cs.Store(ctx) + tree, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } + + addr, err := address.NewFromString("t0122") + if err != nil { + return cid.Undef, xerrors.Errorf("getting address: %w", err) + } + + err = resetMultisigVesting(ctx, store, tree, addr, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + err = resetMultisigVesting(ctx, store, tree, builtin.ReserveAddress, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + err = resetMultisigVesting(ctx, store, tree, builtin.RootVerifierAddress, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + return tree.Flush(ctx) +} + func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) @@ -707,6 +743,7 @@ func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, erro return addr, nil } +// TODO: After the Liftoff epoch, refactor this to use resetMultisigVesting func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, startEpoch abi.ChainEpoch) error { gb, err := sm.cs.GetGenesis() if err != nil { @@ -756,3 +793,34 @@ func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, return nil } + +func resetMultisigVesting(ctx context.Context, store adt0.Store, tree *state.StateTree, addr address.Address, startEpoch abi.ChainEpoch, duration abi.ChainEpoch, balance abi.TokenAmount) error { + act, err := tree.GetActor(addr) + if err != nil { + return xerrors.Errorf("getting actor: %w", err) + } + + if !builtin.IsMultisigActor(act.Code) { + return xerrors.Errorf("actor wasn't msig: %w", err) + } + + var msigState multisig0.State + if err := store.Get(ctx, act.Head, &msigState); err != nil { + return xerrors.Errorf("reading multisig state: %w", err) + } + + msigState.StartEpoch = startEpoch + msigState.UnlockDuration = duration + msigState.InitialBalance = balance + + act.Head, err = store.Put(ctx, &msigState) + if err != nil { + return xerrors.Errorf("writing new multisig state: %w", err) + } + + if err := tree.SetActor(addr, act); err != nil { + return xerrors.Errorf("setting multisig actor: %w", err) + } + + return nil +} From f9f54819d41d333e021b9d2c3b4d32c4d3dbd138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 8 Oct 2020 00:17:24 +0200 Subject: [PATCH 039/125] Env var to control v2 actor migration Env var to control v2 actor migration --- build/params_2k.go | 13 +++++++++++-- build/params_testnet.go | 8 +++++++- chain/stmgr/forks.go | 30 ++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index a9277831d..c6538dc08 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -3,6 +3,9 @@ package build import ( + "math" + "os" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -14,8 +17,9 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 const UpgradeRefuelHeight = -3 -const UpgradeActorsV2Height = 10 -const UpgradeLiftoffHeight = -4 + +var UpgradeActorsV2Height = abi.ChainEpoch(10) +var UpgradeLiftoffHeight = abi.ChainEpoch(-4) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, @@ -26,6 +30,11 @@ func init() { policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) + if os.Getenv("LOTUS_DISABLE_V2_ACTOR_MIGRATION") == "1" { + UpgradeActorsV2Height = math.MaxInt64 + UpgradeLiftoffHeight = 11 + } + BuildType |= Build2k } diff --git a/build/params_testnet.go b/build/params_testnet.go index aef3ff450..fe70deaef 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -5,6 +5,7 @@ package build import ( + "math" "os" "github.com/filecoin-project/go-address" @@ -26,7 +27,8 @@ const UpgradeSmokeHeight = 51000 const UpgradeIgnitionHeight = 94000 const UpgradeRefuelHeight = 130800 -const UpgradeActorsV2Height = 138720 + +var UpgradeActorsV2Height = abi.ChainEpoch(138720) // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. @@ -44,6 +46,10 @@ func init() { SetAddressNetwork(address.Mainnet) } + if os.Getenv("LOTUS_DISABLE_V2_ACTOR_MIGRATION") == "1" { + UpgradeActorsV2Height = math.MaxInt64 + } + Devnet = false } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 989a94870..cbbc351f9 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -56,7 +56,7 @@ type UpgradeSchedule []Upgrade func DefaultUpgradeSchedule() UpgradeSchedule { var us UpgradeSchedule - for _, u := range []Upgrade{{ + updates := []Upgrade{{ Height: build.UpgradeBreezeHeight, Network: network.Version1, Migration: UpgradeFaucetBurnRecovery, @@ -81,7 +81,33 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Height: build.UpgradeLiftoffHeight, Network: network.Version4, Migration: UpgradeLiftoff, - }} { + }} + + if build.UpgradeActorsV2Height == math.MaxInt64 { // disable actors upgrade + updates = []Upgrade{{ + Height: build.UpgradeBreezeHeight, + Network: network.Version1, + Migration: UpgradeFaucetBurnRecovery, + }, { + Height: build.UpgradeSmokeHeight, + Network: network.Version2, + Migration: nil, + }, { + Height: build.UpgradeIgnitionHeight, + Network: network.Version3, + Migration: UpgradeIgnition, + }, { + Height: build.UpgradeRefuelHeight, + Network: network.Version3, + Migration: UpgradeRefuel, + }, { + Height: build.UpgradeLiftoffHeight, + Network: network.Version3, + Migration: UpgradeLiftoff, + }} + } + + for _, u := range updates { if u.Height < 0 { // upgrade disabled continue From cd9af278b46b172e9f918835348f09023bb8b074 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Oct 2020 17:01:01 -0700 Subject: [PATCH 040/125] more info in audit outputs --- chain/types/fil.go | 2 +- cmd/lotus-shed/balances.go | 84 ++++++++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/chain/types/fil.go b/chain/types/fil.go index 64f6a52b3..7eac8ce93 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -18,7 +18,7 @@ func (f FIL) String() string { func (f FIL) Unitless() string { r := new(big.Rat).SetFrac(f.Int, big.NewInt(int64(build.FilecoinPrecision))) if r.Sign() == 0 { - return "0 FIL" + return "0" } return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 3aeb0c879..8aedbedf6 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -7,6 +7,7 @@ import ( "github.com/docker/go-units" lotusbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" @@ -33,16 +34,19 @@ import ( ) type accountInfo struct { - Address address.Address - Balance types.FIL - Type string - Power abi.StoragePower - Worker address.Address - Owner address.Address - InitialPledge types.FIL - PreCommits types.FIL - LockedFunds types.FIL - Sectors uint64 + Address address.Address + Balance types.FIL + Type string + Power abi.StoragePower + Worker address.Address + Owner address.Address + InitialPledge types.FIL + PreCommits types.FIL + LockedFunds types.FIL + Sectors uint64 + VestingStart abi.ChainEpoch + VestingDuration abi.ChainEpoch + VestingAmount types.FIL } var auditsCmd = &cli.Command{ @@ -115,10 +119,8 @@ var chainBalanceCmd = &cli.Command{ infos = append(infos, ai) } - fmt.Printf("Address,Balance,Type,Power,Worker,Owner\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Power, acc.Worker, acc.Owner) - } + printAccountInfos(infos, false) + return nil }, } @@ -196,6 +198,7 @@ var chainBalanceStateCmd = &cli.Command{ LockedFunds: types.FIL(big.NewInt(0)), InitialPledge: types.FIL(big.NewInt(0)), PreCommits: types.FIL(big.NewInt(0)), + VestingAmount: types.FIL(big.NewInt(0)), } if minerInfo && act.IsStorageMinerActor() { @@ -234,6 +237,32 @@ var chainBalanceStateCmd = &cli.Command{ ai.Worker = minfo.Worker ai.Owner = minfo.Owner } + + if act.IsMultisigActor() { + mst, err := multisig.Load(store, act) + if err != nil { + return err + } + + ai.VestingStart, err = mst.StartEpoch() + if err != nil { + return err + } + + ib, err := mst.InitialBalance() + if err != nil { + return err + } + + ai.VestingAmount = types.FIL(ib) + + ai.VestingDuration, err = mst.UnlockDuration() + if err != nil { + return err + } + + } + infos = append(infos, ai) return nil }) @@ -241,22 +270,27 @@ var chainBalanceStateCmd = &cli.Command{ return xerrors.Errorf("failed to loop over actors: %w", err) } - if minerInfo { - fmt.Printf("Address,Balance,Type,Sectors,Worker,Owner,InitialPledge,Locked,PreCommits\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge, acc.LockedFunds, acc.PreCommits) - } - } else { - fmt.Printf("Address,Balance,Type\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type) - } - } + printAccountInfos(infos, minerInfo) return nil }, } +func printAccountInfos(infos []accountInfo, minerInfo bool) { + if minerInfo { + fmt.Printf("Address,Balance,Type,Sectors,Worker,Owner,InitialPledge,Locked,PreCommits,VestingStart,VestingDuration,VestingAmount\n") + for _, acc := range infos { + fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s,%d,%d,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge.Unitless(), acc.LockedFunds.Unitless(), acc.PreCommits.Unitless(), acc.VestingStart, acc.VestingDuration, acc.VestingAmount.Unitless()) + } + } else { + fmt.Printf("Address,Balance,Type\n") + for _, acc := range infos { + fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type) + } + } + +} + var chainPledgeCmd = &cli.Command{ Name: "stateroot-pledge", Description: "Calculate sector pledge numbers", From da34548a05782982b7a152efc0876cd59d8bc336 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 7 Oct 2020 21:45:41 -0400 Subject: [PATCH 041/125] Set head should unmark blocks as valid --- node/impl/full/chain.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index b405fb3dc..aa2ae4df1 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -254,11 +254,31 @@ func (a *ChainAPI) ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) } func (a *ChainAPI) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error { - ts, err := a.Chain.GetTipSetFromKey(tsk) + newHeadTs, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.Chain.SetHead(ts) + + currentTs, err := a.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("getting head: %w", err) + } + + for currentTs.Height() >= newHeadTs.Height() { + for _, blk := range currentTs.Key().Cids() { + err = a.Chain.UnmarkBlockAsValidated(ctx, blk) + if err != nil { + return xerrors.Errorf("unmarking block as validated %s: %w", blk, err) + } + } + + currentTs, err = a.ChainGetTipSet(ctx, currentTs.Parents()) + if err != nil { + return xerrors.Errorf("loading tipset: %w", err) + } + } + + return a.Chain.SetHead(newHeadTs) } func (a *ChainAPI) ChainGetGenesis(ctx context.Context) (*types.TipSet, error) { From dea70426689002cd4fbba6004012e1567328ea8d Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 7 Oct 2020 19:00:56 -0400 Subject: [PATCH 042/125] Lotus version 0.9.0 --- CHANGELOG.md | 95 +++++++++++++++++++++++++++++++++++++++++++++++- build/version.go | 2 +- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6de6ddc2c..4f6f0b3a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,98 @@ # Lotus changelog +# 0.9.0 / 2020-10-07 + +This consensus-breaking release of Lotus upgrades the actors version to v2.0.0. This requires migrating actor state from v0 to v2. The changes that break consensus are: + +- Introducing v2 actors and its migration (https://github.com/filecoin-project/lotus/pull/3936) +- Runtime's Receiver() should only return ID addresses (https://github.com/filecoin-project/lotus/pull/3589) +- Update miner eligibility checks for v2 actors (https://github.com/filecoin-project/lotus/pull/4188) +- Add funds that have left FilReserve to circ supply (https://github.com/filecoin-project/lotus/pull/4160) +- Set WinningPoStSectorSetLookback to finality post-v2 actors (https://github.com/filecoin-project/lotus/pull/4190) +- fix: error when actor panics directly (https://github.com/filecoin-project/lotus/pull/3697) + +## Changes + +#### Dependencies + +- Update go-bitfield (https://github.com/filecoin-project/lotus/pull/4171) +- update the AMT implementation (https://github.com/filecoin-project/lotus/pull/4194) +- Update to actors v0.2.1 (https://github.com/filecoin-project/lotus/pull/4199) + +#### Core Lotus + +- Paych: fix voucher amount verification (https://github.com/filecoin-project/lotus/pull/3821) +- Cap market provider messages (https://github.com/filecoin-project/lotus/pull/4141) +- Run fork function after cron for null block safety (https://github.com/filecoin-project/lotus/pull/4114) +- use bitswap sessions when fetching messages, and cancel them (https://github.com/filecoin-project/lotus/pull/4142) +- relax pubsub IPColocationFactorThreshold to 5 (https://github.com/filecoin-project/lotus/pull/4183) +- Support addresses with mainnet prefixes (https://github.com/filecoin-project/lotus/pull/4186) +- fix: make message signer nonce generation transactional (https://github.com/filecoin-project/lotus/pull/4165) +- build: Env var to keep test address output (https://github.com/filecoin-project/lotus/pull/4213) +- make vm.EnableGasTracing public (https://github.com/filecoin-project/lotus/pull/4214) +- introduce separate state-tree versions (https://github.com/filecoin-project/lotus/pull/4197) +- reject explicit "calls" at the upgrade height (https://github.com/filecoin-project/lotus/pull/4231) +- return an illegal actor error when we see an unsupported actor version (https://github.com/filecoin-project/lotus/pull/4232) +- Set head should unmark blocks as valid (https://gist.github.com/travisperson/3c7cddd77a33979a519ccef4e6515f20) + +#### Mining + +- Increased ExpectedSealDuration and and WaitDealsDelay (https://github.com/filecoin-project/lotus/pull/3743) +- Miner backup/restore commands (https://github.com/filecoin-project/lotus/pull/4133) +- lotus-miner: add more help text to storage / attach (https://github.com/filecoin-project/lotus/pull/3961) +- Reject deals that are > 7 days in the future in the BasicDealFilter (https://github.com/filecoin-project/lotus/pull/4173) +- feat(miner): add miner deadline diffing logic (https://github.com/filecoin-project/lotus/pull/4178) + +#### UX + +- Improve the UX for replacing messages (https://github.com/filecoin-project/lotus/pull/4134) +- Add verified flag to interactive deal creation (https://github.com/filecoin-project/lotus/pull/4145) +- Add command to (slowly) prune lotus chain datastore (https://github.com/filecoin-project/lotus/pull/3876) +- Some helpers for verifreg work (https://github.com/filecoin-project/lotus/pull/4124) +- Always use default 720h for setask duration and hide the duration param option (https://github.com/filecoin-project/lotus/pull/4077) +- Convert ID addresses to key addresses before checking wallet (https://github.com/filecoin-project/lotus/pull/4122) +- add a command to view block space utilization (https://github.com/filecoin-project/lotus/pull/4176) +- allow usage inspection on a chain segment (https://github.com/filecoin-project/lotus/pull/4177) +- Add mpool stats for base fee (https://github.com/filecoin-project/lotus/pull/4170) +- Add verified status to api.DealInfo (https://github.com/filecoin-project/lotus/pull/4153) +- Add a CLI command to set a miner's owner address (https://github.com/filecoin-project/lotus/pull/4189) + +#### Tooling and validation + +- Lotus-pcr: add recover-miners command (https://github.com/filecoin-project/lotus/pull/3714) +- MpoolPushUntrusted API for gateway (https://github.com/filecoin-project/lotus/pull/3915) +- Test lotus-miner info all (https://github.com/filecoin-project/lotus/pull/4166) +- chain export: Error with unfinished exports (https://github.com/filecoin-project/lotus/pull/4179) +- add printf in TestWindowPost (https://github.com/filecoin-project/lotus/pull/4043) +- add trace wdpost (https://github.com/filecoin-project/lotus/pull/4020) +- Fix noncefix (https://github.com/filecoin-project/lotus/pull/4202) +- Lotus-pcr: Limit the fee cap of messages we will process, refund gas fees for windowed post and storage deals (https://github.com/filecoin-project/lotus/pull/4198) +- Fix pond (https://github.com/filecoin-project/lotus/pull/4203) +- allow manual setting of noncefix fee cap (https://github.com/filecoin-project/lotus/pull/4205) +- implement command to get execution traces of any message (https://github.com/filecoin-project/lotus/pull/4200) +- conformance: minor driver refactors (https://github.com/filecoin-project/lotus/pull/4211) +- lotus-pcr: ignore all other messages (https://github.com/filecoin-project/lotus/pull/4218) +- lotus-pcr: zero refund (https://github.com/filecoin-project/lotus/pull/4229) + +## Contributors + +The following contributors had 5 or more commits go into this release. +We are grateful for every contribution! + +| Contributor | Commits | Lines ± | +|--------------------|---------|---------------| +| Stebalien | 84 | +3425/-2287 | +| magik6k | 41 | +2121/-506 | +| arajasek | 39 | +2467/-424 | +| Kubuxu | 25 | +2344/-775 | +| raulk | 21 | +287/-196 | +| whyrusleeping | 13 | +727/-71 | +| hsanjuan | 13 | +5886/-7956 | +| dirkmc | 11 | +2634/-576 | +| travisperson | 8 | +923/-202 | +| ribasushi | 6 | +188/-128 | +| zgfzgf | 5 | +21/-17 | + # 0.8.1 / 2020-09-30 This optional release of Lotus introduces a new version of markets which switches to CBOR-map encodings, and allows datastore migrations. The release also introduces several improvements to the mining process, a few performance optimizations, and a battery of UX additions and enhancements. @@ -117,7 +210,7 @@ We are grateful for every contribution! | vyzo | 22 | +287/-196 | | alanshaw | 15 | +761/-146 | | whyrusleeping | 15 | +736/-52 | -| hannahhoward | 14 | +1237/837- | +| hannahhoward | 14 | +1237/-837 | | anton | 6 | +32/-8 | | travisperson | 5 | +502/-6 | | Frank | 5 | +78/-39 | diff --git a/build/version.go b/build/version.go index 0b317aa17..1602551e4 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.8.1" +const BuildVersion = "0.9.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From cb7d99fcc7075fb565abd2d41278d2b9cae08bd6 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 8 Oct 2020 14:15:09 +0200 Subject: [PATCH 043/125] fix: clash between daemon --api flag and cli tests --- cli/cmd.go | 57 ++++++++++++++++++++++++++----------- cli/paych_test.go | 10 +++---- cli/pprof.go | 3 +- cmd/lotus-shed/consensus.go | 2 +- 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/cli/cmd.go b/cli/cmd.go index 92a366eb5..edcb69adc 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/url" "os" "os/signal" "strings" @@ -47,14 +48,44 @@ func NewCliError(s string) error { type ApiConnector func() api.FullNode type APIInfo struct { - Addr multiaddr.Multiaddr + Addr string Token []byte } func (a APIInfo) DialArgs() (string, error) { - _, addr, err := manet.DialArgs(a.Addr) + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } - return "ws://" + addr + "/rpc/v0", err + return "ws://" + addr + "/rpc/v0", nil + } + + _, err = url.Parse(a.Addr) + if err != nil { + return "", err + } + return a.Addr + "/rpc/v0", nil +} + +func (a APIInfo) Host() (string, error) { + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } + + return addr, nil + } + + spec, err := url.Parse(a.Addr) + if err != nil { + return "", err + } + return spec.Host, nil } func (a APIInfo) AuthHeader() http.Header { @@ -72,11 +103,11 @@ func (a APIInfo) AuthHeader() http.Header { func flagForAPI(t repo.RepoType) string { switch t { case repo.FullNode: - return "api" + return "api-url" case repo.StorageMiner: - return "miner-api" + return "miner-api-url" case repo.Worker: - return "worker-api" + return "worker-api-url" default: panic(fmt.Sprintf("Unknown repo type: %v", t)) } @@ -130,11 +161,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { strma := ctx.String(apiFlag) strma = strings.TrimSpace(strma) - apima, err := multiaddr.NewMultiaddr(strma) - if err != nil { - return APIInfo{}, err - } - return APIInfo{Addr: apima}, nil + return APIInfo{Addr: strma}, nil } envKey := envForRepo(t) @@ -152,12 +179,8 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { if len(sp) != 2 { log.Warnf("invalid env(%s) value, missing token or address", envKey) } else { - ma, err := multiaddr.NewMultiaddr(sp[1]) - if err != nil { - return APIInfo{}, xerrors.Errorf("could not parse multiaddr from env(%s): %w", envKey, err) - } return APIInfo{ - Addr: ma, + Addr: sp[1], Token: []byte(sp[0]), }, nil } @@ -186,7 +209,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } return APIInfo{ - Addr: ma, + Addr: ma.String(), Token: token, }, nil } diff --git a/cli/paych_test.go b/cli/paych_test.go index 35f56d90e..18782b4e8 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -439,12 +439,12 @@ type mockCLI struct { } func newMockCLI(t *testing.T) *mockCLI { - // Create a CLI App with an --api flag so that we can specify which node + // Create a CLI App with an --api-url flag so that we can specify which node // the command should be executed against app := cli.NewApp() app.Flags = []cli.Flag{ &cli.StringFlag{ - Name: "api", + Name: "api-url", Hidden: true, }, } @@ -476,8 +476,8 @@ func (c *mockCLIClient) runCmd(cmd *cli.Command, input []string) string { } func (c *mockCLIClient) runCmdRaw(cmd *cli.Command, input []string) (string, error) { - // prepend --api= - apiFlag := "--api=" + c.addr.String() + // prepend --api-url= + apiFlag := "--api-url=" + c.addr.String() input = append([]string{apiFlag}, input...) fs := c.flagSet(cmd) @@ -493,7 +493,7 @@ func (c *mockCLIClient) runCmdRaw(cmd *cli.Command, input []string) (string, err } func (c *mockCLIClient) flagSet(cmd *cli.Command) *flag.FlagSet { - // Apply app level flags (so we can process --api flag) + // Apply app level flags (so we can process --api-url flag) fs := &flag.FlagSet{} for _, f := range c.cctx.App.Flags { err := f.Apply(fs) diff --git a/cli/pprof.go b/cli/pprof.go index 6819b362a..dccb97f9a 100644 --- a/cli/pprof.go +++ b/cli/pprof.go @@ -9,7 +9,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/node/repo" - manet "github.com/multiformats/go-multiaddr/net" ) var pprofCmd = &cli.Command{ @@ -37,7 +36,7 @@ var PprofGoroutines = &cli.Command{ if err != nil { return xerrors.Errorf("could not get API info: %w", err) } - _, addr, err := manet.DialArgs(ainfo.Addr) + addr, err := ainfo.Host() if err != nil { return err } diff --git a/cmd/lotus-shed/consensus.go b/cmd/lotus-shed/consensus.go index 59d9555df..38a8cd8ef 100644 --- a/cmd/lotus-shed/consensus.go +++ b/cmd/lotus-shed/consensus.go @@ -111,7 +111,7 @@ var consensusCheckCmd = &cli.Command{ if err != nil { return err } - ainfo := lcli.APIInfo{Addr: apima} + ainfo := lcli.APIInfo{Addr: apima.String()} addr, err := ainfo.DialArgs() if err != nil { return err From 4937ca1a96cd2cb70344a54130b7f35d43c511e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 14:55:03 +0100 Subject: [PATCH 044/125] conformance: remove usage of fake syscalls. --- conformance/driver.go | 4 ++-- conformance/stubs.go | 44 ------------------------------------------- 2 files changed, 2 insertions(+), 46 deletions(-) delete mode 100644 conformance/stubs.go diff --git a/conformance/driver.go b/conformance/driver.go index d51798a52..137653141 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -81,7 +81,7 @@ type ExecuteTipsetResult struct { // and reward withdrawal per miner. func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) { var ( - syscalls = mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)) + syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) vmRand = NewFixedRand() cs = store.NewChainStore(bs, ds, syscalls) @@ -173,7 +173,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP StateBase: params.Preroot, Epoch: params.Epoch, Bstore: bs, - Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. + Syscalls: vm.Syscalls(ffiwrapper.ProofVerifier), CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { return params.CircSupply, nil }, diff --git a/conformance/stubs.go b/conformance/stubs.go deleted file mode 100644 index 9307fdd65..000000000 --- a/conformance/stubs.go +++ /dev/null @@ -1,44 +0,0 @@ -package conformance - -import ( - "context" - - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/lotus/chain/state" - "github.com/filecoin-project/lotus/chain/vm" - - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/runtime" - - cbor "github.com/ipfs/go-ipld-cbor" -) - -type testSyscalls struct { - runtime.Syscalls -} - -// TODO VerifySignature this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifySignature(_ crypto.Signature, _ address.Address, _ []byte) error { - return nil -} - -// TODO VerifySeal this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifySeal(_ proof.SealVerifyInfo) error { - return nil -} - -// TODO VerifyPoSt this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifyPoSt(_ proof.WindowPoStVerifyInfo) error { - return nil -} - -func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder { - return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls { - return &testSyscalls{ - base(ctx, cstate, cst), - } - } -} From 1f9446f91fc19473f51408680585035b4dc468ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 15:00:35 +0100 Subject: [PATCH 045/125] upgrade to test vectors schema v0.0.4. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e401af36d..5473dbb59 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a + github.com/filecoin-project/test-vectors/schema v0.0.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 7d6c743ae..45d20d118 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a h1:2jRLaT3/sHyAinWR2asCAkvzcnOAIi13eTlWf3YEVvY= -github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= +github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 67b3a55e30ffb75efc8191d75709fbdfac06f268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 15:44:11 +0100 Subject: [PATCH 046/125] update test-vectors submodule. --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 7471e2805..a8f968ade 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 7471e2805fc3e459e4ee325775633e8ec76cb7c6 +Subproject commit a8f968adeba1995f161f7be0048188affc425079 From cdc2f86f5a33d3e3c0f930287b91c4f36312e6a0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 8 Oct 2020 09:45:07 -0500 Subject: [PATCH 047/125] add more info to chain sync lookback failure --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 655bb4a1d..240d1edef 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -731,7 +731,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, use lbst, _, err := syncer.sm.TipSetState(ctx, lbts) if err != nil { - return xerrors.Errorf("failed to compute lookback tipset state: %w", err) + return xerrors.Errorf("failed to compute lookback tipset state (epoch %d): %w", lbts.Height(), err) } prevBeacon, err := syncer.store.GetLatestBeaconEntry(baseTs) From 98d702fb5a6bff295a5b216b46b9ddd8effd10c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 16:44:20 +0100 Subject: [PATCH 048/125] add signature imports. --- conformance/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conformance/driver.go b/conformance/driver.go index 0451868e4..940c137cf 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -14,6 +14,9 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/lib/blockstore" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures + _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" From 2ad671649c150995d98525999bb3b44547c6454e Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 7 Oct 2020 10:35:57 -0700 Subject: [PATCH 049/125] Use FIL units for `miner set-ask` --- cmd/lotus-storage-miner/market.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 6e3743143..67e694c78 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -154,14 +154,14 @@ var setAskCmd = &cli.Command{ Name: "set-ask", Usage: "Configure the miner's ask", Flags: []cli.Flag{ - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "price", - Usage: "Set the price of the ask for unverified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", + Usage: "Set the price of the ask for unverified deals (specified as FIL / GiB / Epoch) to `PRICE`.", Required: true, }, - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "verified-price", - Usage: "Set the price of the ask for verified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", + Usage: "Set the price of the ask for verified deals (specified as FIL / GiB / Epoch) to `PRICE`", Required: true, }, &cli.StringFlag{ @@ -185,8 +185,15 @@ var setAskCmd = &cli.Command{ } defer closer() - pri := types.NewInt(cctx.Uint64("price")) - vpri := types.NewInt(cctx.Uint64("verified-price")) + pri, err := types.ParseFIL(cctx.String("price")) + if err != nil { + return err + } + + vpri, err := types.ParseFIL(cctx.String("verified-price")) + if err != nil { + return err + } dur, err := time.ParseDuration("720h0m0s") if err != nil { @@ -229,7 +236,7 @@ var setAskCmd = &cli.Command{ return xerrors.Errorf("max piece size (w/bit-padding) %s cannot exceed miner sector size %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) } - return api.MarketSetAsk(ctx, pri, vpri, abi.ChainEpoch(qty), abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) + return api.MarketSetAsk(ctx, types.BigInt(pri), types.BigInt(vpri), abi.ChainEpoch(qty), abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) }, } From 1323e07a2c4913194e45eeb9088ad4eb11840b7e Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 7 Oct 2020 10:45:59 -0700 Subject: [PATCH 050/125] Don't show failed deals by default in `client list-deals`, add option to to show failed deals. --- cli/client.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cli/client.go b/cli/client.go index 7494815bf..caf3ed758 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1039,6 +1039,10 @@ var clientListDeals = &cli.Command{ Usage: "use color in display output", Value: true, }, + &cli.BoolFlag{ + Name: "show-failed", + Usage: "show failed/failing deals", + }, &cli.BoolFlag{ Name: "watch", Usage: "watch deal updates in real-time, rather than a one time list", @@ -1055,6 +1059,7 @@ var clientListDeals = &cli.Command{ verbose := cctx.Bool("verbose") color := cctx.Bool("color") watch := cctx.Bool("watch") + showFailed := cctx.Bool("show-failed") localDeals, err := api.ClientListDeals(ctx) if err != nil { @@ -1071,7 +1076,7 @@ var clientListDeals = &cli.Command{ tm.Clear() tm.MoveCursor(1, 1) - err = outputStorageDeals(ctx, tm.Screen, api, localDeals, verbose, color) + err = outputStorageDeals(ctx, tm.Screen, api, localDeals, verbose, color, showFailed) if err != nil { return err } @@ -1097,7 +1102,7 @@ var clientListDeals = &cli.Command{ } } - return outputStorageDeals(ctx, os.Stdout, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color")) + return outputStorageDeals(ctx, os.Stdout, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color"), showFailed) }, } @@ -1120,7 +1125,7 @@ func dealFromDealInfo(ctx context.Context, full api.FullNode, head *types.TipSet } } -func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, localDeals []api.DealInfo, verbose bool, color bool) error { +func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, localDeals []lapi.DealInfo, verbose bool, color bool, showFailed bool) error { sort.Slice(localDeals, func(i, j int) bool { return localDeals[i].CreationTime.Before(localDeals[j].CreationTime) }) @@ -1132,7 +1137,9 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, l var deals []deal for _, localDeal := range localDeals { - deals = append(deals, dealFromDealInfo(ctx, full, head, localDeal)) + if showFailed || localDeal.State != storagemarket.StorageDealError { + deals = append(deals, dealFromDealInfo(ctx, full, head, localDeal)) + } } if verbose { From f7f229ae54e7782aa12a5ed311626c50857af6f7 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 7 Oct 2020 10:55:07 -0700 Subject: [PATCH 051/125] Show verified status in `client list-deals` --- cli/client.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/client.go b/cli/client.go index caf3ed758..5411965a7 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1144,7 +1144,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, if verbose { w := tabwriter.NewWriter(out, 2, 4, 2, ' ', 0) - fmt.Fprintf(w, "Created\tDealCid\tDealId\tProvider\tState\tOn Chain?\tSlashed?\tPieceCID\tSize\tPrice\tDuration\tMessage\n") + fmt.Fprintf(w, "Created\tDealCid\tDealId\tProvider\tState\tOn Chain?\tSlashed?\tPieceCID\tSize\tPrice\tDuration\tVerified\tMessage\n") for _, d := range deals { onChain := "N" if d.OnChainDealState.SectorStartEpoch != -1 { @@ -1157,7 +1157,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, } price := types.FIL(types.BigMul(d.LocalDeal.PricePerEpoch, types.NewInt(d.LocalDeal.Duration))) - fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", d.LocalDeal.CreationTime.Format(time.Stamp), d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, dealStateString(color, d.LocalDeal.State), onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), price, d.LocalDeal.Duration, d.LocalDeal.Message) + fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%v\t%s\n", d.LocalDeal.CreationTime.Format(time.Stamp), d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, dealStateString(color, d.LocalDeal.State), onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), price, d.LocalDeal.Duration, d.LocalDeal.Verified, d.LocalDeal.Message) } return w.Flush() } @@ -1172,6 +1172,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, tablewriter.Col("Size"), tablewriter.Col("Price"), tablewriter.Col("Duration"), + tablewriter.Col("Verified"), tablewriter.NewLineCol("Message")) for _, d := range deals { @@ -1201,6 +1202,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, "PieceCID": piece, "Size": types.SizeStr(types.NewInt(d.LocalDeal.Size)), "Price": price, + "Verified": d.LocalDeal.Verified, "Duration": d.LocalDeal.Duration, "Message": d.LocalDeal.Message, }) From 8a175a7465aa4115703849339b0259e0fc728e75 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 8 Oct 2020 21:32:10 +0300 Subject: [PATCH 052/125] update go-libp2p-pubsub and go-libp2p-quic-transport --- go.mod | 4 ++-- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index cec47e76b..f2d95d22a 100644 --- a/go.mod +++ b/go.mod @@ -96,8 +96,8 @@ require ( github.com/libp2p/go-libp2p-mplex v0.2.4 github.com/libp2p/go-libp2p-noise v0.1.1 github.com/libp2p/go-libp2p-peerstore v0.2.6 - github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18 - github.com/libp2p/go-libp2p-quic-transport v0.8.0 + github.com/libp2p/go-libp2p-pubsub v0.3.6 + github.com/libp2p/go-libp2p-quic-transport v0.8.2 github.com/libp2p/go-libp2p-record v0.1.3 github.com/libp2p/go-libp2p-routing-helpers v0.2.3 github.com/libp2p/go-libp2p-swarm v0.2.8 diff --git a/go.sum b/go.sum index 55810050f..efb1d2dfb 100644 --- a/go.sum +++ b/go.sum @@ -884,12 +884,12 @@ github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= -github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18 h1:+ae7vHSv/PJ4xGXwLV6LKGj32zjyB8ttJHtyV4TXal0= -github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI= +github.com/libp2p/go-libp2p-pubsub v0.3.6 h1:9oO8W7qIWCYQYyz5z8nUsPcb3rrFehBlkbqvbSVjBxY= +github.com/libp2p/go-libp2p-pubsub v0.3.6/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M= -github.com/libp2p/go-libp2p-quic-transport v0.8.0 h1:mHA94K2+TD0e9XtjWx/P5jGGZn0GdQ4OFYwNllagv4E= -github.com/libp2p/go-libp2p-quic-transport v0.8.0/go.mod h1:F2FG/6Bzz0U6essUVxDzE0s9CrY4XGLbl7QEmDNvU7A= +github.com/libp2p/go-libp2p-quic-transport v0.8.2 h1:FDaXBCBJ1e5hY6gnWEJ4NbYyLk8eezr4J6AY3q3KqwM= +github.com/libp2p/go-libp2p-quic-transport v0.8.2/go.mod h1:L+e0q15ZNaYm3seHgbsXjWP8kXLEqz+elLWKk9l8DhM= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= @@ -1022,8 +1022,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/lucas-clemente/quic-go v0.16.0/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= -github.com/lucas-clemente/quic-go v0.18.0 h1:JhQDdqxdwdmGdKsKgXi1+coHRoGhvU6z0rNzOJqZ/4o= -github.com/lucas-clemente/quic-go v0.18.0/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= +github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys= +github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= From 4cd73f156021b697ac0d5b9ab25de9cb06e13d2d Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 8 Oct 2020 21:40:36 +0300 Subject: [PATCH 053/125] use subscription filter in pubsub --- node/modules/lp2p/pubsub.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 7dca3608f..f36e436d8 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -187,13 +187,14 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { build.MessagesTopic(in.Nn): 1, } + var drandTopic string for _, d := range in.Dr { - topic, err := getDrandTopic(d.Config.ChainInfoJSON) + drandTopic, err = getDrandTopic(d.Config.ChainInfoJSON) if err != nil { return nil, err } - topicParams[topic] = drandTopicParams - pgTopicWeights[topic] = 5 + topicParams[drandTopic] = drandTopicParams + pgTopicWeights[drandTopic] = 5 } options := []pubsub.Option{ @@ -308,6 +309,14 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { } options = append(options, pubsub.WithPeerGater(pgParams)) + options = append(options, + pubsub.WithSubscriptionFilter( + pubsub.WrapLimitSubscriptionFilter( + pubsub.NewAllowlistSubscriptionFilter( + build.BlocksTopic(in.Nn), + build.MessagesTopic(in.Nn), + drandTopic), + 100))) // tracer if in.Cfg.RemoteTracer != "" { @@ -359,14 +368,9 @@ type tracerWrapper struct { topics map[string]struct{} } -func (trw *tracerWrapper) traceMessage(topics []string) bool { - for _, topic := range topics { - _, ok := trw.topics[topic] - if ok { - return true - } - } - return false +func (trw *tracerWrapper) traceMessage(topic string) bool { + _, ok := trw.topics[topic] + return ok } func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) { @@ -379,12 +383,12 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) { switch evt.GetType() { case pubsub_pb.TraceEvent_PUBLISH_MESSAGE: stats.Record(context.TODO(), metrics.PubsubPublishMessage.M(1)) - if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().Topics) { + if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().GetTopic()) { trw.tr.Trace(evt) } case pubsub_pb.TraceEvent_DELIVER_MESSAGE: stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1)) - if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().Topics) { + if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().GetTopic()) { trw.tr.Trace(evt) } case pubsub_pb.TraceEvent_REJECT_MESSAGE: From 7933cf094f3d9d1f1086ef559506a4acd1dda2cb Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 8 Oct 2020 21:48:55 +0300 Subject: [PATCH 054/125] allow all drand topics --- node/modules/lp2p/pubsub.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index f36e436d8..9724eb3b4 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -187,14 +187,15 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { build.MessagesTopic(in.Nn): 1, } - var drandTopic string + var drandTopics []string for _, d := range in.Dr { - drandTopic, err = getDrandTopic(d.Config.ChainInfoJSON) + topic, err := getDrandTopic(d.Config.ChainInfoJSON) if err != nil { return nil, err } - topicParams[drandTopic] = drandTopicParams - pgTopicWeights[drandTopic] = 5 + topicParams[topic] = drandTopicParams + pgTopicWeights[topic] = 5 + drandTopics = append(drandTopics, topic) } options := []pubsub.Option{ @@ -309,13 +310,16 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { } options = append(options, pubsub.WithPeerGater(pgParams)) + + allowTopics := []string{ + build.BlocksTopic(in.Nn), + build.MessagesTopic(in.Nn), + } + allowTopics = append(allowTopics, drandTopics...) options = append(options, pubsub.WithSubscriptionFilter( pubsub.WrapLimitSubscriptionFilter( - pubsub.NewAllowlistSubscriptionFilter( - build.BlocksTopic(in.Nn), - build.MessagesTopic(in.Nn), - drandTopic), + pubsub.NewAllowlistSubscriptionFilter(allowTopics...), 100))) // tracer From c3420c0ceb0023b166ed7a5386b0512ef94e1369 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Thu, 8 Oct 2020 12:08:01 -0700 Subject: [PATCH 055/125] Display price, verified-price with units in `lotus-miner storage-deals get-ask` --- cmd/lotus-storage-miner/market.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 67e694c78..bb1ebd9ec 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -288,7 +288,7 @@ var getAskCmd = &cli.Command{ rem = (time.Second * time.Duration(int64(dlt)*int64(build.BlockDelaySecs))).String() } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%d\n", ask.Price, ask.VerifiedPrice, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%d\n", types.FIL(ask.Price), types.FIL(ask.VerifiedPrice), types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo) return w.Flush() }, From fe5b32026ac725167e38208576eb23d5333475ff Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 8 Oct 2020 15:19:39 -0400 Subject: [PATCH 056/125] Add some new endpoints for querying Msig info --- api/api_full.go | 15 +++++++++++++ api/apistruct/struct.go | 5 +++++ documentation/en/api-methods.md | 32 +++++++++++++++++++++++++++ node/impl/full/state.go | 38 +++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index b6ae77f77..601b14660 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -418,6 +418,9 @@ type FullNode interface { // MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + // MsigGetLockedBalance returns the locked balance of an msig at a vien epoch. + // The return may be greater than the multisig actor's actual balance. + MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (MsigVesting, error) // MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. // It takes the following params: , , MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) @@ -871,3 +874,15 @@ type Fault struct { Miner address.Address Epoch abi.ChainEpoch } + +var EmptyVesting = MsigVesting{ + InitialBalance: types.EmptyInt, + StartEpoch: -1, + UnlockDuration: -1, +} + +type MsigVesting struct { + InitialBalance abi.TokenAmount + StartEpoch abi.ChainEpoch + UnlockDuration abi.ChainEpoch +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a09700eb9..1a3ddda58 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -212,6 +212,7 @@ type FullNodeStruct struct { StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` + MsigGetVestingSchedule func(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error) `perm:"read"` MsigGetVested func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) `perm:"read"` MsigCreate func(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"` MsigPropose func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` @@ -933,6 +934,10 @@ func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address. return c.Internal.MsigGetAvailableBalance(ctx, a, tsk) } +func (c *FullNodeStruct) MsigGetVestingSchedule(ctx context.Context, a address.Address, tsk types.TipSetKey) (api.MsigVesting, error) { + return c.Internal.MsigGetVestingSchedule(ctx, a, tsk) +} + func (c *FullNodeStruct) MsigGetVested(ctx context.Context, a address.Address, sTsk types.TipSetKey, eTsk types.TipSetKey) (types.BigInt, error) { return c.Internal.MsigGetVested(ctx, a, sTsk, eTsk) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index a6ad1ecb2..8a288a4bf 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -87,6 +87,7 @@ * [MsigCreate](#MsigCreate) * [MsigGetAvailableBalance](#MsigGetAvailableBalance) * [MsigGetVested](#MsigGetVested) + * [MsigGetVestingSchedule](#MsigGetVestingSchedule) * [MsigPropose](#MsigPropose) * [MsigSwapApprove](#MsigSwapApprove) * [MsigSwapCancel](#MsigSwapCancel) @@ -2143,6 +2144,37 @@ Inputs: Response: `"0"` +### MsigGetVestingSchedule +MsigGetLockedBalance returns the locked balance of an msig at a vien epoch. +The return may be greater than the multisig actor's actual balance. + + +Perms: read + +Inputs: +```json +[ + "f01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "InitialBalance": "0", + "StartEpoch": 10101, + "UnlockDuration": 10101 +} +``` + ### MsigPropose MsigPropose proposes a multisig message It takes the following params: , , , diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 7d654985a..56dfc0a14 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -820,6 +820,44 @@ func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Add return types.BigSub(act.Balance, locked), nil } +func (a *StateAPI) MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MsigVesting, error) { + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + + act, err := a.StateManager.LoadActor(ctx, addr, ts) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig actor: %w", err) + } + + msas, err := multisig.Load(a.Chain.Store(ctx), act) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig actor state: %w", err) + } + + ib, err := msas.InitialBalance() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig initial balance: %w", err) + } + + se, err := msas.StartEpoch() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig start epoch: %w", err) + } + + ud, err := msas.UnlockDuration() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig unlock duration: %w", err) + } + + return api.MsigVesting{ + InitialBalance: ib, + StartEpoch: se, + UnlockDuration: ud, + }, nil +} + func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { startTs, err := a.Chain.GetTipSetFromKey(start) if err != nil { From 21de538d09fb47b732d5a4709ae429ec1fbb4da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 21:18:33 +0100 Subject: [PATCH 057/125] upgrade go-libp2p-noise to v0.1.2. Improves throughput in large and/or fast transfers and reduces syscalls when data is queued in receive buffer, by using bufio.Reader. --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f2d95d22a..24de7012e 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/libp2p/go-libp2p-discovery v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.8.3 github.com/libp2p/go-libp2p-mplex v0.2.4 - github.com/libp2p/go-libp2p-noise v0.1.1 + github.com/libp2p/go-libp2p-noise v0.1.2 github.com/libp2p/go-libp2p-peerstore v0.2.6 github.com/libp2p/go-libp2p-pubsub v0.3.6 github.com/libp2p/go-libp2p-quic-transport v0.8.2 diff --git a/go.sum b/go.sum index efb1d2dfb..d2b122929 100644 --- a/go.sum +++ b/go.sum @@ -863,6 +863,8 @@ github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLK github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-noise v0.1.1 h1:vqYQWvnIcHpIoWJKC7Al4D6Hgj0H012TuXRhPwSMGpQ= github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM= +github.com/libp2p/go-libp2p-noise v0.1.2 h1:IH9GRihQJTx56obm+GnpdPX4KeVIlvpXrP6xnJ0wxWk= +github.com/libp2p/go-libp2p-noise v0.1.2/go.mod h1:9B10b7ueo7TIxZHHcjcDCo5Hd6kfKT2m77by82SFRfE= github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= From 0f8fc119f32fe312ec96ec94239e8ca2f0b71450 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Thu, 8 Oct 2020 16:45:07 -0400 Subject: [PATCH 058/125] Update address prefix in client cli. --- cli/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/client.go b/cli/client.go index 7494815bf..678c7d7f4 100644 --- a/cli/client.go +++ b/cli/client.go @@ -536,7 +536,7 @@ func interactiveDeal(cctx *cli.Context) error { state = "miner" case "miner": - fmt.Print("Miner Address (t0..): ") + fmt.Print("Miner Address (f0..): ") var maddrStr string _, err := fmt.Scan(&maddrStr) From c60d13ee56342fd4be784574bf05691927f7e4cd Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 8 Oct 2020 13:52:39 -0700 Subject: [PATCH 059/125] feat(markets): update markets v0.7.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 24de7012e..e9687c7b8 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.7 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.7.0 + github.com/filecoin-project/go-fil-markets v0.7.1 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index d2b122929..bab4e929d 100644 --- a/go.sum +++ b/go.sum @@ -244,8 +244,8 @@ github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.7.0 h1:tcEZiUNIYQJ4PBzgVpLwfdJ4ZdC4WCv9LsgvsoCXIls= -github.com/filecoin-project/go-fil-markets v0.7.0/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= +github.com/filecoin-project/go-fil-markets v0.7.1 h1:e0NlpSnaeGyDUhCOzevjcxkSA54kt9BzlXpLRgduUFI= +github.com/filecoin-project/go-fil-markets v0.7.1/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= From 60768f48636633401f58bef5dc9e51af89ebfe94 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 8 Oct 2020 18:38:22 +0200 Subject: [PATCH 060/125] Optimize SearchForMessage and GetReceipt Signed-off-by: Jakub Sztandera --- chain/stmgr/stmgr.go | 57 +++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d81cf1c72..ba3dcd1d8 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -2,6 +2,7 @@ package stmgr import ( "context" + "errors" "fmt" "sync" @@ -507,16 +508,7 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T return nil, fmt.Errorf("failed to load message: %w", err) } - r, _, err := sm.tipsetExecutedMessage(ts, msg, m.VMMessage()) - if err != nil { - return nil, err - } - - if r != nil { - return r, nil - } - - _, r, _, err = sm.searchBackForMsg(ctx, ts, m) + _, r, _, err := sm.searchBackForMsg(ctx, ts, m) if err != nil { return nil, fmt.Errorf("failed to look back through chain for message: %w", err) } @@ -674,6 +666,18 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet, m types.ChainMsg) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { cur := from + curActor, err := sm.LoadActor(ctx, m.VMMessage().From, cur) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("failed to load initital tipset") + } + + mFromId, err := sm.LookupID(ctx, m.VMMessage().From, from) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("looking up From id address: %w", err) + } + + mNonce := m.VMMessage().Nonce + for { if cur.Height() == 0 { // it ain't here! @@ -686,32 +690,37 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet default: } - act, err := sm.LoadActor(ctx, m.VMMessage().From, cur) - if err != nil { - return nil, nil, cid.Cid{}, err - } - // we either have no messages from the sender, or the latest message we found has a lower nonce than the one being searched for, // either way, no reason to lookback, it ain't there - if act.Nonce == 0 || act.Nonce < m.VMMessage().Nonce { + if curActor == nil || curActor.Nonce == 0 || curActor.Nonce < mNonce { return nil, nil, cid.Undef, nil } - ts, err := sm.cs.LoadTipSet(cur.Parents()) + pts, err := sm.cs.LoadTipSet(cur.Parents()) if err != nil { - return nil, nil, cid.Undef, fmt.Errorf("failed to load tipset during msg wait searchback: %w", err) + return nil, nil, cid.Undef, xerrors.Errorf("failed to load tipset during msg wait searchback: %w", err) } - r, foundMsg, err := sm.tipsetExecutedMessage(ts, m.Cid(), m.VMMessage()) - if err != nil { - return nil, nil, cid.Undef, fmt.Errorf("checking for message execution during lookback: %w", err) + act, err := sm.LoadActor(ctx, mFromId, pts) + actorNoExist := errors.Is(err, types.ErrActorNotFound) + if err != nil && !actorNoExist { + return nil, nil, cid.Cid{}, xerrors.Errorf("failed to load the actor: %w", err) } - if r != nil { - return ts, r, foundMsg, nil + // check that between cur and parent tipset the nonce fell into range of our message + if actorNoExist || (curActor.Nonce > mNonce && act.Nonce <= mNonce) { + r, foundMsg, err := sm.tipsetExecutedMessage(cur, m.Cid(), m.VMMessage()) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("checking for message execution during lookback: %w", err) + } + + if r != nil { + return pts, r, foundMsg, nil + } } - cur = ts + cur = pts + curActor = act } } From 973f61bc105acbb7df6b394e092ae400c4c2d911 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 8 Oct 2020 22:49:36 +0200 Subject: [PATCH 061/125] Optimize chain and message sync Signed-off-by: Jakub Sztandera --- chain/sub/incoming.go | 17 +++++----- chain/sub/incoming_test.go | 63 ++++++++++++++++++++++++++++++++++++++ chain/sync.go | 22 ++++++++++--- 3 files changed, 88 insertions(+), 14 deletions(-) create mode 100644 chain/sub/incoming_test.go diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 07b3343d2..d51c481d1 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -172,25 +172,24 @@ func fetchCids( cids []cid.Cid, cb func(int, blocks.Block) error, ) error { - fetchedBlocks := bserv.GetBlocks(ctx, cids) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() cidIndex := make(map[cid.Cid]int) for i, c := range cids { cidIndex[c] = i } + if len(cids) != len(cidIndex) { + return fmt.Errorf("duplicate CIDs in fetchCids input") + } + + fetchedBlocks := bserv.GetBlocks(ctx, cids) for i := 0; i < len(cids); i++ { select { case block, ok := <-fetchedBlocks: if !ok { - // Closed channel, no more blocks fetched, check if we have all - // of the CIDs requested. - // FIXME: Review this check. We don't call the callback on the - // last index? - if i == len(cids)-1 { - break - } - return fmt.Errorf("failed to fetch all messages") } diff --git a/chain/sub/incoming_test.go b/chain/sub/incoming_test.go new file mode 100644 index 000000000..215439209 --- /dev/null +++ b/chain/sub/incoming_test.go @@ -0,0 +1,63 @@ +package sub + +import ( + "context" + "testing" + + address "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" +) + +type getter struct { + msgs []*types.Message +} + +func (g *getter) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { panic("NYI") } + +func (g *getter) GetBlocks(ctx context.Context, ks []cid.Cid) <-chan blocks.Block { + ch := make(chan blocks.Block, len(g.msgs)) + for _, m := range g.msgs { + by, err := m.Serialize() + if err != nil { + panic(err) + } + b, err := blocks.NewBlockWithCid(by, m.Cid()) + if err != nil { + panic(err) + } + ch <- b + } + close(ch) + return ch +} + +func TestFetchCidsWithDedup(t *testing.T) { + msgs := []*types.Message{} + for i := 0; i < 10; i++ { + msgs = append(msgs, &types.Message{ + From: address.TestAddress, + To: address.TestAddress, + + Nonce: uint64(i), + }) + } + cids := []cid.Cid{} + for _, m := range msgs { + cids = append(cids, m.Cid()) + } + g := &getter{msgs} + + // the cids have a duplicate + res, err := FetchMessagesByCids(context.TODO(), g, append(cids, cids[0])) + + t.Logf("err: %+v", err) + t.Logf("res: %+v", res) + if err == nil { + t.Errorf("there should be an error") + } + if err == nil && (res[0] == nil || res[len(res)-1] == nil) { + t.Fatalf("there is a nil message: first %p, last %p", res[0], res[len(res)-1]) + } +} diff --git a/chain/sync.go b/chain/sync.go index 240d1edef..c280e3a40 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -217,6 +217,12 @@ func (syncer *Syncer) Stop() { // This should be called when connecting to new peers, and additionally // when receiving new blocks from the network func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) bool { + defer func() { + if err := recover(); err != nil { + log.Errorf("panic in InformNewHead: ", err) + } + }() + ctx := context.Background() if fts == nil { log.Errorf("got nil tipset in InformNewHead") @@ -1281,9 +1287,11 @@ func (syncer *Syncer) collectHeaders(ctx context.Context, incoming *types.TipSet blockSet := []*types.TipSet{incoming} + // Parent of the new (possibly better) tipset that we need to fetch next. at := incoming.Parents() - // we want to sync all the blocks until the height above the block we have + // we want to sync all the blocks until the height above our + // best tipset so far untilHeight := known.Height() + 1 ss.SetHeight(blockSet[len(blockSet)-1].Height()) @@ -1377,13 +1385,17 @@ loop: } base := blockSet[len(blockSet)-1] - if base.Parents() == known.Parents() { - // common case: receiving a block thats potentially part of the same tipset as our best block + if base.IsChildOf(known) { + // common case: receiving blocks that are building on top of our best tipset return blockSet, nil } - if types.CidArrsEqual(base.Parents().Cids(), known.Cids()) { - // common case: receiving blocks that are building on top of our best tipset + knownParent, err := syncer.store.LoadTipSet(known.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to load next local tipset: %w", err) + } + if base.IsChildOf(knownParent) { + // common case: receiving a block thats potentially part of the same tipset as our best block return blockSet, nil } From 0223582d4106e36d212827fb2f5aa9f4c9189b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 8 Oct 2020 22:48:21 +0200 Subject: [PATCH 062/125] sync wait: Handle processed message offset --- cli/sync.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cli/sync.go b/cli/sync.go index bee87cf70..dea96d14e 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -233,7 +233,13 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { samples := 8 i := 0 - var app, lastApp uint64 + var firstApp, app, lastApp uint64 + + state, err := napi.SyncState(ctx) + if err != nil { + return err + } + firstApp = state.VMApplied for { state, err := napi.SyncState(ctx) @@ -286,10 +292,10 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { if i%samples == 0 { lastApp = app - app = state.VMApplied + app = state.VMApplied - firstApp } if i > 0 { - fmt.Printf("Validated %d messages (%d per second)\n", state.VMApplied, (app-lastApp)*uint64(time.Second/tick)/uint64(samples)) + fmt.Printf("Validated %d messages (%d per second)\n", state.VMApplied-firstApp, (app-lastApp)*uint64(time.Second/tick)/uint64(samples)) lastLines++ } From a23ab123d794dace19775aa0aa2e1686b5552184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=9E=97=E6=AC=A3?= Date: Tue, 1 Sep 2020 11:44:55 +0800 Subject: [PATCH 063/125] fix GetBestMiningCandidate bug Signed-off-by: Jakub Sztandera --- miner/miner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/miner/miner.go b/miner/miner.go index a256e8b3a..aebcb7a25 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -327,6 +327,7 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error) } ltsw, err := m.api.ChainTipSetWeight(ctx, m.lastWork.TipSet.Key()) if err != nil { + m.lastWork = nil return nil, err } From 483510947096928612cadf6e3a91c1d6d466b13e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 00:51:04 +0200 Subject: [PATCH 064/125] wallet: Post-merge fixes --- api/api_wallet.go | 3 ++- chain/messagesigner/messagesigner.go | 10 +++++----- chain/wallet/key.go | 2 +- chain/wallet/wallet.go | 2 +- cmd/lotus-wallet/logged.go | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/api/api_wallet.go b/api/api_wallet.go index 06b00a75b..9c289ca95 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -4,8 +4,9 @@ import ( "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/crypto" ) type WalletAPI interface { diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index ac94d6a3e..1240485b8 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -3,12 +3,12 @@ package messagesigner import ( "bytes" "context" + "github.com/filecoin-project/lotus/api" "sync" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -28,17 +28,17 @@ type mpoolAPI interface { // MessageSigner keeps track of nonces per address, and increments the nonce // when signing a message type MessageSigner struct { - wallet *wallet.Wallet + wallet api.WalletAPI lk sync.Mutex mpool mpoolAPI ds datastore.Batching } -func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { +func NewMessageSigner(wallet api.WalletAPI, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { return newMessageSigner(wallet, mpool, ds) } -func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { +func newMessageSigner(wallet api.WalletAPI, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/")) return &MessageSigner{ wallet: wallet, @@ -61,7 +61,7 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb // Sign the message with the nonce msg.Nonce = nonce - sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes()) + sig, err := ms.wallet.WalletSign(ctx, msg.From, msg.Cid().Bytes()) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } diff --git a/chain/wallet/key.go b/chain/wallet/key.go index 2d4ecafd7..4b746a17a 100644 --- a/chain/wallet/key.go +++ b/chain/wallet/key.go @@ -4,7 +4,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index b4c846d37..5f501307f 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -152,7 +152,7 @@ func (w *LocalWallet) tryFind(addr address.Address) (types.KeyInfo, error) { return ki, nil } -func (w *LocalWallet) Export(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { +func (w *LocalWallet) WalletExport(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { k, err := w.findKey(addr) if err != nil { return nil, xerrors.Errorf("failed to find key to export: %w", err) diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index e73daae90..4cdd62825 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -4,7 +4,7 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" From 114776f2c5e7beccd0dd865bfa9758e8239cdd10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:04:59 +0200 Subject: [PATCH 065/125] wallet: Drop WalletSignMessage from WalletAPI --- api/api_wallet.go | 4 ++-- chain/wallet/wallet.go | 14 -------------- cmd/lotus-wallet/logged.go | 6 ------ node/impl/full/wallet.go | 15 +++++++++++++-- 4 files changed, 15 insertions(+), 24 deletions(-) diff --git a/api/api_wallet.go b/api/api_wallet.go index 9c289ca95..b3dbd5fa9 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -13,9 +13,9 @@ type WalletAPI interface { WalletNew(context.Context, crypto.SigType) (address.Address, error) WalletHas(context.Context, address.Address) (bool, error) WalletList(context.Context) ([]address.Address, error) + WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) - // nonce keeping done by wallet app - WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) + WalletExport(context.Context, address.Address) (*types.KeyInfo, error) WalletImport(context.Context, *types.KeyInfo) (address.Address, error) WalletDelete(context.Context, address.Address) error diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 5f501307f..607380547 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -74,20 +74,6 @@ func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg return sigs.Sign(ActSigType(ki.Type), ki.PrivateKey, msg) } -func (w *LocalWallet) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - mcid := msg.Cid() - - sig, err := w.WalletSign(ctx, k, mcid.Bytes()) - if err != nil { - return nil, xerrors.Errorf("failed to sign message: %w", err) - } - - return &types.SignedMessage{ - Message: *msg, - Signature: *sig, - }, nil -} - func (w *LocalWallet) findKey(addr address.Address) (*Key, error) { w.lk.Lock() defer w.lk.Unlock() diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index 4cdd62825..adafb7cc5 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -43,12 +43,6 @@ func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg [] return c.under.WalletSign(ctx, k, msg) } -func (c *LoggedWallet) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - log.Infow("WalletSignMessage", "address", k) - - return c.under.WalletSignMessage(ctx, k, msg) -} - func (c *LoggedWallet) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { log.Infow("WalletExport", "address", a) diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 6252e12a4..13880a03a 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -3,10 +3,10 @@ package full import ( "context" - "github.com/filecoin-project/go-address" "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" @@ -48,7 +48,18 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.WalletAPI.WalletSignMessage(ctx, keyAddr, msg) + + mcid := msg.Cid() + + sig, err := a.WalletAPI.WalletSign(ctx, k, mcid.Bytes()) + if err != nil { + return nil, xerrors.Errorf("failed to sign message: %w", err) + } + + return &types.SignedMessage{ + Message: *msg, + Signature: *sig, + }, nil } func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { From f3dc730b05c5ff9d7161a731594bbba3fc37b90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:27:38 +0200 Subject: [PATCH 066/125] wallet: Add metadata to WalletAPI.WalletSign --- api/api_wallet.go | 26 +++++++++++++++++++++++++- api/apistruct/struct.go | 6 +++--- chain/gen/gen.go | 16 +++++++++++++--- chain/gen/mining.go | 7 ++++--- chain/messagesigner/messagesigner.go | 11 ++++++++++- chain/wallet/wallet.go | 2 +- cmd/lotus-wallet/logged.go | 4 ++-- markets/storageadapter/client.go | 11 ++++++++--- node/impl/full/state.go | 16 +++++++--------- node/impl/full/wallet.go | 10 ++++++++-- 10 files changed, 81 insertions(+), 28 deletions(-) diff --git a/api/api_wallet.go b/api/api_wallet.go index b3dbd5fa9..82ef726a0 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -9,12 +9,36 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +type MsgType string +const ( + MTUnknown = "unknown" + + // Signing message CID. MsgMeta.Extra contains raw cbor message bytes + MTChainMsg = "message" + + // Signing a blockheader. signing raw cbor block bytes (MsgMeta.Extra is empty) + MTBlock = "block" + + // Signing a deal proposal. signing raw cbor proposal bytes (MsgMeta.Extra is empty) + MTDealProposal = "dealproposal" + + // TODO: Deals, Vouchers, VRF +) + +type MsgMeta struct { + Type MsgType + + // Additional data related to what is signed. Should be verifiable with the + // signed bytes (e.g. CID(Extra).Bytes() == toSign) + Extra []byte +} + type WalletAPI interface { WalletNew(context.Context, crypto.SigType) (address.Address, error) WalletHas(context.Context, address.Address) (bool, error) WalletList(context.Context) ([]address.Address, error) - WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) + WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta MsgMeta) (*crypto.Signature, error) WalletExport(context.Context, address.Address) (*types.KeyInfo, error) WalletImport(context.Context, *types.KeyInfo) (address.Address, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 01de13e61..81fce92b5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -366,7 +366,7 @@ type WalletStruct struct { WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` WalletList func(context.Context) ([]address.Address, error) `perm:"write"` - WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` + WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` @@ -1397,8 +1397,8 @@ func (c *WalletStruct) WalletList(ctx context.Context) ([]address.Address, error return c.Internal.WalletList(ctx) } -func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { - return c.Internal.WalletSign(ctx, k, msg) +func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { + return c.Internal.WalletSign(ctx, k, msg, meta) } func (c *WalletStruct) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { diff --git a/chain/gen/gen.go b/chain/gen/gen.go index ee93ef60e..70c303818 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -374,7 +374,13 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add return nil, nil, nil, xerrors.Errorf("get miner worker: %w", err) } - vrfout, err := ComputeVRF(ctx, cg.w.WalletSign, worker, ticketRand) + sf := func(ctx context.Context, a address.Address, i []byte) (*crypto.Signature, error) { + return cg.w.WalletSign(ctx, a, i, api.MsgMeta{ + Type: api.MTUnknown, + }) + } + + vrfout, err := ComputeVRF(ctx, sf, worker, ticketRand) if err != nil { return nil, nil, nil, xerrors.Errorf("compute VRF: %w", err) } @@ -528,7 +534,9 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) { GasPremium: types.NewInt(0), } - sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes()) + sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes(), api.MsgMeta{ + Type: api.MTUnknown, // testing + }) if err != nil { return nil, err } @@ -588,7 +596,9 @@ func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoc } func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*crypto.Signature, error) { - return mca.w.WalletSign(ctx, a, v) + return mca.w.WalletSign(ctx, a, v, api.MsgMeta{ + Type: api.MTUnknown, + }) } type WinningPoStProver interface { diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 0c2f72590..37e8ca2c0 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -15,11 +15,10 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/sigs/bls" ) -func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.LocalWallet, bt *api.BlockTemplate) (*types.FullBlock, error) { +func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletAPI, bt *api.BlockTemplate) (*types.FullBlock, error) { pts, err := sm.ChainStore().LoadTipSet(bt.Parents) if err != nil { @@ -131,7 +130,9 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Loc return nil, xerrors.Errorf("failed to get signing bytes for block: %w", err) } - sig, err := w.WalletSign(ctx, waddr, nosigbytes) + sig, err := w.WalletSign(ctx, waddr, nosigbytes, api.MsgMeta{ + Type: api.MTBlock, + }) if err != nil { return nil, xerrors.Errorf("failed to sign new block: %w", err) } diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index 1240485b8..c7948e42e 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -61,7 +61,16 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb // Sign the message with the nonce msg.Nonce = nonce - sig, err := ms.wallet.WalletSign(ctx, msg.From, msg.Cid().Bytes()) + + mb, err := msg.ToStorageBlock() + if err != nil { + return nil, xerrors.Errorf("serializing message: %w", err) + } + + sig, err := ms.wallet.WalletSign(ctx, msg.From, mb.Cid().Bytes(), api.MsgMeta{ + Type: api.MTChainMsg, + Extra: mb.RawData(), + }) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 607380547..2dbe94b50 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -62,7 +62,7 @@ func KeyWallet(keys ...*Key) *LocalWallet { } } -func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg []byte) (*crypto.Signature, error) { +func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { ki, err := w.findKey(addr) if err != nil { return nil, err diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index adafb7cc5..9cd729943 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -37,10 +37,10 @@ func (c *LoggedWallet) WalletList(ctx context.Context) ([]address.Address, error return c.under.WalletList(ctx) } -func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { +func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { log.Infow("WalletSign", "address", k) - return c.under.WalletSign(ctx, k, msg) + return c.under.WalletSign(ctx, k, msg, meta) } func (c *LoggedWallet) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 824693dac..22ee483ce 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -5,6 +5,7 @@ package storageadapter import ( "bytes" "context" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/go-state-types/big" miner0 "github.com/filecoin-project/specs-actors/actors/builtin" @@ -422,7 +423,9 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add return nil, err } - sig, err := c.Wallet.WalletSign(ctx, signer, buf) + sig, err := c.Wallet.WalletSign(ctx, signer, buf, api.MsgMeta{ + Type: api.MTDealProposal, + }) if err != nil { return nil, err } @@ -434,7 +437,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add } func (c *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { - addr, err := c.Wallet.GetDefault() + addr, err := c.DefWallet.GetDefault() return addr, err } @@ -475,7 +478,9 @@ func (c *ClientNodeAdapter) SignBytes(ctx context.Context, signer address.Addres return nil, err } - localSignature, err := c.Wallet.WalletSign(ctx, signer, b) + localSignature, err := c.Wallet.WalletSign(ctx, signer, b, api.MsgMeta{ + Type: api.MTUnknown, // TODO: pass type here + }) if err != nil { return nil, err } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 33ee75697..49754265d 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -5,14 +5,6 @@ import ( "context" "strconv" - "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/lotus/chain/actors/policy" - - "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" - - "github.com/filecoin-project/go-state-types/dline" - "github.com/filecoin-project/go-state-types/network" - cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "go.uber.org/fx" @@ -22,14 +14,19 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/state" @@ -47,7 +44,8 @@ type StateAPI struct { // TODO: the wallet here is only needed because we have the MinerCreateBlock // API attached to the state API. It probably should live somewhere better - Wallet *wallet.LocalWallet + Wallet api.WalletAPI + DefWallet wallet.Default ProofVerifier ffiwrapper.Verifier StateManager *stmgr.StateManager diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 13880a03a..2e2c48c5d 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -49,9 +49,15 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - mcid := msg.Cid() + mb, err := msg.ToStorageBlock() + if err != nil { + return nil, xerrors.Errorf("serializing message: %w", err) + } - sig, err := a.WalletAPI.WalletSign(ctx, k, mcid.Bytes()) + sig, err := a.WalletAPI.WalletSign(ctx, k, mb.Cid().Bytes(), api.MsgMeta{ + Type: api.MTChainMsg, + Extra: mb.RawData(), + }) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } From 6e8efb9d2cea1e7d828790a8e42ebd030738654e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:27:49 +0200 Subject: [PATCH 067/125] gofmt --- api/api_wallet.go | 1 + api/apistruct/struct.go | 16 ++++++++-------- chain/gen/gen.go | 6 +++--- chain/gen/mining.go | 2 +- chain/wallet/remotewallet/remote.go | 2 +- cmd/lotus-wallet/logged.go | 2 +- markets/storageadapter/client.go | 4 ++-- node/impl/full/state.go | 2 +- 8 files changed, 18 insertions(+), 17 deletions(-) diff --git a/api/api_wallet.go b/api/api_wallet.go index 82ef726a0..1213ffc1d 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -10,6 +10,7 @@ import ( ) type MsgType string + const ( MTUnknown = "unknown" diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 81fce92b5..d8c2c8094 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -363,14 +363,14 @@ type WorkerStruct struct { type WalletStruct struct { Internal struct { - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` - WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` - WalletList func(context.Context) ([]address.Address, error) `perm:"write"` - WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` - WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` - WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` - WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` - WalletDelete func(context.Context, address.Address) error `perm:"write"` + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` + WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` } } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 70c303818..d76835294 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -376,7 +376,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add sf := func(ctx context.Context, a address.Address, i []byte) (*crypto.Signature, error) { return cg.w.WalletSign(ctx, a, i, api.MsgMeta{ - Type: api.MTUnknown, + Type: api.MTUnknown, }) } @@ -535,7 +535,7 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) { } sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes(), api.MsgMeta{ - Type: api.MTUnknown, // testing + Type: api.MTUnknown, // testing }) if err != nil { return nil, err @@ -597,7 +597,7 @@ func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoc func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*crypto.Signature, error) { return mca.w.WalletSign(ctx, a, v, api.MsgMeta{ - Type: api.MTUnknown, + Type: api.MTUnknown, }) } diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 37e8ca2c0..45a089452 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -131,7 +131,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA } sig, err := w.WalletSign(ctx, waddr, nosigbytes, api.MsgMeta{ - Type: api.MTBlock, + Type: api.MTBlock, }) if err != nil { return nil, xerrors.Errorf("failed to sign new block: %w", err) diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index 712f270b4..ca4e422d5 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -48,4 +48,4 @@ func SetupRemoteWallet(url string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle return &RemoteWallet{wapi}, nil } -} \ No newline at end of file +} diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index 9cd729943..c713c2e87 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -59,4 +59,4 @@ func (c *LoggedWallet) WalletDelete(ctx context.Context, addr address.Address) e log.Infow("WalletDelete", "address", addr) return c.under.WalletDelete(ctx, addr) -} \ No newline at end of file +} diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 22ee483ce..080fe36eb 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -424,7 +424,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add } sig, err := c.Wallet.WalletSign(ctx, signer, buf, api.MsgMeta{ - Type: api.MTDealProposal, + Type: api.MTDealProposal, }) if err != nil { return nil, err @@ -479,7 +479,7 @@ func (c *ClientNodeAdapter) SignBytes(ctx context.Context, signer address.Addres } localSignature, err := c.Wallet.WalletSign(ctx, signer, b, api.MsgMeta{ - Type: api.MTUnknown, // TODO: pass type here + Type: api.MTUnknown, // TODO: pass type here }) if err != nil { return nil, err diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 49754265d..c67b7a887 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -44,7 +44,7 @@ type StateAPI struct { // TODO: the wallet here is only needed because we have the MinerCreateBlock // API attached to the state API. It probably should live somewhere better - Wallet api.WalletAPI + Wallet api.WalletAPI DefWallet wallet.Default ProofVerifier ffiwrapper.Verifier From 7d2f06cfbdd008e8acc41f6a56e7c1ee43c6f12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:33:50 +0200 Subject: [PATCH 068/125] Fix WalletSign in tests --- chain/messagepool/selection_test.go | 20 +++++++++++--------- chain/messagesigner/messagesigner_test.go | 8 ++++---- chain/stmgr/forks_test.go | 19 ++++++++++--------- chain/sync_test.go | 4 ++-- chain/types/mock/chain.go | 3 ++- cmd/lotus-shed/keyinfo.go | 2 +- node/impl/full/wallet.go | 4 +++- 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 72fea4d2c..7dae583ad 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -13,6 +13,10 @@ import ( "sort" "testing" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + logging "github.com/ipfs/go-log" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/build" @@ -21,12 +25,10 @@ import ( "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" + "github.com/filecoin-project/lotus/api" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - logging "github.com/ipfs/go-log" ) func init() { @@ -45,7 +47,7 @@ func makeTestMessage(w *wallet.LocalWallet, from, to address.Address, nonce uint GasFeeCap: types.NewInt(100 + gasPrice), GasPremium: types.NewInt(gasPrice), } - sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{}) if err != nil { panic(err) } @@ -744,7 +746,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -754,7 +756,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -1330,7 +1332,7 @@ readLoop: } actorMap := make(map[address.Address]address.Address) - actorWallets := make(map[address.Address]*wallet.Wallet) + actorWallets := make(map[address.Address]api.WalletAPI) for _, m := range msgs { baseNonce := baseNonces[m.Message.From] @@ -1342,7 +1344,7 @@ readLoop: t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -1360,7 +1362,7 @@ readLoop: m.Message.From = localActor m.Message.Nonce -= baseNonce - sig, err := w.Sign(context.TODO(), localActor, m.Message.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), localActor, m.Message.Cid().Bytes(), api.MsgMeta{}) if err != nil { t.Fatal(err) } diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index 04869ff6d..f50bb2606 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -47,13 +47,13 @@ func TestMessageSignerSignMessage(t *testing.T) { ctx := context.Background() w, _ := wallet.NewWallet(wallet.NewMemKeyStore()) - from1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + from1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - from2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + from2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - to1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + to1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - to2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + to2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) type msgSpec struct { diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 3687e6b34..0388af6ad 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -6,15 +6,21 @@ import ( "io" "testing" + "github.com/ipfs/go-cid" + ipldcbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" + "github.com/stretchr/testify/require" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/specs-actors/actors/builtin" init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/stretchr/testify/require" - "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -25,11 +31,6 @@ import ( "github.com/filecoin-project/lotus/chain/vm" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - - "github.com/ipfs/go-cid" - ipldcbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" - cbg "github.com/whyrusleeping/cbor-gen" ) func init() { @@ -186,7 +187,7 @@ func TestForkHeightTriggers(t *testing.T) { Params: enc, GasLimit: types.TestGasLimit, } - sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes(), api.MsgMeta{}) if err != nil { t.Fatal(err) } @@ -214,7 +215,7 @@ func TestForkHeightTriggers(t *testing.T) { } nonce++ - sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes(), api.MsgMeta{}) if err != nil { return nil, err } diff --git a/chain/sync_test.go b/chain/sync_test.go index 2c9f2f131..82d824e62 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -593,7 +593,7 @@ func TestDuplicateNonce(t *testing.T) { GasPremium: types.NewInt(0), } - sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes(), api.MsgMeta{}) require.NoError(t, err) return &types.SignedMessage{ @@ -685,7 +685,7 @@ func TestBadNonce(t *testing.T) { GasPremium: types.NewInt(0), } - sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes(), api.MsgMeta{}) require.NoError(t, err) return &types.SignedMessage{ diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 7a9c82cba..2e1c0a976 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -33,7 +34,7 @@ func MkMessage(from, to address.Address, nonce uint64, w *wallet.LocalWallet) *t GasPremium: types.NewInt(1), } - sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{}) if err != nil { panic(err) } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index 09be67d1e..fdd1fcb49 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -105,7 +105,7 @@ var keyinfoVerifyCmd = &cli.Command{ return err } - if _, err := w.Import(&keyInfo); err != nil { + if _, err := w.WalletImport(cctx.Context, &keyInfo); err != nil { return err } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 2e2c48c5d..3bbb6556d 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -40,7 +40,9 @@ func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byt if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.WalletSign(ctx, keyAddr, msg) + return a.WalletAPI.WalletSign(ctx, keyAddr, msg, api.MsgMeta{ + Type: api.MTUnknown, + }) } func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { From 82b15d2637655508bc92141db0b739a012ff362c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 8 Oct 2020 15:59:06 -0700 Subject: [PATCH 069/125] cleanup tests on finish --- chain/sync_test.go | 8 ++++---- node/test/builder.go | 17 +++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/chain/sync_test.go b/chain/sync_test.go index 1b06f604b..318f6d82f 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -221,8 +221,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { sourceRepo, genesis, blocks := tu.repoWithChain(tu.t, gen) var out api.FullNode - // TODO: Don't ignore stop - _, err := node.New(tu.ctx, + stop, err := node.New(tu.ctx, node.FullAPI(&out), node.Online(), node.Repo(sourceRepo), @@ -232,6 +231,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { node.Override(new(modules.Genesis), modules.LoadGenesis(genesis)), ) require.NoError(tu.t, err) + tu.t.Cleanup(func() { _ = stop(context.Background()) }) lastTs := blocks[len(blocks)-1].Blocks for _, lastB := range lastTs { @@ -253,8 +253,7 @@ func (tu *syncTestUtil) addClientNode() int { var out api.FullNode - // TODO: Don't ignore stop - _, err := node.New(tu.ctx, + stop, err := node.New(tu.ctx, node.FullAPI(&out), node.Online(), node.Repo(repo.NewMemory(nil)), @@ -264,6 +263,7 @@ func (tu *syncTestUtil) addClientNode() int { node.Override(new(modules.Genesis), modules.LoadGenesis(tu.genesis)), ) require.NoError(tu.t, err) + tu.t.Cleanup(func() { _ = stop(context.Background()) }) tu.nds = append(tu.nds, out) return len(tu.nds) - 1 diff --git a/node/test/builder.go b/node/test/builder.go index a3455f376..d9ec04460 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -101,8 +101,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr var minerapi api.StorageMiner mineBlock := make(chan miner2.MineReq) - // TODO: use stop - _, err = node.New(ctx, + stop, err := node.New(ctx, node.StorageMiner(&minerapi), node.Online(), node.Repo(r), @@ -118,6 +117,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr if err != nil { t.Fatalf("failed to construct node: %v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) /*// Bootstrap with full node remoteAddrs, err := tnd.NetAddrsListen(ctx) @@ -138,7 +138,9 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr } func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mn := mocknet.New(ctx) fulls := make([]test.TestNode, nFull) @@ -214,9 +216,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } - var err error - // TODO: Don't ignore stop - _, err = node.New(ctx, + stop, err := node.New(ctx, node.FullAPI(&fulls[i].FullNode), node.Online(), node.Repo(repo.NewMemory(nil)), @@ -229,7 +229,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. if err != nil { t.Fatal(err) } - + t.Cleanup(func() { _ = stop(context.Background()) }) } for i, def := range storage { @@ -364,7 +364,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options var err error // TODO: Don't ignore stop - _, err = node.New(ctx, + stop, err := node.New(ctx, node.FullAPI(&fulls[i].FullNode), node.Online(), node.Repo(repo.NewMemory(nil)), @@ -379,6 +379,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options if err != nil { t.Fatalf("%+v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) } for i, def := range storage { From 791aa5c6c886fe12def84fc4c2910feb9dd434af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:37:17 +0200 Subject: [PATCH 070/125] Add lotus-wallet to Makefile --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 56ab361ec..95a6af8a9 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,12 @@ lotus-health: .PHONY: lotus-health BINS+=lotus-health +lotus-wallet: + rm -f lotus-wallet + go build -o lotus-stats ./cmd/lotus-wallet +.PHONY: lotus-wallet +BINS+=lotus-wallet + testground: go build -tags testground -o /dev/null ./cmd/lotus .PHONY: testground From 597d065e0510c240b545d30ee8ffe7fc230176ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:39:06 +0200 Subject: [PATCH 071/125] Update go-jsonrpc with http(s) client support --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e9687c7b8..e1bf6cb3b 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-data-transfer v0.6.7 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f github.com/filecoin-project/go-fil-markets v0.7.1 - github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 + github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 diff --git a/go.sum b/go.sum index bab4e929d..ef954e66d 100644 --- a/go.sum +++ b/go.sum @@ -252,8 +252,8 @@ github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CW github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= -github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= -github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= +github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 h1:FSY245KeXFCUgyfFEu+bhrZNk8BGGJyfpSmQl2aiPU8= +github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 h1:+/4aUeUoKr6AKfPE3mBhXA5spIV6UcKdTYDPNU2Tdmg= From 6a10af626c77d083119a00c20dc482376916cccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:50:33 +0200 Subject: [PATCH 072/125] Nicer message logging in lotus-wallet --- cmd/lotus-wallet/logged.go | 33 ++++++++++++++++++++++++++++++++- node/impl/full/wallet.go | 2 +- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index c713c2e87..d3630c241 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -1,7 +1,11 @@ package main import ( + "bytes" "context" + "encoding/hex" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" @@ -38,7 +42,34 @@ func (c *LoggedWallet) WalletList(ctx context.Context) ([]address.Address, error } func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { - log.Infow("WalletSign", "address", k) + switch meta.Type { + case api.MTChainMsg: + var cmsg types.Message + if err := cmsg.UnmarshalCBOR(bytes.NewReader(meta.Extra)); err != nil { + return nil, xerrors.Errorf("unmarshalling message: %w", err) + } + + _, bc, err := cid.CidFromBytes(msg) + if err != nil { + return nil, xerrors.Errorf("getting cid from signing bytes: %w", err) + } + + if !cmsg.Cid().Equals(bc) { + return nil, xerrors.Errorf("cid(meta.Extra).bytes() != msg") + } + + log.Infow("WalletSign", + "address", k, + "type", meta.Type, + "from", cmsg.From, + "to", cmsg.To, + "value", types.FIL(cmsg.Value), + "feecap", types.FIL(cmsg.RequiredFunds()), + "method", cmsg.Method, + "params", hex.EncodeToString(cmsg.Params)) + default: + log.Infow("WalletSign", "address", k, "type", meta.Type) + } return c.under.WalletSign(ctx, k, msg, meta) } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 3bbb6556d..1a32cd422 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -41,7 +41,7 @@ func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byt return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } return a.WalletAPI.WalletSign(ctx, keyAddr, msg, api.MsgMeta{ - Type: api.MTUnknown, + Type: api.MTUnknown, }) } From 6c54d2445df3a307cfb90de5ff6e1fc22a8d08c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:53:12 +0200 Subject: [PATCH 073/125] Appease the linter --- chain/messagesigner/messagesigner.go | 12 +++++++----- chain/wallet/remotewallet/remote.go | 3 ++- markets/storageadapter/client.go | 13 +++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index c7948e42e..e32a78405 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -3,18 +3,20 @@ package messagesigner import ( "bytes" "context" - "github.com/filecoin-project/lotus/api" "sync" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagepool" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) const dsKeyActorNonce = "ActorNextNonce" diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index ca4e422d5..8454ae6c9 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -2,9 +2,10 @@ package remotewallet import ( "context" + "net/http" + "go.uber.org/fx" "golang.org/x/xerrors" - "net/http" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 080fe36eb..e9a260afe 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -5,21 +5,23 @@ package storageadapter import ( "bytes" "context" - "github.com/filecoin-project/lotus/api" - - "github.com/filecoin-project/go-state-types/big" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "golang.org/x/xerrors" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/events" @@ -31,7 +33,6 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" - "github.com/ipfs/go-cid" ) type ClientNodeAdapter struct { From 3cd53ad9d9a17024a1a81846daf358a89313cc85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 02:21:37 +0200 Subject: [PATCH 074/125] Use APIInfo parser for RemoteBacked wallet config --- Makefile | 2 +- chain/wallet/remotewallet/remote.go | 26 ++++++++------------------ cli/cmd.go | 29 ++++++++++++++++++++--------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 95a6af8a9..79f7fa81e 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,7 @@ BINS+=lotus-health lotus-wallet: rm -f lotus-wallet - go build -o lotus-stats ./cmd/lotus-wallet + go build -o lotus-wallet ./cmd/lotus-wallet .PHONY: lotus-wallet BINS+=lotus-wallet diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index 8454ae6c9..61deec7ee 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -2,13 +2,13 @@ package remotewallet import ( "context" - "net/http" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" + lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/node/modules/helpers" ) @@ -16,26 +16,16 @@ type RemoteWallet struct { api.WalletAPI } -func SetupRemoteWallet(url string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { +func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { - /*sp := strings.SplitN(env, ":", 2) - if len(sp) != 2 { - log.Warnf("invalid env(%s) value, missing token or address", envKey) - } else { - ma, err := multiaddr.NewMultiaddr(sp[1]) - if err != nil { - return APIInfo{}, xerrors.Errorf("could not parse multiaddr from env(%s): %w", envKey, err) - } - return APIInfo{ - Addr: ma, - Token: []byte(sp[0]), - }, nil - }*/ + ai := lcli.ParseApiInfo(info) - headers := http.Header{} - /*headers.Add("Authorization", "Bearer "+token)*/ + url, err := ai.DialArgs() + if err != nil { + return nil, err + } - wapi, closer, err := client.NewWalletRPC(mctx, url, headers) + wapi, closer, err := client.NewWalletRPC(mctx, url, ai.AuthHeader()) if err != nil { return nil, xerrors.Errorf("creating jsonrpc client: %w", err) } diff --git a/cli/cmd.go b/cli/cmd.go index edcb69adc..a3736f8d7 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "os/signal" + "regexp" "strings" "syscall" @@ -153,6 +154,24 @@ func envForRepoDeprecation(t repo.RepoType) string { } } +var ( + infoWithToken = regexp.MustCompile("^[a-zA-Z0-9\\-_]+?\\.[a-zA-Z0-9\\-_]+?\\.([a-zA-Z0-9\\-_]+)?:.+$") +) + +func ParseApiInfo(s string) APIInfo { + var tok []byte + if infoWithToken.Match([]byte(s)) { + sp := strings.SplitN(s, ":", 2) + tok = []byte(sp[0]) + s = sp[1] + } + + return APIInfo{ + Addr: s, + Token: tok, + } +} + func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { // Check if there was a flag passed with the listen address of the API // server (only used by the tests) @@ -175,15 +194,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } } if ok { - sp := strings.SplitN(env, ":", 2) - if len(sp) != 2 { - log.Warnf("invalid env(%s) value, missing token or address", envKey) - } else { - return APIInfo{ - Addr: sp[1], - Token: []byte(sp[0]), - }, nil - } + return ParseApiInfo(env), nil } repoFlag := flagForRepo(t) From 6cc7559b046fc3d5410eb27e460e0aa38321751a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 02:57:41 +0200 Subject: [PATCH 075/125] Fix APIInfo import cycle in tests --- chain/wallet/remotewallet/remote.go | 4 +- cli/cmd.go | 88 +++-------------------------- cli/util/apiinfo.go | 83 +++++++++++++++++++++++++++ cmd/lotus-shed/consensus.go | 3 +- 4 files changed, 95 insertions(+), 83 deletions(-) create mode 100644 cli/util/apiinfo.go diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index 61deec7ee..c7192f496 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" - lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/node/modules/helpers" ) @@ -18,7 +18,7 @@ type RemoteWallet struct { func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { - ai := lcli.ParseApiInfo(info) + ai := cliutil.ParseApiInfo(info) url, err := ai.DialArgs() if err != nil { diff --git a/cli/cmd.go b/cli/cmd.go index a3736f8d7..fb0706997 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -4,17 +4,13 @@ import ( "context" "fmt" "net/http" - "net/url" "os" "os/signal" - "regexp" "strings" "syscall" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -22,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/node/repo" ) @@ -48,57 +45,6 @@ func NewCliError(s string) error { // ApiConnector returns API instance type ApiConnector func() api.FullNode -type APIInfo struct { - Addr string - Token []byte -} - -func (a APIInfo) DialArgs() (string, error) { - ma, err := multiaddr.NewMultiaddr(a.Addr) - if err == nil { - _, addr, err := manet.DialArgs(ma) - if err != nil { - return "", err - } - - return "ws://" + addr + "/rpc/v0", nil - } - - _, err = url.Parse(a.Addr) - if err != nil { - return "", err - } - return a.Addr + "/rpc/v0", nil -} - -func (a APIInfo) Host() (string, error) { - ma, err := multiaddr.NewMultiaddr(a.Addr) - if err == nil { - _, addr, err := manet.DialArgs(ma) - if err != nil { - return "", err - } - - return addr, nil - } - - spec, err := url.Parse(a.Addr) - if err != nil { - return "", err - } - return spec.Host, nil -} - -func (a APIInfo) AuthHeader() http.Header { - if len(a.Token) != 0 { - headers := http.Header{} - headers.Add("Authorization", "Bearer "+string(a.Token)) - return headers - } - log.Warn("API Token not set and requested, capabilities might be limited.") - return nil -} - // The flag passed on the command line with the listen address of the API // server (only used by the tests) func flagForAPI(t repo.RepoType) string { @@ -154,25 +100,7 @@ func envForRepoDeprecation(t repo.RepoType) string { } } -var ( - infoWithToken = regexp.MustCompile("^[a-zA-Z0-9\\-_]+?\\.[a-zA-Z0-9\\-_]+?\\.([a-zA-Z0-9\\-_]+)?:.+$") -) - -func ParseApiInfo(s string) APIInfo { - var tok []byte - if infoWithToken.Match([]byte(s)) { - sp := strings.SplitN(s, ":", 2) - tok = []byte(sp[0]) - s = sp[1] - } - - return APIInfo{ - Addr: s, - Token: tok, - } -} - -func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { +func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) { // Check if there was a flag passed with the listen address of the API // server (only used by the tests) apiFlag := flagForAPI(t) @@ -180,7 +108,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { strma := ctx.String(apiFlag) strma = strings.TrimSpace(strma) - return APIInfo{Addr: strma}, nil + return cliutil.APIInfo{Addr: strma}, nil } envKey := envForRepo(t) @@ -194,24 +122,24 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } } if ok { - return ParseApiInfo(env), nil + return cliutil.ParseApiInfo(env), nil } repoFlag := flagForRepo(t) p, err := homedir.Expand(ctx.String(repoFlag)) if err != nil { - return APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err) + return cliutil.APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err) } r, err := repo.NewFS(p) if err != nil { - return APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err) + return cliutil.APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err) } ma, err := r.APIEndpoint() if err != nil { - return APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err) + return cliutil.APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err) } token, err := r.APIToken() @@ -219,7 +147,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err) } - return APIInfo{ + return cliutil.APIInfo{ Addr: ma.String(), Token: token, }, nil diff --git a/cli/util/apiinfo.go b/cli/util/apiinfo.go new file mode 100644 index 000000000..1f9a83769 --- /dev/null +++ b/cli/util/apiinfo.go @@ -0,0 +1,83 @@ +package cliutil + +import ( + "net/http" + "net/url" + "regexp" + "strings" + + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +var log = logging.Logger("cliutil") + +var ( + infoWithToken = regexp.MustCompile("^[a-zA-Z0-9\\-_]+?\\.[a-zA-Z0-9\\-_]+?\\.([a-zA-Z0-9\\-_]+)?:.+$") +) + +type APIInfo struct { + Addr string + Token []byte +} + +func ParseApiInfo(s string) APIInfo { + var tok []byte + if infoWithToken.Match([]byte(s)) { + sp := strings.SplitN(s, ":", 2) + tok = []byte(sp[0]) + s = sp[1] + } + + return APIInfo{ + Addr: s, + Token: tok, + } +} + +func (a APIInfo) DialArgs() (string, error) { + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } + + return "ws://" + addr + "/rpc/v0", nil + } + + _, err = url.Parse(a.Addr) + if err != nil { + return "", err + } + return a.Addr + "/rpc/v0", nil +} + +func (a APIInfo) Host() (string, error) { + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } + + return addr, nil + } + + spec, err := url.Parse(a.Addr) + if err != nil { + return "", err + } + return spec.Host, nil +} + +func (a APIInfo) AuthHeader() http.Header { + if len(a.Token) != 0 { + headers := http.Header{} + headers.Add("Authorization", "Bearer "+string(a.Token)) + return headers + } + log.Warn("API Token not set and requested, capabilities might be limited.") + return nil +} diff --git a/cmd/lotus-shed/consensus.go b/cmd/lotus-shed/consensus.go index 38a8cd8ef..1fe7756c1 100644 --- a/cmd/lotus-shed/consensus.go +++ b/cmd/lotus-shed/consensus.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" @@ -111,7 +112,7 @@ var consensusCheckCmd = &cli.Command{ if err != nil { return err } - ainfo := lcli.APIInfo{Addr: apima.String()} + ainfo := cliutil.APIInfo{Addr: apima.String()} addr, err := ainfo.DialArgs() if err != nil { return err From 356137439f2cd0bcca2d11d8ab616d97b6eae064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 03:00:30 +0200 Subject: [PATCH 076/125] apistruct: Drop unused WalletSignMessage --- api/apistruct/struct.go | 19 +++++++------------ cmd/lotus-wallet/logged.go | 1 + 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index d8c2c8094..0265701d6 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -363,14 +363,13 @@ type WorkerStruct struct { type WalletStruct struct { Internal struct { - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` - WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` - WalletList func(context.Context) ([]address.Address, error) `perm:"write"` - WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` - WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` - WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` - WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` - WalletDelete func(context.Context, address.Address) error `perm:"write"` + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` } } @@ -1401,10 +1400,6 @@ func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg [] return c.Internal.WalletSign(ctx, k, msg, meta) } -func (c *WalletStruct) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - return c.Internal.WalletSignMessage(ctx, k, msg) -} - func (c *WalletStruct) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { return c.Internal.WalletExport(ctx, a) } diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index d3630c241..3bcb3f867 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" From 9c357629662126c43d3b9cab9e40cb522477fc63 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 9 Oct 2020 03:10:40 -0400 Subject: [PATCH 077/125] Fix deletion of addresses from wallet --- chain/wallet/wallet.go | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 9438d2051..f821951b5 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -110,14 +110,12 @@ func (w *Wallet) tryFind(addr address.Address) (types.KeyInfo, error) { // We got an ErrKeyInfoNotFound error // Try again, this time with the testnet prefix - aChars := []rune(addr.String()) - prefixRunes := []rune(address.TestnetPrefix) - if len(prefixRunes) != 1 { - return types.KeyInfo{}, xerrors.Errorf("unexpected prefix length: %d", len(prefixRunes)) + tAddress, err := swapMainnetForTestnetPrefix(addr.String()) + if err != nil { + return types.KeyInfo{}, err } - aChars[0] = prefixRunes[0] - ki, err = w.keystore.Get(KNamePrefix + string(aChars)) + ki, err = w.keystore.Get(KNamePrefix + tAddress) if err != nil { return types.KeyInfo{}, err } @@ -291,6 +289,14 @@ func (w *Wallet) DeleteKey(addr address.Address) error { return xerrors.Errorf("failed to delete key %s: %w", addr, err) } + tAddr, err := swapMainnetForTestnetPrefix(addr.String()) + if err != nil { + return xerrors.Errorf("failed to swap prefixes: %w", err) + } + + // TODO: Does this always error in the not-found case? Just ignoring an error return for now. + _ = w.keystore.Delete(KNamePrefix + tAddr) + return nil } @@ -351,3 +357,14 @@ func ActSigType(typ string) crypto.SigType { return 0 } } + +func swapMainnetForTestnetPrefix(addr string) (string, error) { + aChars := []rune(addr) + prefixRunes := []rune(address.TestnetPrefix) + if len(prefixRunes) != 1 { + return "", xerrors.Errorf("unexpected prefix length: %d", len(prefixRunes)) + } + + aChars[0] = prefixRunes[0] + return string(aChars), nil +} From e19cd9ed01fbd5c633ac6542e8be6d1ffe2c77c4 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 29 Sep 2020 17:25:45 +0200 Subject: [PATCH 078/125] feat: lotus-lite - replace wallet StateManager with thin client to gateway --- api/api_gateway.go | 17 +++++++++++++++++ api/apistruct/struct.go | 32 ++++++++++++++++++++++++++++++++ api/client/client.go | 14 ++++++++++++++ chain/stmgr/stmgr.go | 7 +++++++ cli/cmd.go | 9 +++++++++ cmd/lotus-gateway/api.go | 33 ++++++++++++++++++++++----------- cmd/lotus/daemon.go | 28 ++++++++++++++++++++++++++++ node/builder.go | 1 + node/impl/full/wallet.go | 8 ++++---- node/modules/statemanager.go | 29 +++++++++++++++++++++++++++++ 10 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 api/api_gateway.go create mode 100644 node/modules/statemanager.go diff --git a/api/api_gateway.go b/api/api_gateway.go new file mode 100644 index 000000000..6a220e9ea --- /dev/null +++ b/api/api_gateway.go @@ -0,0 +1,17 @@ +package api + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +type GatewayAPI interface { + StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + ChainHead(ctx context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1a3ddda58..c71b6b560 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -361,6 +361,17 @@ type WorkerStruct struct { } } +type GatewayStruct struct { + Internal struct { + // TODO: does the gateway need perms? + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) + ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + } +} + // CommonStruct func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) { @@ -1372,7 +1383,28 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } +func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + return g.Internal.StateGetActor(ctx, actor, ts) +} + +func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { + return g.Internal.ChainHead(ctx) +} + +func (g GatewayStruct) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + return g.Internal.ChainGetTipSet(ctx, tsk) +} + +func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + return g.Internal.MpoolPush(ctx, sm) +} + +func (g GatewayStruct) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + return g.Internal.StateAccountKey(ctx, addr, tsk) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} var _ api.WorkerAPI = &WorkerStruct{} +var _ api.GatewayAPI = &GatewayStruct{} diff --git a/api/client/client.go b/api/client/client.go index cd915acf0..390ce93d7 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -82,3 +82,17 @@ func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) ( return &res, closer, err } + +// NewGatewayRPC creates a new http jsonrpc client for a gateway node. +func NewGatewayRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.GatewayAPI, jsonrpc.ClientCloser, error) { + var res apistruct.GatewayStruct + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", + []interface{}{ + &res.Internal, + }, + requestHeader, + opts..., + ) + + return &res, closer, err +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ba3dcd1d8..476de1927 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -40,6 +40,11 @@ import ( var log = logging.Logger("statemgr") +type StateManagerAPI interface { + LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) + ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) +} + type versionSpec struct { networkVersion network.Version atOrBelow abi.ChainEpoch @@ -1393,3 +1398,5 @@ func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (m } return actState, nil } + +var _ StateManagerAPI = (*StateManager)(nil) diff --git a/cli/cmd.go b/cli/cmd.go index edcb69adc..e6475934b 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -289,6 +289,15 @@ func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) return client.NewWorkerRPC(ctx.Context, addr, headers) } +func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) { + addr, headers, err := GetRawAPI(ctx, repo.FullNode) + if err != nil { + return nil, nil, err + } + + return client.NewGatewayRPC(ctx.Context, addr, headers) +} + func DaemonContext(cctx *cli.Context) context.Context { if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok { return mtCtx.(context.Context) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 0a6365dbd..edf6525b1 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -49,17 +49,6 @@ func (a *GatewayAPI) checkTipset(ctx context.Context, ts types.TipSetKey) error return nil } -func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - ctx, span := trace.StartSpan(ctx, "StateGetActor") - defer span.End() - - if err := a.checkTipset(ctx, ts); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) - } - - return a.api.StateGetActor(ctx, actor, ts) -} - func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { ctx, span := trace.StartSpan(ctx, "ChainHead") defer span.End() @@ -88,3 +77,25 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci return a.api.MpoolPushUntrusted(ctx, sm) } + +func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + ctx, span := trace.StartSpan(ctx, "StateGetActor") + defer span.End() + + if err := a.checkTipset(ctx, ts); err != nil { + return nil, fmt.Errorf("bad tipset: %w", err) + } + + return a.api.StateGetActor(ctx, actor, ts) +} + +func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + ctx, span := trace.StartSpan(ctx, "StateAccountKey") + defer span.End() + + if err := a.checkTipset(ctx, tsk); err != nil { + return address.Undef, fmt.Errorf("bad tipset: %w", err) + } + + return a.api.StateAccountKey(ctx, addr, tsk) +} diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index a0f754a60..d57dddb79 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,6 +15,8 @@ import ( "runtime/pprof" "strings" + "go.uber.org/fx" + "github.com/filecoin-project/lotus/chain/types" paramfetch "github.com/filecoin-project/go-paramfetch" @@ -114,6 +116,10 @@ var DaemonCmd = &cli.Command{ Name: "halt-after-import", Usage: "halt the process after importing chain from file", }, + &cli.BoolFlag{ + Name: "lite", + Usage: "start lotus in lite mode", + }, &cli.StringFlag{ Name: "pprof", Usage: "specify name of file for writing cpu profile to", @@ -240,6 +246,27 @@ var DaemonCmd = &cli.Command{ shutdownChan := make(chan struct{}) + // If the daemon is started in "lite mode", replace the StateManager + // with a thin client to a gateway server + liteMode := node.Options() + if cctx.Bool("lite") { + gapi, closer, err := lcli.GetGatewayAPI(cctx) + if err != nil { + return err + } + + createRPCStateMgr := func(lc fx.Lifecycle) *modules.RPCStateManager { + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + closer() + return nil + }, + }) + return modules.NewRPCStateManager(gapi) + } + liteMode = node.Override(new(stmgr.StateManagerAPI), createRPCStateMgr) + } + var api api.FullNode stop, err := node.New(ctx, @@ -251,6 +278,7 @@ var DaemonCmd = &cli.Command{ node.Repo(r), genesis, + liteMode, node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, node.Override(node.SetApiEndpointKey, func(lr repo.LockedRepo) error { diff --git a/node/builder.go b/node/builder.go index 0d48ec130..50a20d4bb 100644 --- a/node/builder.go +++ b/node/builder.go @@ -261,6 +261,7 @@ func Online() Option { Override(new(*store.ChainStore), modules.ChainStore), Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index b2ecdebbd..33f924846 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -19,8 +19,8 @@ import ( type WalletAPI struct { fx.In - StateManager *stmgr.StateManager - Wallet *wallet.Wallet + stmgr.StateManagerAPI + Wallet *wallet.Wallet } func (a *WalletAPI) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { @@ -36,7 +36,7 @@ func (a *WalletAPI) WalletList(ctx context.Context) ([]address.Address, error) { } func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) { - act, err := a.StateManager.LoadActorTsk(ctx, addr, types.EmptyTSK) + act, err := a.StateManagerAPI.LoadActorTsk(ctx, addr, types.EmptyTSK) if xerrors.Is(err, types.ErrActorNotFound) { return big.Zero(), nil } else if err != nil { @@ -46,7 +46,7 @@ func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (ty } func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { - keyAddr, err := a.StateManager.ResolveToKeyAddress(ctx, k, nil) + keyAddr, err := a.StateManagerAPI.ResolveToKeyAddress(ctx, k, nil) if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } diff --git a/node/modules/statemanager.go b/node/modules/statemanager.go new file mode 100644 index 000000000..b673f7268 --- /dev/null +++ b/node/modules/statemanager.go @@ -0,0 +1,29 @@ +package modules + +import ( + "context" + + "github.com/filecoin-project/lotus/api" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" +) + +type RPCStateManager struct { + gapi api.GatewayAPI +} + +func NewRPCStateManager(api api.GatewayAPI) *RPCStateManager { + return &RPCStateManager{gapi: api} +} + +func (s *RPCStateManager) LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) { + return s.gapi.StateGetActor(ctx, addr, tsk) +} + +func (s *RPCStateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { + return s.gapi.StateAccountKey(ctx, addr, ts.Key()) +} + +var _ stmgr.StateManagerAPI = (*RPCStateManager)(nil) From 1ffdd7d5b3a83ab313066714f1e305cbe967931a Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 30 Sep 2020 10:43:59 +0200 Subject: [PATCH 079/125] fix: gateway - remove tracing (JsonRPC automatically traces all calls) --- cmd/lotus-gateway/api.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index edf6525b1..22b30544e 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -9,8 +9,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" - - "go.opencensus.io/trace" ) const LookbackCap = time.Hour @@ -50,17 +48,12 @@ func (a *GatewayAPI) checkTipset(ctx context.Context, ts types.TipSetKey) error } func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { - ctx, span := trace.StartSpan(ctx, "ChainHead") - defer span.End() // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) return a.api.ChainHead(ctx) } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - ctx, span := trace.StartSpan(ctx, "ChainGetTipSet") - defer span.End() - if err := a.checkTipset(ctx, tsk); err != nil { return nil, fmt.Errorf("bad tipset: %w", err) } @@ -70,18 +63,12 @@ func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (* } func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { - ctx, span := trace.StartSpan(ctx, "MpoolPush") - defer span.End() - // TODO: additional anti-spam checks return a.api.MpoolPushUntrusted(ctx, sm) } func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - ctx, span := trace.StartSpan(ctx, "StateGetActor") - defer span.End() - if err := a.checkTipset(ctx, ts); err != nil { return nil, fmt.Errorf("bad tipset: %w", err) } @@ -90,9 +77,6 @@ func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, t } func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - ctx, span := trace.StartSpan(ctx, "StateAccountKey") - defer span.End() - if err := a.checkTipset(ctx, tsk); err != nil { return address.Undef, fmt.Errorf("bad tipset: %w", err) } From eec13ff8dc3144606b1a43e21d62f885ef45d8ce Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 30 Sep 2020 16:35:06 +0200 Subject: [PATCH 080/125] refactor: daemon - simplify gateway cleanup --- cmd/lotus/daemon.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index d57dddb79..54dcfe54c 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,8 +15,6 @@ import ( "runtime/pprof" "strings" - "go.uber.org/fx" - "github.com/filecoin-project/lotus/chain/types" paramfetch "github.com/filecoin-project/go-paramfetch" @@ -249,22 +247,24 @@ var DaemonCmd = &cli.Command{ // If the daemon is started in "lite mode", replace the StateManager // with a thin client to a gateway server liteMode := node.Options() - if cctx.Bool("lite") { + isLite := cctx.Bool("lite") + if isLite { gapi, closer, err := lcli.GetGatewayAPI(cctx) if err != nil { return err } - createRPCStateMgr := func(lc fx.Lifecycle) *modules.RPCStateManager { - lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { - closer() - return nil - }, - }) - return modules.NewRPCStateManager(gapi) - } - liteMode = node.Override(new(stmgr.StateManagerAPI), createRPCStateMgr) + defer closer() + + liteMode = node.Options( + node.Override(new(api.GatewayAPI), gapi), + node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + node.Unset(node.RunHelloKey), + node.Unset(node.RunChainExchangeKey), + node.Unset(node.RunPeerMgrKey), + node.Unset(node.HandleIncomingBlocksKey), + node.Unset(node.HandleIncomingMessagesKey), + ) } var api api.FullNode From 00a14de0948ce078437338f2356c04159efd1087 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 30 Sep 2020 16:36:16 +0200 Subject: [PATCH 081/125] feat: gateway - StateLookupID --- api/api_gateway.go | 3 ++- api/apistruct/struct.go | 12 ++++++++---- chain/stmgr/stmgr.go | 1 + cmd/lotus-gateway/api.go | 26 ++++++++++++++++++-------- node/impl/full/multisig.go | 9 +++++---- node/impl/full/state.go | 6 +----- node/modules/statemanager.go | 4 ++++ 7 files changed, 39 insertions(+), 22 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index 6a220e9ea..398a7518f 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -9,9 +9,10 @@ import ( ) type GatewayAPI interface { - StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c71b6b560..268fc4460 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -1383,10 +1383,6 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } -func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - return g.Internal.StateGetActor(ctx, actor, ts) -} - func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { return g.Internal.ChainHead(ctx) } @@ -1403,6 +1399,14 @@ func (g GatewayStruct) StateAccountKey(ctx context.Context, addr address.Address return g.Internal.StateAccountKey(ctx, addr, tsk) } +func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + return g.Internal.StateGetActor(ctx, actor, ts) +} + +func (g GatewayStruct) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + panic("implement me") +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 476de1927..d3129ea20 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -42,6 +42,7 @@ var log = logging.Logger("statemgr") type StateManagerAPI interface { LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) + LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) } diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 22b30544e..642ed4d86 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -68,14 +68,6 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci return a.api.MpoolPushUntrusted(ctx, sm) } -func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - if err := a.checkTipset(ctx, ts); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) - } - - return a.api.StateGetActor(ctx, actor, ts) -} - func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { if err := a.checkTipset(ctx, tsk); err != nil { return address.Undef, fmt.Errorf("bad tipset: %w", err) @@ -83,3 +75,21 @@ func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, return a.api.StateAccountKey(ctx, addr, tsk) } + +func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { + if err := a.checkTipset(ctx, tsk); err != nil { + return nil, fmt.Errorf("bad tipset: %w", err) + } + + return a.api.StateGetActor(ctx, actor, tsk) +} + +func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + if err := a.checkTipset(ctx, tsk); err != nil { + return address.Undef, fmt.Errorf("bad tipset: %w", err) + } + + return a.api.StateLookupID(ctx, addr, tsk) +} + +var _ api.GatewayAPI = &GatewayAPI{} diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 715689edc..489658412 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -3,6 +3,8 @@ package full import ( "context" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-address" @@ -23,9 +25,8 @@ import ( type MsigAPI struct { fx.In - WalletAPI WalletAPI - StateAPI StateAPI - MpoolAPI MpoolAPI + StateManagerAPI stmgr.StateManagerAPI + MpoolAPI MpoolAPI } func (a *MsigAPI) messageBuilder(ctx context.Context, from address.Address) (multisig.MessageBuilder, error) { @@ -152,7 +153,7 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro } if proposer.Protocol() != address.ID { - proposerID, err := a.StateAPI.StateLookupID(ctx, proposer, types.EmptyTSK) + proposerID, err := a.StateManagerAPI.LookupID(ctx, proposer, nil) if err != nil { return cid.Undef, err } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 56dfc0a14..37af5a9a4 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -382,12 +382,8 @@ func (a *StateAPI) StateLookupID(ctx context.Context, addr address.Address, tsk if err != nil { return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - state, err := a.stateForTs(ctx, ts) - if err != nil { - return address.Undef, err - } - return state.LookupID(addr) + return a.StateManager.LookupID(ctx, addr, ts) } func (a *StateAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { diff --git a/node/modules/statemanager.go b/node/modules/statemanager.go index b673f7268..0ed054d45 100644 --- a/node/modules/statemanager.go +++ b/node/modules/statemanager.go @@ -22,6 +22,10 @@ func (s *RPCStateManager) LoadActorTsk(ctx context.Context, addr address.Address return s.gapi.StateGetActor(ctx, addr, tsk) } +func (s *RPCStateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { + return s.gapi.StateLookupID(ctx, addr, ts.Key()) +} + func (s *RPCStateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { return s.gapi.StateAccountKey(ctx, addr, ts.Key()) } From 8fa4c0a970e58bd9cebbd4ba2614b033701b75c5 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 30 Sep 2020 17:59:18 +0200 Subject: [PATCH 082/125] feat: gateway - MpoolPush --- cmd/lotus/daemon.go | 3 +++ node/builder.go | 3 +++ node/impl/full/mpool.go | 7 ++++++- node/modules/pushmessage.go | 25 +++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 node/modules/pushmessage.go diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 54dcfe54c..f83e74118 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,6 +15,8 @@ import ( "runtime/pprof" "strings" + "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/chain/types" paramfetch "github.com/filecoin-project/go-paramfetch" @@ -259,6 +261,7 @@ var DaemonCmd = &cli.Command{ liteMode = node.Options( node.Override(new(api.GatewayAPI), gapi), node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + node.Override(new(full.PushMessageAPI), modules.NewRPCPushMessageAPI), node.Unset(node.RunHelloKey), node.Unset(node.RunChainExchangeKey), node.Unset(node.RunPeerMgrKey), diff --git a/node/builder.go b/node/builder.go index 50a20d4bb..8579131b9 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,6 +6,8 @@ import ( "os" "time" + "github.com/filecoin-project/lotus/node/impl/full" + logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -262,6 +264,7 @@ func Online() Option { Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), + Override(new(full.PushMessageAPI), From(new(full.MpoolAPI))), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 1f093606c..000891bb2 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -15,11 +15,16 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" ) +type PushMessageAPI interface { + MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) +} + type MpoolAPI struct { fx.In WalletAPI GasAPI + PushMessageAPI MessageSigner *messagesigner.MessageSigner @@ -107,7 +112,7 @@ func (a *MpoolAPI) MpoolClear(ctx context.Context, local bool) error { } func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { - return a.Mpool.Push(smsg) + return a.PushMessageAPI.MpoolPush(ctx, smsg) } func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { diff --git a/node/modules/pushmessage.go b/node/modules/pushmessage.go new file mode 100644 index 000000000..59af435a7 --- /dev/null +++ b/node/modules/pushmessage.go @@ -0,0 +1,25 @@ +package modules + +import ( + "context" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/node/impl/full" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/types" +) + +type RPCPushMessageAPI struct { + gapi api.GatewayAPI +} + +func NewRPCPushMessageAPI(api api.GatewayAPI) *RPCPushMessageAPI { + return &RPCPushMessageAPI{gapi: api} +} + +func (p *RPCPushMessageAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return p.gapi.MpoolPush(ctx, smsg) +} + +var _ full.PushMessageAPI = (*RPCPushMessageAPI)(nil) From f1b1d8cec6375f7d3decd47aa1c43f1582c1dfce Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 1 Oct 2020 12:38:14 +0200 Subject: [PATCH 083/125] fix: MpoolAPI DI --- node/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/builder.go b/node/builder.go index 8579131b9..6bdb46ca5 100644 --- a/node/builder.go +++ b/node/builder.go @@ -264,7 +264,7 @@ func Online() Option { Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - Override(new(full.PushMessageAPI), From(new(full.MpoolAPI))), + Override(new(full.PushMessageAPI), new(full.MpoolAPI)), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), From 2719adc1b1e28eb31c561ed0bcfd8610a30abe00 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 1 Oct 2020 17:51:27 +0200 Subject: [PATCH 084/125] feat: lite-mode - thin client for chan & state --- api/apistruct/struct.go | 7 +-- cmd/lotus/daemon.go | 4 +- node/builder.go | 5 +- node/impl/full/chain.go | 25 +++++++-- node/impl/full/mpool.go | 28 +++++++--- node/impl/full/state.go | 52 ++++++++++++++----- node/modules/pushmessage.go | 25 --------- .../{statemanager.go => rpcstatemanager.go} | 0 8 files changed, 93 insertions(+), 53 deletions(-) delete mode 100644 node/modules/pushmessage.go rename node/modules/{statemanager.go => rpcstatemanager.go} (100%) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 268fc4460..7ab50965a 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -364,11 +364,12 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? - StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - ChainHead func(ctx context.Context) (*types.TipSet, error) ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) } } @@ -1404,7 +1405,7 @@ func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, } func (g GatewayStruct) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - panic("implement me") + return g.Internal.StateLookupID(ctx, addr, tsk) } var _ api.Common = &CommonStruct{} diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index f83e74118..1dc9b3c78 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -260,8 +260,10 @@ var DaemonCmd = &cli.Command{ liteMode = node.Options( node.Override(new(api.GatewayAPI), gapi), + node.Override(new(full.ChainModuleAPI), node.From(new(api.GatewayAPI))), + node.Override(new(full.MpoolModuleAPI), node.From(new(api.GatewayAPI))), + node.Override(new(full.StateModuleAPI), node.From(new(api.GatewayAPI))), node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), - node.Override(new(full.PushMessageAPI), modules.NewRPCPushMessageAPI), node.Unset(node.RunHelloKey), node.Unset(node.RunChainExchangeKey), node.Unset(node.RunPeerMgrKey), diff --git a/node/builder.go b/node/builder.go index 6bdb46ca5..e8ce96ec8 100644 --- a/node/builder.go +++ b/node/builder.go @@ -264,10 +264,13 @@ func Online() Option { Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - Override(new(full.PushMessageAPI), new(full.MpoolAPI)), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), + Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), Override(new(dtypes.ChainBitswap), modules.ChainBitswap), diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index aa2ae4df1..810c56c0b 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -39,10 +39,27 @@ import ( var log = logging.Logger("fullnode") +type ChainModuleAPI interface { + ChainHead(context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) +} + +// ChainModule provides a default implementation of ChainModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type ChainModule struct { + fx.In + + Chain *store.ChainStore +} + +var _ ChainModuleAPI = (*ChainModule)(nil) + type ChainAPI struct { fx.In WalletAPI + ChainModuleAPI Chain *store.ChainStore } @@ -51,8 +68,8 @@ func (a *ChainAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, e return a.Chain.SubHeadChanges(ctx), nil } -func (a *ChainAPI) ChainHead(context.Context) (*types.TipSet, error) { - return a.Chain.GetHeaviestTipSet(), nil +func (m *ChainModule) ChainHead(ctx context.Context) (*types.TipSet, error) { + return m.Chain.GetHeaviestTipSet(), nil } func (a *ChainAPI) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) { @@ -77,8 +94,8 @@ func (a *ChainAPI) ChainGetBlock(ctx context.Context, msg cid.Cid) (*types.Block return a.Chain.GetBlock(msg) } -func (a *ChainAPI) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { - return a.Chain.LoadTipSet(key) +func (m *ChainModule) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { + return m.Chain.LoadTipSet(key) } func (a *ChainAPI) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api.BlockMessages, error) { diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 000891bb2..916d5785b 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -4,27 +4,43 @@ import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/chain/messagepool" + + "github.com/filecoin-project/lotus/chain/messagesigner" + + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" ) -type PushMessageAPI interface { +type MpoolModuleAPI interface { MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) } +// MpoolModule provides a default implementation of MpoolModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type MpoolModule struct { + fx.In + + Mpool *messagepool.MessagePool +} + +var _ MpoolModuleAPI = (*MpoolModule)(nil) + type MpoolAPI struct { fx.In + MpoolModuleAPI + WalletAPI GasAPI - PushMessageAPI MessageSigner *messagesigner.MessageSigner @@ -111,8 +127,8 @@ func (a *MpoolAPI) MpoolClear(ctx context.Context, local bool) error { return nil } -func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { - return a.PushMessageAPI.MpoolPush(ctx, smsg) +func (m *MpoolModule) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return m.Mpool.Push(smsg) } func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 37af5a9a4..a391dffa1 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -42,6 +42,24 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" ) +type StateModuleAPI interface { + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) +} + +// StateModule provides a default implementation of StateModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type StateModule struct { + fx.In + + StateManager *stmgr.StateManager + Chain *store.ChainStore +} + +var _ StateModuleAPI = (*StateModule)(nil) + type StateAPI struct { fx.In @@ -49,6 +67,8 @@ type StateAPI struct { // API attached to the state API. It probably should live somewhere better Wallet *wallet.Wallet + StateModuleAPI + ProofVerifier ffiwrapper.Verifier StateManager *stmgr.StateManager Chain *store.ChainStore @@ -349,27 +369,33 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. }, nil } -func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { +func stateForTs(ctx context.Context, ts *types.TipSet, cstore *store.ChainStore, smgr *stmgr.StateManager) (*state.StateTree, error) { if ts == nil { - ts = a.Chain.GetHeaviestTipSet() + ts = cstore.GetHeaviestTipSet() } - st, _, err := a.StateManager.TipSetState(ctx, ts) + st, _, err := smgr.TipSetState(ctx, ts) if err != nil { return nil, err } - buf := bufbstore.NewBufferedBstore(a.Chain.Blockstore()) + buf := bufbstore.NewBufferedBstore(cstore.Blockstore()) cst := cbor.NewCborStore(buf) return state.LoadStateTree(cst, st) } +func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { + return stateForTs(ctx, ts, a.Chain, a.StateManager) +} +func (m *StateModule) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { + return stateForTs(ctx, ts, m.Chain, m.StateManager) +} -func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - state, err := a.stateForTs(ctx, ts) + state, err := m.stateForTs(ctx, ts) if err != nil { return nil, xerrors.Errorf("computing tipset state failed: %w", err) } @@ -377,22 +403,22 @@ func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, tsk return state.GetActor(actor) } -func (a *StateAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.LookupID(ctx, addr, ts) + return m.StateManager.LookupID(ctx, addr, ts) } -func (a *StateAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.ResolveToKeyAddress(ctx, addr, ts) + return m.StateManager.ResolveToKeyAddress(ctx, addr, ts) } func (a *StateAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { diff --git a/node/modules/pushmessage.go b/node/modules/pushmessage.go deleted file mode 100644 index 59af435a7..000000000 --- a/node/modules/pushmessage.go +++ /dev/null @@ -1,25 +0,0 @@ -package modules - -import ( - "context" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/node/impl/full" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/chain/types" -) - -type RPCPushMessageAPI struct { - gapi api.GatewayAPI -} - -func NewRPCPushMessageAPI(api api.GatewayAPI) *RPCPushMessageAPI { - return &RPCPushMessageAPI{gapi: api} -} - -func (p *RPCPushMessageAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { - return p.gapi.MpoolPush(ctx, smsg) -} - -var _ full.PushMessageAPI = (*RPCPushMessageAPI)(nil) diff --git a/node/modules/statemanager.go b/node/modules/rpcstatemanager.go similarity index 100% rename from node/modules/statemanager.go rename to node/modules/rpcstatemanager.go From be09a8a00acc3e41bbe1f08558c24a99785092c7 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 1 Oct 2020 17:54:28 +0200 Subject: [PATCH 085/125] feat: lite-mode - check that gateway API implements required chain, mpool, state methods --- cmd/lotus-gateway/api.go | 4 ++++ cmd/lotus/daemon.go | 6 ++---- node/builder.go | 4 ++-- node/impl/full/chain.go | 2 +- node/impl/full/mpool.go | 9 +++------ node/impl/full/multisig.go | 8 +++----- node/impl/full/wallet.go | 4 ++-- 7 files changed, 17 insertions(+), 20 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 642ed4d86..5f8e9adf9 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" ) @@ -93,3 +94,6 @@ func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, ts } var _ api.GatewayAPI = &GatewayAPI{} +var _ full.ChainModuleAPI = (*GatewayAPI)(nil) +var _ full.MpoolModuleAPI = (*GatewayAPI)(nil) +var _ full.StateModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 1dc9b3c78..c05317458 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,10 +15,6 @@ import ( "runtime/pprof" "strings" - "github.com/filecoin-project/lotus/node/impl/full" - - "github.com/filecoin-project/lotus/chain/types" - paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" @@ -34,6 +30,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -42,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/testing" diff --git a/node/builder.go b/node/builder.go index e8ce96ec8..d513fe626 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,8 +6,6 @@ import ( "os" "time" - "github.com/filecoin-project/lotus/node/impl/full" - logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -61,6 +59,7 @@ import ( "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/impl/common" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -270,6 +269,7 @@ func Online() Option { Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), Override(new(full.StateModuleAPI), From(new(full.StateModule))), Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 810c56c0b..88b84cad2 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -68,7 +68,7 @@ func (a *ChainAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, e return a.Chain.SubHeadChanges(ctx), nil } -func (m *ChainModule) ChainHead(ctx context.Context) (*types.TipSet, error) { +func (m *ChainModule) ChainHead(context.Context) (*types.TipSet, error) { return m.Chain.GetHeaviestTipSet(), nil } diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 916d5785b..fc32aecd2 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -4,19 +4,16 @@ import ( "context" "encoding/json" - "github.com/filecoin-project/lotus/chain/messagepool" - - "github.com/filecoin-project/lotus/chain/messagesigner" - - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagepool" + "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) type MpoolModuleAPI interface { diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 489658412..3b05e4493 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -3,8 +3,6 @@ package full import ( "context" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-address" @@ -25,8 +23,8 @@ import ( type MsigAPI struct { fx.In - StateManagerAPI stmgr.StateManagerAPI - MpoolAPI MpoolAPI + StateAPI StateAPI + MpoolAPI MpoolAPI } func (a *MsigAPI) messageBuilder(ctx context.Context, from address.Address) (multisig.MessageBuilder, error) { @@ -153,7 +151,7 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro } if proposer.Protocol() != address.ID { - proposerID, err := a.StateManagerAPI.LookupID(ctx, proposer, nil) + proposerID, err := a.StateAPI.StateLookupID(ctx, proposer, types.EmptyTSK) if err != nil { return cid.Undef, err } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 33f924846..616c7b46e 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -19,8 +19,8 @@ import ( type WalletAPI struct { fx.In - stmgr.StateManagerAPI - Wallet *wallet.Wallet + StateManagerAPI stmgr.StateManagerAPI + Wallet *wallet.Wallet } func (a *WalletAPI) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { From 767611247c0239f79473d59f02daaedea8a35b0c Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 2 Oct 2020 16:14:30 +0200 Subject: [PATCH 086/125] feat: add RPC for StateWaitMsg --- api/api_full.go | 4 ++ api/api_gateway.go | 3 ++ api/apistruct/struct.go | 27 +++++++--- chain/stmgr/stmgr.go | 28 ++++++++--- cmd/lotus-gateway/api.go | 86 +++++++++++++++++++++++--------- cmd/lotus/daemon.go | 2 +- markets/storageadapter/client.go | 2 +- node/impl/full/chain.go | 7 +-- node/impl/full/state.go | 15 ++++-- 9 files changed, 127 insertions(+), 47 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 601b14660..777e996cb 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -367,6 +367,10 @@ type FullNode interface { // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) + // StateWaitMsgLimited looks back up to limit epochs in the chain for a message. + // If not found, it blocks until the message arrives on chain, and gets to the + // indicated confidence depth. + StateWaitMsgLimited(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch) (*MsgLookup, error) // StateListMiners returns the addresses of every miner that has claimed power in the Power Actor StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) // StateListActors returns the addresses of every actor in the state diff --git a/api/api_gateway.go b/api/api_gateway.go index 398a7518f..3d2293ee1 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -4,6 +4,7 @@ import ( "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" ) @@ -11,8 +12,10 @@ import ( type GatewayAPI interface { ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error) } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7ab50965a..b73379cfa 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -190,6 +190,7 @@ type FullNodeStruct struct { StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` + StateWaitMsgLimited func(context.Context, cid.Cid, uint64, abi.ChainEpoch) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -364,12 +365,14 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? - ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) - ChainHead func(ctx context.Context) (*types.TipSet, error) - MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) - StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) + MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } } @@ -866,6 +869,10 @@ func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confide return c.Internal.StateWaitMsg(ctx, msgc, confidence) } +func (c *FullNodeStruct) StateWaitMsgLimited(ctx context.Context, msgc cid.Cid, confidence uint64, limit abi.ChainEpoch) (*api.MsgLookup, error) { + return c.Internal.StateWaitMsgLimited(ctx, msgc, confidence, limit) +} + func (c *FullNodeStruct) StateSearchMsg(ctx context.Context, msgc cid.Cid) (*api.MsgLookup, error) { return c.Internal.StateSearchMsg(ctx, msgc) } @@ -1392,6 +1399,10 @@ func (g GatewayStruct) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) return g.Internal.ChainGetTipSet(ctx, tsk) } +func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) +} + func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { return g.Internal.MpoolPush(ctx, sm) } @@ -1408,6 +1419,10 @@ func (g GatewayStruct) StateLookupID(ctx context.Context, addr address.Address, return g.Internal.StateLookupID(ctx, addr, tsk) } +func (g GatewayStruct) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return g.Internal.StateWaitMsg(ctx, msg, confidence) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d3129ea20..d6b6f4360 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -38,6 +38,8 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) +const LookbackNoLimit = abi.ChainEpoch(-1) + var log = logging.Logger("statemgr") type StateManagerAPI interface { @@ -514,7 +516,7 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T return nil, fmt.Errorf("failed to load message: %w", err) } - _, r, _, err := sm.searchBackForMsg(ctx, ts, m) + _, r, _, err := sm.searchBackForMsg(ctx, ts, m, LookbackNoLimit) if err != nil { return nil, fmt.Errorf("failed to look back through chain for message: %w", err) } @@ -523,9 +525,9 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T } // WaitForMessage blocks until a message appears on chain. It looks backwards in the chain to see if this has already -// happened. It guarantees that the message has been on chain for at least confidence epochs without being reverted -// before returning. -func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { +// happened, with an optional limit to how many epochs it will search. It guarantees that the message has been on +// chain for at least confidence epochs without being reverted before returning. +func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -563,7 +565,7 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confid var backFm cid.Cid backSearchWait := make(chan struct{}) go func() { - fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head[0].Val, msg) + fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head[0].Val, msg, lookbackLimit) if err != nil { log.Warnf("failed to look back through chain for message: %w", err) return @@ -655,7 +657,7 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty return head, r, foundMsg, nil } - fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg) + fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg, LookbackNoLimit) if err != nil { log.Warnf("failed to look back through chain for message %s", mcid) @@ -669,7 +671,15 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty return fts, r, foundMsg, nil } -func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet, m types.ChainMsg) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { +// searchBackForMsg searches up to limit tipsets backwards from the given +// tipset for a message receipt. +// If limit is +// - 0 then no tipsets are searched +// - 5 then five tipset are searched +// - LookbackNoLimit then there is no limit +func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet, m types.ChainMsg, limit abi.ChainEpoch) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { + limitHeight := from.Height() - limit + noLimit := limit == LookbackNoLimit cur := from curActor, err := sm.LoadActor(ctx, m.VMMessage().From, cur) @@ -685,7 +695,9 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet mNonce := m.VMMessage().Nonce for { - if cur.Height() == 0 { + // If we've reached the genesis block, or we've reached the limit of + // how far back to look + if cur.Height() == 0 || !noLimit && cur.Height() <= limitHeight { // it ain't here! return nil, nil, cid.Undef, nil } diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 5f8e9adf9..98c1a0fea 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -6,13 +6,18 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" ) -const LookbackCap = time.Hour +const ( + LookbackCap = time.Hour + stateWaitLookbackLimit = abi.ChainEpoch(20) +) var ( ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap) @@ -22,26 +27,41 @@ type GatewayAPI struct { api api.FullNode } -func (a *GatewayAPI) getTipsetTimestamp(ctx context.Context, tsk types.TipSetKey) (time.Time, error) { +func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { if tsk.IsEmpty() { - return time.Now(), nil + return nil } ts, err := a.api.ChainGetTipSet(ctx, tsk) - if err != nil { - return time.Time{}, err - } - - return time.Unix(int64(ts.Blocks()[0].Timestamp), 0), nil -} - -func (a *GatewayAPI) checkTipset(ctx context.Context, ts types.TipSetKey) error { - when, err := a.getTipsetTimestamp(ctx, ts) if err != nil { return err } - if time.Since(when) > time.Hour { + return a.checkTipset(ts) +} + +func (a *GatewayAPI) checkTipset(ts *types.TipSet) error { + at := time.Unix(int64(ts.Blocks()[0].Timestamp), 0) + if err := a.checkTimestamp(at); err != nil { + return fmt.Errorf("bad tipset: %w", err) + } + return nil +} + +// TODO: write tests for this check +func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error { + tsBlock := ts.Blocks()[0] + heightDelta := time.Duration(uint64(tsBlock.Height-h)*build.BlockDelaySecs) * time.Second + timeAtHeight := time.Unix(int64(tsBlock.Timestamp), 0).Add(-heightDelta) + + if err := a.checkTimestamp(timeAtHeight); err != nil { + return fmt.Errorf("bad tipset height: %w", err) + } + return nil +} + +func (a *GatewayAPI) checkTimestamp(at time.Time) error { + if time.Since(at) > LookbackCap { return ErrLookbackTooLong } @@ -55,12 +75,26 @@ func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - if err := a.checkTipset(ctx, tsk); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) + return a.api.ChainGetTipSet(ctx, tsk) +} + +func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + ts, err := a.api.ChainGetTipSet(ctx, tsk) + if err != nil { + return nil, err } - // TODO: since we're limiting lookbacks, should just cache this (could really even cache the json response bytes) - return a.api.ChainGetTipSet(ctx, tsk) + // Check if the tipset key refers to a tipset that's too far in the past + if err := a.checkTipset(ts); err != nil { + return nil, err + } + + // Check if the height is too far in the past + if err := a.checkTipsetHeight(ts, h); err != nil { + return nil, err + } + + return a.api.ChainGetTipSetByHeight(ctx, h, tsk) } func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { @@ -70,30 +104,34 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci } func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - if err := a.checkTipset(ctx, tsk); err != nil { - return address.Undef, fmt.Errorf("bad tipset: %w", err) + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return address.Undef, err } return a.api.StateAccountKey(ctx, addr, tsk) } func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { - if err := a.checkTipset(ctx, tsk); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err } return a.api.StateGetActor(ctx, actor, tsk) } func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - if err := a.checkTipset(ctx, tsk); err != nil { - return address.Undef, fmt.Errorf("bad tipset: %w", err) + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return address.Undef, err } return a.api.StateLookupID(ctx, addr, tsk) } -var _ api.GatewayAPI = &GatewayAPI{} +func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) +} + +var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.MpoolModuleAPI = (*GatewayAPI)(nil) var _ full.StateModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index c05317458..04463a839 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -244,7 +244,7 @@ var DaemonCmd = &cli.Command{ shutdownChan := make(chan struct{}) - // If the daemon is started in "lite mode", replace the StateManager + // If the daemon is started in "lite mode", replace key APIs // with a thin client to a gateway server liteMode := node.Options() isLite := cctx.Bool("lite") diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 411c86ec9..647a1675d 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -187,7 +187,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor } // TODO: timeout - _, ret, _, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence) + _, ret, _, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence, stmgr.LookbackNoLimit) if err != nil { return 0, xerrors.Errorf("waiting for deal publish message: %w", err) } diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 88b84cad2..5b4f41114 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -42,6 +42,7 @@ var log = logging.Logger("fullnode") type ChainModuleAPI interface { ChainHead(context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) } // ChainModule provides a default implementation of ChainModuleAPI. @@ -197,12 +198,12 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([] return out, nil } -func (a *ChainAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *ChainModule) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.Chain.GetTipsetByHeight(ctx, h, ts, true) + return m.Chain.GetTipsetByHeight(ctx, h, ts, true) } func (a *ChainAPI) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index a391dffa1..540d959ed 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -46,6 +46,7 @@ type StateModuleAPI interface { StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } // StateModule provides a default implementation of StateModuleAPI. @@ -475,22 +476,28 @@ func (a *StateAPI) MinerCreateBlock(ctx context.Context, bt *api.BlockTemplate) return &out, nil } -func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { - ts, recpt, found, err := a.StateManager.WaitForMessage(ctx, msg, confidence) +func (m *StateModule) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return stateWaitMsgLimited(ctx, m.StateManager, m.Chain, msg, confidence, stmgr.LookbackNoLimit) +} +func (a *StateAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error) { + return stateWaitMsgLimited(ctx, a.StateManager, a.Chain, msg, confidence, lookbackLimit) +} +func stateWaitMsgLimited(ctx context.Context, smgr *stmgr.StateManager, cstore *store.ChainStore, msg cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error) { + ts, recpt, found, err := smgr.WaitForMessage(ctx, msg, confidence, lookbackLimit) if err != nil { return nil, err } var returndec interface{} if recpt.ExitCode == 0 && len(recpt.Return) > 0 { - cmsg, err := a.Chain.GetCMessage(msg) + cmsg, err := cstore.GetCMessage(msg) if err != nil { return nil, xerrors.Errorf("failed to load message after successful receipt search: %w", err) } vmsg := cmsg.VMMessage() - t, err := stmgr.GetReturnType(ctx, a.StateManager, vmsg.To, vmsg.Method, ts) + t, err := stmgr.GetReturnType(ctx, smgr, vmsg.To, vmsg.Method, ts) if err != nil { return nil, xerrors.Errorf("failed to get return type: %w", err) } From b32d25c562875a972714fd29c64bea3aaee636a1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 5 Oct 2020 17:09:21 +0200 Subject: [PATCH 087/125] feat: add RPC for GasEstimateMessageGas --- api/api_gateway.go | 1 + api/apistruct/struct.go | 5 ++ cmd/lotus-gateway/api.go | 9 ++++ cmd/lotus/daemon.go | 1 + node/builder.go | 3 +- node/impl/full/gas.go | 103 +++++++++++++++++++++++++++++++-------- 6 files changed, 100 insertions(+), 22 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index 3d2293ee1..8c06304e9 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -13,6 +13,7 @@ type GatewayAPI interface { ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b73379cfa..38d177aec 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -368,6 +368,7 @@ type GatewayStruct struct { ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainHead func(ctx context.Context) (*types.TipSet, error) + GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) @@ -1403,6 +1404,10 @@ func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEp return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) } +func (g GatewayStruct) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + return g.Internal.GasEstimateMessageGas(ctx, msg, spec, tsk) +} + func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { return g.Internal.MpoolPush(ctx, sm) } diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 98c1a0fea..8192c58ec 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -111,6 +111,14 @@ func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, return a.api.StateAccountKey(ctx, addr, tsk) } +func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + + return a.api.GasEstimateMessageGas(ctx, msg, spec, tsk) +} + func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -133,5 +141,6 @@ func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence u var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) +var _ full.GasModuleAPI = (*GatewayAPI)(nil) var _ full.MpoolModuleAPI = (*GatewayAPI)(nil) var _ full.StateModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 04463a839..495979ea9 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -259,6 +259,7 @@ var DaemonCmd = &cli.Command{ liteMode = node.Options( node.Override(new(api.GatewayAPI), gapi), node.Override(new(full.ChainModuleAPI), node.From(new(api.GatewayAPI))), + node.Override(new(full.GasModuleAPI), node.From(new(api.GatewayAPI))), node.Override(new(full.MpoolModuleAPI), node.From(new(api.GatewayAPI))), node.Override(new(full.StateModuleAPI), node.From(new(api.GatewayAPI))), node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), diff --git a/node/builder.go b/node/builder.go index d513fe626..101ec3dbc 100644 --- a/node/builder.go +++ b/node/builder.go @@ -267,8 +267,9 @@ func Online() Option { Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), - Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(full.GasModuleAPI), From(new(full.GasModule))), Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 3580ca26d..0cb1eb084 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -26,8 +26,27 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +type GasModuleAPI interface { + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) +} + +// GasModule provides a default implementation of GasModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type GasModule struct { + fx.In + Stmgr *stmgr.StateManager + Chain *store.ChainStore + Mpool *messagepool.MessagePool +} + +var _ GasModuleAPI = (*GasModule)(nil) + type GasAPI struct { fx.In + + GasModuleAPI + Stmgr *stmgr.StateManager Chain *store.ChainStore Mpool *messagepool.MessagePool @@ -36,9 +55,24 @@ type GasAPI struct { const MinGasPremium = 100e3 const MaxSpendOnFeeDenom = 100 -func (a *GasAPI) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64, - tsk types.TipSetKey) (types.BigInt, error) { - ts := a.Chain.GetHeaviestTipSet() +func (a *GasAPI) GasEstimateFeeCap( + ctx context.Context, + msg *types.Message, + maxqueueblks int64, + tsk types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateFeeCap(a.Chain, msg, maxqueueblks) +} +func (m *GasModule) GasEstimateFeeCap( + ctx context.Context, + msg *types.Message, + maxqueueblks int64, + tsk types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateFeeCap(m.Chain, msg, maxqueueblks) +} +func gasEstimateFeeCap(cstore *store.ChainStore, msg *types.Message, maxqueueblks int64) (types.BigInt, error) { + ts := cstore.GetHeaviestTipSet() parentBaseFee := ts.Blocks()[0].ParentBaseFee increaseFactor := math.Pow(1.+1./float64(build.BaseFeeMaxChangeDenom), float64(maxqueueblks)) @@ -82,9 +116,25 @@ func medianGasPremium(prices []gasMeta, blocks int) abi.TokenAmount { return premium } -func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, - sender address.Address, gaslimit int64, _ types.TipSetKey) (types.BigInt, error) { - +func (a *GasAPI) GasEstimateGasPremium( + ctx context.Context, + nblocksincl uint64, + sender address.Address, + gaslimit int64, + _ types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateGasPremium(a.Chain, nblocksincl) +} +func (m *GasModule) GasEstimateGasPremium( + ctx context.Context, + nblocksincl uint64, + sender address.Address, + gaslimit int64, + _ types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateGasPremium(m.Chain, nblocksincl) +} +func gasEstimateGasPremium(cstore *store.ChainStore, nblocksincl uint64) (types.BigInt, error) { if nblocksincl == 0 { nblocksincl = 1 } @@ -92,20 +142,20 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, var prices []gasMeta var blocks int - ts := a.Chain.GetHeaviestTipSet() + ts := cstore.GetHeaviestTipSet() for i := uint64(0); i < nblocksincl*2; i++ { if ts.Height() == 0 { break // genesis } - pts, err := a.Chain.LoadTipSet(ts.Parents()) + pts, err := cstore.LoadTipSet(ts.Parents()) if err != nil { return types.BigInt{}, err } blocks += len(pts.Blocks()) - msgs, err := a.Chain.MessagesForTipset(pts) + msgs, err := cstore.MessagesForTipset(pts) if err != nil { return types.BigInt{}, xerrors.Errorf("loading messages: %w", err) } @@ -142,19 +192,30 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, } func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) { - + return gasEstimateGasLimit(ctx, a.Chain, a.Stmgr, a.Mpool, msgIn) +} +func (m *GasModule) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) { + return gasEstimateGasLimit(ctx, m.Chain, m.Stmgr, m.Mpool, msgIn) +} +func gasEstimateGasLimit( + ctx context.Context, + cstore *store.ChainStore, + smgr *stmgr.StateManager, + mpool *messagepool.MessagePool, + msgIn *types.Message, +) (int64, error) { msg := *msgIn msg.GasLimit = build.BlockGasLimit msg.GasFeeCap = types.NewInt(uint64(build.MinimumBaseFee) + 1) msg.GasPremium = types.NewInt(1) - currTs := a.Chain.GetHeaviestTipSet() - fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msgIn.From, currTs) + currTs := cstore.GetHeaviestTipSet() + fromA, err := smgr.ResolveToKeyAddress(ctx, msgIn.From, currTs) if err != nil { return -1, xerrors.Errorf("getting key address: %w", err) } - pending, ts := a.Mpool.PendingFor(fromA) + pending, ts := mpool.PendingFor(fromA) priorMsgs := make([]types.ChainMsg, 0, len(pending)) for _, m := range pending { priorMsgs = append(priorMsgs, m) @@ -163,11 +224,11 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, // Try calling until we find a height with no migration. var res *api.InvocResult for { - res, err = a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) + res, err = smgr.CallWithGas(ctx, &msg, priorMsgs, ts) if err != stmgr.ErrExpensiveFork { break } - ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) + ts, err = cstore.GetTipSetFromKey(ts.Parents()) if err != nil { return -1, xerrors.Errorf("getting parent tipset: %w", err) } @@ -180,7 +241,7 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, } // Special case for PaymentChannel collect, which is deleting actor - st, err := a.Stmgr.ParentState(ts) + st, err := smgr.ParentState(ts) if err != nil { _ = err // somewhat ignore it as it can happen and we just want to detect @@ -206,17 +267,17 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, return res.MsgRct.GasUsed + 76e3, nil } -func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, _ types.TipSetKey) (*types.Message, error) { +func (m *GasModule) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, _ types.TipSetKey) (*types.Message, error) { if msg.GasLimit == 0 { - gasLimit, err := a.GasEstimateGasLimit(ctx, msg, types.TipSetKey{}) + gasLimit, err := m.GasEstimateGasLimit(ctx, msg, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas used: %w", err) } - msg.GasLimit = int64(float64(gasLimit) * a.Mpool.GetConfig().GasLimitOverestimation) + msg.GasLimit = int64(float64(gasLimit) * m.Mpool.GetConfig().GasLimitOverestimation) } if msg.GasPremium == types.EmptyInt || types.BigCmp(msg.GasPremium, types.NewInt(0)) == 0 { - gasPremium, err := a.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) + gasPremium, err := m.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas price: %w", err) } @@ -224,7 +285,7 @@ func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, } if msg.GasFeeCap == types.EmptyInt || types.BigCmp(msg.GasFeeCap, types.NewInt(0)) == 0 { - feeCap, err := a.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK) + feeCap, err := m.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("estimating fee cap: %w", err) } From 7b1bec91ed10aeeea3e93c2801edcaed574592d2 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 6 Oct 2020 11:03:27 +0200 Subject: [PATCH 088/125] feat: add tests for gateway ChainGetTipsetByHeight --- chain/types/mock/chain.go | 3 ++ cmd/lotus-gateway/api.go | 60 +++++++++++++++++++++++---------------- cmd/lotus-gateway/main.go | 2 +- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 559630619..85437079c 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -59,9 +59,11 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types var pcids []cid.Cid var height abi.ChainEpoch weight := types.NewInt(weightInc) + var timestamp uint64 if parents != nil { pcids = parents.Cids() height = parents.Height() + 1 + timestamp = parents.MinTimestamp() + build.BlockDelaySecs weight = types.BigAdd(parents.Blocks()[0].ParentWeight, weight) } @@ -79,6 +81,7 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types ParentWeight: weight, Messages: c, Height: height, + Timestamp: timestamp, ParentStateRoot: pstateRoot, BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS, Data: []byte("boo! im a signature")}, ParentBaseFee: types.NewInt(uint64(build.MinimumBaseFee)), diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 8192c58ec..0400a4a30 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -23,8 +23,20 @@ var ( ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap) ) +type rpcAPI interface { + ChainHead(ctx context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) +} + type GatewayAPI struct { - api api.FullNode + rpc rpcAPI } func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { @@ -32,7 +44,7 @@ func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) er return nil } - ts, err := a.api.ChainGetTipSet(ctx, tsk) + ts, err := a.rpc.ChainGetTipSet(ctx, tsk) if err != nil { return err } @@ -48,7 +60,6 @@ func (a *GatewayAPI) checkTipset(ts *types.TipSet) error { return nil } -// TODO: write tests for this check func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error { tsBlock := ts.Blocks()[0] heightDelta := time.Duration(uint64(tsBlock.Height-h)*build.BlockDelaySecs) * time.Second @@ -71,15 +82,15 @@ func (a *GatewayAPI) checkTimestamp(at time.Time) error { func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) - return a.api.ChainHead(ctx) + return a.rpc.ChainHead(ctx) } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - return a.api.ChainGetTipSet(ctx, tsk) + return a.rpc.ChainGetTipSet(ctx, tsk) } func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.api.ChainGetTipSet(ctx, tsk) + ts, err := a.rpc.ChainGetTipSet(ctx, tsk) if err != nil { return nil, err } @@ -94,21 +105,7 @@ func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoc return nil, err } - return a.api.ChainGetTipSetByHeight(ctx, h, tsk) -} - -func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { - // TODO: additional anti-spam checks - - return a.api.MpoolPushUntrusted(ctx, sm) -} - -func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - if err := a.checkTipsetKey(ctx, tsk); err != nil { - return address.Undef, err - } - - return a.api.StateAccountKey(ctx, addr, tsk) + return a.rpc.ChainGetTipSetByHeight(ctx, h, tsk) } func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { @@ -116,7 +113,20 @@ func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Messa return nil, err } - return a.api.GasEstimateMessageGas(ctx, msg, spec, tsk) + return a.rpc.GasEstimateMessageGas(ctx, msg, spec, tsk) +} + +func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + // TODO: additional anti-spam checks + return a.rpc.MpoolPushUntrusted(ctx, sm) +} + +func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return address.Undef, err + } + + return a.rpc.StateAccountKey(ctx, addr, tsk) } func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { @@ -124,7 +134,7 @@ func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, t return nil, err } - return a.api.StateGetActor(ctx, actor, tsk) + return a.rpc.StateGetActor(ctx, actor, tsk) } func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { @@ -132,11 +142,11 @@ func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, ts return address.Undef, err } - return a.api.StateLookupID(ctx, addr, tsk) + return a.rpc.StateLookupID(ctx, addr, tsk) } func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { - return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) + return a.rpc.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } var _ api.GatewayAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index c19599084..3c8eb4019 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -76,7 +76,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &GatewayAPI{api: api}) + rpcServer.Register("Filecoin", &GatewayAPI{rpc: api}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) From ef73b964fb40a916b921e484ee65111080e12de5 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 7 Oct 2020 11:26:15 +0200 Subject: [PATCH 089/125] feat: add end-to-end test for lite mode --- api/test/ccupgrade.go | 2 +- api/test/deals.go | 8 +- api/test/mining.go | 6 +- api/test/paych.go | 2 +- api/test/test.go | 22 ++- api/test/window_post.go | 4 +- chain/messagesigner/messagesigner.go | 11 +- chain/messagesigner/messagesigner_test.go | 2 +- cli/paych_test.go | 2 +- cmd/lotus-gateway/api_test.go | 183 ++++++++++++++++++++++ cmd/lotus-gateway/endtoend_test.go | 140 +++++++++++++++++ cmd/lotus-storage-miner/allinfo_test.go | 4 +- cmd/lotus/daemon.go | 16 +- node/builder.go | 18 +++ node/impl/full/mpool.go | 2 +- node/modules/mpoolnonceapi.go | 33 ++++ node/test/builder.go | 154 +++++++++++------- 17 files changed, 510 insertions(+), 99 deletions(-) create mode 100644 cmd/lotus-gateway/api_test.go create mode 100644 cmd/lotus-gateway/endtoend_test.go create mode 100644 node/modules/mpoolnonceapi.go diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 97fb665ed..38342d7fe 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -37,7 +37,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) { ctx := context.Background() - n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + n, sn := b(t, OneFull, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ Network: build.ActorUpgradeNetworkVersion, Height: upgradeHeight, Migration: stmgr.UpgradeActorsV2, diff --git a/api/test/deals.go b/api/test/deals.go index aa5bfa716..8b4a7fe8b 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -48,7 +48,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -85,7 +85,7 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -149,7 +149,7 @@ func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Durati _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -204,7 +204,7 @@ func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/api/test/mining.go b/api/test/mining.go index e19774a76..8147c224b 100644 --- a/api/test/mining.go +++ b/api/test/mining.go @@ -25,7 +25,7 @@ var log = logging.Logger("apitest") func (ts *testSuite) testMining(t *testing.T) { ctx := context.Background() - apis, sn := ts.makeNodes(t, 1, OneMiner) + apis, sn := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] newHeads, err := api.ChainNotify(ctx) @@ -54,7 +54,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) { }() ctx := context.Background() - apis, sn := ts.makeNodes(t, 1, OneMiner) + apis, sn := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] newHeads, err := api.ChainNotify(ctx) @@ -93,7 +93,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo // test making a deal with a fresh miner, and see if it starts to mine ctx := context.Background() - n, sn := b(t, 1, []StorageMiner{ + n, sn := b(t, OneFull, []StorageMiner{ {Full: 0, Preseal: PresealGenesis}, {Full: 0, Preseal: 0}, // TODO: Add support for miners on non-first full node }) diff --git a/api/test/paych.go b/api/test/paych.go index e95773b6a..a8ccebdde 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -33,7 +33,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 2, OneMiner) + n, sn := b(t, TwoFull, OneMiner) paymentCreator := n[0] paymentReceiver := n[1] diff --git a/api/test/test.go b/api/test/test.go index 853267eff..63e50bd73 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -40,12 +40,14 @@ type StorageMiner struct { Preseal int } +type OptionGenerator func([]TestNode) node.Option + // APIBuilder is a function which is invoked in test suite to provide // test nodes and networks // // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, full []OptionGenerator, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } @@ -63,13 +65,25 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testMiningReal", ts.testMiningReal) } +func DefaultFullOpts(nFull int) []OptionGenerator { + full := make([]OptionGenerator, nFull) + for i := range full { + full[i] = func(nodes []TestNode) node.Option { + return node.Options() + } + } + return full +} + var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}} +var OneFull = DefaultFullOpts(1) +var TwoFull = DefaultFullOpts(2) func (ts *testSuite) testVersion(t *testing.T) { build.RunningNodeType = build.NodeFull ctx := context.Background() - apis, _ := ts.makeNodes(t, 1, OneMiner) + apis, _ := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] v, err := api.Version(ctx) @@ -81,7 +95,7 @@ func (ts *testSuite) testVersion(t *testing.T) { func (ts *testSuite) testID(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 1, OneMiner) + apis, _ := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] id, err := api.ID(ctx) @@ -93,7 +107,7 @@ func (ts *testSuite) testID(t *testing.T) { func (ts *testSuite) testConnectTwo(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 2, OneMiner) + apis, _ := ts.makeNodes(t, TwoFull, OneMiner) p, err := apis[0].NetPeers(ctx) if err != nil { diff --git a/api/test/window_post.go b/api/test/window_post.go index eadcdbb05..c45d5e1bc 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -34,7 +34,7 @@ func init() { func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -133,7 +133,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + n, sn := b(t, OneFull, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ Network: build.ActorUpgradeNetworkVersion, Height: upgradeHeight, Migration: stmgr.UpgradeActorsV2, diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index ac94d6a3e..1fe8f9565 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -21,7 +20,7 @@ const dsKeyActorNonce = "ActorNextNonce" var log = logging.Logger("messagesigner") -type mpoolAPI interface { +type MpoolNonceAPI interface { GetNonce(address.Address) (uint64, error) } @@ -30,15 +29,11 @@ type mpoolAPI interface { type MessageSigner struct { wallet *wallet.Wallet lk sync.Mutex - mpool mpoolAPI + mpool MpoolNonceAPI ds datastore.Batching } -func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { - return newMessageSigner(wallet, mpool, ds) -} - -func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { +func NewMessageSigner(wallet *wallet.Wallet, mpool MpoolNonceAPI, ds dtypes.MetadataDS) *MessageSigner { ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/")) return &MessageSigner{ wallet: wallet, diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index 04869ff6d..de74b8a48 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -177,7 +177,7 @@ func TestMessageSignerSignMessage(t *testing.T) { t.Run(tt.name, func(t *testing.T) { mpool := newMockMpool() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - ms := newMessageSigner(w, mpool, ds) + ms := NewMessageSigner(w, mpool, ds) for _, m := range tt.msgs { if len(m.mpoolNonce) == 1 { diff --git a/cli/paych_test.go b/cli/paych_test.go index 18782b4e8..2aa5c6009 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -390,7 +390,7 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) { } func startTwoNodesOneMiner(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) { - n, sn := builder.RPCMockSbBuilder(t, 2, test.OneMiner) + n, sn := builder.RPCMockSbBuilder(t, test.TwoFull, test.OneMiner) paymentCreator := n[0] paymentReceiver := n[1] diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go new file mode 100644 index 000000000..ab42c9310 --- /dev/null +++ b/cmd/lotus-gateway/api_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/filecoin-project/lotus/build" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/lotus/chain/types/mock" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { + ctx := context.Background() + + lookbackTimestamp := uint64(time.Now().Unix()) - uint64(LookbackCap.Seconds()) + type args struct { + h abi.ChainEpoch + tskh abi.ChainEpoch + genesisTS uint64 + } + tests := []struct { + name string + args args + expErr bool + }{{ + name: "basic", + args: args{ + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "genesis", + args: args{ + h: abi.ChainEpoch(0), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "same epoch as tipset", + args: args{ + h: abi.ChainEpoch(5), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "tipset too old", + args: args{ + // Tipset height is 5, genesis is at LookbackCap - 10 epochs. + // So resulting tipset height will be 5 epochs earlier than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp - build.BlockDelaySecs*10, + }, + expErr: true, + }, { + name: "lookup height too old", + args: args{ + // Tipset height is 5, lookup height is 1, genesis is at LookbackCap - 3 epochs. + // So + // - lookup height will be 2 epochs earlier than LookbackCap. + // - tipset height will be 2 epochs later than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp - build.BlockDelaySecs*3, + }, + expErr: true, + }, { + name: "tipset and lookup height within acceptable range", + args: args{ + // Tipset height is 5, lookup height is 1, genesis is at LookbackCap. + // So + // - lookup height will be 1 epoch later than LookbackCap. + // - tipset height will be 5 epochs later than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp, + }, + }} + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + mock := &mockRPCAPI{} + a := &GatewayAPI{rpc: mock} + + // Create tipsets from genesis up to tskh and return the highest + ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS) + + got, err := a.ChainGetTipSetByHeight(ctx, tt.args.h, ts.Key()) + if tt.expErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.args.h, got.Height()) + } + }) + } +} + +type mockRPCAPI struct { + lk sync.RWMutex + tipsets []*types.TipSet +} + +func (m *mockRPCAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { + m.lk.RLock() + defer m.lk.RUnlock() + + return m.tipsets[len(m.tipsets)-1], nil +} + +func (m *mockRPCAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + m.lk.RLock() + defer m.lk.RUnlock() + + for _, ts := range m.tipsets { + if ts.Key() == tsk { + return ts, nil + } + } + + return nil, nil +} + +// createTipSets creates tipsets from genesis up to tskh and returns the highest +func (m *mockRPCAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *types.TipSet { + m.lk.Lock() + defer m.lk.Unlock() + + targeth := h + 1 // add one for genesis block + if genesisTimestamp == 0 { + genesisTimestamp = uint64(time.Now().Unix()) - build.BlockDelaySecs*uint64(targeth) + } + var currts *types.TipSet + for currh := abi.ChainEpoch(0); currh < targeth; currh++ { + blks := mock.MkBlock(currts, 1, 1) + if currh == 0 { + blks.Timestamp = genesisTimestamp + } + currts = mock.TipSet(blks) + m.tipsets = append(m.tipsets, currts) + } + + return m.tipsets[len(m.tipsets)-1] +} + +func (m *mockRPCAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + m.lk.Lock() + defer m.lk.Unlock() + + return m.tipsets[h], nil +} + +func (m *mockRPCAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + panic("implement me") +} + +func (m *mockRPCAPI) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + panic("implement me") +} + +func (m *mockRPCAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + panic("implement me") +} + +func (m *mockRPCAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + panic("implement me") +} + +func (m *mockRPCAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + panic("implement me") +} + +func (m *mockRPCAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { + panic("implement me") +} diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go new file mode 100644 index 000000000..e92c3248b --- /dev/null +++ b/cmd/lotus-gateway/endtoend_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/api" + + "github.com/filecoin-project/lotus/node" + + "github.com/filecoin-project/lotus/api/client" + + "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/lotus/chain/wallet" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-state-types/abi" + builder "github.com/filecoin-project/lotus/node/test" + + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/chain/types" +) + +func init() { + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) +} + +func TestEndToEnd(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + + blocktime := 5 * time.Millisecond + ctx := context.Background() + full, lite, closer := startNodes(ctx, t, blocktime) + defer closer() + + // The full node starts with a wallet + fullWalletAddr, err := full.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // Check the full node's wallet balance from the lite node + balance, err := lite.WalletBalance(ctx, fullWalletAddr) + require.NoError(t, err) + fmt.Println(balance) + + // Create a wallet on the lite node + liteWalletAddr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + require.NoError(t, err) + + // Send some funds from the full node to the lite node + err = sendFunds(ctx, t, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + require.NoError(t, err) + + // Send some funds from the lite node back to the full node + err = sendFunds(ctx, t, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) + require.NoError(t, err) + + data := []byte("hello") + sig, err := lite.WalletSign(ctx, liteWalletAddr, data) + require.NoError(t, err) + + ok, err := lite.WalletVerify(ctx, liteWalletAddr, data, sig) + require.NoError(t, err) + require.True(t, ok) +} + +func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { + msg := &types.Message{ + From: fromAddr, + To: toAddr, + Value: amt, + } + + sm, err := fromNode.MpoolPushMessage(ctx, msg, nil) + if err != nil { + return err + } + + res, err := fromNode.StateWaitMsg(ctx, sm.Cid(), 1) + if err != nil { + return err + } + if res.Receipt.ExitCode != 0 { + return xerrors.Errorf("send funds failed with exit code %d", res.Receipt.ExitCode) + } + + return nil +} + +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { + var closer jsonrpc.ClientCloser + + opts := append( + // Full node + test.OneFull, + // Lite node + func(nodes []test.TestNode) node.Option { + fullNode := nodes[0] + + addr, err := builder.WSMultiAddrToString(fullNode.ListenAddr) + require.NoError(t, err) + + // Create a gateway API that connects to the full node + var gapi api.GatewayAPI + gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) + require.NoError(t, err) + return node.LiteModeOverrides(gapi) + }, + ) + n, sn := builder.RPCMockSbBuilderWithOpts(t, opts, test.OneMiner) + + full := n[0] + lite := n[1] + miner := sn[0] + + // Get the listener address for the full node + fullAddr, err := full.NetAddrsListen(ctx) + require.NoError(t, err) + + // Connect the miner and the full node + err = miner.NetConnect(ctx, fullAddr) + require.NoError(t, err) + + // Start mining blocks + bm := test.NewBlockMiner(ctx, t, miner, blocktime) + bm.MineBlocks() + + return full, lite, closer +} diff --git a/cmd/lotus-storage-miner/allinfo_test.go b/cmd/lotus-storage-miner/allinfo_test.go index 8f744c4b3..fbe1d1617 100644 --- a/cmd/lotus-storage-miner/allinfo_test.go +++ b/cmd/lotus-storage-miner/allinfo_test.go @@ -64,8 +64,8 @@ func TestMinerAllInfo(t *testing.T) { require.NoError(t, infoAllCmd.Action(cctx)) } - bp := func(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - n, sn = builder.Builder(t, nFull, storage, opts...) + bp := func(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + n, sn = builder.Builder(t, fullOpts, storage, opts...) t.Run("pre-info-all", run) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 495979ea9..8c154d7d7 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -39,7 +39,6 @@ import ( "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node" - "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/testing" @@ -255,20 +254,7 @@ var DaemonCmd = &cli.Command{ } defer closer() - - liteMode = node.Options( - node.Override(new(api.GatewayAPI), gapi), - node.Override(new(full.ChainModuleAPI), node.From(new(api.GatewayAPI))), - node.Override(new(full.GasModuleAPI), node.From(new(api.GatewayAPI))), - node.Override(new(full.MpoolModuleAPI), node.From(new(api.GatewayAPI))), - node.Override(new(full.StateModuleAPI), node.From(new(api.GatewayAPI))), - node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), - node.Unset(node.RunHelloKey), - node.Unset(node.RunChainExchangeKey), - node.Unset(node.RunPeerMgrKey), - node.Unset(node.HandleIncomingBlocksKey), - node.Unset(node.HandleIncomingMessagesKey), - ) + liteMode = node.LiteModeOverrides(gapi) } var api api.FullNode diff --git a/node/builder.go b/node/builder.go index 101ec3dbc..d521fdb94 100644 --- a/node/builder.go +++ b/node/builder.go @@ -264,6 +264,7 @@ func Online() Option { Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.Wallet), wallet.NewWallet), + Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), @@ -401,6 +402,23 @@ func StorageMiner(out *api.StorageMiner) Option { ) } +func LiteModeOverrides(gapi api.GatewayAPI) Option { + return Options( + Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), + Override(new(api.GatewayAPI), gapi), + Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))), + Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + Unset(RunHelloKey), + Unset(RunChainExchangeKey), + Unset(RunPeerMgrKey), + Unset(HandleIncomingBlocksKey), + Unset(HandleIncomingMessagesKey), + ) +} + // Config sets up constructors based on the provided Config func ConfigCommon(cfg *config.Common) Option { return Options( diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index fc32aecd2..8ad209f3f 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -180,7 +180,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe // Sign and push the message return a.MessageSigner.SignMessage(ctx, msg, func(smsg *types.SignedMessage) error { - if _, err := a.Mpool.Push(smsg); err != nil { + if _, err := a.MpoolModuleAPI.MpoolPush(ctx, smsg); err != nil { return xerrors.Errorf("mpool push: failed to push message: %w", err) } return nil diff --git a/node/modules/mpoolnonceapi.go b/node/modules/mpoolnonceapi.go new file mode 100644 index 000000000..294f4d954 --- /dev/null +++ b/node/modules/mpoolnonceapi.go @@ -0,0 +1,33 @@ +package modules + +import ( + "context" + + "go.uber.org/fx" + + "github.com/filecoin-project/lotus/node/impl/full" + + "github.com/filecoin-project/lotus/chain/messagesigner" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/go-address" +) + +// MpoolNonceAPI substitutes the mpool nonce with an implementation that +// doesn't rely on the mpool - it just gets the nonce from actor state +type MpoolNonceAPI struct { + fx.In + + StateAPI full.StateAPI +} + +// GetNonce gets the nonce from actor state +func (a *MpoolNonceAPI) GetNonce(addr address.Address) (uint64, error) { + act, err := a.StateAPI.StateGetActor(context.Background(), addr, types.EmptyTSK) + if err != nil { + return 0, err + } + return act.Nonce, nil +} + +var _ messagesigner.MpoolNonceAPI = (*MpoolNonceAPI)(nil) diff --git a/node/test/builder.go b/node/test/builder.go index d9ec04460..f38201697 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -7,10 +7,13 @@ import ( "io/ioutil" "net" "net/http/httptest" + "strings" "sync" "testing" "time" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" @@ -117,6 +120,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr if err != nil { t.Fatalf("failed to construct node: %v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) /*// Bootstrap with full node @@ -137,13 +141,33 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { +func Builder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, opts, false) +} + +func MockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, opts, false) +} + +func RPCBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, opts, true) +} + +func RPCMockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) +} + +func RPCMockSbBuilderWithOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) +} + +func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) mn := mocknet.New(ctx) - fulls := make([]test.TestNode, nFull) + fulls := make([]test.TestNode, len(fullOpts)) storers := make([]test.TestStorageNode, len(storage)) pk, _, err := crypto.GenerateEd25519Key(rand.Reader) @@ -208,7 +232,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. // END PRESEAL SECTION - for i := 0; i < nFull; i++ { + for i := 0; i < len(fullOpts); i++ { var genesis node.Option if i == 0 { genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ)) @@ -224,12 +248,18 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. node.Test(), genesis, + fullOpts[i](fulls), node.Options(opts...), ) if err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = stop(context.Background()) }) + + if rpc { + fulls[i] = fullRpc(t, fulls[i]) + } } for i, def := range storage { @@ -261,6 +291,9 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. psd := presealDirs[i] */ + if rpc { + storers[i] = storerRpc(t, storers[i]) + } } if err := mn.LinkAll(); err != nil { @@ -286,11 +319,13 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. return fulls, storers } -func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - ctx := context.Background() +func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mn := mocknet.New(ctx) - fulls := make([]test.TestNode, nFull) + fulls := make([]test.TestNode, len(fullOpts)) storers := make([]test.TestStorageNode, len(storage)) var genbuf bytes.Buffer @@ -354,7 +389,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options // END PRESEAL SECTION - for i := 0; i < nFull; i++ { + for i := 0; i < len(fullOpts); i++ { var genesis node.Option if i == 0 { genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ)) @@ -374,12 +409,18 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), genesis, + fullOpts[i](fulls), node.Options(options...), ) if err != nil { t.Fatalf("%+v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) + + if rpc { + fulls[i] = fullRpc(t, fulls[i]) + } } for i, def := range storage { @@ -414,6 +455,10 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), node.Unset(new(*sectorstorage.Manager)), )) + + if rpc { + storers[i] = storerRpc(t, storers[i]) + } } if err := mn.LinkAll(); err != nil { @@ -438,69 +483,66 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options return fulls, storers } -func RPCBuilder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return rpcWithBuilder(t, Builder, nFull, storage, opts...) +func fullRpc(t *testing.T, nd test.TestNode) test.TestNode { + ma, listenAddr, err := CreateRPCServer(nd) + require.NoError(t, err) + + var full test.TestNode + full.FullNode, _, err = client.NewFullNodeRPC(context.Background(), listenAddr, nil) + require.NoError(t, err) + + full.ListenAddr = ma + return full } -func RPCMockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return rpcWithBuilder(t, MockSbBuilder, nFull, storage, opts...) +func storerRpc(t *testing.T, nd test.TestStorageNode) test.TestStorageNode { + ma, listenAddr, err := CreateRPCServer(nd) + require.NoError(t, err) + + var storer test.TestStorageNode + storer.StorageMiner, _, err = client.NewStorageMinerRPC(context.Background(), listenAddr, nil) + require.NoError(t, err) + + storer.ListenAddr = ma + storer.MineOne = nd.MineOne + return storer } -func rpcWithBuilder(t *testing.T, b test.APIBuilder, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - fullApis, storaApis := b(t, nFull, storage, opts...) - fulls := make([]test.TestNode, nFull) - storers := make([]test.TestStorageNode, len(storage)) +func CreateRPCServer(handler interface{}) (multiaddr.Multiaddr, string, error) { + rpcServer := jsonrpc.NewServer() + rpcServer.Register("Filecoin", handler) + testServ := httptest.NewServer(rpcServer) // todo: close - for i, a := range fullApis { - rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", a) - testServ := httptest.NewServer(rpcServer) // todo: close - - addr := testServ.Listener.Addr() - listenAddr := "ws://" + addr.String() - var err error - fulls[i].FullNode, _, err = client.NewFullNodeRPC(context.Background(), listenAddr, nil) - if err != nil { - t.Fatal(err) - } - ma, err := parseWSSMultiAddr(addr) - if err != nil { - t.Fatal(err) - } - fulls[i].ListenAddr = ma + addr := testServ.Listener.Addr() + listenAddr := "ws://" + addr.String() + ma, err := parseWSMultiAddr(addr) + if err != nil { + return nil, "", err } - - for i, a := range storaApis { - rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", a) - testServ := httptest.NewServer(rpcServer) // todo: close - - addr := testServ.Listener.Addr() - listenAddr := "ws://" + addr.String() - var err error - storers[i].StorageMiner, _, err = client.NewStorageMinerRPC(context.Background(), listenAddr, nil) - if err != nil { - t.Fatal(err) - } - ma, err := parseWSSMultiAddr(addr) - if err != nil { - t.Fatal(err) - } - storers[i].ListenAddr = ma - storers[i].MineOne = a.MineOne - } - - return fulls, storers + return ma, listenAddr, err } -func parseWSSMultiAddr(addr net.Addr) (multiaddr.Multiaddr, error) { +func parseWSMultiAddr(addr net.Addr) (multiaddr.Multiaddr, error) { host, port, err := net.SplitHostPort(addr.String()) if err != nil { return nil, err } - ma, err := multiaddr.NewMultiaddr("/ip4/" + host + "/" + addr.Network() + "/" + port + "/wss") + ma, err := multiaddr.NewMultiaddr("/ip4/" + host + "/" + addr.Network() + "/" + port + "/ws") if err != nil { return nil, err } return ma, nil } + +func WSMultiAddrToString(addr multiaddr.Multiaddr) (string, error) { + parts := strings.Split(addr.String(), "/") + if len(parts) != 6 || parts[0] != "" { + return "", xerrors.Errorf("Malformed ws multiaddr %s", addr) + } + + host := parts[2] + port := parts[4] + proto := parts[5] + + return proto + "://" + host + ":" + port + "/rpc/v0", nil +} From e9e27cbbb0834c93d5b6df949382ac28d07e69ed Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 7 Oct 2020 13:59:32 +0200 Subject: [PATCH 090/125] feat: put gateway in middle of end-to-end test for lite mode --- cmd/lotus-gateway/api.go | 28 +++++++++++++++------------- cmd/lotus-gateway/api_test.go | 26 +++++++++++++------------- cmd/lotus-gateway/endtoend_test.go | 17 ++++++++++++++--- cmd/lotus-gateway/main.go | 2 +- node/test/builder.go | 4 ---- 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 0400a4a30..373c02eaa 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -23,7 +23,9 @@ var ( ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap) ) -type rpcAPI interface { +// gatewayDepsAPI defines the API methods that the GatewayAPI depends on +// (to make it easy to mock for tests) +type gatewayDepsAPI interface { ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) @@ -36,7 +38,7 @@ type rpcAPI interface { } type GatewayAPI struct { - rpc rpcAPI + api gatewayDepsAPI } func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { @@ -44,7 +46,7 @@ func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) er return nil } - ts, err := a.rpc.ChainGetTipSet(ctx, tsk) + ts, err := a.api.ChainGetTipSet(ctx, tsk) if err != nil { return err } @@ -82,15 +84,15 @@ func (a *GatewayAPI) checkTimestamp(at time.Time) error { func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) - return a.rpc.ChainHead(ctx) + return a.api.ChainHead(ctx) } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - return a.rpc.ChainGetTipSet(ctx, tsk) + return a.api.ChainGetTipSet(ctx, tsk) } func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.rpc.ChainGetTipSet(ctx, tsk) + ts, err := a.api.ChainGetTipSet(ctx, tsk) if err != nil { return nil, err } @@ -105,7 +107,7 @@ func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoc return nil, err } - return a.rpc.ChainGetTipSetByHeight(ctx, h, tsk) + return a.api.ChainGetTipSetByHeight(ctx, h, tsk) } func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { @@ -113,12 +115,12 @@ func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Messa return nil, err } - return a.rpc.GasEstimateMessageGas(ctx, msg, spec, tsk) + return a.api.GasEstimateMessageGas(ctx, msg, spec, tsk) } func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { // TODO: additional anti-spam checks - return a.rpc.MpoolPushUntrusted(ctx, sm) + return a.api.MpoolPushUntrusted(ctx, sm) } func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { @@ -126,7 +128,7 @@ func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, return address.Undef, err } - return a.rpc.StateAccountKey(ctx, addr, tsk) + return a.api.StateAccountKey(ctx, addr, tsk) } func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { @@ -134,7 +136,7 @@ func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, t return nil, err } - return a.rpc.StateGetActor(ctx, actor, tsk) + return a.api.StateGetActor(ctx, actor, tsk) } func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { @@ -142,11 +144,11 @@ func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, ts return address.Undef, err } - return a.rpc.StateLookupID(ctx, addr, tsk) + return a.api.StateLookupID(ctx, addr, tsk) } func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { - return a.rpc.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) + return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } var _ api.GatewayAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index ab42c9310..2df2c9ba4 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -87,8 +87,8 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mock := &mockRPCAPI{} - a := &GatewayAPI{rpc: mock} + mock := &mockGatewayDepsAPI{} + a := &GatewayAPI{api: mock} // Create tipsets from genesis up to tskh and return the highest ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS) @@ -104,19 +104,19 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { } } -type mockRPCAPI struct { +type mockGatewayDepsAPI struct { lk sync.RWMutex tipsets []*types.TipSet } -func (m *mockRPCAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { +func (m *mockGatewayDepsAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { m.lk.RLock() defer m.lk.RUnlock() return m.tipsets[len(m.tipsets)-1], nil } -func (m *mockRPCAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { +func (m *mockGatewayDepsAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { m.lk.RLock() defer m.lk.RUnlock() @@ -130,7 +130,7 @@ func (m *mockRPCAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (* } // createTipSets creates tipsets from genesis up to tskh and returns the highest -func (m *mockRPCAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *types.TipSet { +func (m *mockGatewayDepsAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *types.TipSet { m.lk.Lock() defer m.lk.Unlock() @@ -151,33 +151,33 @@ func (m *mockRPCAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *t return m.tipsets[len(m.tipsets)-1] } -func (m *mockRPCAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { +func (m *mockGatewayDepsAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { m.lk.Lock() defer m.lk.Unlock() return m.tipsets[h], nil } -func (m *mockRPCAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { +func (m *mockGatewayDepsAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { panic("implement me") } -func (m *mockRPCAPI) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { +func (m *mockGatewayDepsAPI) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { panic("implement me") } -func (m *mockRPCAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { +func (m *mockGatewayDepsAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { panic("implement me") } -func (m *mockRPCAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { +func (m *mockGatewayDepsAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { panic("implement me") } -func (m *mockRPCAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { +func (m *mockGatewayDepsAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { panic("implement me") } -func (m *mockRPCAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { +func (m *mockGatewayDepsAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { panic("implement me") } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index e92c3248b..317e676d7 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -37,6 +37,8 @@ func init() { policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } +// TestEndToEnd tests that API calls can be made on a lite node that is +// connected through a gateway to a full API node func TestEndToEnd(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") @@ -66,10 +68,12 @@ func TestEndToEnd(t *testing.T) { err = sendFunds(ctx, t, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) require.NoError(t, err) + // Sign some data with the lite node wallet address data := []byte("hello") sig, err := lite.WalletSign(ctx, liteWalletAddr, data) require.NoError(t, err) + // Verify the signature ok, err := lite.WalletVerify(ctx, liteWalletAddr, data, sig) require.NoError(t, err) require.True(t, ok) @@ -101,6 +105,10 @@ func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAd func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { var closer jsonrpc.ClientCloser + // Create one miner and two full nodes. + // - Put a gateway server in front of full node 1 + // - Start full node 2 in lite mode + // - Connect lite node -> gateway server -> full node opts := append( // Full node test.OneFull, @@ -108,17 +116,20 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (tes func(nodes []test.TestNode) node.Option { fullNode := nodes[0] - addr, err := builder.WSMultiAddrToString(fullNode.ListenAddr) + // Create a gateway server in front of the full node + _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) require.NoError(t, err) - // Create a gateway API that connects to the full node + // Create a gateway client API that connects to the gateway server var gapi api.GatewayAPI gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) require.NoError(t, err) + + // Override this node with lite-mode options return node.LiteModeOverrides(gapi) }, ) - n, sn := builder.RPCMockSbBuilderWithOpts(t, opts, test.OneMiner) + n, sn := builder.RPCMockSbBuilder(t, opts, test.OneMiner) full := n[0] lite := n[1] diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index 3c8eb4019..c19599084 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -76,7 +76,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &GatewayAPI{rpc: api}) + rpcServer.Register("Filecoin", &GatewayAPI{api: api}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) diff --git a/node/test/builder.go b/node/test/builder.go index f38201697..ebf96e98f 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -157,10 +157,6 @@ func RPCMockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []t return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) } -func RPCMockSbBuilderWithOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { - return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) -} - func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) From b2834baa4b2818a7773859b47f70e4ccc3975993 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 7 Oct 2020 18:14:12 +0200 Subject: [PATCH 091/125] feat: add msig available balance and vested to lite mode --- api/api_gateway.go | 2 + api/apistruct/struct.go | 28 +++++++++---- cmd/lotus-gateway/api.go | 21 ++++++++++ cmd/lotus-gateway/api_test.go | 8 ++++ cmd/lotus-gateway/endtoend_test.go | 66 ++++++++++++++++++++++++++++++ node/impl/full/state.go | 20 +++++---- 6 files changed, 127 insertions(+), 18 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index 8c06304e9..95d28887d 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -15,6 +15,8 @@ type GatewayAPI interface { ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 38d177aec..df84eac80 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -365,15 +365,17 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? - ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) - ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) - ChainHead func(ctx context.Context) (*types.TipSet, error) - GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) - MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) - StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) + ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) + GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested func(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) + StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } } @@ -1412,6 +1414,14 @@ func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) ( return g.Internal.MpoolPush(ctx, sm) } +func (g GatewayStruct) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + return g.Internal.MsigGetAvailableBalance(ctx, addr, tsk) +} + +func (g GatewayStruct) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + return g.Internal.MsigGetVested(ctx, addr, start, end) +} + func (g GatewayStruct) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { return g.Internal.StateAccountKey(ctx, addr, tsk) } diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 373c02eaa..d5fac0a06 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -31,6 +31,8 @@ type gatewayDepsAPI interface { ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) @@ -123,6 +125,25 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci return a.api.MpoolPushUntrusted(ctx, sm) } +func (a *GatewayAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.NewInt(0), err + } + + return a.api.MsigGetAvailableBalance(ctx, addr, tsk) +} + +func (a *GatewayAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, start); err != nil { + return types.NewInt(0), err + } + if err := a.checkTipsetKey(ctx, end); err != nil { + return types.NewInt(0), err + } + + return a.api.MsigGetVested(ctx, addr, start, end) +} + func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return address.Undef, err diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index 2df2c9ba4..f34f887f5 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -166,6 +166,14 @@ func (m *mockGatewayDepsAPI) MpoolPushUntrusted(ctx context.Context, sm *types.S panic("implement me") } +func (m *mockGatewayDepsAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + panic("implement me") +} + func (m *mockGatewayDepsAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { panic("implement me") } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index 317e676d7..ad62be3fb 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -1,12 +1,17 @@ package main import ( + "bytes" "context" "fmt" "os" "testing" "time" + "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -77,6 +82,67 @@ func TestEndToEnd(t *testing.T) { ok, err := lite.WalletVerify(ctx, liteWalletAddr, data, sig) require.NoError(t, err) require.True(t, ok) + + // Create some wallets on the lite node to use for testing multisig + var walletAddrs []address.Address + for i := 0; i < 4; i++ { + addr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + require.NoError(t, err) + + walletAddrs = append(walletAddrs, addr) + + err = sendFunds(ctx, t, lite, liteWalletAddr, addr, types.NewInt(1e15)) + require.NoError(t, err) + } + + // Create an msig with three of the addresses and threshold of two sigs + msigAddrs := walletAddrs[:3] + amt := types.NewInt(1000) + addProposal, err := lite.MsigCreate(ctx, 2, msigAddrs, abi.ChainEpoch(50), amt, liteWalletAddr, types.NewInt(0)) + require.NoError(t, err) + + res, err := lite.StateWaitMsg(ctx, addProposal, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var execReturn init0.ExecReturn + err = execReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + + // Get available balance of msig: should be greater than zero and less + // than initial amount + msig := execReturn.IDAddress + msigBalance, err := lite.MsigGetAvailableBalance(ctx, msig, types.EmptyTSK) + require.NoError(t, err) + require.Greater(t, msigBalance.Int64(), int64(0)) + require.Less(t, msigBalance.Int64(), amt.Int64()) + + // Propose to add a new address to the msig + addProposal, err = lite.MsigAddPropose(ctx, msig, walletAddrs[0], walletAddrs[3], false) + require.NoError(t, err) + + res, err = lite.StateWaitMsg(ctx, addProposal, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var proposeReturn multisig.ProposeReturn + err = proposeReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + + // Approve proposal (proposer is first (implicit) signer, approver is + // second signer + txnID := uint64(proposeReturn.TxnID) + approval1, err := lite.MsigAddApprove(ctx, msig, walletAddrs[1], txnID, walletAddrs[0], walletAddrs[3], false) + require.NoError(t, err) + + res, err = lite.StateWaitMsg(ctx, approval1, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var approveReturn multisig.ApproveReturn + err = approveReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + require.True(t, approveReturn.Applied) } func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 540d959ed..34724e205 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -47,6 +47,8 @@ type StateModuleAPI interface { StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) } // StateModule provides a default implementation of StateModuleAPI. @@ -828,17 +830,17 @@ func (a *StateAPI) StateCompute(ctx context.Context, height abi.ChainEpoch, msgs }, nil } -func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - act, err := a.StateManager.LoadActor(ctx, addr, ts) + act, err := m.StateManager.LoadActor(ctx, addr, ts) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor: %w", err) } - msas, err := multisig.Load(a.Chain.Store(ctx), act) + msas, err := multisig.Load(m.Chain.Store(ctx), act) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err) } @@ -887,13 +889,13 @@ func (a *StateAPI) MsigGetVestingSchedule(ctx context.Context, addr address.Addr }, nil } -func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { - startTs, err := a.Chain.GetTipSetFromKey(start) +func (m *StateModule) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + startTs, err := m.Chain.GetTipSetFromKey(start) if err != nil { return types.EmptyInt, xerrors.Errorf("loading start tipset %s: %w", start, err) } - endTs, err := a.Chain.GetTipSetFromKey(end) + endTs, err := m.Chain.GetTipSetFromKey(end) if err != nil { return types.EmptyInt, xerrors.Errorf("loading end tipset %s: %w", end, err) } @@ -904,12 +906,12 @@ func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, star return big.Zero(), nil } - act, err := a.StateManager.LoadActor(ctx, addr, endTs) + act, err := m.StateManager.LoadActor(ctx, addr, endTs) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor at end epoch: %w", err) } - msas, err := multisig.Load(a.Chain.Store(ctx), act) + msas, err := multisig.Load(m.Chain.Store(ctx), act) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err) } From d69e4c7cf2fc2c3e64cd83c1a15363e31c9b1ec1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 8 Oct 2020 11:24:31 +0200 Subject: [PATCH 092/125] refactor: lite-mode - simplify organization of dep injection --- api/test/test.go | 20 +++++-- chain/sync_test.go | 4 +- cmd/lotus-gateway/endtoend_test.go | 48 ++++++++--------- cmd/lotus/daemon.go | 21 ++++---- node/builder.go | 87 +++++++++++++++--------------- node/test/builder.go | 24 ++++----- 6 files changed, 105 insertions(+), 99 deletions(-) diff --git a/api/test/test.go b/api/test/test.go index 63e50bd73..014859865 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -35,6 +35,7 @@ var PresealGenesis = -1 const GenesisPreseals = 2 +// Options for setting up a mock storage miner type StorageMiner struct { Full int Preseal int @@ -42,12 +43,19 @@ type StorageMiner struct { type OptionGenerator func([]TestNode) node.Option +// Options for setting up a mock full node +type FullNodeOpts struct { + Lite bool // run node in "lite" mode + Opts OptionGenerator // generate dependency injection options +} + // APIBuilder is a function which is invoked in test suite to provide // test nodes and networks // +// fullOpts array defines options for each full node // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, full []OptionGenerator, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } @@ -65,11 +73,13 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testMiningReal", ts.testMiningReal) } -func DefaultFullOpts(nFull int) []OptionGenerator { - full := make([]OptionGenerator, nFull) +func DefaultFullOpts(nFull int) []FullNodeOpts { + full := make([]FullNodeOpts, nFull) for i := range full { - full[i] = func(nodes []TestNode) node.Option { - return node.Options() + full[i] = FullNodeOpts{ + Opts: func(nodes []TestNode) node.Option { + return node.Options() + }, } } return full diff --git a/chain/sync_test.go b/chain/sync_test.go index 318f6d82f..1fe90a071 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -222,7 +222,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { var out api.FullNode stop, err := node.New(tu.ctx, - node.FullAPI(&out), + node.FullAPI(&out, false), node.Online(), node.Repo(sourceRepo), node.MockHost(tu.mn), @@ -254,7 +254,7 @@ func (tu *syncTestUtil) addClientNode() int { var out api.FullNode stop, err := node.New(tu.ctx, - node.FullAPI(&out), + node.FullAPI(&out, false), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(tu.mn), diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index ad62be3fb..206034968 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -8,32 +8,23 @@ import ( "testing" "time" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - + "github.com/stretchr/testify/require" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/lotus/api" - - "github.com/filecoin-project/lotus/node" - - "github.com/filecoin-project/lotus/api/client" - "github.com/filecoin-project/go-jsonrpc" - - "github.com/filecoin-project/lotus/chain/wallet" - - "github.com/stretchr/testify/require" - "github.com/filecoin-project/go-state-types/abi" - builder "github.com/filecoin-project/lotus/node/test" - + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" "github.com/filecoin-project/lotus/api/test" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node" + builder "github.com/filecoin-project/lotus/node/test" ) func init() { @@ -179,20 +170,23 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (tes // Full node test.OneFull, // Lite node - func(nodes []test.TestNode) node.Option { - fullNode := nodes[0] + test.FullNodeOpts{ + Lite: true, + Opts: func(nodes []test.TestNode) node.Option { + fullNode := nodes[0] - // Create a gateway server in front of the full node - _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) - require.NoError(t, err) + // Create a gateway server in front of the full node + _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) + require.NoError(t, err) - // Create a gateway client API that connects to the gateway server - var gapi api.GatewayAPI - gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) - require.NoError(t, err) + // Create a gateway client API that connects to the gateway server + var gapi api.GatewayAPI + gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) + require.NoError(t, err) - // Override this node with lite-mode options - return node.LiteModeOverrides(gapi) + // Provide the gateway API to dependency injection + return node.Override(new(api.GatewayAPI), gapi) + }, }, ) n, sn := builder.RPCMockSbBuilder(t, opts, test.OneMiner) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 8c154d7d7..084f75ef2 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -136,6 +136,8 @@ var DaemonCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + isLite := cctx.Bool("lite") + err := runmetrics.Enable(runmetrics.RunMetricOptions{ EnableCPU: true, EnableMemory: true, @@ -195,8 +197,10 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil { - return xerrors.Errorf("fetching proof parameters: %w", err) + if !isLite { + if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil { + return xerrors.Errorf("fetching proof parameters: %w", err) + } } var genBytes []byte @@ -243,10 +247,9 @@ var DaemonCmd = &cli.Command{ shutdownChan := make(chan struct{}) - // If the daemon is started in "lite mode", replace key APIs - // with a thin client to a gateway server - liteMode := node.Options() - isLite := cctx.Bool("lite") + // If the daemon is started in "lite mode", provide a GatewayAPI + // for RPC calls + liteModeDeps := node.Options() if isLite { gapi, closer, err := lcli.GetGatewayAPI(cctx) if err != nil { @@ -254,13 +257,13 @@ var DaemonCmd = &cli.Command{ } defer closer() - liteMode = node.LiteModeOverrides(gapi) + liteModeDeps = node.Override(new(api.GatewayAPI), gapi) } var api api.FullNode stop, err := node.New(ctx, - node.FullAPI(&api), + node.FullAPI(&api, isLite), node.Override(new(dtypes.Bootstrapper), isBootstrapper), node.Override(new(dtypes.ShutdownChan), shutdownChan), @@ -268,7 +271,7 @@ var DaemonCmd = &cli.Command{ node.Repo(r), genesis, - liteMode, + liteModeDeps, node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, node.Override(node.SetApiEndpointKey, func(lr repo.LockedRepo) error { diff --git a/node/builder.go b/node/builder.go index d521fdb94..452104ba8 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,6 +6,13 @@ import ( "os" "time" + "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/exchange" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node/hello" + logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -29,9 +36,7 @@ import ( storage2 "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" - "github.com/filecoin-project/lotus/chain/exchange" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/market" @@ -39,10 +44,7 @@ import ( "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -56,7 +58,6 @@ import ( "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/node/impl/full" @@ -159,7 +160,7 @@ type Settings struct { Online bool // Online option applied Config bool // Config option applied - + Lite bool // Start node in "lite" mode } func defaults() []Option { @@ -233,6 +234,10 @@ func isType(t repo.RepoType) func(s *Settings) bool { // Online sets up basic libp2p node func Online() Option { + isFullOrLiteNode := func(s *Settings) bool { return s.nodeType == repo.FullNode } + isFullNode := func(s *Settings) bool { return s.nodeType == repo.FullNode && !s.Lite } + isLiteNode := func(s *Settings) bool { return s.nodeType == repo.FullNode && s.Lite } + return Options( // make sure that online is applied before Config. // This is important because Config overrides some of Online units @@ -246,17 +251,14 @@ func Online() Option { // common Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - // Full node - - ApplyIf(isType(repo.FullNode), + // Full node or lite node + ApplyIf(isFullOrLiteNode, // TODO: Fix offline mode Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), - Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), Override(new(vm.SyscallBuilder), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), @@ -264,15 +266,8 @@ func Online() Option { Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.Wallet), wallet.NewWallet), - Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), - Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), - Override(new(full.GasModuleAPI), From(new(full.GasModule))), - Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), - Override(new(full.StateModuleAPI), From(new(full.StateModule))), - Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), Override(new(dtypes.ChainBitswap), modules.ChainBitswap), @@ -297,12 +292,6 @@ func Online() Option { Override(new(dtypes.Graphsync), modules.Graphsync), Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)), - - Override(RunHelloKey, modules.RunHello), - Override(RunChainExchangeKey, modules.RunChainExchange), - Override(RunPeerMgrKey, modules.RunPeerMgr), - Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), Override(new(discovery.PeerResolver), modules.RetrievalResolver), @@ -321,8 +310,34 @@ func Online() Option { Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels), ), + // Lite node + ApplyIf(isLiteNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), + Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))), + Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + ), + + // Full node + ApplyIf(isFullNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), + Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), + Override(new(full.GasModuleAPI), From(new(full.GasModule))), + Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), + + Override(RunHelloKey, modules.RunHello), + Override(RunChainExchangeKey, modules.RunChainExchange), + Override(RunPeerMgrKey, modules.RunPeerMgr), + Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), + Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), + ), + // miner - ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, + ApplyIf(isType(repo.StorageMiner), Override(new(api.Common), From(new(common.CommonAPI))), Override(new(sectorstorage.StorageAuth), modules.StorageAuth), @@ -402,23 +417,6 @@ func StorageMiner(out *api.StorageMiner) Option { ) } -func LiteModeOverrides(gapi api.GatewayAPI) Option { - return Options( - Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), - Override(new(api.GatewayAPI), gapi), - Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))), - Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))), - Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))), - Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))), - Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), - Unset(RunHelloKey), - Unset(RunChainExchangeKey), - Unset(RunPeerMgrKey), - Unset(HandleIncomingBlocksKey), - Unset(HandleIncomingMessagesKey), - ) -} - // Config sets up constructors based on the provided Config func ConfigCommon(cfg *config.Common) Option { return Options( @@ -533,10 +531,11 @@ func Repo(r repo.Repo) Option { } } -func FullAPI(out *api.FullNode) Option { +func FullAPI(out *api.FullNode, lite bool) Option { return Options( func(s *Settings) error { s.nodeType = repo.FullNode + s.Lite = lite return nil }, func(s *Settings) error { diff --git a/node/test/builder.go b/node/test/builder.go index ebf96e98f..cdb0b43dd 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -141,23 +141,23 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func Builder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { +func Builder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { return mockBuilderOpts(t, fullOpts, storage, opts, false) } -func MockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { +func MockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { return mockSbBuilderOpts(t, fullOpts, storage, opts, false) } -func RPCBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { +func RPCBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { return mockBuilderOpts(t, fullOpts, storage, opts, true) } -func RPCMockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { +func RPCMockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) } -func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { +func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -237,14 +237,15 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []te } stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode), + node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), node.Test(), genesis, - fullOpts[i](fulls), + + fullOpts[i].Opts(fulls), node.Options(opts...), ) if err != nil { @@ -315,7 +316,7 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []te return fulls, storers } -func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { +func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -393,10 +394,8 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage [] genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } - var err error - // TODO: Don't ignore stop stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode), + node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), @@ -405,7 +404,8 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage [] node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), genesis, - fullOpts[i](fulls), + + fullOpts[i].Opts(fulls), node.Options(options...), ) if err != nil { From 200a95f82420757840b5c28a7e7a27c43e347ff1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 8 Oct 2020 12:21:15 +0200 Subject: [PATCH 093/125] refactor: unify test builder full node options --- api/test/ccupgrade.go | 9 +-------- api/test/test.go | 17 ++++++++++++++++- api/test/window_post.go | 8 +------- cmd/lotus-storage-miner/allinfo_test.go | 6 ++---- node/test/builder.go | 20 +++++++++----------- 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 38342d7fe..4a860c661 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -12,10 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" ) @@ -37,11 +34,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) { ctx := context.Background() - n, sn := b(t, OneFull, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: build.ActorUpgradeNetworkVersion, - Height: upgradeHeight, - Migration: stmgr.UpgradeActorsV2, - }})) + n, sn := b(t, []FullNodeOpts{FullNodeWithUpgradeAt(upgradeHeight)}, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/api/test/test.go b/api/test/test.go index 014859865..35b397740 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -4,11 +4,14 @@ import ( "context" "testing" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/miner" @@ -55,7 +58,7 @@ type FullNodeOpts struct { // fullOpts array defines options for each full node // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } @@ -89,6 +92,18 @@ var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}} var OneFull = DefaultFullOpts(1) var TwoFull = DefaultFullOpts(2) +var FullNodeWithUpgradeAt = func(upgradeHeight abi.ChainEpoch) FullNodeOpts { + return FullNodeOpts{ + Opts: func(nodes []TestNode) node.Option { + return node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + Network: build.ActorUpgradeNetworkVersion, + Height: upgradeHeight, + Migration: stmgr.UpgradeActorsV2, + }}) + }, + } +} + func (ts *testSuite) testVersion(t *testing.T) { build.RunningNodeType = build.NodeFull diff --git a/api/test/window_post.go b/api/test/window_post.go index c45d5e1bc..7bc56a562 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -15,11 +15,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/mock" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" bminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl" @@ -133,11 +131,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, OneFull, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: build.ActorUpgradeNetworkVersion, - Height: upgradeHeight, - Migration: stmgr.UpgradeActorsV2, - }})) + n, sn := b(t, []FullNodeOpts{FullNodeWithUpgradeAt(upgradeHeight)}, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/cmd/lotus-storage-miner/allinfo_test.go b/cmd/lotus-storage-miner/allinfo_test.go index fbe1d1617..a458c024b 100644 --- a/cmd/lotus-storage-miner/allinfo_test.go +++ b/cmd/lotus-storage-miner/allinfo_test.go @@ -5,8 +5,6 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/node" - logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" @@ -64,8 +62,8 @@ func TestMinerAllInfo(t *testing.T) { require.NoError(t, infoAllCmd.Action(cctx)) } - bp := func(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - n, sn = builder.Builder(t, fullOpts, storage, opts...) + bp := func(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + n, sn = builder.Builder(t, fullOpts, storage) t.Run("pre-info-all", run) diff --git a/node/test/builder.go b/node/test/builder.go index cdb0b43dd..289fae841 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -141,23 +141,23 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func Builder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return mockBuilderOpts(t, fullOpts, storage, opts, false) +func Builder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, false) } -func MockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return mockSbBuilderOpts(t, fullOpts, storage, opts, false) +func MockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, false) } -func RPCBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return mockBuilderOpts(t, fullOpts, storage, opts, true) +func RPCBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, true) } func RPCMockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { - return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) + return mockSbBuilderOpts(t, fullOpts, storage, true) } -func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { +func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -246,7 +246,6 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test. genesis, fullOpts[i].Opts(fulls), - node.Options(opts...), ) if err != nil { t.Fatal(err) @@ -316,7 +315,7 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test. return fulls, storers } -func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { +func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -406,7 +405,6 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes genesis, fullOpts[i].Opts(fulls), - node.Options(options...), ) if err != nil { t.Fatalf("%+v", err) From 7f7c9e978fc7641eecdb6c7271b468d32f217b4f Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 8 Oct 2020 13:25:51 +0200 Subject: [PATCH 094/125] feat: hide lite mode flag --- cmd/lotus/daemon.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 084f75ef2..298cb7a67 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -114,8 +114,9 @@ var DaemonCmd = &cli.Command{ Usage: "halt the process after importing chain from file", }, &cli.BoolFlag{ - Name: "lite", - Usage: "start lotus in lite mode", + Name: "lite", + Usage: "start lotus in lite mode", + Hidden: true, }, &cli.StringFlag{ Name: "pprof", From 4cca3f19a361a5e02564208f35096722cbcba387 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 9 Oct 2020 09:40:25 -0700 Subject: [PATCH 095/125] add an api for removing multisig signers --- api/api_full.go | 22 ++++++++++-- api/apistruct/struct.go | 16 +++++++-- cli/multisig.go | 4 +-- node/impl/full/multisig.go | 73 ++++++++++++++++++++++++++++++++++---- 4 files changed, 101 insertions(+), 14 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 601b14660..7b854a3e4 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -432,12 +432,21 @@ type FullNode interface { // It takes the following params: , , , // , , MsigPropose(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) - // MsigApprove approves a previously-proposed multisig message + + // MsigApprove approves a previously-proposed multisig message by transaction ID + // It takes the following params: , + MsigApprove(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) + + // MsigApproveTxnHash approves a previously-proposed multisig message, specified + // using both transaction ID and a hash of the parameters used in the + // proposal. This method of approval can be used to ensure you only approve + // exactly the transaction you think you are. // It takes the following params: , , , , , // , , - MsigApprove(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) + MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) + // MsigCancel cancels a previously-proposed multisig message - // It takes the following params: , , , , + // It takes the following params: , , , , // , , MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) // MsigAddPropose proposes adding a signer in the multisig @@ -465,6 +474,13 @@ type FullNode interface { // , MsigSwapCancel(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) + // MsigRemoveSigner proposes the removal of a signer from the multisig. + // It accepts the multisig to make the change on, the proposer address to + // send the message from, the address to be removed, and a boolean + // indicating whether or not the signing threshold should be lowered by one + // along with the address removal. + MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) + MarketEnsureAvailable(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) // MarketFreeBalance diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1a3ddda58..92e6ffe27 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -216,7 +216,8 @@ type FullNodeStruct struct { MsigGetVested func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) `perm:"read"` MsigCreate func(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"` MsigPropose func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` - MsigApprove func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` + MsigApprove func(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) `perm:"sign"` + MsigApproveTxnHash func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` MsigCancel func(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` MsigAddPropose func(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error) `perm:"sign"` MsigAddApprove func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error) `perm:"sign"` @@ -224,6 +225,7 @@ type FullNodeStruct struct { MsigSwapPropose func(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` MsigSwapApprove func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` MsigSwapCancel func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` + MsigRemoveSigner func(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) `perm:"sign"` MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"` @@ -950,8 +952,12 @@ func (c *FullNodeStruct) MsigPropose(ctx context.Context, msig address.Address, return c.Internal.MsigPropose(ctx, msig, to, amt, src, method, params) } -func (c *FullNodeStruct) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return c.Internal.MsigApprove(ctx, msig, txID, proposer, to, amt, src, method, params) +func (c *FullNodeStruct) MsigApprove(ctx context.Context, msig address.Address, txID uint64, signer address.Address) (cid.Cid, error) { + return c.Internal.MsigApprove(ctx, msig, txID, signer) +} + +func (c *FullNodeStruct) MsigApproveTxnHash(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { + return c.Internal.MsigApproveTxnHash(ctx, msig, txID, proposer, to, amt, src, method, params) } func (c *FullNodeStruct) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { @@ -982,6 +988,10 @@ func (c *FullNodeStruct) MsigSwapCancel(ctx context.Context, msig address.Addres return c.Internal.MsigSwapCancel(ctx, msig, src, txID, oldAdd, newAdd) } +func (c *FullNodeStruct) MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) { + return c.Internal.MsigRemoveSigner(ctx, msig, proposer, toRemove, decrease) +} + func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) { return c.Internal.MarketEnsureAvailable(ctx, addr, wallet, amt) } diff --git a/cli/multisig.go b/cli/multisig.go index 6b3867cb3..b692fad5a 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -484,7 +484,7 @@ var msigApproveCmd = &cli.Command{ from = defaddr } - msgCid, err := api.MsigApprove(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) + msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) if err != nil { return err } @@ -1129,7 +1129,7 @@ var msigLockApproveCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigApprove(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) if err != nil { return err } diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 715689edc..d52f29a30 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -95,7 +95,7 @@ func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -122,7 +122,7 @@ func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -134,15 +134,64 @@ func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } -func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return a.msigApproveOrCancel(ctx, api.MsigApprove, msig, txID, proposer, to, amt, src, method, params) +func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { + return a.msigApproveOrCancelSimple(ctx, api.MsigApprove, msig, txID, src) +} + +func (a *MsigAPI) MsigApproveTxnHash(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { + return a.msigApproveOrCancelTxnHash(ctx, api.MsigApprove, msig, txID, proposer, to, amt, src, method, params) } func (a *MsigAPI) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return a.msigApproveOrCancel(ctx, api.MsigCancel, msig, txID, src, to, amt, src, method, params) + return a.msigApproveOrCancelTxnHash(ctx, api.MsigCancel, msig, txID, src, to, amt, src, method, params) } -func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { +func (a *MsigAPI) MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) { + enc, actErr := serializeRemoveParams(toRemove, decrease) + if actErr != nil { + return cid.Undef, actErr + } + + return a.MsigPropose(ctx, msig, msig, types.NewInt(0), proposer, uint64(builtin0.MethodsMultisig.RemoveSigner), enc) +} + +func (a *MsigAPI) msigApproveOrCancelSimple(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { + if msig == address.Undef { + return cid.Undef, xerrors.Errorf("must provide multisig address") + } + + if src == address.Undef { + return cid.Undef, xerrors.Errorf("must provide source address") + } + + mb, err := a.messageBuilder(ctx, src) + if err != nil { + return cid.Undef, err + } + + var msg *types.Message + switch operation { + case api.MsigApprove: + msg, err = mb.Approve(msig, txID, nil) + case api.MsigCancel: + msg, err = mb.Cancel(msig, txID, nil) + default: + return cid.Undef, xerrors.Errorf("Invalid operation for msigApproveOrCancel") + } + if err != nil { + return cid.Undef, err + } + + smsg, err := a.MpoolAPI.MpoolPushMessage(ctx, msg, nil) + if err != nil { + return cid.Undef, err + } + + return smsg.Cid(), nil + +} + +func (a *MsigAPI) msigApproveOrCancelTxnHash(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { if msig == address.Undef { return cid.Undef, xerrors.Errorf("must provide multisig address") } @@ -216,3 +265,15 @@ func serializeSwapParams(old address.Address, new address.Address) ([]byte, erro return enc, nil } + +func serializeRemoveParams(rem address.Address, dec bool) ([]byte, error) { + enc, actErr := actors.SerializeParams(&multisig0.RemoveSignerParams{ + Signer: rem, + Decrease: dec, + }) + if actErr != nil { + return nil, actErr + } + + return enc, nil +} From ab8286fa381c63d13e3e79609ba7659e2974e429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 18:43:22 +0200 Subject: [PATCH 096/125] Fix docsgen, lotus-soup build --- chain/sync_test.go | 4 +-- cmd/lotus/daemon.go | 2 +- documentation/en/api-methods.md | 44 +++++++++++++++++++++++++++++++++ node/builder.go | 13 ++++++++-- node/test/builder.go | 4 +-- 5 files changed, 60 insertions(+), 7 deletions(-) diff --git a/chain/sync_test.go b/chain/sync_test.go index 1fe90a071..318f6d82f 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -222,7 +222,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { var out api.FullNode stop, err := node.New(tu.ctx, - node.FullAPI(&out, false), + node.FullAPI(&out), node.Online(), node.Repo(sourceRepo), node.MockHost(tu.mn), @@ -254,7 +254,7 @@ func (tu *syncTestUtil) addClientNode() int { var out api.FullNode stop, err := node.New(tu.ctx, - node.FullAPI(&out, false), + node.FullAPI(&out), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(tu.mn), diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 298cb7a67..5964a2458 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -264,7 +264,7 @@ var DaemonCmd = &cli.Command{ var api api.FullNode stop, err := node.New(ctx, - node.FullAPI(&api, isLite), + node.FullAPI(&api, node.Lite(isLite)), node.Override(new(dtypes.Bootstrapper), isBootstrapper), node.Override(new(dtypes.ShutdownChan), shutdownChan), diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 8a288a4bf..cc6d92e4f 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -167,6 +167,7 @@ * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) * [StateVerifierStatus](#StateVerifierStatus) * [StateWaitMsg](#StateWaitMsg) + * [StateWaitMsgLimited](#StateWaitMsgLimited) * [Sync](#Sync) * [SyncCheckBad](#SyncCheckBad) * [SyncCheckpoint](#SyncCheckpoint) @@ -4322,6 +4323,49 @@ Response: } ``` +### StateWaitMsgLimited +StateWaitMsgLimited looks back up to limit epochs in the chain for a message. +If not found, it blocks until the message arrives on chain, and gets to the +indicated confidence depth. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + 42, + 10101 +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ReturnDec": {}, + "TipSet": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Height": 10101 +} +``` + ## Sync The Sync method group contains methods for interacting with and observing the lotus sync service. diff --git a/node/builder.go b/node/builder.go index 452104ba8..71295dbc8 100644 --- a/node/builder.go +++ b/node/builder.go @@ -531,13 +531,22 @@ func Repo(r repo.Repo) Option { } } -func FullAPI(out *api.FullNode, lite bool) Option { +type FullOption = Option + +func Lite(enable bool) FullOption { + return func(s *Settings) error { + s.Lite = enable + return nil + } +} + +func FullAPI(out *api.FullNode, fopts ...FullOption) Option { return Options( func(s *Settings) error { s.nodeType = repo.FullNode - s.Lite = lite return nil }, + Options(fopts...), func(s *Settings) error { resAPI := &impl.FullNodeAPI{} s.invokes[ExtractApiKey] = fx.Populate(resAPI) diff --git a/node/test/builder.go b/node/test/builder.go index 289fae841..4aa8a55ea 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -237,7 +237,7 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test. } stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite), + node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), @@ -394,7 +394,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes } stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite), + node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), From f87d325b6fecab5a50bce36dc69d3c8f0e1b09db Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Fri, 9 Oct 2020 17:23:40 +0100 Subject: [PATCH 097/125] fix: spec actors 2 reward shim uses correct field for CumsumRealized --- chain/actors/builtin/reward/v0.go | 2 +- chain/actors/builtin/reward/v2.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 0efd0b482..6a6e6d12e 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -57,7 +57,7 @@ func (s *state0) CumsumBaseline() (abi.StoragePower, error) { } func (s *state0) CumsumRealized() (abi.StoragePower, error) { - return s.State.CumsumBaseline, nil + return s.State.CumsumRealized, nil } func (s *state0) InitialPledgeForPower(sectorWeight abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { diff --git a/chain/actors/builtin/reward/v2.go b/chain/actors/builtin/reward/v2.go index ec0709c39..b7cb49102 100644 --- a/chain/actors/builtin/reward/v2.go +++ b/chain/actors/builtin/reward/v2.go @@ -60,7 +60,7 @@ func (s *state2) CumsumBaseline() (abi.StoragePower, error) { } func (s *state2) CumsumRealized() (abi.StoragePower, error) { - return s.State.CumsumBaseline, nil + return s.State.CumsumRealized, nil } func (s *state2) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { From 105aa40007f92bfd1b2e057731da76381d2ed6a2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 11:06:49 -0700 Subject: [PATCH 098/125] simplify message syncing logic 1. Allow duplicate blocks from bitswap. This shouldn't happen, but there's no reason to bail (just log loudly). 2. Simplify logic to very explicitly check to make sure we're fetching every block. --- chain/sub/incoming.go | 49 +++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index d51c481d1..68ee5e20c 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -120,12 +120,6 @@ func FetchMessagesByCids( return err } - // FIXME: We already sort in `fetchCids`, we are duplicating too much work, - // we don't need to pass the index. - if out[i] != nil { - return fmt.Errorf("received duplicate message") - } - out[i] = msg return nil }) @@ -149,10 +143,6 @@ func FetchSignedMessagesByCids( return err } - if out[i] != nil { - return fmt.Errorf("received duplicate message") - } - out[i] = smsg return nil }) @@ -184,24 +174,29 @@ func fetchCids( return fmt.Errorf("duplicate CIDs in fetchCids input") } - fetchedBlocks := bserv.GetBlocks(ctx, cids) - - for i := 0; i < len(cids); i++ { - select { - case block, ok := <-fetchedBlocks: - if !ok { - return fmt.Errorf("failed to fetch all messages") - } - - ix, ok := cidIndex[block.Cid()] - if !ok { - return fmt.Errorf("received message we didnt ask for") - } - - if err := cb(ix, block); err != nil { - return err - } + for block := range bserv.GetBlocks(ctx, cids) { + ix, ok := cidIndex[block.Cid()] + if !ok { + // Ignore duplicate/unexpected blocks. This shouldn't + // happen, but we can be safe. + log.Errorw("received duplicate/unexpected block when syncing", "cid", block.Cid()) + continue } + + // Record that we've received the block. + delete(cidIndex, block.Cid()) + + if err := cb(ix, block); err != nil { + return err + } + } + + if len(cidIndex) > 0 { + err := ctx.Err() + if err == nil { + err = fmt.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex)) + } + return err } return nil From 9f9d5cb2a5645e16e269160f32852537c9502094 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 11:50:16 -0700 Subject: [PATCH 099/125] fix a race in the payment channel status test We should not be clobbering the cmd variable. --- cli/paych_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/paych_test.go b/cli/paych_test.go index 2aa5c6009..862ca2e74 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -121,13 +121,13 @@ func TestPaymentChannelStatus(t *testing.T) { create := make(chan string) go func() { // creator: paych add-funds - cmd = []string{creatorAddr.String(), receiverAddr.String(), fmt.Sprintf("%d", channelAmt)} + cmd := []string{creatorAddr.String(), receiverAddr.String(), fmt.Sprintf("%d", channelAmt)} create <- creatorCLI.runCmd(paychAddFundsCmd, cmd) }() // Wait for the output to stop being "Channel does not exist" for regexp.MustCompile(noChannelState).MatchString(out) { - cmd = []string{creatorAddr.String(), receiverAddr.String()} + cmd := []string{creatorAddr.String(), receiverAddr.String()} out = creatorCLI.runCmd(paychStatusByFromToCmd, cmd) } fmt.Println(out) From d80e7eda932a2a28ec6c5cd6fdb0acdc1c22c966 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 9 Oct 2020 13:18:43 -0700 Subject: [PATCH 100/125] Revert cached "available" amount if the AddFunds message send fails --- chain/market/fundmgr.go | 6 ++++-- chain/market/fundmgr_test.go | 38 +++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go index aef3b98eb..8b5f85a12 100644 --- a/chain/market/fundmgr.go +++ b/chain/market/fundmgr.go @@ -120,9 +120,10 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add return cid.Undef, err } fm.lk.Lock() + defer fm.lk.Unlock() + bal, err := fm.api.StateMarketBalance(ctx, addr, types.EmptyTSK) if err != nil { - fm.lk.Unlock() return cid.Undef, err } @@ -138,7 +139,6 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add toAdd = types.NewInt(0) } fm.available[idAddr] = big.Add(avail, toAdd) - fm.lk.Unlock() log.Infof("Funds operation w/ Expected Balance: %s, In State: %s, Requested: %s, Adding: %s", avail.String(), stateAvail.String(), amt.String(), toAdd.String()) @@ -148,6 +148,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add params, err := actors.SerializeParams(&addr) if err != nil { + fm.available[idAddr] = avail return cid.Undef, err } @@ -159,6 +160,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add Params: params, }, nil) if err != nil { + fm.available[idAddr] = avail return cid.Undef, err } diff --git a/chain/market/fundmgr_test.go b/chain/market/fundmgr_test.go index b05db55d8..f5936f73d 100644 --- a/chain/market/fundmgr_test.go +++ b/chain/market/fundmgr_test.go @@ -57,9 +57,10 @@ func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Add } type expectedResult struct { - addAmt abi.TokenAmount - shouldAdd bool - err error + addAmt abi.TokenAmount + shouldAdd bool + err error + cachedAvailable abi.TokenAmount } func TestAddFunds(t *testing.T) { @@ -88,8 +89,9 @@ func TestAddFunds(t *testing.T) { addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)}, expectedResults: []expectedResult{ { - shouldAdd: false, - err: nil, + shouldAdd: false, + err: nil, + cachedAvailable: abi.NewTokenAmount(100), }, }, }, @@ -102,18 +104,21 @@ func TestAddFunds(t *testing.T) { err: nil, }, { - addAmt: abi.NewTokenAmount(100), - shouldAdd: true, - err: nil, + addAmt: abi.NewTokenAmount(100), + shouldAdd: true, + err: nil, + cachedAvailable: abi.NewTokenAmount(200), }, { - addAmt: abi.NewTokenAmount(50), - shouldAdd: true, - err: nil, + addAmt: abi.NewTokenAmount(50), + shouldAdd: true, + err: nil, + cachedAvailable: abi.NewTokenAmount(250), }, { - shouldAdd: false, - err: nil, + shouldAdd: false, + err: nil, + cachedAvailable: abi.NewTokenAmount(250), }, }, }, @@ -132,7 +137,8 @@ func TestAddFunds(t *testing.T) { addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)}, expectedResults: []expectedResult{ { - err: errors.New("something went wrong"), + err: errors.New("something went wrong"), + cachedAvailable: abi.NewTokenAmount(0), }, }, }, @@ -183,6 +189,10 @@ func TestAddFunds(t *testing.T) { } else { require.EqualError(t, err, expected.err.Error()) } + + if !expected.cachedAvailable.Nil() { + require.Equal(t, expected.cachedAvailable, fundMgr.available[addr]) + } } }) } From 748d2e82a766e4db46e1d6aabc4f6778ddc063e7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 12:52:04 -0700 Subject: [PATCH 101/125] unshare the journal Motivation: * Run lotus with the race detector enabled (primary motivation). * Allow multiple lotus nodes in a process (not a high priority). Previously, the journal was shared between all lotus instances, but it was initialized for every new node. This caused safety problems in tests (at a minimum). This patch explicitly passes the journal to all services that need it. --- chain/gen/gen.go | 6 ++++-- chain/gen/genesis/genesis.go | 8 ++++++-- chain/messagepool/messagepool.go | 18 ++++++++++++------ chain/messagepool/messagepool_test.go | 18 +++++++++--------- chain/messagepool/repub.go | 3 +-- chain/messagepool/repub_test.go | 2 +- chain/messagepool/selection_test.go | 2 +- chain/store/index_test.go | 2 +- chain/store/store.go | 11 ++++++++--- chain/store/store_test.go | 4 ++-- cmd/lotus-bench/import.go | 2 +- cmd/lotus-shed/balances.go | 4 ++-- cmd/lotus-shed/export.go | 2 +- cmd/lotus-shed/genesis-verify.go | 2 +- cmd/lotus-shed/pruning.go | 2 +- cmd/lotus-storage-miner/init.go | 7 +++---- cmd/lotus/daemon.go | 7 ++++++- conformance/driver.go | 2 +- journal/env.go | 19 +++++++++++++++++++ journal/global.go | 9 --------- markets/journal.go | 16 ++++++++-------- miner/miner.go | 8 +++++--- miner/testminer.go | 2 ++ node/builder.go | 18 +----------------- node/modules/chain.go | 9 +++++---- node/modules/client.go | 12 ++++++------ node/modules/storageminer.go | 22 ++++++++++++---------- node/modules/testing/genesis.go | 13 +++++++------ storage/miner.go | 9 ++++++--- storage/wdpost_run.go | 13 ++++++------- storage/wdpost_run_test.go | 2 ++ storage/wdpost_sched.go | 14 ++++++++------ 32 files changed, 148 insertions(+), 120 deletions(-) create mode 100644 journal/env.go delete mode 100644 journal/global.go diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 3ebd127f3..a1d9a3f7e 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -39,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/repo" @@ -122,6 +123,7 @@ var DefaultRemainderAccountActor = genesis.Actor{ } func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { + j := journal.NilJournal() // TODO: we really shouldn't modify a global variable here. policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) @@ -229,12 +231,12 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { Timestamp: uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()), } - genb, err := genesis2.MakeGenesisBlock(context.TODO(), bs, sys, tpl) + genb, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, sys, tpl) if err != nil { return nil, xerrors.Errorf("make genesis block failed: %w", err) } - cs := store.NewChainStore(bs, ds, sys) + cs := store.NewChainStore(bs, ds, sys, j) genfb := &types.FullBlock{Header: genb.Genesis} gents := store.NewFullTipSet([]*types.FullBlock{genfb}) diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index 9f15ecaed..6a1090784 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/journal" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -466,7 +467,10 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci return st, nil } -func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallBuilder, template genesis.Template) (*GenesisBootstrap, error) { +func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blockstore, sys vm.SyscallBuilder, template genesis.Template) (*GenesisBootstrap, error) { + if j == nil { + j = journal.NilJournal() + } st, keyIDs, err := MakeInitialStateTree(ctx, bs, template) if err != nil { return nil, xerrors.Errorf("make initial state tree failed: %w", err) @@ -478,7 +482,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB } // temp chainstore - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys, j) // Verify PreSealed Data stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 83aa5c6b7..d3c638b22 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -159,6 +159,7 @@ type MessagePool struct { sigValCache *lru.TwoQueueCache evtTypes [3]journal.EventType + journal journal.Journal } type msgSet struct { @@ -316,7 +317,7 @@ func (ms *msgSet) getRequiredFunds(nonce uint64) types.BigInt { return types.BigInt{Int: requiredFunds} } -func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*MessagePool, error) { +func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) { cache, _ := lru.New2Q(build.BlsSignatureCacheSize) verifcache, _ := lru.New2Q(build.VerifSigCacheSize) @@ -325,6 +326,10 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa return nil, xerrors.Errorf("error loading mpool config: %w", err) } + if j == nil { + j = journal.NilJournal() + } + mp := &MessagePool{ ds: ds, addSema: make(chan struct{}, 1), @@ -344,10 +349,11 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa netName: netName, cfg: cfg, evtTypes: [...]journal.EventType{ - evtTypeMpoolAdd: journal.J.RegisterEventType("mpool", "add"), - evtTypeMpoolRemove: journal.J.RegisterEventType("mpool", "remove"), - evtTypeMpoolRepub: journal.J.RegisterEventType("mpool", "repub"), + evtTypeMpoolAdd: j.RegisterEventType("mpool", "add"), + evtTypeMpoolRemove: j.RegisterEventType("mpool", "remove"), + evtTypeMpoolRepub: j.RegisterEventType("mpool", "repub"), }, + journal: j, } // enable initial prunes @@ -744,7 +750,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage, strict, untrusted bool) Message: m, }, localUpdates) - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolAdd], func() interface{} { + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolAdd], func() interface{} { return MessagePoolEvt{ Action: "add", Messages: []MessagePoolEvtMessage{{Message: m.Message, CID: m.Cid()}}, @@ -865,7 +871,7 @@ func (mp *MessagePool) remove(from address.Address, nonce uint64, applied bool) Message: m, }, localUpdates) - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRemove], func() interface{} { + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolRemove], func() interface{} { return MessagePoolEvt{ Action: "remove", Messages: []MessagePoolEvtMessage{{Message: m.Message, CID: m.Cid()}}} diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index a4aa059ca..afb1efde2 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -225,7 +225,7 @@ func TestMessagePool(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -266,7 +266,7 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -315,7 +315,7 @@ func TestRevertMessages(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -378,7 +378,7 @@ func TestPruningSimple(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -422,7 +422,7 @@ func TestLoadLocal(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -465,7 +465,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - mp, err = New(tma, ds, "mptest") + mp, err = New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -494,7 +494,7 @@ func TestClearAll(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -548,7 +548,7 @@ func TestClearNonLocal(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -609,7 +609,7 @@ func TestUpdates(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index 672119ba9..cdd169e1d 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" "github.com/ipfs/go-cid" ) @@ -148,7 +147,7 @@ loop: } if len(msgs) > 0 { - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { msgsEv := make([]MessagePoolEvtMessage, 0, len(msgs)) for _, m := range msgs { msgsEv = append(msgsEv, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) diff --git a/chain/messagepool/repub_test.go b/chain/messagepool/repub_test.go index 491f484f5..5cb99f836 100644 --- a/chain/messagepool/repub_test.go +++ b/chain/messagepool/repub_test.go @@ -21,7 +21,7 @@ func TestRepubMessages(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index ea19dad9c..ca95b9621 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -58,7 +58,7 @@ func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, g func makeTestMpool() (*MessagePool, *testMpoolAPI) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "test") + mp, err := New(tma, ds, "test", nil) if err != nil { panic(err) } diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 63e08070c..5283d10dc 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -31,7 +31,7 @@ func TestIndexSeeks(t *testing.T) { ctx := context.TODO() nbs := blockstore.NewTemporarySync() - cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) + cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil) _, err = cs.Import(bytes.NewReader(gencar)) if err != nil { diff --git a/chain/store/store.go b/chain/store/store.go index aac28e5d3..39f3d5c48 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -124,11 +124,15 @@ type ChainStore struct { vmcalls vm.SyscallBuilder evtTypes [1]journal.EventType + journal journal.Journal } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder) *ChainStore { +func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore { c, _ := lru.NewARC(DefaultMsgMetaCacheSize) tsc, _ := lru.NewARC(DefaultTipSetCacheSize) + if j == nil { + j = journal.NilJournal() + } cs := &ChainStore{ bs: bs, ds: ds, @@ -137,10 +141,11 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallB mmCache: c, tsCache: tsc, vmcalls: vmcalls, + journal: j, } cs.evtTypes = [1]journal.EventType{ - evtTypeHeadChange: journal.J.RegisterEventType("sync", "head_change"), + evtTypeHeadChange: j.RegisterEventType("sync", "head_change"), } ci := NewChainIndex(cs.LoadTipSet) @@ -379,7 +384,7 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo continue } - journal.J.RecordEvent(cs.evtTypes[evtTypeHeadChange], func() interface{} { + cs.journal.RecordEvent(cs.evtTypes[evtTypeHeadChange], func() interface{} { return HeadChangeEvt{ From: r.old.Key(), FromHeight: r.old.Height(), diff --git a/chain/store/store_test.go b/chain/store/store_test.go index b7adfb595..326899f90 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -62,7 +62,7 @@ func BenchmarkGetRandomness(b *testing.B) { bs := blockstore.NewBlockstore(bds) - cs := store.NewChainStore(bs, mds, nil) + cs := store.NewChainStore(bs, mds, nil, nil) b.ResetTimer() @@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) { } nbs := blockstore.NewTemporary() - cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil) + cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, nil) root, err := cs.Import(buf) if err != nil { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 3d93b0e5e..3f99d0453 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -183,7 +183,7 @@ var importBenchCmd = &cli.Command{ return nil } - cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) + cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier), nil) stm := stmgr.NewStateManager(cs) if cctx.Bool("global-profile") { diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 26450bdb8..c2099bb2d 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -173,7 +173,7 @@ var chainBalanceStateCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) @@ -343,7 +343,7 @@ var chainPledgeCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index c12cbd82d..3be49f0e0 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -83,7 +83,7 @@ var exportChainCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, nil) + cs := store.NewChainStore(bs, mds, nil, nil) if err := cs.Load(); err != nil { return err } diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index 9a47d6561..4b197c58f 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -52,7 +52,7 @@ var genesisVerifyCmd = &cli.Command{ } bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil, nil) cf := cctx.Args().Get(0) f, err := os.Open(cf) diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index 79158c3a3..6cf4f8c6f 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -162,7 +162,7 @@ var stateTreePruneCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) if err := cs.Load(); err != nil { return fmt.Errorf("loading chainstore: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 9218faa77..0f830023f 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -464,13 +464,12 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - if jrnl, err := journal.OpenFSJournal(lr, journal.DefaultDisabledEvents); err == nil { - journal.J = jrnl - } else { + j, err := journal.OpenFSJournal(lr, journal.EnvDisabledEvents()) + if err != nil { return fmt.Errorf("failed to open filesystem journal: %w", err) } - m := miner.NewMiner(api, epp, a, slashfilter.New(mds)) + m := miner.NewMiner(api, epp, a, slashfilter.New(mds), j) { if err := m.Start(ctx); err != nil { return xerrors.Errorf("failed to start up genesis miner: %w", err) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 5964a2458..a1f301572 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -34,6 +34,7 @@ import ( "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/peermgr" "github.com/filecoin-project/lotus/lib/ulimit" @@ -410,7 +411,11 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { bs := blockstore.NewBlockstore(ds) - cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + j, err := journal.OpenFSJournal(lr, journal.EnvDisabledEvents()) + if err != nil { + return xerrors.Errorf("failed to open journal: %w", err) + } + cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), j) log.Infof("importing chain from %s...", fname) diff --git a/conformance/driver.go b/conformance/driver.go index 940c137cf..f49022b9c 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -87,7 +87,7 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) vmRand = NewFixedRand() - cs = store.NewChainStore(bs, ds, syscalls) + cs = store.NewChainStore(bs, ds, syscalls, nil) sm = stmgr.NewStateManager(cs) ) diff --git a/journal/env.go b/journal/env.go new file mode 100644 index 000000000..68380844e --- /dev/null +++ b/journal/env.go @@ -0,0 +1,19 @@ +package journal + +import ( + "os" +) + +// envJournalDisabledEvents is the environment variable through which disabled +// journal events can be customized. +const envDisabledEvents = "LOTUS_JOURNAL_DISABLED_EVENTS" + +func EnvDisabledEvents() DisabledEvents { + if env, ok := os.LookupEnv(envDisabledEvents); ok { + if ret, err := ParseDisabledEvents(env); err == nil { + return ret + } + } + // fallback if env variable is not set, or if it failed to parse. + return DefaultDisabledEvents +} diff --git a/journal/global.go b/journal/global.go deleted file mode 100644 index b4d0e0a1b..000000000 --- a/journal/global.go +++ /dev/null @@ -1,9 +0,0 @@ -package journal - -var ( - // J is a globally accessible Journal. It starts being NilJournal, and early - // during the Lotus initialization routine, it is reset to whichever Journal - // is configured (by default, the filesystem journal). Components can safely - // record in the journal by calling: journal.J.RecordEvent(...). - J Journal = NilJournal() // nolint -) diff --git a/markets/journal.go b/markets/journal.go index a73d28011..9c9c5be9c 100644 --- a/markets/journal.go +++ b/markets/journal.go @@ -28,9 +28,9 @@ type RetrievalProviderEvt struct { } // StorageClientJournaler records journal events from the storage client. -func StorageClientJournaler(evtType journal.EventType) func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { +func StorageClientJournaler(j journal.Journal, evtType journal.EventType) func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { return func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return StorageClientEvt{ Event: storagemarket.ClientEvents[event], Deal: deal, @@ -40,9 +40,9 @@ func StorageClientJournaler(evtType journal.EventType) func(event storagemarket. } // StorageProviderJournaler records journal events from the storage provider. -func StorageProviderJournaler(evtType journal.EventType) func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { +func StorageProviderJournaler(j journal.Journal, evtType journal.EventType) func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { return func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return StorageProviderEvt{ Event: storagemarket.ProviderEvents[event], Deal: deal, @@ -52,9 +52,9 @@ func StorageProviderJournaler(evtType journal.EventType) func(event storagemarke } // RetrievalClientJournaler records journal events from the retrieval client. -func RetrievalClientJournaler(evtType journal.EventType) func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { +func RetrievalClientJournaler(j journal.Journal, evtType journal.EventType) func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { return func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return RetrievalClientEvt{ Event: retrievalmarket.ClientEvents[event], Deal: deal, @@ -64,9 +64,9 @@ func RetrievalClientJournaler(evtType journal.EventType) func(event retrievalmar } // RetrievalProviderJournaler records journal events from the retrieval provider. -func RetrievalProviderJournaler(evtType journal.EventType) func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { +func RetrievalProviderJournaler(j journal.Journal, evtType journal.EventType) func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { return func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return RetrievalProviderEvt{ Event: retrievalmarket.ProviderEvents[event], Deal: deal, diff --git a/miner/miner.go b/miner/miner.go index aebcb7a25..73985a649 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -49,7 +49,7 @@ func randTimeOffset(width time.Duration) time.Duration { return val - (width / 2) } -func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter) *Miner { +func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter, j journal.Journal) *Miner { arc, err := lru.NewARC(10000) if err != nil { panic(err) @@ -74,8 +74,9 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf: sf, minedBlockHeights: arc, evtTypes: [...]journal.EventType{ - evtTypeBlockMined: journal.J.RegisterEventType("miner", "block_mined"), + evtTypeBlockMined: j.RegisterEventType("miner", "block_mined"), }, + journal: j, } } @@ -97,6 +98,7 @@ type Miner struct { minedBlockHeights *lru.ARCCache evtTypes [1]journal.EventType + journal journal.Journal } func (m *Miner) Address() address.Address { @@ -239,7 +241,7 @@ minerLoop: onDone(b != nil, h, nil) if b != nil { - journal.J.RecordEvent(m.evtTypes[evtTypeBlockMined], func() interface{} { + m.journal.RecordEvent(m.evtTypes[evtTypeBlockMined], func() interface{} { return map[string]interface{}{ "parents": base.TipSet.Cids(), "nulls": base.NullRounds, diff --git a/miner/testminer.go b/miner/testminer.go index 64e3b3a62..5f461d884 100644 --- a/miner/testminer.go +++ b/miner/testminer.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" + "github.com/filecoin-project/lotus/journal" ) type MineReq struct { @@ -32,6 +33,7 @@ func NewTestMiner(nextCh <-chan MineReq, addr address.Address) func(api.FullNode minedBlockHeights: arc, address: addr, sf: slashfilter.New(ds.NewMapDatastore()), + journal: journal.NilJournal(), } if err := m.Start(context.TODO()); err != nil { diff --git a/node/builder.go b/node/builder.go index 71295dbc8..a2e3c981e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,7 +3,6 @@ package node import ( "context" "errors" - "os" "time" "github.com/filecoin-project/lotus/chain" @@ -73,10 +72,6 @@ import ( "github.com/filecoin-project/lotus/storage/sectorblocks" ) -// EnvJournalDisabledEvents is the environment variable through which disabled -// journal events can be customized. -const EnvJournalDisabledEvents = "LOTUS_JOURNAL_DISABLED_EVENTS" - //nolint:deadcode,varcheck var log = logging.Logger("builder") @@ -166,19 +161,8 @@ type Settings struct { func defaults() []Option { return []Option{ // global system journal. - Override(new(journal.DisabledEvents), func() journal.DisabledEvents { - if env, ok := os.LookupEnv(EnvJournalDisabledEvents); ok { - if ret, err := journal.ParseDisabledEvents(env); err == nil { - return ret - } - } - // fallback if env variable is not set, or if it failed to parse. - return journal.DefaultDisabledEvents - }), + Override(new(journal.DisabledEvents), journal.EnvDisabledEvents), Override(new(journal.Journal), modules.OpenFilesystemJournal), - Override(InitJournalKey, func(j journal.Journal) { - journal.J = j // eagerly sets the global journal through fx.Invoke. - }), Override(new(helpers.MetricsCtx), context.Background), Override(new(record.Validator), modules.RecordValidator), diff --git a/node/modules/chain.go b/node/modules/chain.go index 66f54a76a..f563b4cdd 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -17,6 +17,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" @@ -51,9 +52,9 @@ func ChainBitswap(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt r return exch } -func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName) (*messagepool.MessagePool, error) { +func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName, j journal.Journal) (*messagepool.MessagePool, error) { mpp := messagepool.NewProvider(sm, ps) - mp, err := messagepool.New(mpp, ds, nn) + mp, err := messagepool.New(mpp, ds, nn, j) if err != nil { return nil, xerrors.Errorf("constructing mpool: %w", err) } @@ -88,8 +89,8 @@ func ChainBlockService(bs dtypes.ChainBlockstore, rem dtypes.ChainBitswap) dtype return blockservice.New(bs, rem) } -func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder) *store.ChainStore { - chain := store.NewChainStore(bs, ds, syscalls) +func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore { + chain := store.NewChainStore(bs, ds, syscalls, j) if err := chain.Load(); err != nil { log.Warnf("loading chain state from disk: %s", err) diff --git a/node/modules/client.go b/node/modules/client.go index 6972ca36e..d012e4539 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -113,7 +113,7 @@ func NewClientDealFunds(ds dtypes.MetadataDS) (ClientDealFunds, error) { return funds.NewDealFunds(ds, datastore.NewKey("/marketfunds/client")) } -func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, dealFunds ClientDealFunds) (storagemarket.StorageClient, error) { +func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, dealFunds ClientDealFunds, j journal.Journal) (storagemarket.StorageClient, error) { net := smnet.NewFromLibp2pHost(h) c, err := storageimpl.NewClient(net, ibs, mds, dataTransfer, discovery, deals, scn, dealFunds, storageimpl.DealPollingInterval(time.Second)) if err != nil { @@ -124,8 +124,8 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md OnStart: func(ctx context.Context) error { c.SubscribeToEvents(marketevents.StorageClientLogger) - evtType := journal.J.RegisterEventType("markets/storage/client", "state_change") - c.SubscribeToEvents(markets.StorageClientJournaler(evtType)) + evtType := j.RegisterEventType("markets/storage/client", "state_change") + c.SubscribeToEvents(markets.StorageClientJournaler(j, evtType)) return c.Start(ctx) }, @@ -137,7 +137,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md } // RetrievalClient creates a new retrieval client attached to the client blockstore -func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver, ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI) (retrievalmarket.RetrievalClient, error) { +func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver, ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, j journal.Journal) (retrievalmarket.RetrievalClient, error) { adapter := retrievaladapter.NewRetrievalClientNode(payAPI, chainAPI, stateAPI) network := rmnet.NewFromLibp2pHost(h) sc := storedcounter.New(ds, datastore.NewKey("/retr")) @@ -150,8 +150,8 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, OnStart: func(ctx context.Context) error { client.SubscribeToEvents(marketevents.RetrievalClientLogger) - evtType := journal.J.RegisterEventType("markets/retrieval/client", "state_change") - client.SubscribeToEvents(markets.RetrievalClientJournaler(evtType)) + evtType := j.RegisterEventType("markets/retrieval/client", "state_change") + client.SubscribeToEvents(markets.RetrievalClientJournaler(j, evtType)) return client.Start(ctx) }, diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 95bfb6c11..9011c4821 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -161,6 +161,7 @@ type StorageMinerParams struct { SectorIDCounter sealing.SectorIDCounter Verifier ffiwrapper.Verifier GetSealingConfigFn dtypes.GetSealingConfigFunc + Journal journal.Journal } func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*storage.Miner, error) { @@ -175,6 +176,7 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st sc = params.SectorIDCounter verif = params.Verifier gsd = params.GetSealingConfigFn + j = params.Journal ) maddr, err := minerAddrFromDS(ds) @@ -194,12 +196,12 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st return nil, err } - fps, err := storage.NewWindowedPoStScheduler(api, fc, sealer, sealer, maddr, worker) + fps, err := storage.NewWindowedPoStScheduler(api, fc, sealer, sealer, j, maddr, worker) if err != nil { return nil, err } - sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd, fc) + sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd, fc, j) if err != nil { return nil, err } @@ -216,15 +218,15 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st } } -func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider) { +func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider, j journal.Journal) { m.OnReady(marketevents.ReadyLogger("retrieval provider")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { m.SubscribeToEvents(marketevents.RetrievalProviderLogger) - evtType := journal.J.RegisterEventType("markets/retrieval/provider", "state_change") - m.SubscribeToEvents(markets.RetrievalProviderJournaler(evtType)) + evtType := j.RegisterEventType("markets/retrieval/provider", "state_change") + m.SubscribeToEvents(markets.RetrievalProviderJournaler(j, evtType)) return m.Start(ctx) }, @@ -234,15 +236,15 @@ func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.Retrieva }) } -func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider) { +func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider, j journal.Journal) { ctx := helpers.LifecycleCtx(mctx, lc) h.OnReady(marketevents.ReadyLogger("storage provider")) lc.Append(fx.Hook{ OnStart: func(context.Context) error { h.SubscribeToEvents(marketevents.StorageProviderLogger) - evtType := journal.J.RegisterEventType("markets/storage/provider", "state_change") - h.SubscribeToEvents(markets.StorageProviderJournaler(evtType)) + evtType := j.RegisterEventType("markets/storage/provider", "state_change") + h.SubscribeToEvents(markets.StorageProviderJournaler(j, evtType)) return h.Start(ctx) }, @@ -354,13 +356,13 @@ func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Stagi return gs } -func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter) (*miner.Miner, error) { +func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter, j journal.Journal) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - m := miner.NewMiner(api, epp, minerAddr, sf) + m := miner.NewMiner(api, epp, minerAddr, sf, j) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index 40df68bf3..fa9e0cff7 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -23,17 +23,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" ) var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { +func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") - b, err := genesis2.MakeGenesisBlock(context.TODO(), bs, syscalls, template) + b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template) if err != nil { return nil, xerrors.Errorf("make genesis block failed: %w", err) } @@ -50,8 +51,8 @@ func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.Cha } } -func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { +func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") genesisTemplate, err := homedir.Expand(genesisTemplate) @@ -73,7 +74,7 @@ func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore template.Timestamp = uint64(build.Clock.Now().Unix()) } - b, err := genesis2.MakeGenesisBlock(context.TODO(), bs, syscalls, template) + b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template) if err != nil { return nil, xerrors.Errorf("make genesis block: %w", err) } diff --git a/storage/miner.go b/storage/miner.go index c1b50fe89..74a048c8e 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -57,6 +57,8 @@ type Miner struct { sealing *sealing.Sealing sealingEvtType journal.EventType + + journal journal.Journal } // SealingStateEvt is a journal event that records a sector state transition. @@ -110,7 +112,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingConfigFunc, feeCfg config.MinerFeeConfig) (*Miner, error) { +func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingConfigFunc, feeCfg config.MinerFeeConfig, journal journal.Journal) (*Miner, error) { m := &Miner{ api: api, feeCfg: feeCfg, @@ -123,7 +125,8 @@ func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, d maddr: maddr, worker: worker, getSealConfig: gsd, - sealingEvtType: journal.J.RegisterEventType("storage", "sealing_states"), + journal: journal, + sealingEvtType: journal.RegisterEventType("storage", "sealing_states"), } return m, nil @@ -156,7 +159,7 @@ func (m *Miner) Run(ctx context.Context) error { } func (m *Miner) handleSealingNotifications(before, after sealing.SectorInfo) { - journal.J.RecordEvent(m.sealingEvtType, func() interface{} { + m.journal.RecordEvent(m.sealingEvtType, func() interface{} { return SealingStateEvt{ SectorNumber: before.SectorNumber, SectorType: before.SectorType, diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 6aa3d5188..d4ed4d64c 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -26,11 +26,10 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" ) func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dline.Info) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { c := evtCommon{Error: err} if ts != nil { c.Deadline = deadline @@ -54,7 +53,7 @@ func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dl // recordProofsEvent records a successful proofs_processed event in the // journal, even if it was a noop (no partitions). func (s *WindowPoStScheduler) recordProofsEvent(partitions []miner.PoStPartition, mcid cid.Cid) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { return &WdPoStProofsProcessedEvt{ evtCommon: s.getEvtCommon(nil), Partitions: partitions, @@ -74,7 +73,7 @@ func (s *WindowPoStScheduler) startGeneratePoST( go func() { defer abort() - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), State: SchedulerStateStarted, @@ -125,7 +124,7 @@ func (s *WindowPoStScheduler) startSubmitPoST( err := s.runSubmitPoST(ctx, ts, deadline, posts) if err == nil { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), State: SchedulerStateSucceeded, @@ -439,7 +438,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Errorf("checking sector recoveries: %v", err) } - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStRecoveries], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStRecoveries], func() interface{} { j := WdPoStRecoveriesProcessedEvt{ evtCommon: s.getEvtCommon(err), Declarations: recoveries, @@ -458,7 +457,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Errorf("checking sector faults: %v", err) } - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStFaults], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStFaults], func() interface{} { return WdPoStFaultsProcessedEvt{ evtCommon: s.getEvtCommon(err), Declarations: faults, diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 09b9aee5c..dd7ac4c24 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" ) type mockStorageMinerAPI struct { @@ -168,6 +169,7 @@ func TestWDPostDoPost(t *testing.T) { proofType: proofType, actor: postAct, worker: workerAct, + journal: journal.NilJournal(), } di := &dline.Info{ diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index ee380fbaf..99bea2e66 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -35,12 +35,13 @@ type WindowPoStScheduler struct { worker address.Address evtTypes [4]journal.EventType + journal journal.Journal // failed abi.ChainEpoch // eps // failLk sync.Mutex } -func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb storage.Prover, ft sectorstorage.FaultTracker, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) { +func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb storage.Prover, ft sectorstorage.FaultTracker, j journal.Journal, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) { mi, err := api.StateMinerInfo(context.TODO(), actor, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("getting sector size: %w", err) @@ -62,11 +63,12 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb actor: actor, worker: worker, evtTypes: [...]journal.EventType{ - evtTypeWdPoStScheduler: journal.J.RegisterEventType("wdpost", "scheduler"), - evtTypeWdPoStProofs: journal.J.RegisterEventType("wdpost", "proofs_processed"), - evtTypeWdPoStRecoveries: journal.J.RegisterEventType("wdpost", "recoveries_processed"), - evtTypeWdPoStFaults: journal.J.RegisterEventType("wdpost", "faults_processed"), + evtTypeWdPoStScheduler: j.RegisterEventType("wdpost", "scheduler"), + evtTypeWdPoStProofs: j.RegisterEventType("wdpost", "proofs_processed"), + evtTypeWdPoStRecoveries: j.RegisterEventType("wdpost", "recoveries_processed"), + evtTypeWdPoStFaults: j.RegisterEventType("wdpost", "faults_processed"), }, + journal: j, }, nil } @@ -166,7 +168,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, revert, apply *types.T // onAbort is called when generating proofs or submitting proofs is aborted func (s *WindowPoStScheduler) onAbort(ts *types.TipSet, deadline *dline.Info) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { c := evtCommon{} if ts != nil { c.Deadline = deadline From c0182f8301b9773f3441f25c5dc9b938d4e08aa8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 13:59:56 -0700 Subject: [PATCH 102/125] return the correct err from iterFullTipset --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index c280e3a40..369c65d33 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1526,7 +1526,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS ss.SetStage(api.StageMessages) if batchErr != nil { - return xerrors.Errorf("failed to fetch messages: %w", err) + return xerrors.Errorf("failed to fetch messages: %w", batchErr) } for bsi := 0; bsi < len(bstout); bsi++ { From f773ab4b2824b2549f7768dc83ae200aa00ddaa2 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 9 Oct 2020 17:14:21 -0400 Subject: [PATCH 103/125] Correct docs --- api/api_full.go | 3 +-- documentation/en/api-methods.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 777e996cb..7ebc8791a 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -422,8 +422,7 @@ type FullNode interface { // MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) - // MsigGetLockedBalance returns the locked balance of an msig at a vien epoch. - // The return may be greater than the multisig actor's actual balance. + // MsigGetVestingSchedule returns the vesting details of a given multisig. MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (MsigVesting, error) // MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. // It takes the following params: , , diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index cc6d92e4f..b3a9627e0 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -2146,8 +2146,7 @@ Inputs: Response: `"0"` ### MsigGetVestingSchedule -MsigGetLockedBalance returns the locked balance of an msig at a vien epoch. -The return may be greater than the multisig actor's actual balance. +MsigGetVestingSchedule returns the vesting details of a given multisig. Perms: read From 283fd054e89ba9119088e85b30bd27f71992ecd9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 15:35:44 -0700 Subject: [PATCH 104/125] fix a race when retrieving pieces We'd read the deal ID without synchronizing. This could (and probably did given the history of flaky deal tests) cause us to miss events. This patch also makes sure to always unsubscribe from events, even on error. --- node/impl/client/client.go | 40 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index f146dcea3..fe83fda7b 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -462,13 +462,19 @@ type retrievalSubscribeEvent struct { state rm.ClientDealState } -func readSubscribeEvents(ctx context.Context, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { +func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { for { var subscribeEvent retrievalSubscribeEvent select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") case subscribeEvent = <-subscribeEvents: + if subscribeEvent.state.ID != dealID { + // we can't check the deal ID ahead of time because: + // 1. We need to subscribe before retrieving. + // 2. We won't know the deal ID until after retrieving. + continue + } } select { @@ -531,19 +537,6 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return err }*/ - var dealID retrievalmarket.DealID - subscribeEvents := make(chan retrievalSubscribeEvent, 1) - subscribeCtx, cancel := context.WithCancel(ctx) - defer cancel() - unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { - if state.PayloadCID.Equals(order.Root) && state.ID == dealID { - select { - case <-subscribeCtx.Done(): - case subscribeEvents <- retrievalSubscribeEvent{event, state}: - } - } - }) - ppb := types.BigDiv(order.Total, types.NewInt(order.Size)) params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, shared.AllSelector(), order.Piece, order.UnsealPrice) @@ -562,7 +555,21 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref _ = a.RetrievalStoreMgr.ReleaseStore(store) }() - dealID, err = a.Retrieval.Retrieve( + // Subscribe to events before retrieving to avoid losing events. + subscribeEvents := make(chan retrievalSubscribeEvent, 1) + subscribeCtx, cancel := context.WithCancel(ctx) + defer cancel() + unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { + // We'll check the deal IDs inside readSubscribeEvents. + if state.PayloadCID.Equals(order.Root) { + select { + case <-subscribeCtx.Done(): + case subscribeEvents <- retrievalSubscribeEvent{event, state}: + } + } + }) + + dealID, err := a.Retrieval.Retrieve( ctx, order.Root, params, @@ -573,11 +580,12 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref store.StoreID()) if err != nil { + unsubscribe() finish(xerrors.Errorf("Retrieve failed: %w", err)) return } - err = readSubscribeEvents(ctx, subscribeEvents, events) + err = readSubscribeEvents(ctx, dealID, subscribeEvents, events) unsubscribe() if err != nil { From 83dfc460d490721dafa237717c96c8918cf1262e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 15:39:41 -0700 Subject: [PATCH 105/125] fix race in unseal 1. Remove an invalid error check. 2. Make sure to shadow the outer error type from within the goroutine instead or reading the outer type. This may have been causing test issues (caught in TestMinerAllInfo with the race detector). --- extern/sector-storage/ffiwrapper/sealer_cgo.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index d75501838..bae6cafb4 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -290,10 +290,6 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s defer opr.Close() // nolint padwriter := fr32.NewPadWriter(out) - if err != nil { - perr = xerrors.Errorf("creating new padded writer: %w", err) - return - } bsize := uint64(size.Padded()) if bsize > uint64(runtime.NumCPU())*fr32.MTTresh { @@ -302,7 +298,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s bw := bufio.NewWriterSize(padwriter, int(abi.PaddedPieceSize(bsize).Unpadded())) - _, err = io.CopyN(bw, opr, int64(size)) + _, err := io.CopyN(bw, opr, int64(size)) if err != nil { perr = xerrors.Errorf("copying data: %w", err) return From f32b3bf99b78821ff15dc66e92e3e73ee817996c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 17:21:50 -0700 Subject: [PATCH 106/125] fix a race in tipset cache usage This tipset cache is shared between multiple services and is called from multiple places. --- chain/events/tscache.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/chain/events/tscache.go b/chain/events/tscache.go index d47c71480..44699684e 100644 --- a/chain/events/tscache.go +++ b/chain/events/tscache.go @@ -2,6 +2,7 @@ package events import ( "context" + "sync" "github.com/filecoin-project/go-state-types/abi" "golang.org/x/xerrors" @@ -17,6 +18,8 @@ type tsCacheAPI interface { // tipSetCache implements a simple ring-buffer cache to keep track of recent // tipsets type tipSetCache struct { + mu sync.RWMutex + cache []*types.TipSet start int len int @@ -35,6 +38,9 @@ func newTSCache(cap abi.ChainEpoch, storage tsCacheAPI) *tipSetCache { } func (tsc *tipSetCache) add(ts *types.TipSet) error { + tsc.mu.Lock() + defer tsc.mu.Unlock() + if tsc.len > 0 { if tsc.cache[tsc.start].Height() >= ts.Height() { return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.cache[tsc.start].Height()+1, ts.Height()) @@ -65,6 +71,13 @@ func (tsc *tipSetCache) add(ts *types.TipSet) error { } func (tsc *tipSetCache) revert(ts *types.TipSet) error { + tsc.mu.Lock() + defer tsc.mu.Unlock() + + return tsc.revertUnlocked(ts) +} + +func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error { if tsc.len == 0 { return nil // this can happen, and it's fine } @@ -77,7 +90,7 @@ func (tsc *tipSetCache) revert(ts *types.TipSet) error { tsc.start = normalModulo(tsc.start-1, len(tsc.cache)) tsc.len-- - _ = tsc.revert(nil) // revert null block gap + _ = tsc.revertUnlocked(nil) // revert null block gap return nil } @@ -95,7 +108,10 @@ func (tsc *tipSetCache) getNonNull(height abi.ChainEpoch) (*types.TipSet, error) } func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { + tsc.mu.RLock() + if tsc.len == 0 { + tsc.mu.RUnlock() log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height) return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, types.EmptyTSK) } @@ -103,6 +119,7 @@ func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { headH := tsc.cache[tsc.start].Height() if height > headH { + tsc.mu.RUnlock() return nil, xerrors.Errorf("tipSetCache.get: requested tipset not in cache (req: %d, cache head: %d)", height, headH) } @@ -116,15 +133,20 @@ func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { } if height < tail.Height() { + tsc.mu.RUnlock() log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tail.Height()) return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, tail.Key()) } - return tsc.cache[normalModulo(tsc.start-int(headH-height), clen)], nil + ts := tsc.cache[normalModulo(tsc.start-int(headH-height), clen)] + tsc.mu.RUnlock() + return ts, nil } func (tsc *tipSetCache) best() (*types.TipSet, error) { + tsc.mu.RLock() best := tsc.cache[tsc.start] + tsc.mu.RUnlock() if best == nil { return tsc.storage.ChainHead(context.TODO()) } From 6610a247cfebb4a213e92fc2bb85dcd23e87c189 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Fri, 9 Oct 2020 22:41:34 +0200 Subject: [PATCH 107/125] Dump the block validation cache whenever we perform an import This solves a problem with folks resurrecting long-out-of-sync nodes via snapshot imports. The interface switch to Batching is necessary: startup is too long otherwise ( 8 minutes just to clear everything on a relatively old node ) --- chain/store/store.go | 42 +++++++++++++++++++++++++++++++++++++++++- cmd/lotus/daemon.go | 6 +++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 39f3d5c48..22b305afb 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -8,6 +8,7 @@ import ( "io" "os" "strconv" + "strings" "sync" "golang.org/x/sync/errgroup" @@ -38,7 +39,9 @@ import ( lru "github.com/hashicorp/golang-lru" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" dstore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" car "github.com/ipld/go-car" @@ -102,7 +105,7 @@ type HeadChangeEvt struct { // 2. a block => messages references cache. type ChainStore struct { bs bstore.Blockstore - ds dstore.Datastore + ds dstore.Batching heaviestLk sync.Mutex heaviest *types.TipSet @@ -446,6 +449,43 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet) return nil } +// FlushValidationCache removes all results of block validation from the +// chain metadata store. Usually the first step after a new chain import. +func (cs *ChainStore) FlushValidationCache() error { + log.Infof("clearing block validation cache...") + + dsWalk, err := cs.ds.Query(query.Query{KeysOnly: true}) + if err != nil { + return xerrors.Errorf("failed to initialize key listing query: %w", err) + } + + allKeys, err := dsWalk.Rest() + if err != nil { + return xerrors.Errorf("failed to run key listing query: %w", err) + } + + batch, err := cs.ds.Batch() + if err != nil { + return xerrors.Errorf("failed to open a DS batch: %w", err) + } + + delCnt := 0 + for _, k := range allKeys { + if strings.HasPrefix(k.Key, blockValidationCacheKeyPrefix.String()) { + delCnt++ + batch.Delete(datastore.RawKey(k.Key)) + } + } + + if err := batch.Commit(); err != nil { + return xerrors.Errorf("failed to commit the DS batch: %w", err) + } + + log.Infof("%d block validation entries cleared.", delCnt) + + return nil +} + // SetHead sets the chainstores current 'best' head node. // This should only be called if something is broken and needs fixing func (cs *ChainStore) SetHead(ts *types.TipSet) error { diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index a1f301572..4ff63be11 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -436,6 +436,10 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { return xerrors.Errorf("importing chain failed: %w", err) } + if err := cst.FlushValidationCache(); err != nil { + return xerrors.Errorf("flushing validation cache failed: %w", err) + } + gb, err := cst.GetTipsetByHeight(context.TODO(), 0, ts, true) if err != nil { return err @@ -455,7 +459,7 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { } } - log.Info("accepting %s as new head", ts.Cids()) + log.Infof("accepting %s as new head", ts.Cids()) if err := cst.SetHead(ts); err != nil { return err } From 85abca4b16231c7693b7b3ca142113e737c19240 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 10 Oct 2020 04:00:12 +0200 Subject: [PATCH 108/125] Delint --- chain/store/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/store/store.go b/chain/store/store.go index 22b305afb..6fb689b3e 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -473,7 +473,7 @@ func (cs *ChainStore) FlushValidationCache() error { for _, k := range allKeys { if strings.HasPrefix(k.Key, blockValidationCacheKeyPrefix.String()) { delCnt++ - batch.Delete(datastore.RawKey(k.Key)) + batch.Delete(datastore.RawKey(k.Key)) // nolint:errcheck } } From f3abd1ae82d0dbd500b4fa1ccc32e6a6fe54d020 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 20:32:09 -0700 Subject: [PATCH 109/125] make pledge test pass with the race detector --- api/test/window_post.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 7bc56a562..28639cda8 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -3,6 +3,7 @@ package test import ( "context" "fmt" + "sync/atomic" "os" "strings" @@ -31,7 +32,9 @@ func init() { } func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -46,11 +49,11 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect } build.Clock.Sleep(time.Second) - mine := true + mine := int64(1) done := make(chan struct{}) go func() { defer close(done) - for mine { + for atomic.LoadInt64(&mine) != 0 { build.Clock.Sleep(blocktime) if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, abi.ChainEpoch, error) { @@ -62,7 +65,7 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect pledgeSectors(t, ctx, miner, nSectors, 0, nil) - mine = false + atomic.StoreInt64(&mine, 0) <-done } From d84f5dababc6b795e8b118eda7e77d47d1877767 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 21:38:47 -0700 Subject: [PATCH 110/125] remove upgrade invariant checks These checks may be _too_ strict and could cause the upgrade to fail unnecessarily. --- chain/stmgr/forks.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index a61f70b44..fba92ee3f 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -24,7 +24,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/migration/nv3" m2 "github.com/filecoin-project/specs-actors/v2/actors/migration" - states2 "github.com/filecoin-project/specs-actors/v2/actors/states" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -580,19 +579,6 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("upgrading to actors v2: %w", err) } - newStateTree, err := states2.LoadTree(store, newHamtRoot) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to load new state tree: %w", err) - } - - // Check all state-tree invariants. - if msgs, err := states2.CheckStateInvariants(newStateTree, types.TotalFilecoinInt); err != nil { - return cid.Undef, xerrors.Errorf("failed to check new state tree: %w", err) - } else if !msgs.IsEmpty() { - // This error is going to be really nasty. - return cid.Undef, xerrors.Errorf("network upgrade failed: %v", msgs.Messages()) - } - newRoot, err := store.Put(ctx, &types.StateRoot{ Version: types.StateTreeVersion1, Actors: newHamtRoot, From 10d464a1ebb90fe612164645885704b17ef54b47 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 21:39:55 -0700 Subject: [PATCH 111/125] upgrade to specs-actors v2.0.3 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e1bf6cb3b..eb79a496c 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.0.1 + github.com/filecoin-project/specs-actors/v2 v2.0.3 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index ef954e66d..e13359f6c 100644 --- a/go.sum +++ b/go.sum @@ -276,8 +276,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.1 h1:bf08x6tqCDfClzrv2q/rmt/A/UbBOy1KgaoogQEcLhU= -github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= +github.com/filecoin-project/specs-actors/v2 v2.0.3 h1:Niy6xncgi8bI8aBCt1McdZfATBfG4Uxytt8KW4s3bAc= +github.com/filecoin-project/specs-actors/v2 v2.0.3/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= From f417d39a0831568d7c9fd0d84353009ab82b0c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 08:55:33 +0200 Subject: [PATCH 112/125] Bump the version to 0.9.1 --- CHANGELOG.md | 20 ++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f6f0b3a0..d397762a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Lotus changelog +# 0.9.1 / 2020-10-10 + +This release fixes an issue which may cause the actors v2 migration to compute the state incorrectly when more than one migration is running in parallel. + +## Changes + +- Make concurrent actor migrations safe (https://github.com/filecoin-project/lotus/pull/4293) +- Remote wallet backends (https://github.com/filecoin-project/lotus/pull/3583) +- Track funds in FundMgr correctly in case of AddFunds failing (https://github.com/filecoin-project/lotus/pull/4273) +- Partial lite-node mode (https://github.com/filecoin-project/lotus/pull/4095) +- Fix potential infinite loop in GetBestMiningCandidate (https://github.com/filecoin-project/lotus/pull/3444) +- sync wait: Handle processed message offset (https://github.com/filecoin-project/lotus/pull/4253) +- Add some new endpoints for querying Msig info (https://github.com/filecoin-project/lotus/pull/4250) +- Update markets v0.7.1 (https://github.com/filecoin-project/lotus/pull/4254) +- Optimize SearchForMessage and GetReceipt (https://github.com/filecoin-project/lotus/pull/4246) +- Use FIL instead of attoFIL in CLI more consistently (https://github.com/filecoin-project/lotus/pull/4249) +- fix: clash between daemon --api flag and cli tests (https://github.com/filecoin-project/lotus/pull/4241) +- add more info to chain sync lookback failure (https://github.com/filecoin-project/lotus/pull/4245) +- Add message counts to inspect chain output (https://github.com/filecoin-project/lotus/pull/4230) + # 0.9.0 / 2020-10-07 This consensus-breaking release of Lotus upgrades the actors version to v2.0.0. This requires migrating actor state from v0 to v2. The changes that break consensus are: diff --git a/build/version.go b/build/version.go index 1602551e4..5baed1fb7 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.9.0" +const BuildVersion = "0.9.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 18e58467f8cbfb9e12d74524c03ae4d139800ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 10:26:42 +0200 Subject: [PATCH 113/125] sync unmark-bad --all --- api/api_full.go | 3 +++ api/apistruct/struct.go | 5 +++++ chain/badtscache.go | 4 ++++ chain/sync.go | 4 ++++ cli/sync.go | 10 ++++++++++ node/impl/full/sync.go | 6 ++++++ 6 files changed, 32 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index 7ebc8791a..1311dac6a 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -172,6 +172,9 @@ type FullNode interface { // SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error + // SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad + SyncUnmarkAllBad(ctx context.Context) error + // SyncCheckBad checks if a block was marked as bad, and if it was, returns // the reason. SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 348e8a6de..6bfbf43cb 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -111,6 +111,7 @@ type FullNodeStruct struct { SyncCheckpoint func(ctx context.Context, key types.TipSetKey) error `perm:"admin"` SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncUnmarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` + SyncUnmarkAllBad func(ctx context.Context) error `perm:"admin"` SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"` SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"` @@ -780,6 +781,10 @@ func (c *FullNodeStruct) SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error return c.Internal.SyncUnmarkBad(ctx, bcid) } +func (c *FullNodeStruct) SyncUnmarkAllBad(ctx context.Context) error { + return c.Internal.SyncUnmarkAllBad(ctx) +} + func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) { return c.Internal.SyncCheckBad(ctx, bcid) } diff --git a/chain/badtscache.go b/chain/badtscache.go index 3c5bf05ef..42564128e 100644 --- a/chain/badtscache.go +++ b/chain/badtscache.go @@ -60,6 +60,10 @@ func (bts *BadBlockCache) Remove(c cid.Cid) { bts.badBlocks.Remove(c) } +func (bts *BadBlockCache) Purge() { + bts.badBlocks.Purge() +} + func (bts *BadBlockCache) Has(c cid.Cid) (BadBlockReason, bool) { rval, ok := bts.badBlocks.Get(c) if !ok { diff --git a/chain/sync.go b/chain/sync.go index 369c65d33..23688ecb7 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1740,6 +1740,10 @@ func (syncer *Syncer) UnmarkBad(blk cid.Cid) { syncer.bad.Remove(blk) } +func (syncer *Syncer) UnmarkAllBad() { + syncer.bad.Purge() +} + func (syncer *Syncer) CheckBadBlockCache(blk cid.Cid) (string, bool) { bbr, ok := syncer.bad.Has(blk) return bbr.String(), ok diff --git a/cli/sync.go b/cli/sync.go index dea96d14e..a5178441d 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -124,6 +124,12 @@ var syncMarkBadCmd = &cli.Command{ var syncUnmarkBadCmd = &cli.Command{ Name: "unmark-bad", Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "all", + Usage: "drop the entire bad block cache", + }, + }, ArgsUsage: "[blockCid]", Action: func(cctx *cli.Context) error { napi, closer, err := GetFullNodeAPI(cctx) @@ -133,6 +139,10 @@ var syncUnmarkBadCmd = &cli.Command{ defer closer() ctx := ReqContext(cctx) + if cctx.Bool("all") { + return napi.SyncUnmarkAllBad(ctx) + } + if !cctx.Args().Present() { return fmt.Errorf("must specify block cid to unmark") } diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index 1bd3af415..05d4c9cb7 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -118,6 +118,12 @@ func (a *SyncAPI) SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error { return nil } +func (a *SyncAPI) SyncUnmarkAllBad(ctx context.Context) error { + log.Warnf("Dropping bad block cache") + a.Syncer.UnmarkAllBad() + return nil +} + func (a *SyncAPI) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) { reason, ok := a.Syncer.CheckBadBlockCache(bcid) if !ok { From 388c9afce0f6a43be99454bb6229eadbddb1fa23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 12:28:59 +0200 Subject: [PATCH 114/125] docsgen --- documentation/en/api-methods.md | 59 +++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 8a288a4bf..e921d3e45 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -83,12 +83,14 @@ * [MsigAddCancel](#MsigAddCancel) * [MsigAddPropose](#MsigAddPropose) * [MsigApprove](#MsigApprove) + * [MsigApproveTxnHash](#MsigApproveTxnHash) * [MsigCancel](#MsigCancel) * [MsigCreate](#MsigCreate) * [MsigGetAvailableBalance](#MsigGetAvailableBalance) * [MsigGetVested](#MsigGetVested) * [MsigGetVestingSchedule](#MsigGetVestingSchedule) * [MsigPropose](#MsigPropose) + * [MsigRemoveSigner](#MsigRemoveSigner) * [MsigSwapApprove](#MsigSwapApprove) * [MsigSwapCancel](#MsigSwapCancel) * [MsigSwapPropose](#MsigSwapPropose) @@ -2006,7 +2008,33 @@ Response: ``` ### MsigApprove -MsigApprove approves a previously-proposed multisig message +MsigApprove approves a previously-proposed multisig message by transaction ID +It takes the following params: , + + +Perms: sign + +Inputs: +```json +[ + "f01234", + 42, + "f01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigApproveTxnHash +MsigApproveTxnHash approves a previously-proposed multisig message, specified +using both transaction ID and a hash of the parameters used in the +proposal. This method of approval can be used to ensure you only approve +exactly the transaction you think you are. It takes the following params: , , , , , , , @@ -2036,7 +2064,7 @@ Response: ### MsigCancel MsigCancel cancels a previously-proposed multisig message -It takes the following params: , , , , +It takes the following params: , , , , , , @@ -2202,6 +2230,33 @@ Response: } ``` +### MsigRemoveSigner +MsigRemoveSigner proposes the removal of a signer from the multisig. +It accepts the multisig to make the change on, the proposer address to +send the message from, the address to be removed, and a boolean +indicating whether or not the signing threshold should be lowered by one +along with the address removal. + + +Perms: sign + +Inputs: +```json +[ + "f01234", + "f01234", + "f01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + ### MsigSwapApprove MsigSwapApprove approves a previously proposed SwapSigner It takes the following params: , , , From 23f5a99117226fba70fea235c421f1d7b6f26745 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 10 Oct 2020 15:36:32 +0200 Subject: [PATCH 115/125] Add comment clarifying current codepath --- chain/store/store.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/chain/store/store.go b/chain/store/store.go index 6fb689b3e..c6fc0cbef 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -454,7 +454,17 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet) func (cs *ChainStore) FlushValidationCache() error { log.Infof("clearing block validation cache...") - dsWalk, err := cs.ds.Query(query.Query{KeysOnly: true}) + dsWalk, err := cs.ds.Query(query.Query{ + // Potential TODO: the validation cache is not a namespace on its own + // but is rather constructed as prefixed-key `foo:bar` via .Instance(), which + // in turn does not work with the filter, which can match only on `foo/bar` + // + // If this is addressed (blockcache goes into its own sub-namespace) then + // strings.HasPrefix(...) below can be skipped + // + //Prefix: blockValidationCacheKeyPrefix.String() + KeysOnly: true, + }) if err != nil { return xerrors.Errorf("failed to initialize key listing query: %w", err) } From a80e98dc051bca1ff87e7569c8750fbeecebb42a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 15:38:14 +0200 Subject: [PATCH 116/125] cli: Don't output errors to stdout --- cli/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/helper.go b/cli/helper.go index 70a168145..9398ead71 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -35,7 +35,7 @@ func RunApp(app *cli.App) { if os.Getenv("LOTUS_DEV") != "" { log.Warnf("%+v", err) } else { - fmt.Printf("ERROR: %s\n\n", err) + fmt.Fprintf(os.Stderr, "ERROR: %s\n\n", err) // nolint:errcheck } var phe *PrintHelpErr if xerrors.As(err, &phe) { From 4fa6afad132704c699b351b7c2b05ec48057c629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 15:44:22 +0200 Subject: [PATCH 117/125] Fix panic in wallet export when key is not found --- chain/wallet/wallet.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 3efbf8ab7..57e63625b 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -141,6 +141,9 @@ func (w *LocalWallet) WalletExport(ctx context.Context, addr address.Address) (* if err != nil { return nil, xerrors.Errorf("failed to find key to export: %w", err) } + if k == nil { + return nil, xerrors.Errorf("key not found") + } return &k.KeyInfo, nil } @@ -274,6 +277,9 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er if err != nil { return xerrors.Errorf("failed to delete key %s : %w", addr, err) } + if k == nil { + return nil // already not there + } if err := w.keystore.Put(KTrashPrefix+k.Address.String(), k.KeyInfo); err != nil { return xerrors.Errorf("failed to mark key %s as trashed: %w", addr, err) From 7245ac2b699263d91321c4286f9b1a51eac4a91c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 10 Oct 2020 08:31:04 -0700 Subject: [PATCH 118/125] fix a race in the sync manager 1. SyncerState contains a mutex and should never be copied. Honestly, this case was probably fine but it's just as easy to create a separate snapshot type and easier to reason about. 2. We need to initialize the syncStates array once at start, before accessing it. By each syncer state inside each worker, we were racing with calls to `State()`. Again, this was probably benign, but I don't trust optimizing compilers. --- chain/sync.go | 2 +- chain/sync_manager.go | 15 ++++++++------ chain/syncstate.go | 46 ++++++++++++++++++++----------------------- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 369c65d33..f71100621 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1726,7 +1726,7 @@ func VerifyElectionPoStVRF(ctx context.Context, worker address.Address, rand []b return gen.VerifyVRF(ctx, worker, rand, evrf) } -func (syncer *Syncer) State() []SyncerState { +func (syncer *Syncer) State() []SyncerStateSnapshot { return syncer.syncmgr.State() } diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 811092bc7..c7fdea726 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -38,7 +38,7 @@ type SyncManager interface { SetPeerHead(ctx context.Context, p peer.ID, ts *types.TipSet) // State retrieves the state of the sync workers. - State() []SyncerState + State() []SyncerStateSnapshot } type syncManager struct { @@ -79,7 +79,7 @@ type syncResult struct { const syncWorkerCount = 3 func NewSyncManager(sync SyncFunc) SyncManager { - return &syncManager{ + sm := &syncManager{ bspThresh: 1, peerHeads: make(map[peer.ID]*types.TipSet), syncTargets: make(chan *types.TipSet), @@ -90,6 +90,10 @@ func NewSyncManager(sync SyncFunc) SyncManager { doSync: sync, stop: make(chan struct{}), } + for i := range sm.syncStates { + sm.syncStates[i] = new(SyncerState) + } + return sm } func (sm *syncManager) Start() { @@ -128,8 +132,8 @@ func (sm *syncManager) SetPeerHead(ctx context.Context, p peer.ID, ts *types.Tip sm.incomingTipSets <- ts } -func (sm *syncManager) State() []SyncerState { - ret := make([]SyncerState, 0, len(sm.syncStates)) +func (sm *syncManager) State() []SyncerStateSnapshot { + ret := make([]SyncerStateSnapshot, 0, len(sm.syncStates)) for _, s := range sm.syncStates { ret = append(ret, s.Snapshot()) } @@ -405,8 +409,7 @@ func (sm *syncManager) scheduleWorkSent() { } func (sm *syncManager) syncWorker(id int) { - ss := &SyncerState{} - sm.syncStates[id] = ss + ss := sm.syncStates[id] for { select { case ts, ok := <-sm.syncTargets: diff --git a/chain/syncstate.go b/chain/syncstate.go index 06cd5d91e..26f9f1c39 100644 --- a/chain/syncstate.go +++ b/chain/syncstate.go @@ -11,8 +11,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -type SyncerState struct { - lk sync.Mutex +type SyncerStateSnapshot struct { Target *types.TipSet Base *types.TipSet Stage api.SyncStateStage @@ -22,6 +21,11 @@ type SyncerState struct { End time.Time } +type SyncerState struct { + lk sync.Mutex + data SyncerStateSnapshot +} + func (ss *SyncerState) SetStage(v api.SyncStateStage) { if ss == nil { return @@ -29,9 +33,9 @@ func (ss *SyncerState) SetStage(v api.SyncStateStage) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Stage = v + ss.data.Stage = v if v == api.StageSyncComplete { - ss.End = build.Clock.Now() + ss.data.End = build.Clock.Now() } } @@ -42,13 +46,13 @@ func (ss *SyncerState) Init(base, target *types.TipSet) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Target = target - ss.Base = base - ss.Stage = api.StageHeaders - ss.Height = 0 - ss.Message = "" - ss.Start = build.Clock.Now() - ss.End = time.Time{} + ss.data.Target = target + ss.data.Base = base + ss.data.Stage = api.StageHeaders + ss.data.Height = 0 + ss.data.Message = "" + ss.data.Start = build.Clock.Now() + ss.data.End = time.Time{} } func (ss *SyncerState) SetHeight(h abi.ChainEpoch) { @@ -58,7 +62,7 @@ func (ss *SyncerState) SetHeight(h abi.ChainEpoch) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Height = h + ss.data.Height = h } func (ss *SyncerState) Error(err error) { @@ -68,21 +72,13 @@ func (ss *SyncerState) Error(err error) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Message = err.Error() - ss.Stage = api.StageSyncErrored - ss.End = build.Clock.Now() + ss.data.Message = err.Error() + ss.data.Stage = api.StageSyncErrored + ss.data.End = build.Clock.Now() } -func (ss *SyncerState) Snapshot() SyncerState { +func (ss *SyncerState) Snapshot() SyncerStateSnapshot { ss.lk.Lock() defer ss.lk.Unlock() - return SyncerState{ - Base: ss.Base, - Target: ss.Target, - Stage: ss.Stage, - Height: ss.Height, - Message: ss.Message, - Start: ss.Start, - End: ss.End, - } + return ss.data } From c463582528150150ae5571901e1005f5475d89ff Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 10 Oct 2020 08:33:06 -0700 Subject: [PATCH 119/125] fix a race and optimize hello messages LifecycleCtx can _only_ be called during startup as it appends an fx hook. Worse, this was causing us to append an fx hook on every single hello message, leaking memory (and probably causing other shutdown issues...). --- node/modules/services.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/node/modules/services.go b/node/modules/services.go index 4ee0abacc..e0a7c2eda 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -42,11 +42,13 @@ func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello. return xerrors.Errorf("failed to subscribe to event bus: %w", err) } + ctx := helpers.LifecycleCtx(mctx, lc) + go func() { for evt := range sub.Out() { pic := evt.(event.EvtPeerIdentificationCompleted) go func() { - if err := svc.SayHello(helpers.LifecycleCtx(mctx, lc), pic.Peer); err != nil { + if err := svc.SayHello(ctx, pic.Peer); err != nil { protos, _ := h.Peerstore().GetProtocols(pic.Peer) agent, _ := h.Peerstore().Get(pic.Peer, "AgentVersion") if protosContains(protos, hello.ProtocolID) { From 32d95fc97a01eb02d17665dfe329c89f6f4d6ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 20:46:06 +0200 Subject: [PATCH 120/125] docsgen, gofmt --- api/apistruct/struct.go | 2 +- cli/sync.go | 6 +++--- documentation/en/api-methods.md | 11 +++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6bfbf43cb..5ff83ed09 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -111,7 +111,7 @@ type FullNodeStruct struct { SyncCheckpoint func(ctx context.Context, key types.TipSetKey) error `perm:"admin"` SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncUnmarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` - SyncUnmarkAllBad func(ctx context.Context) error `perm:"admin"` + SyncUnmarkAllBad func(ctx context.Context) error `perm:"admin"` SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"` SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"` diff --git a/cli/sync.go b/cli/sync.go index a5178441d..ea066cbda 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -122,11 +122,11 @@ var syncMarkBadCmd = &cli.Command{ } var syncUnmarkBadCmd = &cli.Command{ - Name: "unmark-bad", - Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it", + Name: "unmark-bad", + Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "all", + Name: "all", Usage: "drop the entire bad block cache", }, }, diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index b3a9627e0..0774db1c8 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -175,6 +175,7 @@ * [SyncMarkBad](#SyncMarkBad) * [SyncState](#SyncState) * [SyncSubmitBlock](#SyncSubmitBlock) + * [SyncUnmarkAllBad](#SyncUnmarkAllBad) * [SyncUnmarkBad](#SyncUnmarkBad) * [SyncValidateTipset](#SyncValidateTipset) * [Wallet](#Wallet) @@ -4546,6 +4547,16 @@ Inputs: Response: `{}` +### SyncUnmarkAllBad +SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad + + +Perms: admin + +Inputs: `null` + +Response: `{}` + ### SyncUnmarkBad SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. From 23470bc38ff8502660c107418d323554d9d6fb83 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sun, 11 Oct 2020 20:27:36 +0000 Subject: [PATCH 121/125] lotus-stats: optmize getting miner power --- tools/stats/metrics.go | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index aee61b2aa..8dc79ea44 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -16,6 +16,9 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" @@ -269,22 +272,47 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) pl.AddPoint(p) - miners, err := api.StateListMiners(ctx, tipset.Key()) + powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, tipset.Key()) if err != nil { return err } - for _, addr := range miners { - mp, err := api.StateMinerPower(ctx, addr, tipset.Key()) + powerRaw, err := api.ChainReadObj(ctx, powerActor.Head) + if err != nil { + return err + } + + var powerActorState power.State + + if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil { + return fmt.Errorf("failed to unmarshal power actor state: %w", err) + } + + s := &ApiIpldStore{ctx, api} + mp, err := adt.AsMap(s, powerActorState.Claims) + if err != nil { + return err + } + + var claim power.Claim + err = mp.ForEach(&claim, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) if err != nil { return err } - if !mp.MinerPower.QualityAdjPower.IsZero() { - p = NewPoint("chain.miner_power", mp.MinerPower.QualityAdjPower.Int64()) - p.AddTag("miner", addr.String()) - pl.AddPoint(p) + if claim.QualityAdjPower.Int64() == 0 { + return nil } + + p = NewPoint("chain.miner_power", claim.QualityAdjPower.Int64()) + p.AddTag("miner", addr.String()) + pl.AddPoint(p) + + return nil + }) + if err != nil { + return err } return nil From ae366972a9e8170d616d8c1f36a819dd27d78c3f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 05:11:04 +0200 Subject: [PATCH 122/125] Chain is love Signed-off-by: Jakub Sztandera --- cli/chain.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 763752f23..bc0d7b508 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -521,8 +521,9 @@ var chainInspectUsage = &cli.Command{ } var chainListCmd = &cli.Command{ - Name: "list", - Usage: "View a segment of the chain", + Name: "list", + Aliases: []string{"love"}, + Usage: "View a segment of the chain", Flags: []cli.Flag{ &cli.Uint64Flag{Name: "height"}, &cli.IntFlag{Name: "count", Value: 30}, From c296e1bbcac98225f1d5ef7b34e9790c3dbed31e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 11 Oct 2020 20:31:20 -0700 Subject: [PATCH 123/125] use actor abstraction in metrics command --- chain/actors/builtin/power/power.go | 1 + chain/actors/builtin/power/v0.go | 19 +++++++++++++++++ chain/actors/builtin/power/v2.go | 19 +++++++++++++++++ tools/stats/metrics.go | 33 ++++------------------------- 4 files changed, 43 insertions(+), 29 deletions(-) diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index e683cfd96..bafb14de0 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -51,6 +51,7 @@ type State interface { MinerPower(address.Address) (Claim, bool, error) MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error) ListAllMiners() ([]address.Address, error) + ForEachClaim(func(miner address.Address, claim Claim) error) error } type Claim struct { diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index e2a9cf382..3f9a65777 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -96,3 +96,22 @@ func (s *state0) ListAllMiners() ([]address.Address, error) { return miners, nil } + +func (s *state0) ForEachClaim(cb func(miner address.Address, claim Claim) error) error { + claims, err := adt0.AsMap(s.store, s.Claims) + if err != nil { + return err + } + + var claim power0.Claim + return claims.ForEach(&claim, func(k string) error { + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + return cb(a, Claim{ + RawBytePower: claim.RawBytePower, + QualityAdjPower: claim.QualityAdjPower, + }) + }) +} diff --git a/chain/actors/builtin/power/v2.go b/chain/actors/builtin/power/v2.go index 6346a09b6..0c15f0669 100644 --- a/chain/actors/builtin/power/v2.go +++ b/chain/actors/builtin/power/v2.go @@ -96,3 +96,22 @@ func (s *state2) ListAllMiners() ([]address.Address, error) { return miners, nil } + +func (s *state2) ForEachClaim(cb func(miner address.Address, claim Claim) error) error { + claims, err := adt2.AsMap(s.store, s.Claims) + if err != nil { + return err + } + + var claim power2.Claim + return claims.ForEach(&claim, func(k string) error { + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + return cb(a, Claim{ + RawBytePower: claim.RawBytePower, + QualityAdjPower: claim.QualityAdjPower, + }) + }) +} diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 8dc79ea44..795203c40 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -13,12 +13,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" @@ -272,35 +270,17 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) pl.AddPoint(p) - powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, tipset.Key()) + powerActor, err := api.StateGetActor(ctx, power.Address, tipset.Key()) if err != nil { return err } - powerRaw, err := api.ChainReadObj(ctx, powerActor.Head) + powerActorState, err := power.Load(&ApiIpldStore{ctx, api}, powerActor) if err != nil { return err } - var powerActorState power.State - - if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil { - return fmt.Errorf("failed to unmarshal power actor state: %w", err) - } - - s := &ApiIpldStore{ctx, api} - mp, err := adt.AsMap(s, powerActorState.Claims) - if err != nil { - return err - } - - var claim power.Claim - err = mp.ForEach(&claim, func(key string) error { - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - + return powerActorState.ForEachClaim(func(addr address.Address, claim power.Claim) error { if claim.QualityAdjPower.Int64() == 0 { return nil } @@ -311,11 +291,6 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis return nil }) - if err != nil { - return err - } - - return nil } type msgTag struct { From 92177b573812cd19e56f5873921b7ed8ec685906 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 11 Oct 2020 22:31:46 -0700 Subject: [PATCH 124/125] implement tape upgrade Upgrade to specs-actors v2.1 and network version 5. This fixes the bug where prove commits were not accepted. --- build/params_2k.go | 3 ++- build/params_shared_vals.go | 2 +- build/params_testground.go | 5 +++-- build/params_testnet.go | 2 ++ chain/actors/version.go | 2 +- chain/stmgr/forks.go | 5 ++++- documentation/en/api-methods.md | 2 +- go.mod | 4 ++-- go.sum | 6 ++++-- 9 files changed, 20 insertions(+), 11 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index c6538dc08..b09b60fae 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -17,9 +17,10 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 const UpgradeRefuelHeight = -3 +const UpgradeTapeHeight = -4 var UpgradeActorsV2Height = abi.ChainEpoch(10) -var UpgradeLiftoffHeight = abi.ChainEpoch(-4) +var UpgradeLiftoffHeight = abi.ChainEpoch(-5) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 722590575..ede40c0e3 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -25,7 +25,7 @@ const UnixfsLinksPerLevel = 1024 // Consensus / Network const AllowableClockDriftSecs = uint64(1) -const NewestNetworkVersion = network.Version4 +const NewestNetworkVersion = network.Version5 const ActorUpgradeNetworkVersion = network.Version4 // Epochs diff --git a/build/params_testground.go b/build/params_testground.go index 6109cbc04..7ef034234 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -83,14 +83,15 @@ var ( UpgradeSmokeHeight abi.ChainEpoch = -1 UpgradeIgnitionHeight abi.ChainEpoch = -2 UpgradeRefuelHeight abi.ChainEpoch = -3 + UpgradeTapeHeight abi.ChainEpoch = -4 UpgradeActorsV2Height abi.ChainEpoch = 10 - UpgradeLiftoffHeight abi.ChainEpoch = -4 + UpgradeLiftoffHeight abi.ChainEpoch = -5 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, } - NewestNetworkVersion = network.Version4 + NewestNetworkVersion = network.Version5 ActorUpgradeNetworkVersion = network.Version4 Devnet = true diff --git a/build/params_testnet.go b/build/params_testnet.go index fe70deaef..54f50ac6e 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -30,6 +30,8 @@ const UpgradeRefuelHeight = 130800 var UpgradeActorsV2Height = abi.ChainEpoch(138720) +const UpgradeTapeHeight = 140760 + // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. // We still have upgrades and state changes to do, but can happen after signaling timing here. diff --git a/chain/actors/version.go b/chain/actors/version.go index 17af8b08b..2efd903bb 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -18,7 +18,7 @@ func VersionForNetwork(version network.Version) Version { switch version { case network.Version0, network.Version1, network.Version2, network.Version3: return Version0 - case network.Version4: + case network.Version4, network.Version5: return Version2 default: panic(fmt.Sprintf("unsupported network version %d", version)) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index fba92ee3f..6ebe8a5b9 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -79,9 +79,12 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Network: network.Version4, Expensive: true, Migration: UpgradeActorsV2, + }, { + Height: build.UpgradeTapeHeight, + Network: network.Version5, }, { Height: build.UpgradeLiftoffHeight, - Network: network.Version4, + Network: network.Version5, Migration: UpgradeLiftoff, }} diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ec8071b57..76dce56db 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3975,7 +3975,7 @@ Inputs: ] ``` -Response: `4` +Response: `5` ### StateReadState StateReadState returns the indicated actor's state. diff --git a/go.mod b/go.mod index eb79a496c..ef43a5274 100644 --- a/go.mod +++ b/go.mod @@ -34,12 +34,12 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab + github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.0.3 + github.com/filecoin-project/specs-actors/v2 v2.1.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index e13359f6c..a9ac12ff7 100644 --- a/go.sum +++ b/go.sum @@ -265,6 +265,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b h1:bMUfG6Sy6YSMbsjQAO1Q2vEZldbSdsbRy/FX3OlTck0= +github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -276,8 +278,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.3 h1:Niy6xncgi8bI8aBCt1McdZfATBfG4Uxytt8KW4s3bAc= -github.com/filecoin-project/specs-actors/v2 v2.0.3/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= +github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT+dGVWJTr5yDevU00g= +github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= From b273bfc5c38ae3d531e6bd3bcb1cfe696461ac8c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 12 Oct 2020 02:15:39 -0400 Subject: [PATCH 125/125] Set tape upgrade to 15000 --- build/params_testnet.go | 3 +- documentation/en/api-methods.md | 372 ++++++++++++++++---------------- 2 files changed, 188 insertions(+), 187 deletions(-) diff --git a/build/params_testnet.go b/build/params_testnet.go index 15b7f8c46..eef274e0d 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -22,10 +22,11 @@ const BreezeGasTampingDuration = 120 const UpgradeSmokeHeight = -2 const UpgradeIgnitionHeight = -3 +const UpgradeRefuelHeight = -4 const UpgradeActorsV2Height = 500 -const UpgradeTapeHeight = 140760 +const UpgradeTapeHeight = 15000 // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 76dce56db..9bde9b9f9 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -357,7 +357,7 @@ Inputs: Response: ```json { - "Miner": "f01234", + "Miner": "t01234", "Ticket": { "VRFProof": "Ynl0ZSBhcnJheQ==" }, @@ -454,8 +454,8 @@ Response: ```json { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -947,7 +947,7 @@ Response: }, "State": 42, "Message": "string value", - "Provider": "f01234", + "Provider": "t01234", "DataRef": { "TransferType": "string value", "Root": { @@ -984,7 +984,7 @@ Response: }, "State": 42, "Message": "string value", - "Provider": "f01234", + "Provider": "t01234", "DataRef": { "TransferType": "string value", "Root": { @@ -1087,7 +1087,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, @@ -1108,9 +1108,9 @@ Response: "UnsealPrice": "0", "PaymentInterval": 42, "PaymentIntervalIncrease": 42, - "Miner": "f01234", + "Miner": "t01234", "MinerPeer": { - "Address": "f01234", + "Address": "t01234", "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", "PieceCID": null } @@ -1127,7 +1127,7 @@ Inputs: ```json [ "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "f01234" + "t01234" ] ``` @@ -1138,7 +1138,7 @@ Response: "VerifiedPrice": "0", "MinPieceSize": 1032, "MaxPieceSize": 1032, - "Miner": "f01234", + "Miner": "t01234", "Timestamp": 10101, "Expiry": 10101, "SeqNo": 42 @@ -1179,10 +1179,10 @@ Inputs: "UnsealPrice": "0", "PaymentInterval": 42, "PaymentIntervalIncrease": 42, - "Client": "f01234", - "Miner": "f01234", + "Client": "t01234", + "Miner": "t01234", "MinerPeer": { - "Address": "f01234", + "Address": "t01234", "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", "PieceCID": null } @@ -1206,7 +1206,7 @@ Perms: write Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -1232,10 +1232,10 @@ Inputs: "UnsealPrice": "0", "PaymentInterval": 42, "PaymentIntervalIncrease": 42, - "Client": "f01234", - "Miner": "f01234", + "Client": "t01234", + "Miner": "t01234", "MinerPeer": { - "Address": "f01234", + "Address": "t01234", "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", "PieceCID": null } @@ -1276,8 +1276,8 @@ Inputs: "PieceCid": null, "PieceSize": 1024 }, - "Wallet": "f01234", - "Miner": "f01234", + "Wallet": "t01234", + "Miner": "t01234", "EpochPrice": "0", "MinBlocksDuration": 42, "ProviderCollateral": "0", @@ -1325,8 +1325,8 @@ Inputs: [ { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1361,8 +1361,8 @@ Inputs: [ { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1395,7 +1395,7 @@ Inputs: ```json [ 42, - "f01234", + "t01234", 9, [ { @@ -1421,8 +1421,8 @@ Inputs: [ { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1449,8 +1449,8 @@ Response: ```json { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1512,8 +1512,8 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", + "t01234", + "t01234", "0" ] ``` @@ -1537,7 +1537,7 @@ Inputs: ```json [ { - "Miner": "f01234", + "Miner": "t01234", "Parents": [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -1566,7 +1566,7 @@ Response: ```json { "Header": { - "Miner": "f01234", + "Miner": "t01234", "Ticket": { "VRFProof": "Ynl0ZSBhcnJheQ==" }, @@ -1613,7 +1613,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", 10101, [ { @@ -1632,7 +1632,7 @@ Response: "MinerPower": "0", "NetworkPower": "0", "Sectors": null, - "WorkerKey": "f01234", + "WorkerKey": "t01234", "SectorSize": 34359738368, "PrevBeaconEntry": { "Round": 42, @@ -1693,7 +1693,7 @@ Perms: read Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -1733,8 +1733,8 @@ Inputs: { "Message": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1774,8 +1774,8 @@ Inputs: [ { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1795,8 +1795,8 @@ Response: { "Message": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1824,8 +1824,8 @@ Inputs: { "Message": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1908,8 +1908,8 @@ Response: "Message": { "Message": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1942,11 +1942,11 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", + "t01234", + "t01234", 42, - "f01234", - "f01234", + "t01234", + "t01234", true ] ``` @@ -1969,10 +1969,10 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", + "t01234", + "t01234", 42, - "f01234", + "t01234", true ] ``` @@ -1995,9 +1995,9 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", - "f01234", + "t01234", + "t01234", + "t01234", true ] ``` @@ -2019,9 +2019,9 @@ Perms: sign Inputs: ```json [ - "f01234", + "t01234", 42, - "f01234" + "t01234" ] ``` @@ -2046,12 +2046,12 @@ Perms: sign Inputs: ```json [ - "f01234", + "t01234", 42, - "f01234", - "f01234", + "t01234", + "t01234", "0", - "f01234", + "t01234", 42, "Ynl0ZSBhcnJheQ==" ] @@ -2075,11 +2075,11 @@ Perms: sign Inputs: ```json [ - "f01234", + "t01234", 42, - "f01234", + "t01234", "0", - "f01234", + "t01234", 42, "Ynl0ZSBhcnJheQ==" ] @@ -2107,7 +2107,7 @@ Inputs: null, 10101, "0", - "f01234", + "t01234", "0" ] ``` @@ -2128,7 +2128,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -2152,7 +2152,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -2183,7 +2183,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -2215,10 +2215,10 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", + "t01234", + "t01234", "0", - "f01234", + "t01234", 42, "Ynl0ZSBhcnJheQ==" ] @@ -2244,9 +2244,9 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", - "f01234", + "t01234", + "t01234", + "t01234", true ] ``` @@ -2269,12 +2269,12 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", + "t01234", + "t01234", 42, - "f01234", - "f01234", - "f01234" + "t01234", + "t01234", + "t01234" ] ``` @@ -2296,11 +2296,11 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", + "t01234", + "t01234", 42, - "f01234", - "f01234" + "t01234", + "t01234" ] ``` @@ -2322,10 +2322,10 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", - "f01234", - "f01234" + "t01234", + "t01234", + "t01234", + "t01234" ] ``` @@ -2533,7 +2533,7 @@ Perms: sign Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -2547,7 +2547,7 @@ Perms: sign Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -2555,8 +2555,8 @@ Response: ```json { "Channel": "\u003cempty\u003e", - "From": "f01234", - "To": "f01234", + "From": "t01234", + "To": "t01234", "ConfirmedAmt": "0", "PendingAmt": "0", "PendingWaitSentinel": null, @@ -2573,8 +2573,8 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234" + "t01234", + "t01234" ] ``` @@ -2582,8 +2582,8 @@ Response: ```json { "Channel": "\u003cempty\u003e", - "From": "f01234", - "To": "f01234", + "From": "t01234", + "To": "t01234", "ConfirmedAmt": "0", "PendingAmt": "0", "PendingWaitSentinel": null, @@ -2600,7 +2600,7 @@ Perms: sign Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -2619,8 +2619,8 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", + "t01234", + "t01234", "0" ] ``` @@ -2628,7 +2628,7 @@ Inputs: Response: ```json { - "Channel": "f01234", + "Channel": "t01234", "WaitSentinel": { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" } @@ -2649,7 +2649,7 @@ Inputs: ] ``` -Response: `"f01234"` +Response: `"t01234"` ### PaychList There are not yet any comments for this method. @@ -2668,8 +2668,8 @@ Perms: sign Inputs: ```json [ - "f01234", - "f01234", + "t01234", + "t01234", null ] ``` @@ -2677,7 +2677,7 @@ Inputs: Response: ```json { - "Channel": "f01234", + "Channel": "t01234", "WaitSentinel": { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, @@ -2693,7 +2693,7 @@ Perms: sign Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -2712,14 +2712,14 @@ Perms: read Inputs: ```json [ - "f01234" + "t01234" ] ``` Response: ```json { - "ControlAddr": "f01234", + "ControlAddr": "t01234", "Direction": 1 } ``` @@ -2732,14 +2732,14 @@ Perms: write Inputs: ```json [ - "f01234", + "t01234", { - "ChannelAddr": "f01234", + "ChannelAddr": "t01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "f01234", + "Actor": "t01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2768,14 +2768,14 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", { - "ChannelAddr": "f01234", + "ChannelAddr": "t01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "f01234", + "Actor": "t01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2804,14 +2804,14 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", { - "ChannelAddr": "f01234", + "ChannelAddr": "t01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "f01234", + "Actor": "t01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2838,7 +2838,7 @@ Perms: sign Inputs: ```json [ - "f01234", + "t01234", "0", 42 ] @@ -2848,12 +2848,12 @@ Response: ```json { "Voucher": { - "ChannelAddr": "f01234", + "ChannelAddr": "t01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "f01234", + "Actor": "t01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2879,7 +2879,7 @@ Perms: write Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -2893,14 +2893,14 @@ Perms: sign Inputs: ```json [ - "f01234", + "t01234", { - "ChannelAddr": "f01234", + "ChannelAddr": "t01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "f01234", + "Actor": "t01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2941,7 +2941,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -2953,7 +2953,7 @@ Inputs: ] ``` -Response: `"f01234"` +Response: `"t01234"` ### StateAllMinerFaults StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset @@ -2989,8 +2989,8 @@ Inputs: [ { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -3015,8 +3015,8 @@ Response: { "Msg": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -3033,8 +3033,8 @@ Response: "ExecutionTrace": { "Msg": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -3197,7 +3197,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3288,8 +3288,8 @@ Inputs: [ { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -3343,7 +3343,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3355,7 +3355,7 @@ Inputs: ] ``` -Response: `"f01234"` +Response: `"t01234"` ### StateMarketBalance StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market @@ -3366,7 +3366,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3416,8 +3416,8 @@ Response: }, "PieceSize": 1032, "VerifiedDeal": true, - "Client": "f01234", - "Provider": "f01234", + "Client": "t01234", + "Provider": "t01234", "Label": "string value", "StartEpoch": 10101, "EndEpoch": 10101, @@ -3494,8 +3494,8 @@ Response: }, "PieceSize": 1032, "VerifiedDeal": true, - "Client": "f01234", - "Provider": "f01234", + "Client": "t01234", + "Provider": "t01234", "Label": "string value", "StartEpoch": 10101, "EndEpoch": 10101, @@ -3520,7 +3520,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3543,7 +3543,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3566,7 +3566,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3589,7 +3589,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3618,7 +3618,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3633,9 +3633,9 @@ Inputs: Response: ```json { - "Owner": "f01234", - "Worker": "f01234", - "NewWorker": "f01234", + "Owner": "t01234", + "Worker": "t01234", + "NewWorker": "t01234", "ControlAddresses": null, "WorkerChangeEpoch": 10101, "PeerId": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", @@ -3656,7 +3656,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", { "SealProof": 3, "SectorNumber": 9, @@ -3693,7 +3693,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", 42, [ { @@ -3717,7 +3717,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3753,7 +3753,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", { "SealProof": 3, "SectorNumber": 9, @@ -3791,7 +3791,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3830,7 +3830,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3859,7 +3859,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3889,7 +3889,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ 0 ], @@ -3986,7 +3986,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -4034,8 +4034,8 @@ Response: { "Msg": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -4052,8 +4052,8 @@ Response: "ExecutionTrace": { "Msg": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -4125,7 +4125,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", 9, [ { @@ -4157,7 +4157,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", 9, [ { @@ -4198,7 +4198,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", 9, [ { @@ -4228,7 +4228,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", 9, [ { @@ -4276,7 +4276,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -4310,7 +4310,7 @@ Inputs: ] ``` -Response: `"f01234"` +Response: `"t01234"` ### StateVerifierStatus StateVerifierStatus returns the data cap for the given address. @@ -4323,7 +4323,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -4478,7 +4478,7 @@ Inputs: `null` Response: ```json { - "Miner": "f01234", + "Miner": "t01234", "Ticket": { "VRFProof": "Ynl0ZSBhcnJheQ==" }, @@ -4560,7 +4560,7 @@ Inputs: [ { "Header": { - "Miner": "f01234", + "Miner": "t01234", "Ticket": { "VRFProof": "Ynl0ZSBhcnJheQ==" }, @@ -4663,7 +4663,7 @@ Perms: read Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -4677,7 +4677,7 @@ Perms: write Inputs: `null` -Response: `"f01234"` +Response: `"t01234"` ### WalletDelete WalletDelete deletes an address from the wallet. @@ -4688,7 +4688,7 @@ Perms: write Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -4703,7 +4703,7 @@ Perms: admin Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -4724,7 +4724,7 @@ Perms: write Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -4746,7 +4746,7 @@ Inputs: ] ``` -Response: `"f01234"` +Response: `"t01234"` ### WalletList WalletList lists all the addresses in the wallet. @@ -4771,7 +4771,7 @@ Inputs: ] ``` -Response: `"f01234"` +Response: `"t01234"` ### WalletSetDefault WalletSetDefault marks the given address as as the default one. @@ -4782,7 +4782,7 @@ Perms: admin Inputs: ```json [ - "f01234" + "t01234" ] ``` @@ -4797,7 +4797,7 @@ Perms: sign Inputs: ```json [ - "f01234", + "t01234", "Ynl0ZSBhcnJheQ==" ] ``` @@ -4819,11 +4819,11 @@ Perms: sign Inputs: ```json [ - "f01234", + "t01234", { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -4840,8 +4840,8 @@ Response: { "Message": { "Version": 42, - "To": "f01234", - "From": "f01234", + "To": "t01234", + "From": "t01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -4870,7 +4870,7 @@ Inputs: ] ``` -Response: `"f01234"` +Response: `"t01234"` ### WalletVerify WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. @@ -4882,7 +4882,7 @@ Perms: read Inputs: ```json [ - "f01234", + "t01234", "Ynl0ZSBhcnJheQ==", { "Type": 2,