From e0dcb0ec4930b465948af8ec04dc9ce6835c8087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 9 Oct 2020 22:20:15 +0100 Subject: [PATCH 001/106] tvx: trace puts to blockstore for inclusion in CAR. --- cmd/tvx/stores.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index 93e0d215f..4f574c175 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -87,7 +87,7 @@ type proxyingBlockstore struct { ctx context.Context api api.FullNode - lk sync.RWMutex + lk sync.Mutex tracing bool traced map[cid.Cid]struct{} @@ -113,11 +113,11 @@ func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} { } func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { - pb.lk.RLock() + pb.lk.Lock() if pb.tracing { pb.traced[cid] = struct{}{} } - pb.lk.RUnlock() + pb.lk.Unlock() if block, err := pb.Blockstore.Get(cid); err == nil { return block, err @@ -140,3 +140,12 @@ func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { return block, nil } + +func (pb *proxyingBlockstore) Put(block blocks.Block) error { + pb.lk.Lock() + if pb.tracing { + pb.traced[block.Cid()] = struct{}{} + } + pb.lk.Unlock() + return pb.Blockstore.Put(block) +} From 85be11fb7515d997460b12cb91ed212d7e3dc3ac Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 11 Oct 2020 18:10:02 +0200 Subject: [PATCH 002/106] Add propose remove Signed-off-by: Jakub Sztandera --- chain/actors/builtin/multisig/message.go | 1 + cli/multisig.go | 79 ++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/chain/actors/builtin/multisig/message.go b/chain/actors/builtin/multisig/message.go index b19287432..5beb5319d 100644 --- a/chain/actors/builtin/multisig/message.go +++ b/chain/actors/builtin/multisig/message.go @@ -45,6 +45,7 @@ type MessageBuilder interface { // this type is the same between v0 and v2 type ProposalHashData = multisig2.ProposalHashData +type ProposeReturn = multisig2.ProposeReturn func txnParams(id uint64, data *ProposalHashData) ([]byte, error) { params := multisig2.TxnIDParams{ID: multisig2.TxnID(id)} diff --git a/cli/multisig.go b/cli/multisig.go index b692fad5a..90074c64d 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -41,6 +41,7 @@ var multisigCmd = &cli.Command{ msigCreateCmd, msigInspectCmd, msigProposeCmd, + msigRemoveProposeCmd, msigApproveCmd, msigAddProposeCmd, msigAddApproveCmd, @@ -504,6 +505,84 @@ var msigApproveCmd = &cli.Command{ }, } +var msigRemoveProposeCmd = &cli.Command{ + Name: "propose-remove", + Usage: "Propose to remove a signer", + ArgsUsage: "[multisigAddress signer]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "decrease-threshold", + Usage: "whether the number of required signers should be decreased", + }, + &cli.StringFlag{ + Name: "from", + Usage: "account to send the propose message from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 2 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address and signer address")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + addr, err := address.NewFromString(cctx.Args().Get(1)) + if err != nil { + return err + } + + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr + } + + msgCid, err := api.MsigRemoveSigner(ctx, msig, from, addr, cctx.Bool("decrease-threshold")) + if err != nil { + return err + } + + fmt.Println("sent remove proposal in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("add proposal returned exit %d", wait.Receipt.ExitCode) + } + + var ret multisig.ProposeReturn + err = ret.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)) + if err != nil { + return xerrors.Errorf("decoding proposal return: %w", err) + } + fmt.Printf("TxnID: %d", ret.TxnID) + + return nil + }, +} + var msigAddProposeCmd = &cli.Command{ Name: "add-propose", Usage: "Propose to add a signer", From b35f9b56b42ae7415284f5ced424af21923d693e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 9 Oct 2020 20:17:04 -0700 Subject: [PATCH 003/106] WIP: adding in ledger support Signed-off-by: Jakub Sztandera --- chain/messagepool/messagepool.go | 3 + chain/wallet/ledger/ledger.go | 139 +++++++++++++++++++++ cmd/lotus-shed/ledger.go | 205 +++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + cmd/lotus-wallet/main.go | 11 +- go.mod | 1 + go.sum | 7 ++ 7 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 chain/wallet/ledger/ledger.go create mode 100644 cmd/lotus-shed/ledger.go diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index d3c638b22..2a4704c3b 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -1249,6 +1249,9 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err } func (mp *MessagePool) loadLocal() error { + return nil + // TODO: this causes super slow startup... + res, err := mp.localMsgs.Query(query.Query{}) if err != nil { return xerrors.Errorf("query local messages: %w", err) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go new file mode 100644 index 000000000..36b0fb040 --- /dev/null +++ b/chain/wallet/ledger/ledger.go @@ -0,0 +1,139 @@ +package ledgerwallet + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" +) + +type LedgerWallet struct { + ds datastore.Datastore +} + +func NewWallet(ds datastore.Datastore) *LedgerWallet { + return &LedgerWallet{ds} +} + +type LedgerKeyInfo struct { + Address address.Address + Path []uint32 +} + +var _ (api.WalletAPI) = (*LedgerWallet)(nil) + +func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta api.MsgMeta) (*crypto.Signature, error) { + ki, err := lw.getKeyInfo(signer) + if err != nil { + return nil, err + } + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return nil, err + } + defer fl.Close() + if meta.Type != api.MTChainMsg { + return nil, fmt.Errorf("ledger can only sign chain messages") + } + + // TODO: assert meta matches the 'toSign' bits + + sig, err := fl.SignSECP256K1(ki.Path, meta.Extra) + if err != nil { + return nil, err + } + + return &crypto.Signature{ + Type: crypto.SigTypeSecp256k1, + Data: sig.SignatureBytes(), + }, nil +} + +func (lw LedgerWallet) getKeyInfo(addr address.Address) (*LedgerKeyInfo, error) { + kib, err := lw.ds.Get(keyForAddr(addr)) + if err != nil { + return nil, err + } + + var out LedgerKeyInfo + if err := json.Unmarshal(kib, &out); err != nil { + return nil, err + } + + return &out, nil +} + +func (lw LedgerWallet) WalletDelete(ctx context.Context, k address.Address) error { + return lw.ds.Delete(keyForAddr(k)) +} + +func (lw LedgerWallet) WalletExport(ctx context.Context, k address.Address) (*types.KeyInfo, error) { + return nil, fmt.Errorf("cannot export keys from ledger wallets") +} + +func (lw LedgerWallet) WalletHas(ctx context.Context, k address.Address) (bool, error) { + _, err := lw.ds.Get(keyForAddr(k)) + if err == nil { + return true, nil + } + if err == datastore.ErrNotFound { + return false, nil + } + return false, err +} + +func (lw LedgerWallet) WalletImport(ctx context.Context, kinfo *types.KeyInfo) (address.Address, error) { + var ki LedgerKeyInfo + if err := json.Unmarshal(kinfo.PrivateKey, &ki); err != nil { + return address.Undef, err + } + + if ki.Address == address.Undef { + return address.Undef, fmt.Errorf("no address given in imported key info") + } + + if err := lw.ds.Put(keyForAddr(ki.Address), kinfo.PrivateKey); err != nil { + return address.Undef, err + } + + return ki.Address, nil +} + +func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error) { + res, err := lw.ds.Query(query.Query{Prefix: "ledgerkey/"}) + if err != nil { + return nil, err + } + + var out []address.Address + for { + res, ok := res.NextSync() + if !ok { + break + } + + var ki LedgerKeyInfo + if err := json.Unmarshal(res.Value, &ki); err != nil { + return nil, err + } + + out = append(out, ki.Address) + } + return out, nil +} + +func (lw LedgerWallet) WalletNew(ctx context.Context, t crypto.SigType) (address.Address, error) { + return address.Undef, fmt.Errorf("cannot create new address on ledger") +} + +func keyForAddr(addr address.Address) datastore.Key { + return datastore.NewKey("ledgerkey/" + addr.String()) +} diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go new file mode 100644 index 000000000..4bb61e5e0 --- /dev/null +++ b/cmd/lotus-shed/ledger.go @@ -0,0 +1,205 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" + "github.com/urfave/cli/v2" + ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" +) + +var ledgerCmd = &cli.Command{ + Name: "ledger", + Usage: "Ledger interactions", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + ledgerListAddressesCmd, + ledgerKeyInfoCmd, + ledgerSignTestCmd, + }, +} + +var ledgerListAddressesCmd = &cli.Command{ + Name: "list", + Action: func(cctx *cli.Context) error { + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return err + } + + for i := 0; i < 20; i++ { + p := []uint32{0x80000000 + 44, 0x80000000 + 461, 0x80000000, 0, uint32(i)} + pubk, err := fl.GetPublicKeySECP256K1(p) + if err != nil { + return err + } + + addr, err := address.NewSecp256k1Address(pubk) + if err != nil { + return err + } + + fmt.Printf("%s: %s\n", addr, printHDPath(p)) + } + + return nil + }, +} + +func parseHDPath(s string) ([]uint32, error) { + parts := strings.Split(s, "/") + if parts[0] != "m" { + return nil, fmt.Errorf("expected HD path to start with 'm'") + } + + var out []uint32 + for _, p := range parts[1:] { + var hard bool + if strings.HasSuffix(p, "'") { + p = p[:len(p)-1] + hard = true + } + + v, err := strconv.ParseUint(p, 10, 32) + if err != nil { + return nil, err + } + if v >= 0x80000000 { + return nil, fmt.Errorf("path element %s too large", p) + } + + if hard { + v += 0x80000000 + } + out = append(out, uint32(v)) + } + return out, nil +} + +func printHDPath(pth []uint32) string { + s := "m/" + for _, p := range pth { + var hard bool + if p >= 0x80000000 { + p -= 0x80000000 + } + s += fmt.Sprint(p) + if hard { + s += "'" + } + s += "/" + } + + return strings.TrimRight(s, "/") +} + +func numlist(p []uint32) string { + var out []string + for _, v := range p { + out = append(out, fmt.Sprint(v)) + } + return strings.Join(out, ",") +} + +var ledgerKeyInfoCmd = &cli.Command{ + Name: "key-info", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return err + } + + p, err := parseHDPath(cctx.Args().First()) + if err != nil { + return err + } + + pubk, _, addr, err := fl.GetAddressPubKeySECP256K1(p) + if err != nil { + return err + } + fmt.Println(addr) + fmt.Println(pubk) + + a, err := address.NewFromString(addr) + if err != nil { + return err + } + + var pd ledgerwallet.LedgerKeyInfo + pd.Address = a + pd.Path = p + + b, err := json.Marshal(pd) + if err != nil { + return err + } + + var ki types.KeyInfo + ki.Type = wallet.KTSecp256k1 + ki.PrivateKey = b + + out, err := json.Marshal(ki) + if err != nil { + return err + } + + fmt.Println(string(out)) + + return nil + }, +} + +var ledgerSignTestCmd = &cli.Command{ + Name: "sign", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return err + } + + p, err := parseHDPath(cctx.Args().First()) + if err != nil { + return err + } + + addr, err := address.NewFromString("f1xc3hws5n6y5m3m44gzb3gyjzhups6wzmhe663ji") + if err != nil { + return err + } + + m := &types.Message{ + To: addr, + From: addr, + } + + b, err := m.ToStorageBlock() + if err != nil { + return err + } + + sig, err := fl.SignSECP256K1(p, b.RawData()) + if err != nil { + return err + } + + fmt.Println(sig.SignatureBytes()) + + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 4542551db..29a105355 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -41,6 +41,7 @@ func main() { syncCmd, stateTreePruneCmd, datastoreCmd, + ledgerCmd, } app := &cli.App{ diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index ae3580a59..fbedffed0 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/wallet" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/repo" @@ -98,6 +99,14 @@ var runCmd = &cli.Command{ if err != nil { return err } + _ = w + + ds, err := lr.Datastore("/metadata") + if err != nil { + return err + } + + lw := ledgerwallet.NewWallet(ds) address := cctx.String("listen") mux := mux.NewRouter() @@ -105,7 +114,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &LoggedWallet{under: w}) + rpcServer.Register("Filecoin", &LoggedWallet{under: lw}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof diff --git a/go.mod b/go.mod index eb79a496c..d6a4768b7 100644 --- a/go.mod +++ b/go.mod @@ -120,6 +120,7 @@ require ( github.com/urfave/cli/v2 v2.2.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163 + github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 diff --git a/go.sum b/go.sum index e13359f6c..fe7aacbf9 100644 --- a/go.sum +++ b/go.sum @@ -1422,6 +1422,8 @@ github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDb github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/go-smux-yamux v2.0.9+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= +github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 h1:NwiwjQDB3CzQ5XH0rdMh1oQqzJH7O2PSLWxif/w3zsY= +github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4/go.mod h1:K+EVq8d5QcQ2At5VECsA+SNZvWefyBXh8TnIsxo1OvQ= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= @@ -1443,6 +1445,10 @@ github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZD github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zondax/hid v0.9.0 h1:eiT3P6vNxAEVxXMw66eZUAAnU2zD33JBkfG/EnfAKl8= +github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= +github.com/zondax/ledger-go v0.12.1 h1:hYRcyznPRJp+5mzF2sazTLP2nGvGjYDD2VzhHhFomLU= +github.com/zondax/ledger-go v0.12.1/go.mod h1:KatxXrVDzgWwbssUWsF5+cOJHXPvzQ09YSlzGNuhOEo= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= @@ -1515,6 +1521,7 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= From 99791a695373c78ec93d985adb5895358406585b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 13:03:39 +0200 Subject: [PATCH 004/106] lotus-wallet: ledger flag Signed-off-by: Jakub Sztandera --- cmd/lotus-wallet/main.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index fbedffed0..2d87998a4 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "github.com/filecoin-project/lotus/api" "net" "net/http" "os" @@ -61,6 +62,10 @@ var runCmd = &cli.Command{ Usage: "host address and port the wallet api will listen on", Value: "0.0.0.0:1777", }, + &cli.BoolFlag{ + Name: "ledger", + Usage: "use a ledger device instead of an on-disk wallet", + }, }, Action: func(cctx *cli.Context) error { log.Info("Starting lotus wallet") @@ -95,18 +100,21 @@ var runCmd = &cli.Command{ return err } - w, err := wallet.NewWallet(ks) - if err != nil { - return err - } - _ = w + var w api.WalletAPI + if !cctx.Bool("ledger") { + w, err = wallet.NewWallet(ks) + if err != nil { + return err + } + } else { + ds, err := lr.Datastore("/metadata") + if err != nil { + return err + } - ds, err := lr.Datastore("/metadata") - if err != nil { - return err + w = ledgerwallet.NewWallet(ds) } - lw := ledgerwallet.NewWallet(ds) address := cctx.String("listen") mux := mux.NewRouter() @@ -114,7 +122,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &LoggedWallet{under: lw}) + rpcServer.Register("Filecoin", &LoggedWallet{under: w}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof From 66d6113340e943fe9aca2b0227312ac521b059e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 13:06:02 +0200 Subject: [PATCH 005/106] ledgerwallet: Validate signing bytes Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 21 +++++++++++++++++++-- cmd/lotus-wallet/main.go | 6 +++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 36b0fb040..71ec067c8 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -1,9 +1,12 @@ package ledgerwallet import ( + "bytes" "context" "encoding/json" "fmt" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" @@ -27,7 +30,7 @@ type LedgerKeyInfo struct { Path []uint32 } -var _ (api.WalletAPI) = (*LedgerWallet)(nil) +var _ api.WalletAPI = (*LedgerWallet)(nil) func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta api.MsgMeta) (*crypto.Signature, error) { ki, err := lw.getKeyInfo(signer) @@ -44,7 +47,21 @@ func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, t return nil, fmt.Errorf("ledger can only sign chain messages") } - // TODO: assert meta matches the 'toSign' bits + { + var cmsg types.Message + if err := cmsg.UnmarshalCBOR(bytes.NewReader(meta.Extra)); err != nil { + return nil, xerrors.Errorf("unmarshalling message: %w", err) + } + + _, bc, err := cid.CidFromBytes(toSign) + if err != nil { + return nil, xerrors.Errorf("getting cid from signing bytes: %w", err) + } + + if !cmsg.Cid().Equals(bc) { + return nil, xerrors.Errorf("cid(meta.Extra).bytes() != toSign") + } + } sig, err := fl.SignSECP256K1(ki.Path, meta.Extra) if err != nil { diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index 2d87998a4..ffaddc7e3 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "github.com/filecoin-project/lotus/api" "net" "net/http" "os" @@ -12,6 +11,8 @@ import ( "github.com/urfave/cli/v2" "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/wallet" ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" @@ -63,7 +64,7 @@ var runCmd = &cli.Command{ Value: "0.0.0.0:1777", }, &cli.BoolFlag{ - Name: "ledger", + Name: "ledger", Usage: "use a ledger device instead of an on-disk wallet", }, }, @@ -115,7 +116,6 @@ var runCmd = &cli.Command{ w = ledgerwallet.NewWallet(ds) } - address := cctx.String("listen") mux := mux.NewRouter() From 2c9215723344c106f9508df7e7c13fde6def76d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 13:41:19 +0200 Subject: [PATCH 006/106] shed ledger: Some UX improvements Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/ledger.go | 55 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index 4bb61e5e0..fd7ac1b51 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -7,11 +7,15 @@ import ( "strings" "github.com/filecoin-project/go-address" + "github.com/urfave/cli/v2" + ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" - "github.com/urfave/cli/v2" - ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" + lcli "github.com/filecoin-project/lotus/cli" ) var ledgerCmd = &cli.Command{ @@ -27,7 +31,26 @@ var ledgerCmd = &cli.Command{ var ledgerListAddressesCmd = &cli.Command{ Name: "list", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "print-balances", + Usage: "print balances", + Aliases: []string{"b"}, + }, + }, Action: func(cctx *cli.Context) error { + var api api.FullNode + if cctx.Bool("print-balances") { + a, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + + api = a + + defer closer() + } + ctx := lcli.ReqContext(cctx) fl, err := ledgerfil.FindLedgerFilecoinApp() if err != nil { @@ -35,6 +58,10 @@ var ledgerListAddressesCmd = &cli.Command{ } for i := 0; i < 20; i++ { + if err := ctx.Err(); err != nil { + return err + } + p := []uint32{0x80000000 + 44, 0x80000000 + 461, 0x80000000, 0, uint32(i)} pubk, err := fl.GetPublicKeySECP256K1(p) if err != nil { @@ -46,7 +73,17 @@ var ledgerListAddressesCmd = &cli.Command{ return err } - fmt.Printf("%s: %s\n", addr, printHDPath(p)) + if cctx.Bool("print-balances") && api != nil { // api check makes linter happier + b, err := api.WalletBalance(ctx, addr) + if err != nil { + return xerrors.Errorf("getting balance: %w", err) + } + + fmt.Printf("%s %s %s\n", addr, printHDPath(p), types.FIL(b)) + } else { + fmt.Printf("%s %s\n", addr, printHDPath(p)) + } + } return nil @@ -110,6 +147,12 @@ func numlist(p []uint32) string { var ledgerKeyInfoCmd = &cli.Command{ Name: "key-info", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + }, + }, Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { return cli.ShowCommandHelp(cctx, cctx.Command.Name) @@ -129,8 +172,10 @@ var ledgerKeyInfoCmd = &cli.Command{ if err != nil { return err } - fmt.Println(addr) - fmt.Println(pubk) + if cctx.Bool("verbose") { + fmt.Println(addr) + fmt.Println(pubk) + } a, err := address.NewFromString(addr) if err != nil { From 8ae66d6c6a2aa27105566bbdf41dd569b2e4a109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 14:06:03 +0200 Subject: [PATCH 007/106] Fix lint warnings Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 12 +++++++----- cmd/lotus-shed/ledger.go | 14 +++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 71ec067c8..94a828603 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -5,16 +5,18 @@ import ( "context" "encoding/json" "fmt" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" - ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" ) type LedgerWallet struct { @@ -42,7 +44,7 @@ func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, t if err != nil { return nil, err } - defer fl.Close() + defer fl.Close() // nolint:errcheck if meta.Type != api.MTChainMsg { return nil, fmt.Errorf("ledger can only sign chain messages") } @@ -82,7 +84,7 @@ func (lw LedgerWallet) getKeyInfo(addr address.Address) (*LedgerKeyInfo, error) var out LedgerKeyInfo if err := json.Unmarshal(kib, &out); err != nil { - return nil, err + return nil, xerrors.Errorf("unmarshalling ledger key info: %w", err) } return &out, nil diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index fd7ac1b51..8111cc6bf 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -33,8 +33,8 @@ var ledgerListAddressesCmd = &cli.Command{ Name: "list", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "print-balances", - Usage: "print balances", + Name: "print-balances", + Usage: "print balances", Aliases: []string{"b"}, }, }, @@ -137,19 +137,11 @@ func printHDPath(pth []uint32) string { return strings.TrimRight(s, "/") } -func numlist(p []uint32) string { - var out []string - for _, v := range p { - out = append(out, fmt.Sprint(v)) - } - return strings.Join(out, ",") -} - var ledgerKeyInfoCmd = &cli.Command{ Name: "key-info", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "verbose", + Name: "verbose", Aliases: []string{"v"}, }, }, From bb6a354b6269949529bc9791b62ec34c7562d74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 14:17:44 +0200 Subject: [PATCH 008/106] chain cli: decode params command Signed-off-by: Jakub Sztandera --- cli/chain.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/cli/chain.go b/cli/chain.go index 763752f23..4dfcf3f36 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "context" + "encoding/hex" "encoding/json" "fmt" "os" @@ -53,6 +54,7 @@ var chainCmd = &cli.Command{ slashConsensusFault, chainGasPriceCmd, chainInspectUsage, + chainDecodeCmd, }, } @@ -1233,3 +1235,68 @@ var chainGasPriceCmd = &cli.Command{ return nil }, } + +var chainDecodeCmd = &cli.Command{ + Name: "decode", + Usage: "decode various types", + Subcommands: []*cli.Command{ + chainDecodeParamsCmd, + }, +} + +var chainDecodeParamsCmd = &cli.Command{ + Name: "params", + Usage: "Decode message params", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "tipset", + }, + }, + ArgsUsage: "[toAddr method hexParams]", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + if cctx.Args().Len() != 3 { + return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) + } + + to, err := address.NewFromString(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("parsing toAddr: %w", err) + } + + method, err := strconv.ParseInt(cctx.Args().Get(1), 10, 64) + if err != nil { + return xerrors.Errorf("parsing method id: %w", err) + } + + params, err := hex.DecodeString(cctx.Args().Get(2)) + if err != nil { + return xerrors.Errorf("parsing hex params: %w", err) + } + + ts, err := LoadTipSet(ctx, cctx, api) + if err != nil { + return err + } + + act, err := api.StateGetActor(ctx, to, ts.Key()) + if err != nil { + return xerrors.Errorf("getting actor: %w", err) + } + + pstr, err := jsonParams(act.Code, abi.MethodNum(method), params) + if err != nil { + return err + } + + fmt.Println(pstr) + + return nil + }, +} From 753d7b71a9e3ac330ec29fd9c41a9eba824bd8ec Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 10 Oct 2020 22:59:40 +0200 Subject: [PATCH 009/106] Fix hardened path printing, show 20 addresses after empty Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/ledger.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index 8111cc6bf..84d582dea 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -29,6 +29,8 @@ var ledgerCmd = &cli.Command{ }, } +const hdHard = 0x80000000 + var ledgerListAddressesCmd = &cli.Command{ Name: "list", Flags: []cli.Flag{ @@ -57,12 +59,13 @@ var ledgerListAddressesCmd = &cli.Command{ return err } - for i := 0; i < 20; i++ { + end := 20 + for i := 0; i < end; i++ { if err := ctx.Err(); err != nil { return err } - p := []uint32{0x80000000 + 44, 0x80000000 + 461, 0x80000000, 0, uint32(i)} + p := []uint32{hdHard | 44, hdHard | 461, hdHard | 0, 0, uint32(i)} pubk, err := fl.GetPublicKeySECP256K1(p) if err != nil { return err @@ -78,6 +81,9 @@ var ledgerListAddressesCmd = &cli.Command{ if err != nil { return xerrors.Errorf("getting balance: %w", err) } + if !b.IsZero() { + end = i + 21 // BIP32 spec, stop after 20 empty addresses + } fmt.Printf("%s %s %s\n", addr, printHDPath(p), types.FIL(b)) } else { @@ -108,12 +114,12 @@ func parseHDPath(s string) ([]uint32, error) { if err != nil { return nil, err } - if v >= 0x80000000 { + if v >= hdHard { return nil, fmt.Errorf("path element %s too large", p) } if hard { - v += 0x80000000 + v += hdHard } out = append(out, uint32(v)) } @@ -121,20 +127,20 @@ func parseHDPath(s string) ([]uint32, error) { } func printHDPath(pth []uint32) string { - s := "m/" + s := "m" for _, p := range pth { - var hard bool - if p >= 0x80000000 { - p -= 0x80000000 - } + s += "/" + + hard := p&hdHard != 0 + p &^= hdHard // remove hdHard bit + s += fmt.Sprint(p) if hard { s += "'" } - s += "/" } - return strings.TrimRight(s, "/") + return s } var ledgerKeyInfoCmd = &cli.Command{ @@ -164,6 +170,7 @@ var ledgerKeyInfoCmd = &cli.Command{ if err != nil { return err } + if cctx.Bool("verbose") { fmt.Println(addr) fmt.Println(pubk) From 7ee6bafd2312f8894eee66bd2c57fe3015118ede Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 10 Oct 2020 23:13:22 +0200 Subject: [PATCH 010/106] Fix lint warn Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/ledger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index 84d582dea..f8267ce12 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -65,7 +65,7 @@ var ledgerListAddressesCmd = &cli.Command{ return err } - p := []uint32{hdHard | 44, hdHard | 461, hdHard | 0, 0, uint32(i)} + p := []uint32{hdHard | 44, hdHard | 461, hdHard, 0, uint32(i)} pubk, err := fl.GetPublicKeySECP256K1(p) if err != nil { return err From 6a232e7214ae6e21c2b6c7f5ac5735b694cc47a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 11 Oct 2020 00:08:12 +0200 Subject: [PATCH 011/106] Basic multi-wallet support Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 17 ++- chain/wallet/multi.go | 160 ++++++++++++++++++++++++++++ chain/wallet/remotewallet/remote.go | 8 ++ chain/wallet/wallet.go | 21 ++++ node/builder.go | 13 ++- node/config/def.go | 2 + 6 files changed, 215 insertions(+), 6 deletions(-) create mode 100644 chain/wallet/multi.go diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 94a828603..231b3faae 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -17,13 +17,14 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) type LedgerWallet struct { ds datastore.Datastore } -func NewWallet(ds datastore.Datastore) *LedgerWallet { +func NewWallet(ds dtypes.MetadataDS) *LedgerWallet { return &LedgerWallet{ds} } @@ -127,7 +128,7 @@ func (lw LedgerWallet) WalletImport(ctx context.Context, kinfo *types.KeyInfo) ( } func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error) { - res, err := lw.ds.Query(query.Query{Prefix: "ledgerkey/"}) + res, err := lw.ds.Query(query.Query{Prefix: "/ledgerkey/"}) if err != nil { return nil, err } @@ -153,6 +154,14 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t crypto.SigType) (address return address.Undef, fmt.Errorf("cannot create new address on ledger") } -func keyForAddr(addr address.Address) datastore.Key { - return datastore.NewKey("ledgerkey/" + addr.String()) +func (lw *LedgerWallet) Get() api.WalletAPI { + if lw == nil { + return nil + } + + return lw +} + +func keyForAddr(addr address.Address) datastore.Key { + return datastore.NewKey("/ledgerkey/" + addr.String()) } diff --git a/chain/wallet/multi.go b/chain/wallet/multi.go new file mode 100644 index 000000000..bff8d7003 --- /dev/null +++ b/chain/wallet/multi.go @@ -0,0 +1,160 @@ +package wallet + +import ( + "context" + + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" + "github.com/filecoin-project/lotus/chain/wallet/remotewallet" +) + +type MultiWallet struct { + fx.In // "constructed" with fx.In instead of normal constructor + + Local *LocalWallet `optional:"true"` + Remote *remotewallet.RemoteWallet `optional:"true"` + Ledger *ledgerwallet.LedgerWallet `optional:"true"` +} + +type getif interface { + api.WalletAPI + + // workaround for the fact that iface(*struct(nil)) != nil + Get() api.WalletAPI +} + +func firstNonNil(wallets ...getif) api.WalletAPI { + for _, w := range wallets { + if w.Get() != nil { + return w + } + } + + return nil +} + +func nonNil(wallets ...getif) []api.WalletAPI { + var out []api.WalletAPI + for _, w := range wallets { + if w.Get() == nil { + continue + } + + out = append(out, w) + } + + return out +} + +func (m MultiWallet) find(ctx context.Context, address address.Address, wallets ...getif) (api.WalletAPI, error) { + ws := nonNil(wallets...) + + for _, w := range ws { + have, err := w.WalletHas(ctx, address) + if err != nil { + return nil, err + } + + if have { + return w, nil + } + } + + return nil, nil +} + +func (m MultiWallet) WalletNew(ctx context.Context, sigType crypto.SigType) (address.Address, error) { + w := firstNonNil(m.Remote, m.Local) + if w == nil { + return address.Undef, xerrors.Errorf("no wallet backends configured") + } + + return w.WalletNew(ctx, sigType) +} + +func (m MultiWallet) WalletHas(ctx context.Context, address address.Address) (bool, error) { + w, err := m.find(ctx, address, m.Remote, m.Ledger, m.Local) + return w != nil, err +} + +func (m MultiWallet) WalletList(ctx context.Context) ([]address.Address, error) { + var out []address.Address + seen := map[address.Address]struct{}{} + + ws := nonNil(m.Remote, m.Ledger, m.Local) + for _, w := range ws { + l, err := w.WalletList(ctx) + if err != nil { + return nil, err + } + + for _, a := range l { + if _, ok := seen[a]; ok { + continue + } + seen[a] = struct{}{} + + out = append(out, a) + } + } + + return out, nil +} + +func (m MultiWallet) WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta api.MsgMeta) (*crypto.Signature, error) { + w, err := m.find(ctx, signer, m.Remote, m.Ledger, m.Local) + if err != nil { + return nil, err + } + if w == nil { + return nil, xerrors.Errorf("key not found") + } + + return w.WalletSign(ctx, signer, toSign, meta) +} + +func (m MultiWallet) WalletExport(ctx context.Context, address address.Address) (*types.KeyInfo, error) { + w, err := m.find(ctx, address, m.Remote, m.Local) + if err != nil { + return nil, err + } + if w == nil { + return nil, xerrors.Errorf("key not found") + } + + return w.WalletExport(ctx, address) +} + +func (m MultiWallet) WalletImport(ctx context.Context, info *types.KeyInfo) (address.Address, error) { + w := firstNonNil(m.Remote, m.Ledger, m.Local) + if w == nil { + return address.Undef, xerrors.Errorf("no wallet backends configured") + } + + return w.WalletImport(ctx, info) +} + +func (m MultiWallet) WalletDelete(ctx context.Context, address address.Address) error { + for { + w, err := m.find(ctx, address, m.Remote, m.Ledger, m.Local) + if err != nil { + return err + } + if w == nil { + return nil + } + + if err := w.WalletDelete(ctx, address); err != nil { + return err + } + } +} + +var _ api.WalletAPI = MultiWallet{} diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index c7192f496..aa4427132 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -40,3 +40,11 @@ func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycl return &RemoteWallet{wapi}, nil } } + +func (w *RemoteWallet) Get() api.WalletAPI { + if w == nil { + return nil + } + + return w +} diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 57e63625b..66529d889 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -300,6 +300,14 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er return nil } +func (w *LocalWallet) Get() api.WalletAPI { + if w == nil { + return nil + } + + return w +} + var _ api.WalletAPI = &LocalWallet{} func swapMainnetForTestnetPrefix(addr string) (string, error) { @@ -312,3 +320,16 @@ func swapMainnetForTestnetPrefix(addr string) (string, error) { aChars[0] = prefixRunes[0] return string(aChars), nil } + +type nilDefault struct{} + +func (n nilDefault) GetDefault() (address.Address, error) { + return address.Undef, nil +} + +func (n nilDefault) SetDefault(a address.Address) error { + return xerrors.Errorf("not supported; local wallet disabled") +} + +var NilDefault nilDefault +var _ Default = NilDefault diff --git a/node/builder.go b/node/builder.go index b91172386..f96505eec 100644 --- a/node/builder.go +++ b/node/builder.go @@ -44,6 +44,7 @@ import ( "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" "github.com/filecoin-project/lotus/chain/wallet/remotewallet" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -251,8 +252,8 @@ func Online() Option { Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.LocalWallet), wallet.NewWallet), - Override(new(api.WalletAPI), From(new(*wallet.LocalWallet))), Override(new(wallet.Default), From(new(*wallet.LocalWallet))), + Override(new(api.WalletAPI), From(new(wallet.MultiWallet))), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), @@ -459,8 +460,16 @@ func ConfigFullNode(c interface{}) Option { If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), ), + If(cfg.Wallet.RemoteBackend != "", - Override(new(api.WalletAPI), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + Override(new(*remotewallet.RemoteWallet), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + ), + If(cfg.Wallet.EnableLedger, + Override(new(*ledgerwallet.LedgerWallet), ledgerwallet.NewWallet), + ), + If(cfg.Wallet.DisableLocal, + Unset(new(*wallet.LocalWallet)), + Override(new(wallet.Default), wallet.NilDefault), ), ) } diff --git a/node/config/def.go b/node/config/def.go index 63340cfd5..7eb7e19f0 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -110,6 +110,8 @@ type Client struct { type Wallet struct { RemoteBackend string + EnableLedger bool + DisableLocal bool } func defCommon() Common { From 2b21fdef3348dcc5799a9ecc21c04cfdd40e9cc1 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 11 Oct 2020 20:12:01 +0200 Subject: [PATCH 012/106] Refactor from crypto.SigType to types.KeyType Signed-off-by: Jakub Sztandera --- api/api_full.go | 4 +- api/api_wallet.go | 2 +- api/apistruct/struct.go | 8 ++-- api/docgen/docgen.go | 1 + api/test/paych.go | 3 +- chain/gen/gen.go | 4 +- chain/messagepool/messagepool_test.go | 25 ++++++------ chain/messagepool/repub_test.go | 6 +-- chain/messagepool/selection_test.go | 43 +++++++++++---------- chain/messagesigner/messagesigner_test.go | 9 ++--- chain/types/keystore.go | 46 ++++++++++++++++++++++- chain/vectors/gen/main.go | 5 +-- chain/wallet/key.go | 30 ++++++++------- chain/wallet/ledger/ledger.go | 2 +- chain/wallet/multi.go | 11 ++++-- chain/wallet/wallet.go | 4 +- cli/paych_test.go | 3 +- cli/wallet.go | 7 ++-- cmd/chain-noise/main.go | 4 +- cmd/lotus-gateway/endtoend_test.go | 5 +-- cmd/lotus-keygen/main.go | 8 ++-- cmd/lotus-seed/seed/seed.go | 3 +- cmd/lotus-shed/keyinfo.go | 20 +++++----- cmd/lotus-shed/ledger.go | 3 +- cmd/lotus-storage-miner/init.go | 3 +- cmd/lotus-wallet/logged.go | 9 +---- cmd/lotus-wallet/main.go | 19 ++++++---- documentation/en/api-methods.md | 8 ++-- node/modules/lp2p/libp2p.go | 4 +- scripts/dev/api.bash | 4 +- storage/mockstorage/preseal.go | 3 +- 31 files changed, 173 insertions(+), 133 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index a2fe94ee9..f42c8d03f 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -229,7 +229,9 @@ type FullNode interface { // MethodGroup: Wallet // WalletNew creates a new address in the wallet with the given sigType. - WalletNew(context.Context, crypto.SigType) (address.Address, error) + // Available key types: bls, secp256k1, secp256k1-ledger + // Support for numerical types: 1 - secp256k1, 2 - BLS is deprecated + WalletNew(context.Context, types.KeyType) (address.Address, error) // WalletHas indicates whether the given address is in the wallet. WalletHas(context.Context, address.Address) (bool, error) // WalletList lists all the addresses in the wallet. diff --git a/api/api_wallet.go b/api/api_wallet.go index 1213ffc1d..88ad8f43a 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -35,7 +35,7 @@ type MsgMeta struct { } type WalletAPI interface { - WalletNew(context.Context, crypto.SigType) (address.Address, error) + WalletNew(context.Context, types.KeyType) (address.Address, error) WalletHas(context.Context, address.Address) (bool, error) WalletList(context.Context) ([]address.Address, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 22d50e726..cef335939 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -133,7 +133,7 @@ type FullNodeStruct struct { MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"` MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error) `perm:"write"` - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletNew func(context.Context, types.KeyType) (address.Address, error) `perm:"write"` WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` WalletList func(context.Context) ([]address.Address, error) `perm:"write"` WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"` @@ -384,7 +384,7 @@ type GatewayStruct struct { type WalletStruct struct { Internal struct { - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletNew func(context.Context, types.KeyType) (address.Address, error) `perm:"write"` WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` WalletList func(context.Context) ([]address.Address, error) `perm:"write"` WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` @@ -631,7 +631,7 @@ func (c *FullNodeStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.Chain return c.Internal.ChainGetTipSetByHeight(ctx, h, tsk) } -func (c *FullNodeStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { +func (c *FullNodeStruct) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) { return c.Internal.WalletNew(ctx, typ) } @@ -1465,7 +1465,7 @@ func (g GatewayStruct) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence return g.Internal.StateWaitMsg(ctx, msg, confidence) } -func (c *WalletStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { +func (c *WalletStruct) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) { return c.Internal.WalletNew(ctx, typ) } diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index ced536cc3..dc6004121 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -87,6 +87,7 @@ func init() { addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1) addExample(abi.ChainEpoch(10101)) addExample(crypto.SigTypeBLS) + addExample(types.KTBLS) addExample(int64(9)) addExample(12.3) addExample(123) diff --git a/api/test/paych.go b/api/test/paych.go index a8ccebdde..43401554d 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -26,7 +26,6 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" ) func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { @@ -58,7 +57,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { bm.MineBlocks() // send some funds to register the receiver - receiverAddr, err := paymentReceiver.WalletNew(ctx, wallet.ActSigType("secp256k1")) + receiverAddr, err := paymentReceiver.WalletNew(ctx, types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 9133f14b4..288c84219 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -155,14 +155,14 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("creating memrepo wallet failed: %w", err) } - banker, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + banker, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { return nil, xerrors.Errorf("failed to generate banker key: %w", err) } receievers := make([]address.Address, msgsPerBlock) for r := range receievers { - receievers[r], err = w.WalletNew(context.Background(), crypto.SigTypeBLS) + receievers[r], err = w.WalletNew(context.Background(), types.KTBLS) if err != nil { return nil, xerrors.Errorf("failed to generate receiver key: %w", err) } diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 063e0bbab..8f6613cad 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -8,7 +8,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" @@ -232,7 +231,7 @@ func TestMessagePool(t *testing.T) { a := tma.nextBlock() - sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -273,7 +272,7 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) { a := tma.nextBlock() - sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { t.Fatal(err) } @@ -323,7 +322,7 @@ func TestRevertMessages(t *testing.T) { a := tma.nextBlock() b := tma.nextBlock() - sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { t.Fatal(err) } @@ -386,7 +385,7 @@ func TestPruningSimple(t *testing.T) { a := tma.nextBlock() tma.applyBlock(t, a) - sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { t.Fatal(err) } @@ -433,7 +432,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -443,7 +442,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -505,7 +504,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -515,7 +514,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -559,7 +558,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -569,7 +568,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -620,7 +619,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -630,7 +629,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/repub_test.go b/chain/messagepool/repub_test.go index 3e1252eec..9cadf24c7 100644 --- a/chain/messagepool/repub_test.go +++ b/chain/messagepool/repub_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-datastore" @@ -33,7 +33,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -43,7 +43,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 9e4fe39e5..37c2f2dc0 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -18,7 +18,6 @@ import ( logging "github.com/ipfs/go-log" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" @@ -77,7 +76,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -87,7 +86,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -315,7 +314,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -325,7 +324,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -391,7 +390,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -401,7 +400,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -535,7 +534,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -545,7 +544,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -598,7 +597,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -608,7 +607,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -677,7 +676,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -687,7 +686,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -746,7 +745,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -756,7 +755,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -843,7 +842,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -853,7 +852,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -910,7 +909,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -920,7 +919,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -994,7 +993,7 @@ func TestOptimalMessageSelection3(t *testing.T) { t.Fatal(err) } - a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -1074,7 +1073,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu t.Fatal(err) } - a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -1344,7 +1343,7 @@ readLoop: t.Fatal(err) } - a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index 0dafce9a1..5eebd36da 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/go-state-types/crypto" "github.com/stretchr/testify/require" ds_sync "github.com/ipfs/go-datastore/sync" @@ -47,13 +46,13 @@ func TestMessageSignerSignMessage(t *testing.T) { ctx := context.Background() w, _ := wallet.NewWallet(wallet.NewMemKeyStore()) - from1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) + from1, err := w.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) - from2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) + from2, err := w.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) - to1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) + to1, err := w.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) - to2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) + to2, err := w.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) type msgSpec struct { diff --git a/chain/types/keystore.go b/chain/types/keystore.go index 76eb5f296..107c1fbe3 100644 --- a/chain/types/keystore.go +++ b/chain/types/keystore.go @@ -1,7 +1,10 @@ package types import ( + "encoding/json" "fmt" + + "github.com/filecoin-project/go-state-types/crypto" ) var ( @@ -9,9 +12,50 @@ var ( ErrKeyExists = fmt.Errorf("key already exists") ) +// KeyType defines a type of a key +type KeyType string + +func (kt *KeyType) UnmarshalJSON(bb []byte) error { + { + // first option, try unmarshaling as string + var s string + err := json.Unmarshal(bb, &s) + if err == nil { + *kt = KeyType(s) + return nil + } + } + + { + var b byte + err := json.Unmarshal(bb, &b) + if err != nil { + return fmt.Errorf("could not unmarshal KeyType either as string nor integer: %w", err) + } + bst := crypto.SigType(b) + + switch bst { + case crypto.SigTypeBLS: + *kt = KTBLS + case crypto.SigTypeSecp256k1: + *kt = KTSecp256k1 + default: + return fmt.Errorf("unknown sigtype: %d", bst) + } + log.Warnf("deprecation: integer style 'KeyType' is deprecated, switch to string style") + return nil + } +} + +const ( + KTBLS KeyType = "bls" + KTSecp256k1 KeyType = "secp256k1" + KTSecp256k1Ledger KeyType = "secp256k1-ledger" +) + // KeyInfo is used for storing keys in KeyStore type KeyInfo struct { - Type string + Type KeyType PrivateKey []byte } diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index 096548e04..757227d0d 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -11,7 +11,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" @@ -62,7 +61,7 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { panic(err) } - blsk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + blsk, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { panic(err) } @@ -86,7 +85,7 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { Signature: &bmsg.Signature, } - secpk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + secpk, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { panic(err) } diff --git a/chain/wallet/key.go b/chain/wallet/key.go index 4b746a17a..ac9fc436f 100644 --- a/chain/wallet/key.go +++ b/chain/wallet/key.go @@ -10,13 +10,17 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" ) -func GenerateKey(typ crypto.SigType) (*Key, error) { - pk, err := sigs.Generate(typ) +func GenerateKey(typ types.KeyType) (*Key, error) { + ctyp := ActSigType(typ) + if ctyp == crypto.SigTypeUnknown { + return nil, xerrors.Errorf("unknown key type: %s", typ) + } + pk, err := sigs.Generate(ctyp) if err != nil { return nil, err } ki := types.KeyInfo{ - Type: kstoreSigType(typ), + Type: typ, PrivateKey: pk, } return NewKey(ki) @@ -41,41 +45,41 @@ func NewKey(keyinfo types.KeyInfo) (*Key, error) { } switch k.Type { - case KTSecp256k1: + case types.KTSecp256k1: k.Address, err = address.NewSecp256k1Address(k.PublicKey) if err != nil { return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) } - case KTBLS: + case types.KTBLS: k.Address, err = address.NewBLSAddress(k.PublicKey) if err != nil { return nil, xerrors.Errorf("converting BLS to address: %w", err) } default: - return nil, xerrors.Errorf("unknown key type") + return nil, xerrors.Errorf("unknown key type: %s", k.Type) } return k, nil } -func kstoreSigType(typ crypto.SigType) string { +func kstoreSigType(typ crypto.SigType) types.KeyType { switch typ { case crypto.SigTypeBLS: - return KTBLS + return types.KTBLS case crypto.SigTypeSecp256k1: - return KTSecp256k1 + return types.KTSecp256k1 default: return "" } } -func ActSigType(typ string) crypto.SigType { +func ActSigType(typ types.KeyType) crypto.SigType { switch typ { - case KTBLS: + case types.KTBLS: return crypto.SigTypeBLS - case KTSecp256k1: + case types.KTSecp256k1: return crypto.SigTypeSecp256k1 default: - return 0 + return crypto.SigTypeUnknown } } diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 231b3faae..b6b5683cd 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -150,7 +150,7 @@ func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error return out, nil } -func (lw LedgerWallet) WalletNew(ctx context.Context, t crypto.SigType) (address.Address, error) { +func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address.Address, error) { return address.Undef, fmt.Errorf("cannot create new address on ledger") } diff --git a/chain/wallet/multi.go b/chain/wallet/multi.go index bff8d7003..6eac03367 100644 --- a/chain/wallet/multi.go +++ b/chain/wallet/multi.go @@ -18,7 +18,7 @@ import ( type MultiWallet struct { fx.In // "constructed" with fx.In instead of normal constructor - Local *LocalWallet `optional:"true"` + Local *LocalWallet `optional:"true"` Remote *remotewallet.RemoteWallet `optional:"true"` Ledger *ledgerwallet.LedgerWallet `optional:"true"` } @@ -70,7 +70,7 @@ func (m MultiWallet) find(ctx context.Context, address address.Address, wallets return nil, nil } -func (m MultiWallet) WalletNew(ctx context.Context, sigType crypto.SigType) (address.Address, error) { +func (m MultiWallet) WalletNew(ctx context.Context, sigType types.KeyType) (address.Address, error) { w := firstNonNil(m.Remote, m.Local) if w == nil { return address.Undef, xerrors.Errorf("no wallet backends configured") @@ -133,7 +133,12 @@ func (m MultiWallet) WalletExport(ctx context.Context, address address.Address) } func (m MultiWallet) WalletImport(ctx context.Context, info *types.KeyInfo) (address.Address, error) { - w := firstNonNil(m.Remote, m.Ledger, m.Local) + var local getif = m.Local + if info.Type == types.KTSecp256k1Ledger { + local = m.Ledger + } + + w := firstNonNil(m.Remote, local) if w == nil { return address.Undef, xerrors.Errorf("no wallet backends configured") } diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 66529d889..925547ea8 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -26,8 +26,6 @@ const ( KNamePrefix = "wallet-" KTrashPrefix = "trash-" KDefault = "default" - KTBLS = "bls" - KTSecp256k1 = "secp256k1" ) type LocalWallet struct { @@ -236,7 +234,7 @@ func (w *LocalWallet) SetDefault(a address.Address) error { return nil } -func (w *LocalWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { +func (w *LocalWallet) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() diff --git a/cli/paych_test.go b/cli/paych_test.go index 862ca2e74..b94ba27e7 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -29,7 +29,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" builder "github.com/filecoin-project/lotus/node/test" ) @@ -415,7 +414,7 @@ func startTwoNodesOneMiner(ctx context.Context, t *testing.T, blocktime time.Dur bm.MineBlocks() // Send some funds to register the receiver - receiverAddr, err := paymentReceiver.WalletNew(ctx, wallet.ActSigType("secp256k1")) + receiverAddr, err := paymentReceiver.WalletNew(ctx, types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/cli/wallet.go b/cli/wallet.go index aa5b9bed3..3d6abc357 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" types "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/tablewriter" ) @@ -55,7 +54,7 @@ var walletNew = &cli.Command{ t = "secp256k1" } - nk, err := api.WalletNew(ctx, wallet.ActSigType(t)) + nk, err := api.WalletNew(ctx, types.KeyType(t)) if err != nil { return err } @@ -329,9 +328,9 @@ var walletImport = &cli.Command{ ki.PrivateKey = gk.PrivateKey switch gk.SigType { case 1: - ki.Type = wallet.KTSecp256k1 + ki.Type = types.KTSecp256k1 case 2: - ki.Type = wallet.KTBLS + ki.Type = types.KTBLS default: return fmt.Errorf("unrecognized key type: %d", gk.SigType) } diff --git a/cmd/chain-noise/main.go b/cmd/chain-noise/main.go index 7b9824016..81586e1b2 100644 --- a/cmd/chain-noise/main.go +++ b/cmd/chain-noise/main.go @@ -7,8 +7,6 @@ import ( "os" "time" - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -61,7 +59,7 @@ var runCmd = &cli.Command{ func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Address, rate int) error { var sendSet []address.Address for i := 0; i < 20; i++ { - naddr, err := api.WalletNew(ctx, crypto.SigTypeSecp256k1) + naddr, err := api.WalletNew(ctx, types.KTSecp256k1) if err != nil { return err } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index 206034968..eebf56a9f 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -22,7 +22,6 @@ import ( "github.com/filecoin-project/lotus/api/test" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node" builder "github.com/filecoin-project/lotus/node/test" ) @@ -53,7 +52,7 @@ func TestEndToEnd(t *testing.T) { fmt.Println(balance) // Create a wallet on the lite node - liteWalletAddr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) // Send some funds from the full node to the lite node @@ -77,7 +76,7 @@ func TestEndToEnd(t *testing.T) { // Create some wallets on the lite node to use for testing multisig var walletAddrs []address.Address for i := 0; i < 4; i++ { - addr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + addr, err := lite.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) walletAddrs = append(walletAddrs, addr) diff --git a/cmd/lotus-keygen/main.go b/cmd/lotus-keygen/main.go index 4b971cf48..d296cb5da 100644 --- a/cmd/lotus-keygen/main.go +++ b/cmd/lotus-keygen/main.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" @@ -30,12 +30,12 @@ func main() { return err } - var kt crypto.SigType + var kt types.KeyType switch cctx.String("type") { case "bls": - kt = crypto.SigTypeBLS + kt = types.KTBLS case "secp256k1": - kt = crypto.SigTypeSecp256k1 + kt = types.KTSecp256k1 default: return fmt.Errorf("unrecognized key type: %q", cctx.String("type")) } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 5e911991d..cd0c4e7ab 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -21,7 +21,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" "github.com/filecoin-project/specs-actors/actors/builtin/market" @@ -93,7 +92,7 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect return nil, nil, err } } else { - minerAddr, err = wallet.GenerateKey(crypto.SigTypeBLS) + minerAddr, err = wallet.GenerateKey(types.KTBLS) if err != nil { return nil, nil, err } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index fdd1fcb49..4dcd10cbf 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -32,10 +32,10 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) -var validTypes = []string{wallet.KTBLS, wallet.KTSecp256k1, lp2p.KTLibp2pHost} +var validTypes = []types.KeyType{types.KTBLS, types.KTSecp256k1, lp2p.KTLibp2pHost} type keyInfoOutput struct { - Type string + Type types.KeyType Address string PublicKey string } @@ -86,7 +86,7 @@ var keyinfoVerifyCmd = &cli.Command{ return xerrors.Errorf("decoding key: '%s': %w", fileName, err) } - if string(name) != keyInfo.Type { + if types.KeyType(name) != keyInfo.Type { return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) } case modules.KTJwtHmacSecret: @@ -98,7 +98,7 @@ var keyinfoVerifyCmd = &cli.Command{ if string(name) != modules.JWTSecretName { return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) } - case wallet.KTSecp256k1, wallet.KTBLS: + case types.KTSecp256k1, types.KTBLS: keystore := wallet.NewMemKeyStore() w, err := wallet.NewWallet(keystore) if err != nil { @@ -214,7 +214,7 @@ var keyinfoImportCmd = &cli.Command{ fmt.Printf("%s\n", peerid.String()) break - case wallet.KTSecp256k1, wallet.KTBLS: + case types.KTSecp256k1, types.KTBLS: w, err := wallet.NewWallet(keystore) if err != nil { return err @@ -317,7 +317,7 @@ var keyinfoInfoCmd = &cli.Command{ kio.PublicKey = base64.StdEncoding.EncodeToString(pkBytes) break - case wallet.KTSecp256k1, wallet.KTBLS: + case types.KTSecp256k1, types.KTBLS: kio.Type = keyInfo.Type key, err := wallet.NewKey(keyInfo) @@ -366,7 +366,7 @@ var keyinfoNewCmd = &cli.Command{ return fmt.Errorf("please specify a type to generate") } - keyType := cctx.Args().First() + keyType := types.KeyType(cctx.Args().First()) flagOutput := cctx.String("output") if i := SliceIndex(len(validTypes), func(i int) bool { @@ -404,8 +404,8 @@ var keyinfoNewCmd = &cli.Command{ keyInfo = ki break - case wallet.KTSecp256k1, wallet.KTBLS: - key, err := wallet.GenerateKey(wallet.ActSigType(keyType)) + case types.KTSecp256k1, types.KTBLS: + key, err := wallet.GenerateKey(keyType) if err != nil { return err } @@ -418,7 +418,7 @@ var keyinfoNewCmd = &cli.Command{ filename := flagOutput filename = strings.ReplaceAll(filename, "", keyAddr) - filename = strings.ReplaceAll(filename, "", keyType) + filename = strings.ReplaceAll(filename, "", string(keyType)) file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index f8267ce12..f64916534 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" lcli "github.com/filecoin-project/lotus/cli" ) @@ -191,7 +190,7 @@ var ledgerKeyInfoCmd = &cli.Command{ } var ki types.KeyInfo - ki.Type = wallet.KTSecp256k1 + ki.Type = types.KTSecp256k1Ledger ki.PrivateKey = b out, err := json.Marshal(ki) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 0f830023f..ccea707af 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -27,7 +27,6 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" - crypto2 "github.com/filecoin-project/go-state-types/crypto" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -624,7 +623,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, if cctx.String("worker") != "" { worker, err = address.NewFromString(cctx.String("worker")) } else if cctx.Bool("create-worker-key") { // TODO: Do we need to force this if owner is Secpk? - worker, err = api.WalletNew(ctx, crypto2.SigTypeBLS) + worker, err = api.WalletNew(ctx, types.KTBLS) } // TODO: Transfer some initial funds to worker if err != nil { diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index 3bcb3f867..272a8d10b 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -19,13 +19,8 @@ type LoggedWallet struct { under api.WalletAPI } -func (c *LoggedWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { - n, err := typ.Name() - if err != nil { - return address.Address{}, err - } - - log.Infow("WalletNew", "type", n) +func (c *LoggedWallet) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) { + log.Infow("WalletNew", "type", typ) return c.under.WalletNew(ctx, typ) } diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index ffaddc7e3..3285b13e7 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -101,19 +101,22 @@ var runCmd = &cli.Command{ return err } - var w api.WalletAPI - if !cctx.Bool("ledger") { - w, err = wallet.NewWallet(ks) - if err != nil { - return err - } - } else { + lw, err := wallet.NewWallet(ks) + if err != nil { + return err + } + + var w api.WalletAPI = lw + if cctx.Bool("ledger") { ds, err := lr.Datastore("/metadata") if err != nil { return err } - w = ledgerwallet.NewWallet(ds) + w = wallet.MultiWallet{ + Local: lw, + Ledger: ledgerwallet.NewWallet(ds), + } } address := cctx.String("listen") diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ec8071b57..6d201b435 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -4710,7 +4710,7 @@ Inputs: Response: ```json { - "Type": "string value", + "Type": "bls", "PrivateKey": "Ynl0ZSBhcnJheQ==" } ``` @@ -4740,7 +4740,7 @@ Inputs: ```json [ { - "Type": "string value", + "Type": "bls", "PrivateKey": "Ynl0ZSBhcnJheQ==" } ] @@ -4760,6 +4760,8 @@ Response: `null` ### WalletNew WalletNew creates a new address in the wallet with the given sigType. +Available key types: bls, secp256k1, secp256k1-ledger +Support for numerical types: 1 - secp256k1, 2 - BLS is deprecated Perms: write @@ -4767,7 +4769,7 @@ Perms: write Inputs: ```json [ - 2 + "bls" ] ``` diff --git a/node/modules/lp2p/libp2p.go b/node/modules/lp2p/libp2p.go index 5a1666cb6..51749c4d6 100644 --- a/node/modules/lp2p/libp2p.go +++ b/node/modules/lp2p/libp2p.go @@ -20,8 +20,8 @@ import ( var log = logging.Logger("p2pnode") const ( - KLibp2pHost = "libp2p-host" - KTLibp2pHost = KLibp2pHost + KLibp2pHost = "libp2p-host" + KTLibp2pHost types.KeyType = KLibp2pHost ) type Libp2pOpts struct { diff --git a/scripts/dev/api.bash b/scripts/dev/api.bash index 5539e4fef..ac9eb4e66 100644 --- a/scripts/dev/api.bash +++ b/scripts/dev/api.bash @@ -1,11 +1,11 @@ #!/bin/bash # vim: set expandtab ts=2 sw=2: -token=$(lotus auth create-token --perm admin) +_lotus_token=$(./lotus auth create-token --perm admin) runAPI() { curl -X POST \ -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","id":2,"method":"Filecoin.'"$1"'","params":'"${2:-null}"'}' \ - 'http://127.0.0.1:1234/rpc/v0?token='"$token" + 'http://127.0.0.1:1234/rpc/v0?token='"$_lotus_token" } diff --git a/storage/mockstorage/preseal.go b/storage/mockstorage/preseal.go index 8ca789ba6..090aacc3c 100644 --- a/storage/mockstorage/preseal.go +++ b/storage/mockstorage/preseal.go @@ -7,7 +7,6 @@ import ( commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/mock" market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" @@ -20,7 +19,7 @@ import ( ) func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis.Miner, *types.KeyInfo, error) { - k, err := wallet.GenerateKey(crypto.SigTypeBLS) + k, err := wallet.GenerateKey(types.KTBLS) if err != nil { return nil, nil, err } From d6c8276c7a3e76e32484aaae745a0ccb2e6fef53 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 00:48:23 +0200 Subject: [PATCH 013/106] Add support for WalletNew to ledger Signed-off-by: Jakub Sztandera --- chain/wallet/key.go | 15 +------ chain/wallet/ledger/ledger.go | 76 +++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/chain/wallet/key.go b/chain/wallet/key.go index ac9fc436f..1b191cc4b 100644 --- a/chain/wallet/key.go +++ b/chain/wallet/key.go @@ -13,7 +13,7 @@ import ( func GenerateKey(typ types.KeyType) (*Key, error) { ctyp := ActSigType(typ) if ctyp == crypto.SigTypeUnknown { - return nil, xerrors.Errorf("unknown key type: %s", typ) + return nil, xerrors.Errorf("unknown sig type: %s", typ) } pk, err := sigs.Generate(ctyp) if err != nil { @@ -56,23 +56,12 @@ func NewKey(keyinfo types.KeyInfo) (*Key, error) { return nil, xerrors.Errorf("converting BLS to address: %w", err) } default: - return nil, xerrors.Errorf("unknown key type: %s", k.Type) + return nil, xerrors.Errorf("unsupported key type: %s", k.Type) } return k, nil } -func kstoreSigType(typ crypto.SigType) types.KeyType { - switch typ { - case crypto.SigTypeBLS: - return types.KTBLS - case crypto.SigTypeSecp256k1: - return types.KTSecp256k1 - default: - return "" - } -} - func ActSigType(typ types.KeyType) crypto.SigType { switch typ { case types.KTBLS: diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index b6b5683cd..9a0d13419 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -115,12 +115,22 @@ func (lw LedgerWallet) WalletImport(ctx context.Context, kinfo *types.KeyInfo) ( if err := json.Unmarshal(kinfo.PrivateKey, &ki); err != nil { return address.Undef, err } + return lw.importKey(ki) +} +func (lw LedgerWallet) importKey(ki LedgerKeyInfo) (address.Address, error) { if ki.Address == address.Undef { return address.Undef, fmt.Errorf("no address given in imported key info") } + if len(ki.Path) != filHdPathLen { + return address.Undef, fmt.Errorf("bad hd path len: %d, expected: %d", len(ki.Path), filHdPathLen) + } + bb, err := json.Marshal(ki) + if err != nil { + return address.Undef, xerrors.Errorf("marshaling key info: %w", err) + } - if err := lw.ds.Put(keyForAddr(ki.Address), kinfo.PrivateKey); err != nil { + if err := lw.ds.Put(keyForAddr(ki.Address), bb); err != nil { return address.Undef, err } @@ -128,10 +138,11 @@ func (lw LedgerWallet) WalletImport(ctx context.Context, kinfo *types.KeyInfo) ( } func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error) { - res, err := lw.ds.Query(query.Query{Prefix: "/ledgerkey/"}) + res, err := lw.ds.Query(query.Query{Prefix: dsLedgerPrefix}) if err != nil { return nil, err } + defer res.Close() var out []address.Address for { @@ -150,8 +161,63 @@ func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error return out, nil } +const hdHard = 0x80000000 + +var filHDBasePath = []uint32{hdHard | 44, hdHard | 461, hdHard, 0} +var filHdPathLen = 5 + func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address.Address, error) { - return address.Undef, fmt.Errorf("cannot create new address on ledger") + if t != types.KTSecp256k1Ledger { + return address.Undef, fmt.Errorf("unsupported key type: '%s', only '%s' supported", + t, types.KTSecp256k1Ledger) + } + + res, err := lw.ds.Query(query.Query{Prefix: dsLedgerPrefix}) + if err != nil { + return address.Undef, err + } + defer res.Close() + + var maxi int64 = -1 + for { + res, ok := res.NextSync() + if !ok { + break + } + + var ki LedgerKeyInfo + if err := json.Unmarshal(res.Value, &ki); err != nil { + return address.Undef, err + } + if i := ki.Path[filHdPathLen-1]; maxi == -1 || maxi < int64(i) { + maxi = int64(i) + } + } + if maxi == -1 { + maxi = 0 + } + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return address.Undef, xerrors.Errorf("finding ledger: %w", err) + } + + path := append(append([]uint32(nil), filHDBasePath...), uint32(maxi)) + _, _, addr, err := fl.GetAddressPubKeySECP256K1(path) + if err != nil { + return address.Undef, xerrors.Errorf("getting public key from ledger: %w", err) + } + + a, err := address.NewFromString(addr) + if err != nil { + return address.Undef, fmt.Errorf("parsing address: %w", err) + } + + var lki LedgerKeyInfo + lki.Address = a + lki.Path = path + + return lw.importKey(lki) } func (lw *LedgerWallet) Get() api.WalletAPI { @@ -162,6 +228,8 @@ func (lw *LedgerWallet) Get() api.WalletAPI { return lw } +var dsLedgerPrefix = "/ledgerkey/" + func keyForAddr(addr address.Address) datastore.Key { - return datastore.NewKey("/ledgerkey/" + addr.String()) + return datastore.NewKey(dsLedgerPrefix + addr.String()) } From 7b4a5fc7ca0cd0fd5fd6068c93005adbec3167db Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 00:50:29 +0200 Subject: [PATCH 014/106] Fixup off by one Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 9a0d13419..14c75b356 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -193,16 +193,13 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address. maxi = int64(i) } } - if maxi == -1 { - maxi = 0 - } fl, err := ledgerfil.FindLedgerFilecoinApp() if err != nil { return address.Undef, xerrors.Errorf("finding ledger: %w", err) } - path := append(append([]uint32(nil), filHDBasePath...), uint32(maxi)) + path := append(append([]uint32(nil), filHDBasePath...), uint32(maxi+1)) _, _, addr, err := fl.GetAddressPubKeySECP256K1(path) if err != nil { return address.Undef, xerrors.Errorf("getting public key from ledger: %w", err) From fda8b932bab6899e28089b72ef18ea366408fbf0 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 01:27:15 +0200 Subject: [PATCH 015/106] Ask for confirmation when creating a key Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 9 +++++++++ chain/wallet/multi.go | 15 ++++++++++----- go.mod | 3 ++- go.sum | 9 +++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 14c75b356..34307d192 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" + logging "github.com/ipfs/go-log" ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" "golang.org/x/xerrors" @@ -20,6 +21,8 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" ) +var log = logging.Logger("wallet-ledger") + type LedgerWallet struct { ds datastore.Datastore } @@ -205,6 +208,12 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address. return address.Undef, xerrors.Errorf("getting public key from ledger: %w", err) } + log.Warnf("creating key: %s, accept the key in ledger device", addr) + _, _, addr, err = fl.ShowAddressPubKeySECP256K1(path) + if err != nil { + return address.Undef, xerrors.Errorf("verifying public key with ledger: %w", err) + } + a, err := address.NewFromString(addr) if err != nil { return address.Undef, fmt.Errorf("parsing address: %w", err) diff --git a/chain/wallet/multi.go b/chain/wallet/multi.go index 6eac03367..532ad217b 100644 --- a/chain/wallet/multi.go +++ b/chain/wallet/multi.go @@ -70,13 +70,18 @@ func (m MultiWallet) find(ctx context.Context, address address.Address, wallets return nil, nil } -func (m MultiWallet) WalletNew(ctx context.Context, sigType types.KeyType) (address.Address, error) { - w := firstNonNil(m.Remote, m.Local) - if w == nil { - return address.Undef, xerrors.Errorf("no wallet backends configured") +func (m MultiWallet) WalletNew(ctx context.Context, keyType types.KeyType) (address.Address, error) { + var local getif = m.Local + if keyType == types.KTSecp256k1Ledger { + local = m.Ledger } - return w.WalletNew(ctx, sigType) + w := firstNonNil(m.Remote, local) + if w == nil { + return address.Undef, xerrors.Errorf("no wallet backends supporting key type: %s", keyType) + } + + return w.WalletNew(ctx, keyType) } func (m MultiWallet) WalletHas(ctx context.Context, address address.Address) (bool, error) { diff --git a/go.mod b/go.mod index d6a4768b7..922e4b6b5 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/drand/kyber v1.1.2 github.com/dustin/go-humanize v1.0.0 github.com/elastic/go-sysinfo v1.3.0 - github.com/fatih/color v1.8.0 + github.com/fatih/color v1.9.0 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d github.com/filecoin-project/go-address v0.0.4 github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect @@ -104,6 +104,7 @@ require ( github.com/libp2p/go-libp2p-tls v0.1.3 github.com/libp2p/go-libp2p-yamux v0.2.8 github.com/libp2p/go-maddr-filter v0.1.0 + github.com/mattn/go-colorable v0.1.6 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 diff --git a/go.sum b/go.sum index fe7aacbf9..6dfc9aac6 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:Jp github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-address v0.0.3 h1:eVfbdjEbpbzIrbiSa+PiGUY+oDK9HnUn+M1R/ggoHf8= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= @@ -1045,11 +1047,16 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1665,11 +1672,13 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From df7ee610cd426bbd20bdf6ccdeae4b9b4e89e3fe Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 01:29:19 +0200 Subject: [PATCH 016/106] Re-enable repub Signed-off-by: Jakub Sztandera --- chain/messagepool/messagepool.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 2a4704c3b..d3c638b22 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -1249,9 +1249,6 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err } func (mp *MessagePool) loadLocal() error { - return nil - // TODO: this causes super slow startup... - res, err := mp.localMsgs.Query(query.Query{}) if err != nil { return xerrors.Errorf("query local messages: %w", err) From b4215e5f46003284599f4736eb6f7003f9785ae5 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 11 Oct 2020 19:49:57 -0400 Subject: [PATCH 017/106] Add a CLI tool for miner's to repay debt --- chain/actors/builtin/miner/miner.go | 14 ++++ cmd/lotus-storage-miner/actor.go | 105 ++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 8649d4351..281068827 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -148,6 +148,20 @@ type MinerInfo struct { ConsensusFaultElapsed abi.ChainEpoch } +func (mi MinerInfo) IsController(addr address.Address) bool { + if addr == mi.Owner || addr == mi.Worker { + return true + } + + for _, ca := range mi.ControlAddresses { + if addr == ca { + return true + } + } + + return false +} + type SectorExpiration struct { OnTime abi.ChainEpoch diff --git a/cmd/lotus-storage-miner/actor.go b/cmd/lotus-storage-miner/actor.go index fa320289e..35e2a268e 100644 --- a/cmd/lotus-storage-miner/actor.go +++ b/cmd/lotus-storage-miner/actor.go @@ -5,6 +5,11 @@ import ( "os" "strings" + "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/filecoin-project/lotus/build" builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" @@ -34,6 +39,7 @@ var actorCmd = &cli.Command{ Subcommands: []*cli.Command{ actorSetAddrsCmd, actorWithdrawCmd, + actorRepayDebtCmd, actorSetPeeridCmd, actorSetOwnerCmd, actorControl, @@ -254,6 +260,105 @@ var actorWithdrawCmd = &cli.Command{ }, } +var actorRepayDebtCmd = &cli.Command{ + Name: "repay-debt", + Usage: "pay down a miner's debt", + ArgsUsage: "[amount (FIL)]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "optionally specify the account to send funds from", + }, + }, + Action: func(cctx *cli.Context) error { + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + api, acloser, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer acloser() + + ctx := lcli.ReqContext(cctx) + + maddr, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + + mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + var amount abi.TokenAmount + if cctx.Args().Present() { + f, err := types.ParseFIL(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("parsing 'amount' argument: %w", err) + } + + amount = abi.TokenAmount(f) + } else { + mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + store := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(api))) + + mst, err := miner.Load(store, mact) + if err != nil { + return err + } + + amount, err = mst.FeeDebt() + if err != nil { + return err + } + + } + + fromAddr := mi.Worker + if from := cctx.String("from"); from != "" { + addr, err := address.NewFromString(from) + if err != nil { + return err + } + + fromAddr = addr + } + + fromId, err := api.StateLookupID(ctx, fromAddr, types.EmptyTSK) + if err != nil { + return err + } + + if !mi.IsController(fromId) { + return xerrors.Errorf("sender isn't a controller of miner: %s", fromId) + } + + smsg, err := api.MpoolPushMessage(ctx, &types.Message{ + To: maddr, + From: fromId, + Value: amount, + Method: builtin2.MethodsMiner.RepayDebt, + Params: nil, + }, nil) + if err != nil { + return err + } + + fmt.Printf("Sent repay debt message %s\n", smsg.Cid()) + + return nil + }, +} + var actorControl = &cli.Command{ Name: "control", Usage: "Manage control addresses", From a2e0832a12f220678ac73bf96542c747814b02bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 06:17:00 +0200 Subject: [PATCH 018/106] Fix lint --- chain/wallet/ledger/ledger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 34307d192..c1757fd0a 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -145,7 +145,7 @@ func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error if err != nil { return nil, err } - defer res.Close() + defer res.Close() // nolint:errcheck var out []address.Address for { @@ -179,7 +179,7 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address. if err != nil { return address.Undef, err } - defer res.Close() + defer res.Close() // nolint:errcheck var maxi int64 = -1 for { From 4ac2bee81ce92676e966959299a4cd1a18b3a6ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 06:34:45 +0200 Subject: [PATCH 019/106] Update oni --- extern/oni | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/oni b/extern/oni index dbee44e4f..10ed9ef57 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit dbee44e4f940a502971f17116ccbba61ceaf2674 +Subproject commit 10ed9ef576836186de3b8513c03cdc3fb18c44ed From 160f209f6a857eb4b16a8d96b48b0c0d1a0ac404 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 12 Oct 2020 02:08:38 -0400 Subject: [PATCH 020/106] Lotus version 0.10.0 --- CHANGELOG.md | 19 +++++++++++++++++++ build/version.go | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d397762a6..16d8d15b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Lotus changelog +# 0.10.0 / 2020-10-12 + +This is a consensus-breaking hotfix that addresses an issue in specs-actors v2.0.3 that made it impossible to pledge new 32GiB sectors. The change in Lotus is to update to actors v2.1.0, behind the new network version 5. + +## Changes + +- make pledge test pass with the race detector (https://github.com/filecoin-project/lotus/pull/4291) +- fix a race in tipset cache usage (https://github.com/filecoin-project/lotus/pull/4282) +- add an api for removing multisig signers (https://github.com/filecoin-project/lotus/pull/4274) +- cli: Don't output errors to stdout (https://github.com/filecoin-project/lotus/pull/4298) +- Fix panic in wallet export when key is not found (https://github.com/filecoin-project/lotus/pull/4299) +- Dump the block validation cache whenever we perform an import (https://github.com/filecoin-project/lotus/pull/4287) +- Fix two races (https://github.com/filecoin-project/lotus/pull/4301) +- sync unmark-bad --all (https://github.com/filecoin-project/lotus/pull/4296) +- decode parameters for multisig transactions in inspect (https://github.com/filecoin-project/lotus/pull/4312) +- Chain is love (https://github.com/filecoin-project/lotus/pull/4321) +- lotus-stats: optmize getting miner power (https://github.com/filecoin-project/lotus/pull/4315) +- implement tape upgrade (https://github.com/filecoin-project/lotus/pull/4322) + # 0.9.1 / 2020-10-10 This release fixes an issue which may cause the actors v2 migration to compute the state incorrectly when more than one migration is running in parallel. diff --git a/build/version.go b/build/version.go index 5baed1fb7..baeafdfe7 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.9.1" +const BuildVersion = "0.10.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From dc4e73c73724115f6fff6c17df608860c03de510 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 11 Oct 2020 23:31:23 -0700 Subject: [PATCH 021/106] Test the tape upgrade --- api/test/tape.go | 114 +++++++++++++++++++++++++++++ api/test/test.go | 4 +- extern/sector-storage/mock/mock.go | 9 ++- node/node_test.go | 10 +++ 4 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 api/test/tape.go diff --git a/api/test/tape.go b/api/test/tape.go new file mode 100644 index 000000000..466bdd829 --- /dev/null +++ b/api/test/tape.go @@ -0,0 +1,114 @@ +package test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/stmgr" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/node" + "github.com/filecoin-project/lotus/node/impl" + "github.com/stretchr/testify/require" +) + +func TestTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration) { + t.Run("before", func(t *testing.T) { testTapeFix(t, b, blocktime, false) }) + t.Run("after", func(t *testing.T) { testTapeFix(t, b, blocktime, true) }) +} +func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + upgradeSchedule := stmgr.UpgradeSchedule{{ + Network: build.ActorUpgradeNetworkVersion, + Height: 1, + Migration: stmgr.UpgradeActorsV2, + }} + if after { + upgradeSchedule = append(upgradeSchedule, stmgr.Upgrade{ + Network: network.Version5, + Height: 2, + }) + } + + n, sn := b(t, []FullNodeOpts{{Opts: func(_ []TestNode) node.Option { + return node.Override(new(stmgr.UpgradeSchedule), upgradeSchedule) + }}}, OneMiner) + + client := n[0].FullNode.(*impl.FullNodeAPI) + miner := sn[0] + + addrinfo, err := client.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrinfo); err != nil { + t.Fatal(err) + } + build.Clock.Sleep(time.Second) + + done := make(chan struct{}) + go func() { + defer close(done) + for ctx.Err() == nil { + build.Clock.Sleep(blocktime) + if err := sn[0].MineOne(ctx, MineNext); err != nil { + if ctx.Err() != nil { + // context was canceled, ignore the error. + return + } + t.Error(err) + } + } + }() + defer func() { + cancel() + <-done + }() + + err = miner.PledgeSector(ctx) + require.NoError(t, err) + + // Wait till done. + var sectorNo abi.SectorNumber + for { + s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM + require.NoError(t, err) + fmt.Printf("Sectors: %d\n", len(s)) + if len(s) == 1 { + sectorNo = s[0] + break + } + + build.Clock.Sleep(100 * time.Millisecond) + } + + fmt.Printf("All sectors is fsm\n") + + // If before, we expect the precommit to fail + successState := api.SectorState(sealing.CommitFailed) + failureState := api.SectorState(sealing.Proving) + if after { + // otherwise, it should succeed. + successState, failureState = failureState, successState + } + + for { + st, err := miner.SectorsStatus(ctx, sectorNo, false) + require.NoError(t, err) + if st.State == successState { + break + } + require.NotEqual(t, failureState, st.State) + build.Clock.Sleep(100 * time.Millisecond) + fmt.Println("WaitSeal") + } + +} diff --git a/api/test/test.go b/api/test/test.go index 35b397740..947f2bef4 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/miner" @@ -96,7 +97,8 @@ var FullNodeWithUpgradeAt = func(upgradeHeight abi.ChainEpoch) FullNodeOpts { return FullNodeOpts{ Opts: func(nodes []TestNode) node.Option { return node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: build.ActorUpgradeNetworkVersion, + // Skip directly to tape height so precommits work. + Network: network.Version5, Height: upgradeHeight, Migration: stmgr.UpgradeActorsV2, }}) diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 64207e66d..001c7159c 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -230,8 +230,8 @@ func (mgr *SectorMgr) SealCommit1(ctx context.Context, sid abi.SectorID, ticket } func (mgr *SectorMgr) SealCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { - var out [32]byte - for i := range out { + var out [1920]byte + for i := range out[:len(phase1Out)] { out[i] = phase1Out[i] ^ byte(sid.Number&0xff) } @@ -407,11 +407,12 @@ func (mgr *SectorMgr) CheckProvable(ctx context.Context, spt abi.RegisteredSealP } func (m mockVerif) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { - if len(svi.Proof) != 32 { // Real ones are longer, but this should be fine + if len(svi.Proof) != 1920 { return false, nil } - for i, b := range svi.Proof { + // only the first 32 bytes, the rest are 0. + for i, b := range svi.Proof[:32] { if b != svi.UnsealedCID.Bytes()[i]+svi.SealedCID.Bytes()[31-i]-svi.InteractiveRandomness[i]*svi.Randomness[i] { return false, nil } diff --git a/node/node_test.go b/node/node_test.go index 001b99c04..e553e83b2 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -117,6 +117,16 @@ func TestPledgeSectors(t *testing.T) { }) } +func TestTapeFix(t *testing.T) { + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") + + test.TestTapeFix(t, builder.MockSbBuilder, 2*time.Millisecond) +} + func TestWindowedPost(t *testing.T) { if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") From c0961fb63d7311d7c41f4c40340e0de013382703 Mon Sep 17 00:00:00 2001 From: Dan Shao Date: Mon, 12 Oct 2020 15:07:56 +0800 Subject: [PATCH 022/106] Improve the UX for `lotus-miner sealing workers`: add total number of cores --- cmd/lotus-storage-miner/sealing.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/sealing.go b/cmd/lotus-storage-miner/sealing.go index 5cc5c419a..507e842c3 100644 --- a/cmd/lotus-storage-miner/sealing.go +++ b/cmd/lotus-storage-miner/sealing.go @@ -80,7 +80,8 @@ var sealingWorkersCmd = &cli.Command{ cpuBars := int(stat.CpuUse * barCols / stat.Info.Resources.CPUs) cpuBar := strings.Repeat("|", cpuBars) + strings.Repeat(" ", int(barCols)-cpuBars) - fmt.Printf("\tCPU: [%s] %d core(s) in use\n", color.GreenString(cpuBar), stat.CpuUse) + fmt.Printf("\tCPU: [%s] %d/%d core(s) in use\n", + color.GreenString(cpuBar), stat.CpuUse, stat.Info.Resources.CPUs) ramBarsRes := int(stat.Info.Resources.MemReserved * barCols / stat.Info.Resources.MemPhysical) ramBarsUsed := int(stat.MemUsedMin * barCols / stat.Info.Resources.MemPhysical) From 8b648cf77cba845ae1bf0f0c46a906b51ffd1ec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 09:21:42 +0200 Subject: [PATCH 023/106] ledger: Close device in WalletNew --- chain/wallet/ledger/ledger.go | 1 + 1 file changed, 1 insertion(+) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index c1757fd0a..07f92e7ff 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -201,6 +201,7 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address. if err != nil { return address.Undef, xerrors.Errorf("finding ledger: %w", err) } + defer fl.Close() // nolint:errcheck path := append(append([]uint32(nil), filHDBasePath...), uint32(maxi+1)) _, _, addr, err := fl.GetAddressPubKeySECP256K1(path) From 8001ed30308f3e81bd695e8f329a45b162f4b46d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Oct 2020 11:02:57 +0200 Subject: [PATCH 024/106] Drand: add cloudflare. Use HTTP for PL nodes. Adding Cloudflare ensures additional redundancy. Switching to HTTP for PL nodes reduces latency and cost. Randomness is verified after being received. --- build/drand.go | 11 +---------- chain/beacon/drand/drand_test.go | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/build/drand.go b/build/drand.go index 73299249a..3b976ac92 100644 --- a/build/drand.go +++ b/build/drand.go @@ -35,6 +35,7 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{ "https://api.drand.sh", "https://api2.drand.sh", "https://api3.drand.sh", + "https://drand.cloudflare.com", }, Relays: []string{ "/dnsaddr/api.drand.sh/", @@ -68,16 +69,6 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{ ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`, }, DrandIncentinet: { - Servers: []string{ - "https://pl-eu.incentinet.drand.sh", - "https://pl-us.incentinet.drand.sh", - "https://pl-sin.incentinet.drand.sh", - }, - Relays: []string{ - "/dnsaddr/pl-eu.incentinet.drand.sh/", - "/dnsaddr/pl-us.incentinet.drand.sh/", - "/dnsaddr/pl-sin.incentinet.drand.sh/", - }, ChainInfoJSON: `{"public_key":"8cad0c72c606ab27d36ee06de1d5b2db1faf92e447025ca37575ab3a8aac2eaae83192f846fc9e158bc738423753d000","period":30,"genesis_time":1595873820,"hash":"80c8b872c714f4c00fdd3daa465d5514049f457f01f85a4caf68cdcd394ba039","groupHash":"d9406aaed487f7af71851b4399448e311f2328923d454e971536c05398ce2d9b"}`, }, } diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index 0cb9c2ba8..d66ee7b54 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -12,7 +12,7 @@ import ( ) func TestPrintGroupInfo(t *testing.T) { - server := build.DrandConfigs[build.DrandIncentinet].Servers[0] + server := build.DrandConfigs[build.DrandDevnet].Servers[0] c, err := hclient.New(server, nil, nil) assert.NoError(t, err) cg := c.(interface { From 79c5d2c6d2524a064af1a715fd56bb10af000bb5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Oct 2020 11:04:46 -0700 Subject: [PATCH 025/106] Rename params_testnet to params_mainnet --- build/{params_testnet.go => params_mainnet.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build/{params_testnet.go => params_mainnet.go} (100%) diff --git a/build/params_testnet.go b/build/params_mainnet.go similarity index 100% rename from build/params_testnet.go rename to build/params_mainnet.go From 6c0812062881bb26681bf84633af9e454d7824c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 20:46:34 +0200 Subject: [PATCH 026/106] types: Add CID fields to messages in json marshalers --- chain/types/message.go | 15 ++++++++ chain/types/message_test.go | 66 ++++++++++++++++++++++++++++++++++++ chain/types/signedmessage.go | 15 ++++++++ 3 files changed, 96 insertions(+) diff --git a/chain/types/message.go b/chain/types/message.go index 4fead44bc..cbc7c1dad 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -2,6 +2,7 @@ package types import ( "bytes" + "encoding/json" "fmt" "github.com/filecoin-project/go-state-types/abi" @@ -106,6 +107,20 @@ func (m *Message) Cid() cid.Cid { return b.Cid() } +type mCid struct { + *RawMessage + CID cid.Cid +} + +type RawMessage Message + +func (m *Message) MarshalJSON() ([]byte, error) { + return json.Marshal(&mCid{ + RawMessage: (*RawMessage)(m), + CID: m.Cid(), + }) +} + func (m *Message) RequiredFunds() BigInt { return BigMul(m.GasFeeCap, NewInt(uint64(m.GasLimit))) } diff --git a/chain/types/message_test.go b/chain/types/message_test.go index f57385a09..dab8e0939 100644 --- a/chain/types/message_test.go +++ b/chain/types/message_test.go @@ -1,11 +1,14 @@ package types import ( + "encoding/json" + "fmt" "testing" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin" ) @@ -70,3 +73,66 @@ func TestEqualCall(t *testing.T) { require.True(t, m1.EqualCall(m3)) require.False(t, m1.EqualCall(m4)) } + +func TestMessageJson(t *testing.T) { + m := &Message{ + To: builtin.StoragePowerActorAddr, + From: builtin.SystemActorAddr, + Nonce: 34, + Value: big.Zero(), + + GasLimit: 123, + GasFeeCap: big.NewInt(234), + GasPremium: big.NewInt(234), + + Method: 6, + Params: []byte("hai"), + } + + b, err := json.Marshal(m) + require.NoError(t, err) + + exp := []byte("{\"Version\":0,\"To\":\"f04\",\"From\":\"f00\",\"Nonce\":34,\"Value\":\"0\",\"GasLimit\":123,\"GasFeeCap\":\"234\",\"GasPremium\":\"234\",\"Method\":6,\"Params\":\"aGFp\",\"CID\":{\"/\":\"bafy2bzaced5rdpz57e64sc7mdwjn3blicglhpialnrph2dlbufhf6iha63dmc\"}}") + fmt.Println(string(b)) + + require.Equal(t, exp, b) + + var um Message + require.NoError(t, json.Unmarshal(b, &um)) + + require.EqualValues(t, *m, um) +} + +func TestSignedMessageJson(t *testing.T) { + m := Message{ + To: builtin.StoragePowerActorAddr, + From: builtin.SystemActorAddr, + Nonce: 34, + Value: big.Zero(), + + GasLimit: 123, + GasFeeCap: big.NewInt(234), + GasPremium: big.NewInt(234), + + Method: 6, + Params: []byte("hai"), + } + + sm := &SignedMessage{ + Message: m, + Signature: crypto.Signature{}, + } + + b, err := json.Marshal(sm) + require.NoError(t, err) + + exp := []byte("{\"Message\":{\"Version\":0,\"To\":\"f04\",\"From\":\"f00\",\"Nonce\":34,\"Value\":\"0\",\"GasLimit\":123,\"GasFeeCap\":\"234\",\"GasPremium\":\"234\",\"Method\":6,\"Params\":\"aGFp\",\"CID\":{\"/\":\"bafy2bzaced5rdpz57e64sc7mdwjn3blicglhpialnrph2dlbufhf6iha63dmc\"}},\"Signature\":{\"Type\":0,\"Data\":null},\"CID\":{\"/\":\"bafy2bzacea5ainifngxj3rygaw2hppnyz2cw72x5pysqty2x6dxmjs5qg2uus\"}}") + fmt.Println(string(b)) + + require.Equal(t, exp, b) + + var um SignedMessage + require.NoError(t, json.Unmarshal(b, &um)) + + require.EqualValues(t, *sm, um) +} diff --git a/chain/types/signedmessage.go b/chain/types/signedmessage.go index 17d2f5d94..7532bea35 100644 --- a/chain/types/signedmessage.go +++ b/chain/types/signedmessage.go @@ -2,6 +2,7 @@ package types import ( "bytes" + "encoding/json" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" @@ -62,6 +63,20 @@ func (sm *SignedMessage) Serialize() ([]byte, error) { return buf.Bytes(), nil } +type smCid struct { + *RawSignedMessage + CID cid.Cid +} + +type RawSignedMessage SignedMessage + +func (sm *SignedMessage) MarshalJSON() ([]byte, error) { + return json.Marshal(&smCid{ + RawSignedMessage: (*RawSignedMessage)(sm), + CID: sm.Cid(), + }) +} + func (sm *SignedMessage) ChainLength() int { ser, err := sm.Serialize() if err != nil { From 61661fb1f06986d9fbfe25f31dfb15ea6743c9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 20:47:09 +0200 Subject: [PATCH 027/106] docsgen --- documentation/en/api-methods.md | 105 ++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 18 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 50a6a160d..97d10b064 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -462,7 +462,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } } ``` @@ -1333,7 +1336,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, 9, [ @@ -1369,7 +1375,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, [ { @@ -1429,7 +1438,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, { "MaxFee": "0" @@ -1457,7 +1469,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } } ``` @@ -1741,11 +1756,17 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } ] @@ -1782,7 +1803,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, { "MaxFee": "0" @@ -1803,11 +1827,17 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } ``` @@ -1832,11 +1862,17 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } ] @@ -1916,11 +1952,17 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } } @@ -2997,7 +3039,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, [ { @@ -3023,7 +3068,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "MsgRct": { "ExitCode": 0, @@ -3041,7 +3089,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "MsgRct": { "ExitCode": 0, @@ -3296,7 +3347,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, [ { @@ -4042,7 +4096,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "MsgRct": { "ExitCode": 0, @@ -4060,7 +4117,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "MsgRct": { "ExitCode": 0, @@ -4832,7 +4892,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } } ] ``` @@ -4850,11 +4913,17 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } ``` From 96e1dfd8d7f5778b5d534f3b29bac2c8a50f0118 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 11 Oct 2020 18:17:28 -0400 Subject: [PATCH 028/106] Add an endpoint for precise circulating supply --- api/api_full.go | 8 ++- api/apistruct/struct.go | 9 ++- chain/actors/builtin/builtin.go | 9 ++- chain/actors/builtin/miner/miner.go | 5 ++ chain/actors/builtin/miner/v0.go | 12 +++- chain/actors/builtin/miner/v2.go | 12 +++- chain/stmgr/call.go | 4 +- chain/stmgr/forks.go | 26 +++---- chain/stmgr/stmgr.go | 103 ++++++++++++++++++++++++++-- chain/stmgr/utils.go | 2 +- cli/state.go | 37 +++++++--- cmd/lotus-chainwatch/syncer/sync.go | 2 +- cmd/lotus-shed/balances.go | 2 +- cmd/tvx/extract.go | 2 +- documentation/en/api-methods.md | 47 ++++++++++--- node/impl/full/state.go | 21 ++++-- 16 files changed, 236 insertions(+), 65 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index f42c8d03f..b7cdf9524 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -416,8 +416,12 @@ type FullNode interface { // can issue. It takes the deal size and verified status as parameters. StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error) - // StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset - StateCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error) + // StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset. + // This is not used anywhere in the protocol itself, and is only for external consumption. + StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) + // StateVMCirculatingSupply returns an approximation of the circulating supply of Filecoin at the given tipset. + // This is the value reported by the runtime interface to actors code. + StateVMCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error) // StateNetworkVersion returns the network version at the given tipset StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index cef335939..a938653ce 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -210,7 +210,8 @@ type FullNodeStruct struct { StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` StateVerifiedRegistryRootKey func(ctx context.Context, tsk types.TipSetKey) (address.Address, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` - StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` + StateCirculatingSupply func(context.Context, types.TipSetKey) (abi.TokenAmount, error) `perm:"read"` + StateVMCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` @@ -963,10 +964,14 @@ func (c *FullNodeStruct) StateDealProviderCollateralBounds(ctx context.Context, return c.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk) } -func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { +func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) { return c.Internal.StateCirculatingSupply(ctx, tsk) } +func (c *FullNodeStruct) StateVMCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { + return c.Internal.StateVMCirculatingSupply(ctx, tsk) +} + func (c *FullNodeStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) { return c.Internal.StateNetworkVersion(ctx, tsk) } diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index cb24a2c33..350b3a9a6 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -2,25 +2,28 @@ package builtin import ( "github.com/filecoin-project/go-address" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + smoothing2 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" "github.com/ipfs/go-cid" "golang.org/x/xerrors" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - smoothing2 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" ) var SystemActorAddr = builtin0.SystemActorAddr var BurntFundsActorAddr = builtin0.BurntFundsActorAddr +var CronActorAddr = builtin0.CronActorAddr +var SaftAddress = makeAddress("t0122") var ReserveAddress = makeAddress("t090") var RootVerifierAddress = makeAddress("t080") diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 281068827..3cca39326 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -1,6 +1,7 @@ package miner import ( + "github.com/filecoin-project/go-state-types/big" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" @@ -196,3 +197,7 @@ type LockedFunds struct { InitialPledgeRequirement abi.TokenAmount PreCommitDeposits abi.TokenAmount } + +func (lf LockedFunds) TotalLockedFunds() abi.TokenAmount { + return big.Add(lf.VestingFunds, big.Add(lf.InitialPledgeRequirement, lf.PreCommitDeposits)) +} diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 7e71c7611..14718d002 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -47,8 +47,16 @@ type partition0 struct { store adt.Store } -func (s *state0) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { - return s.GetAvailableBalance(bal), nil +func (s *state0) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmount, err error) { + defer func() { + if r := recover(); r != nil { + err = xerrors.Errorf("failed to get available balance: %w", r) + available = abi.NewTokenAmount(0) + } + }() + // this panics if the miner doesnt have enough funds to cover their locked pledge + available = s.GetAvailableBalance(bal) + return available, err } func (s *state0) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index eed82257f..9e599f891 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -45,8 +45,16 @@ type partition2 struct { store adt.Store } -func (s *state2) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { - return s.GetAvailableBalance(bal) +func (s *state2) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmount, err error) { + defer func() { + if r := recover(); r != nil { + err = xerrors.Errorf("failed to get available balance: %w", r) + available = abi.NewTokenAmount(0) + } + }() + // this panics if the miner doesnt have enough funds to cover their locked pledge + available, err = s.GetAvailableBalance(bal) + return available, err } func (s *state2) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index df3bfa357..8d0f6f6c1 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -61,7 +61,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. Rand: store.NewChainRand(sm.cs, ts.Cids()), Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, + CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: types.NewInt(0), } @@ -174,7 +174,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri Rand: r, Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, + CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 6ebe8a5b9..7f2b57546 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -265,11 +265,6 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal return cid.Undef, xerrors.Errorf("loading state tree failed: %w", err) } - ReserveAddress, err := address.NewFromString("t090") - if err != nil { - return cid.Undef, xerrors.Errorf("failed to parse reserve address: %w", err) - } - tree, err := sm.StateTree(root) if err != nil { return cid.Undef, xerrors.Errorf("getting state tree: %w", err) @@ -295,7 +290,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if !sysAcc { transfers = append(transfers, transfer{ From: addr, - To: ReserveAddress, + To: builtin.ReserveAddress, Amt: act.Balance, }) } @@ -319,7 +314,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal transfers = append(transfers, transfer{ From: addr, - To: ReserveAddress, + To: builtin.ReserveAddress, Amt: available, }) } @@ -370,7 +365,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal nbalance := big.Min(prevBalance, AccountCap) if nbalance.Sign() != 0 { transfersBack = append(transfersBack, transfer{ - From: ReserveAddress, + From: builtin.ReserveAddress, To: addr, Amt: nbalance, }) @@ -397,7 +392,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal mfunds := minerFundsAlloc(power, totalPower) transfersBack = append(transfersBack, transfer{ - From: ReserveAddress, + From: builtin.ReserveAddress, To: minfo.Worker, Amt: mfunds, }) @@ -417,7 +412,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if lbsectors.Length() > 0 { transfersBack = append(transfersBack, transfer{ - From: ReserveAddress, + From: builtin.ReserveAddress, To: minfo.Worker, Amt: BaseMinerBalance, }) @@ -444,7 +439,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if err != nil { return cid.Undef, xerrors.Errorf("failed to load burnt funds actor: %w", err) } - if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil { + if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, builtin.ReserveAddress, burntAct.Balance); err != nil { return cid.Undef, xerrors.Errorf("failed to unburn funds: %w", err) } @@ -460,7 +455,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } difference := types.BigSub(DesiredReimbursementBalance, reimb.Balance) - if err := doTransfer(cb, tree, ReserveAddress, reimbAddr, difference); err != nil { + if err := doTransfer(cb, tree, builtin.ReserveAddress, reimbAddr, difference); err != nil { return cid.Undef, xerrors.Errorf("failed to top up reimbursement account: %w", err) } @@ -545,12 +540,7 @@ func UpgradeRefuel(ctx context.Context, sm *StateManager, cb ExecCallback, root return cid.Undef, xerrors.Errorf("getting state tree: %w", err) } - addr, err := address.NewFromString("t0122") - if err != nil { - return cid.Undef, xerrors.Errorf("getting address: %w", err) - } - - err = resetMultisigVesting(ctx, store, tree, addr, 0, 0, big.Zero()) + err = resetMultisigVesting(ctx, store, tree, builtin.SaftAddress, 0, 0, big.Zero()) if err != nil { return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d6b6f4360..b470cfe83 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -7,6 +7,11 @@ import ( "sync" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" + + _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" + + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -229,7 +234,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp Rand: r, Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, + CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: baseFee, } @@ -1294,7 +1299,16 @@ func GetFilBurnt(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err return burnt.Balance, nil } -func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (api.CirculatingSupply, error) { +func (sm *StateManager) GetVMCirculatingSupply(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) { + cs, err := sm.GetVMCirculatingSupplyDetailed(ctx, height, st) + if err != nil { + return types.EmptyInt, err + } + + return cs.FilCirculating, err +} + +func (sm *StateManager) GetVMCirculatingSupplyDetailed(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (api.CirculatingSupply, error) { sm.genesisMsigLk.Lock() defer sm.genesisMsigLk.Unlock() if sm.preIgnitionGenInfos == nil { @@ -1357,12 +1371,91 @@ func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height } func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) { - csi, err := sm.GetCirculatingSupplyDetailed(ctx, height, st) + circ := big.Zero() + unCirc := big.Zero() + err := st.ForEach(func(a address.Address, actor *types.Actor) error { + switch { + case actor.Balance.IsZero(): + // Do nothing for zero-balance actors + break + case a == _init.Address || + a == reward.Address || + a == verifreg.Address || + // The power actor itself should never receive funds + a == power.Address || + a == builtin.SystemActorAddr || + a == builtin.CronActorAddr || + a == builtin.BurntFundsActorAddr || + a == builtin.SaftAddress || + a == builtin.ReserveAddress: + + unCirc = big.Add(unCirc, actor.Balance) + + case a == market.Address: + mst, err := market.Load(sm.cs.Store(ctx), actor) + if err != nil { + return err + } + + lb, err := mst.TotalLocked() + if err != nil { + return err + } + + circ = big.Add(circ, big.Sub(actor.Balance, lb)) + unCirc = big.Add(unCirc, lb) + + case builtin.IsAccountActor(actor.Code) || builtin.IsPaymentChannelActor(actor.Code): + circ = big.Add(circ, actor.Balance) + + case builtin.IsStorageMinerActor(actor.Code): + mst, err := miner.Load(sm.cs.Store(ctx), actor) + if err != nil { + return err + } + + ab, err := mst.AvailableBalance(actor.Balance) + + if err == nil { + circ = big.Add(circ, ab) + unCirc = big.Add(unCirc, big.Sub(actor.Balance, ab)) + } else { + // Assume any error is because the miner state is "broken" (lower actor balance than locked funds) + // In this case, the actor's entire balance is considered "uncirculating" + unCirc = big.Add(unCirc, actor.Balance) + } + + case builtin.IsMultisigActor(actor.Code): + mst, err := multisig.Load(sm.cs.Store(ctx), actor) + if err != nil { + return err + } + + lb, err := mst.LockedBalance(height) + if err != nil { + return err + } + + ab := big.Sub(actor.Balance, lb) + circ = big.Add(circ, big.Max(ab, big.Zero())) + unCirc = big.Add(unCirc, big.Min(actor.Balance, lb)) + default: + return xerrors.Errorf("unexpected actor: %s", a) + } + + return nil + }) + if err != nil { - return big.Zero(), err + return types.EmptyInt, err } - return csi.FilCirculating, nil + total := big.Add(circ, unCirc) + if !total.Equals(types.TotalFilecoinInt) { + return types.EmptyInt, xerrors.Errorf("total filecoin didn't add to expected amount: %s != %s", total, types.TotalFilecoinInt) + } + + return circ, nil } func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index c0f0c4d2f..cbc78ca5b 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -383,7 +383,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, Rand: r, Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, + CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } diff --git a/cli/state.go b/cli/state.go index 453cde77f..5acdf940f 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1688,7 +1688,14 @@ func parseParamsForMethod(act cid.Cid, method uint64, args []string) ([]byte, er var stateCircSupplyCmd = &cli.Command{ Name: "circulating-supply", - Usage: "Get the current circulating supply of filecoin", + Usage: "Get the exact current circulating supply of Filecoin", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "vm-supply", + Usage: "calculates the approximation of the circulating supply used internally by the VM (instead of the exact amount)", + Value: false, + }, + }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -1703,16 +1710,26 @@ var stateCircSupplyCmd = &cli.Command{ return err } - circ, err := api.StateCirculatingSupply(ctx, ts.Key()) - if err != nil { - return err - } + if cctx.IsSet("vm-supply") { + circ, err := api.StateVMCirculatingSupply(ctx, ts.Key()) + if err != nil { + return err + } - fmt.Println("Circulating supply: ", types.FIL(circ.FilCirculating)) - fmt.Println("Mined: ", types.FIL(circ.FilMined)) - fmt.Println("Vested: ", types.FIL(circ.FilVested)) - fmt.Println("Burnt: ", types.FIL(circ.FilBurnt)) - fmt.Println("Locked: ", types.FIL(circ.FilLocked)) + fmt.Println("Circulating supply: ", types.FIL(circ.FilCirculating)) + fmt.Println("Mined: ", types.FIL(circ.FilMined)) + fmt.Println("Vested: ", types.FIL(circ.FilVested)) + fmt.Println("Burnt: ", types.FIL(circ.FilBurnt)) + fmt.Println("Locked: ", types.FIL(circ.FilLocked)) + } else { + circ, err := api.StateCirculatingSupply(ctx, ts.Key()) + if err != nil { + return err + } + + fmt.Println("Exact circulating supply: ", types.FIL(circ)) + return nil + } return nil }, diff --git a/cmd/lotus-chainwatch/syncer/sync.go b/cmd/lotus-chainwatch/syncer/sync.go index 609b71088..d0931a978 100644 --- a/cmd/lotus-chainwatch/syncer/sync.go +++ b/cmd/lotus-chainwatch/syncer/sync.go @@ -316,7 +316,7 @@ limit 1 } func (s *Syncer) storeCirculatingSupply(ctx context.Context, tipset *types.TipSet) error { - supply, err := s.node.StateCirculatingSupply(ctx, tipset.Key()) + supply, err := s.node.StateVMCirculatingSupply(ctx, tipset.Key()) if err != nil { return err } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index c2099bb2d..dfdaf9fa9 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -372,7 +372,7 @@ var chainPledgeCmd = &cli.Command{ pledgeCollateral = c } - circ, err := sm.GetCirculatingSupplyDetailed(ctx, abi.ChainEpoch(epoch), state) + circ, err := sm.GetVMCirculatingSupplyDetailed(ctx, abi.ChainEpoch(epoch), state) if err != nil { return err } diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index b0ed574df..cc4494d0f 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -135,7 +135,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { } // get the circulating supply before the message was executed. - circSupplyDetail, err := fapi.StateCirculatingSupply(ctx, incTs.Key()) + circSupplyDetail, err := fapi.StateVMCirculatingSupply(ctx, incTs.Key()) if err != nil { return fmt.Errorf("failed while fetching circulating supply: %w", err) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 50a6a160d..f5dfcf815 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -165,6 +165,7 @@ * [StateSectorGetInfo](#StateSectorGetInfo) * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) + * [StateVMCirculatingSupply](#StateVMCirculatingSupply) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) * [StateVerifierStatus](#StateVerifierStatus) @@ -3094,7 +3095,8 @@ Response: ``` ### StateCirculatingSupply -StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset +StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset. +This is not used anywhere in the protocol itself, and is only for external consumption. Perms: read @@ -3113,16 +3115,7 @@ Inputs: ] ``` -Response: -```json -{ - "FilVested": "0", - "FilMined": "0", - "FilBurnt": "0", - "FilLocked": "0", - "FilCirculating": "0" -} -``` +Response: `"0"` ### StateCompute StateCompute is a flexible command that applies the given messages on the given tipset. @@ -4265,6 +4258,38 @@ Response: } ``` +### StateVMCirculatingSupply +StateVMCirculatingSupply returns an approximation of the circulating supply of Filecoin at the given tipset. +This is the value reported by the runtime interface to actors code. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "FilVested": "0", + "FilMined": "0", + "FilBurnt": "0", + "FilLocked": "0", + "FilCirculating": "0" +} +``` + ### StateVerifiedClientStatus StateVerifiedClientStatus returns the data cap for the given address. Returns nil if there is no entry in the data cap table for the diff --git a/node/impl/full/state.go b/node/impl/full/state.go index db91433aa..62636e4fa 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1049,7 +1049,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) } - circSupply, err := a.StateCirculatingSupply(ctx, ts.Key()) + circSupply, err := a.StateVMCirculatingSupply(ctx, ts.Key()) if err != nil { return big.Zero(), xerrors.Errorf("getting circulating supply: %w", err) } @@ -1203,7 +1203,7 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor state: %w", err) } - circ, err := a.StateCirculatingSupply(ctx, ts.Key()) + circ, err := a.StateVMCirculatingSupply(ctx, ts.Key()) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("getting total circulating supply: %w", err) } @@ -1231,7 +1231,20 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a }, nil } -func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { +func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) { + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + + sTree, err := a.stateForTs(ctx, ts) + if err != nil { + return types.EmptyInt, err + } + return a.StateManager.GetCirculatingSupply(ctx, ts.Height(), sTree) +} + +func (a *StateAPI) StateVMCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return api.CirculatingSupply{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -1241,7 +1254,7 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK if err != nil { return api.CirculatingSupply{}, err } - return a.StateManager.GetCirculatingSupplyDetailed(ctx, ts.Height(), sTree) + return a.StateManager.GetVMCirculatingSupplyDetailed(ctx, ts.Height(), sTree) } func (a *StateAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { From 83624a8858d1983a28e0cd523c5ec353a5bbdaee Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 12 Oct 2020 16:41:27 -0400 Subject: [PATCH 029/106] Rename StateVMCirculatingSupply to StateVMCirculatingSupplyInternal --- api/api_full.go | 4 ++-- api/apistruct/struct.go | 6 +++--- cli/state.go | 2 +- cmd/lotus-chainwatch/syncer/sync.go | 2 +- cmd/tvx/extract.go | 2 +- documentation/en/api-methods.md | 6 +++--- node/impl/full/state.go | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index b7cdf9524..657e945ca 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -419,9 +419,9 @@ type FullNode interface { // StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset. // This is not used anywhere in the protocol itself, and is only for external consumption. StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) - // StateVMCirculatingSupply returns an approximation of the circulating supply of Filecoin at the given tipset. + // StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset. // This is the value reported by the runtime interface to actors code. - StateVMCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error) + StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error) // StateNetworkVersion returns the network version at the given tipset StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a938653ce..7896d566e 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -211,7 +211,7 @@ type FullNodeStruct struct { StateVerifiedRegistryRootKey func(ctx context.Context, tsk types.TipSetKey) (address.Address, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` StateCirculatingSupply func(context.Context, types.TipSetKey) (abi.TokenAmount, error) `perm:"read"` - StateVMCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` + StateVMCirculatingSupplyInternal func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` @@ -968,8 +968,8 @@ func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.T return c.Internal.StateCirculatingSupply(ctx, tsk) } -func (c *FullNodeStruct) StateVMCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { - return c.Internal.StateVMCirculatingSupply(ctx, tsk) +func (c *FullNodeStruct) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { + return c.Internal.StateVMCirculatingSupplyInternal(ctx, tsk) } func (c *FullNodeStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) { diff --git a/cli/state.go b/cli/state.go index 5acdf940f..1e7e24ddb 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1711,7 +1711,7 @@ var stateCircSupplyCmd = &cli.Command{ } if cctx.IsSet("vm-supply") { - circ, err := api.StateVMCirculatingSupply(ctx, ts.Key()) + circ, err := api.StateVMCirculatingSupplyInternal(ctx, ts.Key()) if err != nil { return err } diff --git a/cmd/lotus-chainwatch/syncer/sync.go b/cmd/lotus-chainwatch/syncer/sync.go index d0931a978..37af9cce0 100644 --- a/cmd/lotus-chainwatch/syncer/sync.go +++ b/cmd/lotus-chainwatch/syncer/sync.go @@ -316,7 +316,7 @@ limit 1 } func (s *Syncer) storeCirculatingSupply(ctx context.Context, tipset *types.TipSet) error { - supply, err := s.node.StateVMCirculatingSupply(ctx, tipset.Key()) + supply, err := s.node.StateVMCirculatingSupplyInternal(ctx, tipset.Key()) if err != nil { return err } diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index cc4494d0f..4bb391af6 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -135,7 +135,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { } // get the circulating supply before the message was executed. - circSupplyDetail, err := fapi.StateVMCirculatingSupply(ctx, incTs.Key()) + circSupplyDetail, err := fapi.StateVMCirculatingSupplyInternal(ctx, incTs.Key()) if err != nil { return fmt.Errorf("failed while fetching circulating supply: %w", err) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index f5dfcf815..cbec60c10 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -165,7 +165,7 @@ * [StateSectorGetInfo](#StateSectorGetInfo) * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) - * [StateVMCirculatingSupply](#StateVMCirculatingSupply) + * [StateVMCirculatingSupplyInternal](#StateVMCirculatingSupplyInternal) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) * [StateVerifierStatus](#StateVerifierStatus) @@ -4258,8 +4258,8 @@ Response: } ``` -### StateVMCirculatingSupply -StateVMCirculatingSupply returns an approximation of the circulating supply of Filecoin at the given tipset. +### StateVMCirculatingSupplyInternal +StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset. This is the value reported by the runtime interface to actors code. diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 62636e4fa..1c39b59da 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1049,7 +1049,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) } - circSupply, err := a.StateVMCirculatingSupply(ctx, ts.Key()) + circSupply, err := a.StateVMCirculatingSupplyInternal(ctx, ts.Key()) if err != nil { return big.Zero(), xerrors.Errorf("getting circulating supply: %w", err) } @@ -1203,7 +1203,7 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor state: %w", err) } - circ, err := a.StateVMCirculatingSupply(ctx, ts.Key()) + circ, err := a.StateVMCirculatingSupplyInternal(ctx, ts.Key()) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("getting total circulating supply: %w", err) } @@ -1244,7 +1244,7 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK return a.StateManager.GetCirculatingSupply(ctx, ts.Height(), sTree) } -func (a *StateAPI) StateVMCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { +func (a *StateAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return api.CirculatingSupply{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) From 49155ace62ab22ea13202ce60c755ad266d04581 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Mon, 12 Oct 2020 13:45:37 -0700 Subject: [PATCH 030/106] Use seal-duration in calculating the earliest StartEpoch that we will consider (#4337) --- node/modules/storageminer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 9011c4821..4dbce4cdb 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -473,7 +473,7 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline // Reject if it's more than 7 days in the future // TODO: read from cfg - maxStartEpoch := ht + abi.ChainEpoch(7*builtin.EpochsInDay) + maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.EpochsInDay) if deal.Proposal.StartEpoch > maxStartEpoch { return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil } From be74a137365bc250b592718aa2a7436e1c25eae0 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Mon, 12 Oct 2020 21:41:48 +0000 Subject: [PATCH 031/106] lotus-pcr: ignore all other market messages --- cmd/lotus-pcr/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 5491e4af2..f88a0a9dc 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -884,6 +884,8 @@ func (r *refunder) processTipsetStorageMarketActor(ctx context.Context, tipset * } refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) + default: + return false, messageMethod, types.NewInt(0), nil } return true, messageMethod, refundValue, nil From 9b4cac961229ca4a9396a10862c9a80470c67466 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Mon, 12 Oct 2020 21:14:16 -0300 Subject: [PATCH 032/106] fix(sync state): set state height to actual tipset height --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 9040e3f05..dd83bf319 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1478,7 +1478,7 @@ func (syncer *Syncer) syncFork(ctx context.Context, incoming *types.TipSet, know func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []*types.TipSet) error { ss := extractSyncState(ctx) - ss.SetHeight(0) + ss.SetHeight(headers[len(headers)-1].Height()) return syncer.iterFullTipsets(ctx, headers, func(ctx context.Context, fts *store.FullTipSet) error { log.Debugw("validating tipset", "height", fts.TipSet().Height(), "size", len(fts.TipSet().Cids())) From 07d0b6762812881fc85295aa5a1b45b14cf063ac Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Tue, 13 Oct 2020 10:46:41 +0800 Subject: [PATCH 033/106] optimize code use ts.Cids --- chain/stmgr/stmgr.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b470cfe83..b381b2d59 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -435,12 +435,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, ts *types.TipSet parentEpoch = parent.Height } - cids := make([]cid.Cid, len(blks)) - for i, v := range blks { - cids[i] = v.Cid() - } - - r := store.NewChainRand(sm.cs, cids) + r := store.NewChainRand(sm.cs, ts.Cids()) blkmsgs, err := sm.cs.BlockMsgsForTipset(ts) if err != nil { From 4edebcec2b9e5c13b69743d812db604bd05398da Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 13 Oct 2020 03:37:00 -0700 Subject: [PATCH 034/106] feat(markets): update markets 0.9.0 and add data transfer restart command --- api/api_full.go | 3 ++ api/apistruct/struct.go | 6 ++++ cli/client.go | 62 +++++++++++++++++++++++++++++++++ documentation/en/api-methods.md | 18 ++++++++++ go.mod | 6 ++-- go.sum | 16 ++++----- node/impl/client/client.go | 10 +++++- node/modules/client.go | 1 + node/modules/storageminer.go | 1 + 9 files changed, 110 insertions(+), 13 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 657e945ca..5b1316730 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" @@ -298,6 +299,8 @@ type FullNode interface { // ClientListTransfers returns the status of all ongoing transfers of data ClientListDataTransfers(ctx context.Context) ([]DataTransferChannel, error) ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) + // ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer + ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error // ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel // which are stuck due to insufficient funds ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7896d566e..7723e0b1f 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" @@ -165,6 +166,7 @@ type FullNodeStruct struct { ClientDealSize func(ctx context.Context, root cid.Cid) (api.DataSize, error) `perm:"read"` ClientListDataTransfers func(ctx context.Context) ([]api.DataTransferChannel, error) `perm:"write"` ClientDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error) `perm:"write"` + ClientRestartDataTransfer func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error `perm:"write"` ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error `perm:"write"` StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` @@ -552,6 +554,10 @@ func (c *FullNodeStruct) ClientDataTransferUpdates(ctx context.Context) (<-chan return c.Internal.ClientDataTransferUpdates(ctx) } +func (c *FullNodeStruct) ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error { + return c.Internal.ClientRestartDataTransfer(ctx, transferID, otherPeer, isInitiator) +} + func (c *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error { return c.Internal.ClientRetrieveTryRestartInsufficientFunds(ctx, paymentChannel) } diff --git a/cli/client.go b/cli/client.go index 34d151ace..7035fb64b 100644 --- a/cli/client.go +++ b/cli/client.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -82,6 +83,7 @@ var clientCmd = &cli.Command{ WithCategory("util", clientCarGenCmd), WithCategory("util", clientInfoCmd), WithCategory("util", clientListTransfers), + WithCategory("util", clientRestartTransfer), }, } @@ -1326,6 +1328,66 @@ var clientInfoCmd = &cli.Command{ }, } +var clientRestartTransfer = &cli.Command{ + Name: "restart-transfer", + Usage: "Force restart a stalled data transfer", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "peerid", + Usage: "narrow to transfer with specific peer", + }, + &cli.BoolFlag{ + Name: "initiator", + Usage: "specify only transfers where peer is/is not initiator", + Value: true, + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + transferUint, err := strconv.ParseUint(cctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("Error reading transfer ID: %w", err) + } + transferID := datatransfer.TransferID(transferUint) + initiator := cctx.Bool("initiator") + var other peer.ID + if pidstr := cctx.String("peerid"); pidstr != "" { + p, err := peer.Decode(pidstr) + if err != nil { + return err + } + other = p + } else { + channels, err := api.ClientListDataTransfers(ctx) + if err != nil { + return err + } + found := false + for _, channel := range channels { + if channel.IsInitiator == initiator && channel.TransferID == transferID { + other = channel.OtherPeer + found = true + break + } + } + if !found { + return errors.New("unable to find matching data transfer") + } + } + + return api.ClientRestartDataTransfer(ctx, transferID, other, initiator) + }, +} + var clientListTransfers = &cli.Command{ Name: "list-transfers", Usage: "List ongoing data transfers for deals", diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index cbec60c10..c5d612fbe 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -46,6 +46,7 @@ * [ClientMinerQueryOffer](#ClientMinerQueryOffer) * [ClientQueryAsk](#ClientQueryAsk) * [ClientRemoveImport](#ClientRemoveImport) + * [ClientRestartDataTransfer](#ClientRestartDataTransfer) * [ClientRetrieve](#ClientRetrieve) * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) @@ -1161,6 +1162,23 @@ Inputs: Response: `{}` +### ClientRestartDataTransfer +ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer + + +Perms: write + +Inputs: +```json +[ + 3, + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + true +] +``` + +Response: `{}` + ### ClientRetrieve ClientRetrieve initiates the retrieval of a file, as specified in the order. diff --git a/go.mod b/go.mod index adff07d32..75926117a 100644 --- a/go.mod +++ b/go.mod @@ -27,9 +27,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.1 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.7 + github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.7.1 + github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -66,7 +66,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.2.1 + github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 576690c10..50e85dfd5 100644 --- a/go.sum +++ b/go.sum @@ -240,14 +240,14 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.7 h1:Kacr5qz2YWtd3sensU6aXFtES7joeapVDeXApeUD35I= -github.com/filecoin-project/go-data-transfer v0.6.7/go.mod h1:C++k1U6+jMQODOaen5OPDo9XQbth9Yq3ie94vNjBJbk= +github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5 h1:kgmVHDP+rqcPqytpC215MzRRXrW9FqTfn0VFWhB2oog= +github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5/go.mod h1:1zaap881YZLMeIxhvLM0zA9qfUkh3bjOLSRaOoO9WRM= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.7.1 h1:e0NlpSnaeGyDUhCOzevjcxkSA54kt9BzlXpLRgduUFI= -github.com/filecoin-project/go-fil-markets v0.7.1/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= +github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a h1:jC3yO5R0+jGnVAHVqIHXb3BqCN/crslxTALz4I5gw1o= +github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a/go.mod h1:HqbtLIwJ4Lonp0BWq4ajmdXZ2UTh2+YK3wZSXXAE8yA= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -264,12 +264,10 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 h github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= -github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b h1:bMUfG6Sy6YSMbsjQAO1Q2vEZldbSdsbRy/FX3OlTck0= github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= @@ -277,9 +275,9 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= -github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= +github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT+dGVWJTr5yDevU00g= github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= @@ -540,8 +538,8 @@ github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPi github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= -github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= -github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c h1:De/AZGvRa3WMyw5zdMMhcvRcho46BVo+C0NRud+T4io= +github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= diff --git a/node/impl/client/client.go b/node/impl/client/client.go index fe83fda7b..f04d310ce 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -8,7 +8,6 @@ import ( "github.com/filecoin-project/go-state-types/dline" - datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/big" "golang.org/x/xerrors" @@ -33,6 +32,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/discovery" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -850,6 +850,14 @@ func (a *API) ClientDataTransferUpdates(ctx context.Context) (<-chan api.DataTra return channels, nil } +func (a *API) ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error { + selfPeer := a.Host.ID() + if isInitiator { + return a.DataTransfer.RestartDataTransferChannel(ctx, datatransfer.ChannelID{Initiator: selfPeer, Responder: otherPeer, ID: transferID}) + } + return a.DataTransfer.RestartDataTransferChannel(ctx, datatransfer.ChannelID{Initiator: otherPeer, Responder: selfPeer, ID: transferID}) +} + func newDealInfo(v storagemarket.ClientDeal) api.DealInfo { return api.DealInfo{ ProposalCid: v.ProposalCid, diff --git a/node/modules/client.go b/node/modules/client.go index d012e4539..eab309445 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -91,6 +91,7 @@ func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Grap return nil, err } + dt.OnReady(marketevents.ReadyLogger("provider data transfer")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return dt.Start(ctx) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4dbce4cdb..613203a60 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -267,6 +267,7 @@ func NewProviderDAGServiceDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.S return nil, err } + dt.OnReady(marketevents.ReadyLogger("provider data transfer")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return dt.Start(ctx) From 1eb2d0ae5059604be9d878cab370e71eeba288ea Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Tue, 13 Oct 2020 22:34:07 +0800 Subject: [PATCH 035/106] optimize map use struct{} --- chain/stmgr/stmgr.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b470cfe83..e3bd5f89d 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -313,7 +313,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } var receipts []cbg.CBORMarshaler - processedMsgs := map[cid.Cid]bool{} + processedMsgs := make(map[cid.Cid]struct{}) for _, b := range bms { penalty := types.NewInt(0) gasReward := big.Zero() @@ -337,7 +337,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, err } } - processedMsgs[m.Cid()] = true + processedMsgs[m.Cid()] = struct{}{} } params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{ From 87cd8c672571ec22ae36b9eef1a42557e8546436 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 13 Oct 2020 17:48:54 +0200 Subject: [PATCH 036/106] Fix off by one tipset in searchBackForMsg This way we will return the tipset the message was executed in Signed-off-by: Jakub Sztandera --- chain/stmgr/stmgr.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b470cfe83..0cb17fcf3 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -738,7 +738,7 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet } if r != nil { - return pts, r, foundMsg, nil + return cur, r, foundMsg, nil } } From 8987defb9d8ba19ecc9bc6cb675bb3740cbb6923 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 13 Oct 2020 19:20:11 +0200 Subject: [PATCH 037/106] Fix BLS message ChainLength add more detail to ValidForBlockInclusion Signed-off-by: Jakub Sztandera --- chain/types/message.go | 2 +- chain/types/signedmessage.go | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/chain/types/message.go b/chain/types/message.go index cbc7c1dad..c53ecc7c1 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -195,7 +195,7 @@ func (m *Message) ValidForBlockInclusion(minGas int64) error { // since prices might vary with time, this is technically semantic validation if m.GasLimit < minGas { - return xerrors.New("'GasLimit' field cannot be less than the cost of storing a message on chain") + return xerrors.Errorf("'GasLimit' field cannot be less than the cost of storing a message on chain %d < %d", m.GasLimit, minGas) } return nil diff --git a/chain/types/signedmessage.go b/chain/types/signedmessage.go index 7532bea35..c539ac240 100644 --- a/chain/types/signedmessage.go +++ b/chain/types/signedmessage.go @@ -78,7 +78,14 @@ func (sm *SignedMessage) MarshalJSON() ([]byte, error) { } func (sm *SignedMessage) ChainLength() int { - ser, err := sm.Serialize() + var ser []byte + var err error + if sm.Signature.Type == crypto.SigTypeBLS { + // BLS chain message length doesn't include signature + ser, err = sm.Message.Serialize() + } else { + ser, err = sm.Serialize() + } if err != nil { panic(err) } From ce548c8f95aa7f3459ec58b8df27902579b6ef4e Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 13 Oct 2020 18:25:49 +0200 Subject: [PATCH 038/106] Add test Signed-off-by: Jakub Sztandera --- api/test/test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/api/test/test.go b/api/test/test.go index 947f2bef4..4074ce4a6 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -3,8 +3,10 @@ package test import ( "context" "testing" + "time" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" "github.com/multiformats/go-multiaddr" @@ -12,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -75,6 +78,7 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testConnectTwo", ts.testConnectTwo) t.Run("testMining", ts.testMining) t.Run("testMiningReal", ts.testMiningReal) + t.Run("testSearchMsg", ts.testSearchMsg) } func DefaultFullOpts(nFull int) []FullNodeOpts { @@ -120,6 +124,49 @@ func (ts *testSuite) testVersion(t *testing.T) { require.Equal(t, v.Version, build.BuildVersion) } +func (ts *testSuite) testSearchMsg(t *testing.T) { + apis, miners := ts.makeNodes(t, OneFull, OneMiner) + + api := apis[0] + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + senderAddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + t.Fatal(err) + } + + msg := &types.Message{ + From: senderAddr, + To: senderAddr, + Value: big.Zero(), + } + bm := NewBlockMiner(ctx, t, miners[0], 100*time.Millisecond) + bm.MineBlocks() + defer bm.Stop() + + sm, err := api.MpoolPushMessage(ctx, msg, nil) + if err != nil { + t.Fatal(err) + } + res, err := api.StateWaitMsg(ctx, sm.Cid(), 1) + if err != nil { + t.Fatal(err) + } + if res.Receipt.ExitCode != 0 { + t.Fatal("did not successfully send message") + } + + searchRes, err := api.StateSearchMsg(ctx, sm.Cid()) + if err != nil { + t.Fatal(err) + } + + if searchRes.TipSet != res.TipSet { + t.Fatalf("search ts: %s, different from wait ts: %s", searchRes.TipSet, res.TipSet) + } + +} + func (ts *testSuite) testID(t *testing.T) { ctx := context.Background() apis, _ := ts.makeNodes(t, OneFull, OneMiner) From 98c1aa7988750b172bce86ce13554bc218af33ad Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 13 Oct 2020 13:31:20 -0700 Subject: [PATCH 039/106] add WalletVerify to lotus-gateway --- cmd/lotus-gateway/api.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index d5fac0a06..1e680daac 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -7,9 +7,13 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" ) @@ -172,6 +176,10 @@ func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence u return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } +func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { + return sigs.Verify(sig, k, msg) == nil, nil +} + var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.GasModuleAPI = (*GatewayAPI)(nil) From 3d80c3806473c02984d51de61de903615f61a876 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 13 Oct 2020 14:23:08 -0700 Subject: [PATCH 040/106] support stateReadState in gateway --- cmd/lotus-gateway/api.go | 5 +++++ cmd/lotus-gateway/api_test.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 1e680daac..381dd931b 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -41,6 +41,7 @@ type gatewayDepsAPI interface { StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) + StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) } type GatewayAPI struct { @@ -180,6 +181,10 @@ func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg [] return sigs.Verify(sig, k, msg) == nil, nil } +func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { + return a.api.StateReadState(ctx, actor, tsk) +} + var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.GasModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index f34f887f5..6719c57c5 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -189,3 +189,7 @@ func (m *mockGatewayDepsAPI) StateLookupID(ctx context.Context, addr address.Add func (m *mockGatewayDepsAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { panic("implement me") } + +func (m *mockGatewayDepsAPI) StateReadState(ctx context.Context, act address.Address, ts types.TipSetKey) (*api.ActorState, error) { + panic("implement me") +} From b61281ba4bea27b334343e2a1ebc65663e736146 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 13 Oct 2020 15:06:47 -0700 Subject: [PATCH 041/106] fix a panic on startup when we fail to load the tipset There may be a deeper issue here, but we need to handle this more gracefully somehow. See #4358. --- chain/messagepool/messagepool.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index d3c638b22..83b1a7f29 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -670,6 +670,10 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { curTs := mp.curTs + if curTs == nil { + return xerrors.Errorf("current tipset not loaded") + } + snonce, err := mp.getStateNonce(m.Message.From, curTs) if err != nil { return xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure) From 24267008212f8330bf67ba8d064d2784ae557e4a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 12 Oct 2020 21:50:11 -0400 Subject: [PATCH 042/106] Don't transfer zero FIL during rebalancing fork --- chain/stmgr/forks.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 7f2b57546..36b0da83c 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -312,11 +312,13 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal available = st.GetAvailableBalance(act.Balance) } - transfers = append(transfers, transfer{ - From: addr, - To: builtin.ReserveAddress, - Amt: available, - }) + if !available.IsZero() { + transfers = append(transfers, transfer{ + From: addr, + To: builtin.ReserveAddress, + Amt: available, + }) + } } return nil }) From 7dbc8965bea3c17e22c643b34e4d91fc3091a38a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 13 Oct 2020 01:05:28 -0400 Subject: [PATCH 043/106] Apply nonces to implicit messages --- chain/stmgr/stmgr.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 0cb17fcf3..113304de1 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -247,17 +247,12 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err) } - runCron := func() error { - // TODO: this nonce-getting is a tiny bit ugly - ca, err := vmi.StateTree().GetActor(builtin0.SystemActorAddr) - if err != nil { - return err - } + runCron := func(epoch abi.ChainEpoch) error { cronMsg := &types.Message{ To: builtin0.CronActorAddr, From: builtin0.SystemActorAddr, - Nonce: ca.Nonce, + Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), @@ -284,7 +279,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp for i := parentEpoch; i < epoch; i++ { if i > parentEpoch { // run cron for null rounds if any - if err := runCron(); err != nil { + if err := runCron(i); err != nil { return cid.Undef, cid.Undef, err } @@ -350,15 +345,10 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) } - sysAct, actErr := vmi.StateTree().GetActor(builtin0.SystemActorAddr) - if actErr != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to get system actor: %w", actErr) - } - rwMsg := &types.Message{ From: builtin0.SystemActorAddr, To: reward.Address, - Nonce: sysAct.Nonce, + Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), @@ -381,7 +371,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } } - if err := runCron(); err != nil { + if err := runCron(epoch); err != nil { return cid.Cid{}, cid.Cid{}, err } From 58662b79b38be538d76ec06c7ab06dcaef854ac7 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 13 Oct 2020 19:50:13 -0700 Subject: [PATCH 044/106] fix(deps): use tagged go-fil-markets 0.9.0 --- go.mod | 6 +++--- go.sum | 12 ++++++------ node/modules/client.go | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 75926117a..0118baa88 100644 --- a/go.mod +++ b/go.mod @@ -27,9 +27,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.1 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5 + github.com/filecoin-project/go-data-transfer v0.9.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a + github.com/filecoin-project/go-fil-markets v0.9.0 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -66,7 +66,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c + github.com/ipfs/go-graphsync v0.3.0 github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 50e85dfd5..2b0859bfd 100644 --- a/go.sum +++ b/go.sum @@ -240,14 +240,14 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5 h1:kgmVHDP+rqcPqytpC215MzRRXrW9FqTfn0VFWhB2oog= -github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5/go.mod h1:1zaap881YZLMeIxhvLM0zA9qfUkh3bjOLSRaOoO9WRM= +github.com/filecoin-project/go-data-transfer v0.9.0 h1:nTT8j7Hu3TM0wRWrGy83/ctawG7sleJGdFWtIsUsKgY= +github.com/filecoin-project/go-data-transfer v0.9.0/go.mod h1:i2CqUy7TMQGKukj9BgqIxiP8nDHDXU2VLd771KVaCaQ= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a h1:jC3yO5R0+jGnVAHVqIHXb3BqCN/crslxTALz4I5gw1o= -github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a/go.mod h1:HqbtLIwJ4Lonp0BWq4ajmdXZ2UTh2+YK3wZSXXAE8yA= +github.com/filecoin-project/go-fil-markets v0.9.0 h1:RVdMKeOdRtpHawB14+uvJhvq243/UfZ4g3yFa/6qekk= +github.com/filecoin-project/go-fil-markets v0.9.0/go.mod h1:h+bJ/IUnYjnW5HMKyt9JQSnhslqetkpuzwwugc3K8vM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -538,8 +538,8 @@ github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPi github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= -github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c h1:De/AZGvRa3WMyw5zdMMhcvRcho46BVo+C0NRud+T4io= -github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.3.0 h1:I6Y20kSuCWkUvPoUWo4V3am704/9QjgDVVkf0zIV8+8= +github.com/ipfs/go-graphsync v0.3.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= diff --git a/node/modules/client.go b/node/modules/client.go index eab309445..f1380bc97 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -91,7 +91,7 @@ func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Grap return nil, err } - dt.OnReady(marketevents.ReadyLogger("provider data transfer")) + dt.OnReady(marketevents.ReadyLogger("client data transfer")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return dt.Start(ctx) From 458ea277fcb4cb9f0e52453da4b50399632040e6 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 13 Oct 2020 01:20:49 -0400 Subject: [PATCH 045/106] Aggregate irregular state transitions into a single EexcutionTrace --- chain/stmgr/forks.go | 118 +++++++++++++++++++++++++++++++++---------- 1 file changed, 92 insertions(+), 26 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 36b0da83c..156c68783 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -168,7 +168,7 @@ func (sm *StateManager) hasExpensiveFork(ctx context.Context, height abi.ChainEp return ok } -func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { +func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmount, cb func(trace types.ExecutionTrace)) error { fromAct, err := tree.GetActor(from) if err != nil { return xerrors.Errorf("failed to get 'from' actor for transfer: %w", err) @@ -201,7 +201,6 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, From: from, To: to, Value: amt, - Nonce: math.MaxUint64, } fakeRct := &types.MessageReceipt{ ExitCode: 0, @@ -209,22 +208,14 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, GasUsed: 0, } - if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ - MessageReceipt: *fakeRct, - ActorErr: nil, - ExecutionTrace: types.ExecutionTrace{ - Msg: fakeMsg, - MsgRct: fakeRct, - Error: "", - Duration: 0, - GasCharges: nil, - Subcalls: nil, - }, - Duration: 0, - GasCosts: vm.ZeroGasOutputs(), - }); err != nil { - return xerrors.Errorf("recording transfer: %w", err) - } + cb(types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: nil, + }) } return nil @@ -277,6 +268,10 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } var transfers []transfer + subcalls := make([]types.ExecutionTrace, 0) + transferCb := func(trace types.ExecutionTrace) { + subcalls = append(subcalls, trace) + } // Take all excess funds away, put them into the reserve account err = tree.ForEach(func(addr address.Address, act *types.Actor) error { @@ -328,7 +323,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal // Execute transfers from previous step for _, t := range transfers { - if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + if err := doTransfer(tree, t.From, t.To, t.Amt, transferCb); err != nil { return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } @@ -431,7 +426,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } for _, t := range transfersBack { - if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + if err := doTransfer(tree, t.From, t.To, t.Amt, transferCb); err != nil { return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } @@ -441,7 +436,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if err != nil { return cid.Undef, xerrors.Errorf("failed to load burnt funds actor: %w", err) } - if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, builtin.ReserveAddress, burntAct.Balance); err != nil { + if err := doTransfer(tree, builtin0.BurntFundsActorAddr, builtin.ReserveAddress, burntAct.Balance, transferCb); err != nil { return cid.Undef, xerrors.Errorf("failed to unburn funds: %w", err) } @@ -457,7 +452,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } difference := types.BigSub(DesiredReimbursementBalance, reimb.Balance) - if err := doTransfer(cb, tree, builtin.ReserveAddress, reimbAddr, difference); err != nil { + if err := doTransfer(tree, builtin.ReserveAddress, reimbAddr, difference, transferCb); err != nil { return cid.Undef, xerrors.Errorf("failed to top up reimbursement account: %w", err) } @@ -476,6 +471,39 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal return cid.Undef, xerrors.Errorf("resultant state tree account balance was not correct: %s", total) } + if cb != nil { + // record the transfer in execution traces + + fakeMsg := &types.Message{ + From: builtin.SystemActorAddr, + To: builtin.SystemActorAddr, + Value: big.Zero(), + Nonce: uint64(epoch), + } + fakeRct := &types.MessageReceipt{ + ExitCode: 0, + Return: nil, + GasUsed: 0, + } + + if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ + MessageReceipt: *fakeRct, + ActorErr: nil, + ExecutionTrace: types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: subcalls, + }, + Duration: 0, + GasCosts: vm.ZeroGasOutputs(), + }); err != nil { + return cid.Undef, xerrors.Errorf("recording transfers: %w", err) + } + } + return tree.Flush(ctx) } @@ -516,12 +544,12 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err) } - err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50) + err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50, epoch) if err != nil { return cid.Undef, xerrors.Errorf("splitting first msig: %w", err) } - err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50) + err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50, epoch) if err != nil { return cid.Undef, xerrors.Errorf("splitting second msig: %w", err) } @@ -647,7 +675,7 @@ func setNetworkName(ctx context.Context, store adt.Store, tree *state.StateTree, return nil } -func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64) error { +func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64, epoch abi.ChainEpoch) error { if portions < 1 { return xerrors.Errorf("cannot split into 0 portions") } @@ -716,6 +744,11 @@ func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Add } i := uint64(0) + subcalls := make([]types.ExecutionTrace, 0, portions) + transferCb := func(trace types.ExecutionTrace) { + subcalls = append(subcalls, trace) + } + for i < portions { keyAddr, err := makeKeyAddr(addr, i) if err != nil { @@ -732,13 +765,46 @@ func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Add return xerrors.Errorf("setting new msig actor state: %w", err) } - if err := doTransfer(cb, tree, addr, idAddr, newIbal); err != nil { + if err := doTransfer(tree, addr, idAddr, newIbal, transferCb); err != nil { return xerrors.Errorf("transferring split msig balance: %w", err) } i++ } + if cb != nil { + // record the transfer in execution traces + + fakeMsg := &types.Message{ + From: builtin.SystemActorAddr, + To: addr, + Value: big.Zero(), + Nonce: uint64(epoch), + } + fakeRct := &types.MessageReceipt{ + ExitCode: 0, + Return: nil, + GasUsed: 0, + } + + if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ + MessageReceipt: *fakeRct, + ActorErr: nil, + ExecutionTrace: types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: subcalls, + }, + Duration: 0, + GasCosts: vm.ZeroGasOutputs(), + }); err != nil { + return xerrors.Errorf("recording transfers: %w", err) + } + } + return nil } From ce541102289ee8db4017c84c9fb88250e9ce4fbe Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 13 Oct 2020 23:45:47 -0400 Subject: [PATCH 046/106] Add message CID to InvocResult --- api/api_full.go | 1 + chain/stmgr/call.go | 2 ++ chain/stmgr/stmgr.go | 1 + documentation/en/api-methods.md | 6 ++++++ node/impl/full/state.go | 1 + 5 files changed, 11 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index 657e945ca..22a7380ac 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -734,6 +734,7 @@ type RetrievalOrder struct { } type InvocResult struct { + MsgCid cid.Cid Msg *types.Message MsgRct *types.MessageReceipt ExecutionTrace types.ExecutionTrace diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 8d0f6f6c1..86035140b 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -113,6 +113,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. } return &api.InvocResult{ + MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, ExecutionTrace: ret.ExecutionTrace, @@ -228,6 +229,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri } return &api.InvocResult{ + MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, ExecutionTrace: ret.ExecutionTrace, diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 0cb17fcf3..e5accc0c9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -200,6 +200,7 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { return func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { ir := &api.InvocResult{ + MsgCid: mcid, Msg: msg, MsgRct: &ret.MessageReceipt, ExecutionTrace: ret.ExecutionTrace, diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index eefc53ab7..2ed000c59 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3059,6 +3059,9 @@ Inputs: Response: ```json { + "MsgCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, "Msg": { "Version": 42, "To": "f01234", @@ -4079,6 +4082,9 @@ Inputs: Response: ```json { + "MsgCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, "Msg": { "Version": 42, "To": "f01234", diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 1c39b59da..08ac62f2e 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -362,6 +362,7 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. } return &api.InvocResult{ + MsgCid: mc, Msg: m, MsgRct: &r.MessageReceipt, ExecutionTrace: r.ExecutionTrace, From 2e7cd024df10d9c0ea9840dd17f4ec6b4c980698 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 14 Oct 2020 02:10:26 -0400 Subject: [PATCH 047/106] Lotus version 0.10.1 --- CHANGELOG.md | 23 +++++++++++++++++++++++ build/version.go | 4 ++-- documentation/en/api-methods.md | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d8d15b8..7c94e387c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Lotus changelog +# 0.10.1 / 2020-10-14 + +This is an optional release of Lotus that updates markets to 0.9.0, which adds the ability to restart data transfers. This release also introduces Ledger support, and various UX improvements. + +## Changes + +- Test the tape upgrade (https://github.com/filecoin-project/lotus/pull/4328) +- Adding in Ledger support (https://github.com/filecoin-project/lotus/pull/4290) +- Improve the UX for lotus-miner sealing workers (https://github.com/filecoin-project/lotus/pull/4329) +- Add a CLI tool for miner's to repay debt (https://github.com/filecoin-project/lotus/pull/4319) +- Rename params_testnet to params_mainnet (https://github.com/filecoin-project/lotus/pull/4336) +- Use seal-duration in calculating the earliest StartEpoch (https://github.com/filecoin-project/lotus/pull/4337) +- Reject deals that are > 7 days in the future in the BasicDealFilter (https://github.com/filecoin-project/lotus/pull/4173) +- Add an API endpoint to calculate the exact circulating supply (https://github.com/filecoin-project/lotus/pull/4148) +- lotus-pcr: ignore all other market messages (https://github.com/filecoin-project/lotus/pull/4341) +- Add message CID to InvocResult (https://github.com/filecoin-project/lotus/pull/4382) +- types: Add CID fields to messages in json marshalers (https://github.com/filecoin-project/lotus/pull/4338) +- fix(sync state): set state height to actual tipset height (https://github.com/filecoin-project/lotus/pull/4347) +- Fix off by one tipset in searchBackForMsg (https://github.com/filecoin-project/lotus/pull/4367) +- fix a panic on startup when we fail to load the tipset (https://github.com/filecoin-project/lotus/pull/4376) +- Avoid having the same message CID show up in execution traces (https://github.com/filecoin-project/lotus/pull/4350) +- feat(markets): update markets 0.9.0 and add data transfer restart (https://github.com/filecoin-project/lotus/pull/4363) + # 0.10.0 / 2020-10-12 This is a consensus-breaking hotfix that addresses an issue in specs-actors v2.0.3 that made it impossible to pledge new 32GiB sectors. The change in Lotus is to update to actors v2.1.0, behind the new network version 5. diff --git a/build/version.go b/build/version.go index baeafdfe7..39f9d48ff 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.10.0" +const BuildVersion = "0.10.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit @@ -83,7 +83,7 @@ func VersionForType(nodeType NodeType) (Version, error) { // semver versions of the rpc api exposed var ( - FullAPIVersion = newVer(0, 16, 0) + FullAPIVersion = newVer(0, 17, 0) MinerAPIVersion = newVer(0, 15, 0) WorkerAPIVersion = newVer(0, 15, 0) ) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 1c8a3a24f..a905ef2e3 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -228,7 +228,7 @@ Response: ```json { "Version": "string value", - "APIVersion": 4096, + "APIVersion": 4352, "BlockDelay": 42 } ``` From 30dd5d9644d88c157b0f025607df84a50e7ad388 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Wed, 14 Oct 2020 11:17:44 +0530 Subject: [PATCH 048/106] use updated stored ask api --- node/modules/storageminer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 613203a60..adacf5405 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -393,13 +393,13 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat if err != nil { return nil, err } - storedAsk, err := storedask.NewStoredAsk(namespace.Wrap(providerDs, datastore.NewKey("/storage-ask")), datastore.NewKey("latest"), spn, address.Address(minerAddress)) + storedAsk, err := storedask.NewStoredAsk(namespace.Wrap(providerDs, datastore.NewKey("/storage-ask")), datastore.NewKey("latest"), spn, address.Address(minerAddress), + storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) if err != nil { return nil, err } - // Hacky way to set max piece size to the sector size a := storedAsk.GetAsk().Ask - err = storedAsk.SetAsk(a.Price, a.VerifiedPrice, a.Expiry-a.Timestamp, storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) + err = storedAsk.SetAsk(a.Price, a.VerifiedPrice, a.Expiry-a.Timestamp) if err != nil { return storedAsk, err } From c6ed6a0d65bbf294c1d634161b2ea4288630bddf Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 14 Oct 2020 14:53:11 +0200 Subject: [PATCH 049/106] Add support for /https, /http, /wss API multiaddresses. Lotus API endpoint are by default expressed as multiaddresses (i.e. lotus auth api-info) which end in /http, although they can be provided as standard URLs too. There is an inconsistency here because despite the "http" part, Lotus will use "ws" protocol. This lets lotus default to "ws" but honor whatever the user puts in the multiaddress (i.e. /dns4/my.lotus.node/tcp/443/https) would work now using https, where before it uses "ws". --- cli/util/apiinfo.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/cli/util/apiinfo.go b/cli/util/apiinfo.go index 1f9a83769..dec4c2e19 100644 --- a/cli/util/apiinfo.go +++ b/cli/util/apiinfo.go @@ -44,7 +44,22 @@ func (a APIInfo) DialArgs() (string, error) { return "", err } - return "ws://" + addr + "/rpc/v0", nil + protocol := "ws" + + // If the user specifies the multiaddress as + // /something/tcp/1234/http or/something/tcp/1234/https + // or /something/tcp/1234/wss then honor that. + for _, p := range []int{ + multiaddr.P_HTTP, + multiaddr.P_HTTPS, + multiaddr.P_WSS, + } { + if _, err := ma.ValueForProtocol(p); err == nil { + protocol = multiaddr.ProtocolWithCode(p).Name + break + } + } + return protocol + "://" + addr + "/rpc/v0", nil } _, err = url.Parse(a.Addr) From 00a50831460c41727cb9d1bc67a1ef4f78b949a0 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 14 Oct 2020 15:11:30 +0200 Subject: [PATCH 050/106] Update SECURITY.md --- SECURITY.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index ecb600deb..592206bc5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,9 +2,7 @@ ## Reporting a Vulnerability -For *critical* bugs, please send an email to security@filecoin.org. - -The bug reporting process differs between bugs that are critical and may crash the network, and others that are unlikely to cause problems if malicious parties know about it. For non-critical bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). +For *critical* bugs, please consult our Security Policy and Responsible Disclosure Program information at https://github.com/filecoin-project/community/blob/master/SECURITY.md Please try to provide a clear description of any bugs reported, along with how to reproduce the bug if possible. More detailed bug reports (especially those with a PoC included) will help us move forward much faster. Additionally, please avoid reporting bugs that already have open issues. Take a moment to search the issue list of the related GitHub repositories before writing up a new report. @@ -20,10 +18,6 @@ Here are some examples of bugs we would consider 'critical': This is not an exhaustive list, but should provide some idea of what we consider 'critical'. -## Supported Versions +## Reporting a non security bug -* TODO: This should be defined and set up by Mainnet launch. - -| Version | Supported | -| ------- | ------------------ | -| Testnet | :white_check_mark: | +For non-critical bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). From 45cd510da11807351ef87cc47c4169581eb2779b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 13 Oct 2020 23:00:01 +0100 Subject: [PATCH 051/106] conformance: support multiple protocol versions. This PR introduces support for running multiple variants of a vector, each of which targets a unique protocol version. tvx tooling has been adapted to produce and parse the new version of the schema. --- cmd/tvx/codenames.go | 40 ++++++++++++++++++++++++++++++++++++++ cmd/tvx/codenames_test.go | 28 ++++++++++++++++++++++++++ cmd/tvx/exec.go | 30 +++++++++++++++------------- cmd/tvx/extract.go | 14 ++++++++++++- conformance/corpus_test.go | 22 ++++++++++++++------- conformance/driver.go | 5 ++--- conformance/runner.go | 26 +++++++++++++------------ go.mod | 2 +- go.sum | 4 ++-- 9 files changed, 132 insertions(+), 39 deletions(-) create mode 100644 cmd/tvx/codenames.go create mode 100644 cmd/tvx/codenames_test.go diff --git a/cmd/tvx/codenames.go b/cmd/tvx/codenames.go new file mode 100644 index 000000000..1cb1b32fb --- /dev/null +++ b/cmd/tvx/codenames.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/build" +) + +// ProtocolCodenames is a table that summarises the protocol codenames that +// will be set on extracted vectors, depending on the original execution height. +// +// Implementers rely on these names to filter the vectors they can run through +// their implementations, based on their support level +var ProtocolCodenames = []struct { + firstEpoch abi.ChainEpoch + name string +}{ + {0, "genesis"}, + // TODO there is some off-by-one trickery in GetNtwkVersion. Not sure if the + // protocol version really kicks in at the designated height, or at the + // following epoch. + {build.UpgradeBreezeHeight + 1, "breeze"}, + {build.UpgradeSmokeHeight + 1, "smoke"}, + {build.UpgradeIgnitionHeight + 1, "ignition"}, + {build.UpgradeRefuelHeight + 1, "refuel"}, + {build.UpgradeActorsV2Height + 1, "actorsv2"}, + {build.UpgradeTapeHeight + 1, "tape"}, + {build.UpgradeLiftoffHeight + 1, "liftoff"}, +} + +// GetProtocolCodename gets the protocol codename associated with a height. +func GetProtocolCodename(height abi.ChainEpoch) string { + for i, v := range ProtocolCodenames { + if height < v.firstEpoch { + // found the cutoff, return previous. + return ProtocolCodenames[i-1].name + } + } + return ProtocolCodenames[len(ProtocolCodenames)-1].name +} diff --git a/cmd/tvx/codenames_test.go b/cmd/tvx/codenames_test.go new file mode 100644 index 000000000..0b1de9fc2 --- /dev/null +++ b/cmd/tvx/codenames_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "math" + "testing" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/build" +) + +func TestProtocolCodenames(t *testing.T) { + if height := abi.ChainEpoch(100); GetProtocolCodename(height) != "genesis" { + t.Fatal("expected genesis codename") + } + + if height := abi.ChainEpoch(build.UpgradeBreezeHeight + 1); GetProtocolCodename(height) != "breeze" { + t.Fatal("expected breeze codename") + } + + if height := abi.ChainEpoch(build.UpgradeActorsV2Height + 1); GetProtocolCodename(height) != "actorsv2" { + t.Fatal("expected actorsv2 codename") + } + + if height := abi.ChainEpoch(math.MaxInt64); GetProtocolCodename(height) != ProtocolCodenames[len(ProtocolCodenames)-1].name { + t.Fatal("expected last codename") + } +} diff --git a/cmd/tvx/exec.go b/cmd/tvx/exec.go index 9ec6f9e2b..de7d2a398 100644 --- a/cmd/tvx/exec.go +++ b/cmd/tvx/exec.go @@ -72,20 +72,24 @@ func runExecLotus(_ *cli.Context) error { func executeTestVector(tv schema.TestVector) error { log.Println("executing test vector:", tv.Meta.ID) - r := new(conformance.LogReporter) - switch class := tv.Class; class { - case "message": - conformance.ExecuteMessageVector(r, &tv) - case "tipset": - conformance.ExecuteTipsetVector(r, &tv) - default: - return fmt.Errorf("test vector class %s not supported", class) - } - if r.Failed() { - log.Println(color.HiRedString("❌ test vector failed")) - } else { - log.Println(color.GreenString("✅ test vector succeeded")) + for _, v := range tv.Pre.Variants { + r := new(conformance.LogReporter) + + switch class := tv.Class; class { + case "message": + conformance.ExecuteMessageVector(r, &tv, &v) + case "tipset": + conformance.ExecuteTipsetVector(r, &tv, &v) + default: + return fmt.Errorf("test vector class %s not supported", class) + } + + if r.Failed() { + log.Println(color.HiRedString("❌ test vector failed for variant %s", v.ID)) + } else { + log.Println(color.GreenString("✅ test vector succeeded for variant %s", v.ID)) + } } return nil diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 4bb391af6..4a8dd162c 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -347,6 +347,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { return err } + nv, err := fapi.StateNetworkVersion(ctx, execTs.Key()) + if err != nil { + return err + } + + codename := GetProtocolCodename(execTs.Height()) + // Write out the test vector. vector := schema.TestVector{ Class: schema.ClassMessage, @@ -363,10 +370,15 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, + Selector: schema.Selector{ + schema.SelectorMinProtocolVersion: codename, + }, Randomness: recordingRand.Recorded(), CAR: out.Bytes(), Pre: &schema.Preconditions{ - Epoch: int64(execTs.Height()), + Variants: []schema.Variant{ + {ID: codename, Epoch: int64(execTs.Height()), NetworkVersion: uint(nv)}, + }, CircSupply: circSupply.Int, BaseFee: basefee.Int, StateTree: &schema.StateTree{ diff --git a/conformance/corpus_test.go b/conformance/corpus_test.go index 3d447570d..a09f9a8d3 100644 --- a/conformance/corpus_test.go +++ b/conformance/corpus_test.go @@ -11,6 +11,11 @@ import ( "github.com/filecoin-project/test-vectors/schema" ) +var invokees = map[schema.Class]func(Reporter, *schema.TestVector, *schema.Variant){ + schema.ClassMessage: ExecuteMessageVector, + schema.ClassTipset: ExecuteTipsetVector, +} + const ( // EnvSkipConformance, if 1, skips the conformance test suite. EnvSkipConformance = "SKIP_CONFORMANCE" @@ -120,13 +125,16 @@ func TestConformance(t *testing.T) { } // dispatch the execution depending on the vector class. - switch vector.Class { - case "message": - ExecuteMessageVector(t, &vector) - case "tipset": - ExecuteTipsetVector(t, &vector) - default: - t.Fatalf("test vector class not supported: %s", vector.Class) + invokee, ok := invokees[vector.Class] + if !ok { + t.Fatalf("unsupported test vector class: %s", vector.Class) + } + + for _, variant := range vector.Pre.Variants { + variant := variant + t.Run(variant.ID, func(t *testing.T) { + invokee(t, &vector, &variant) + }) } }) } diff --git a/conformance/driver.go b/conformance/driver.go index f49022b9c..fbb7dda6d 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -82,7 +82,7 @@ type ExecuteTipsetResult struct { // This method returns the the receipts root, the poststate root, and the VM // message results. The latter _include_ implicit messages, such as cron ticks // and reward withdrawal per miner. -func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) { +func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset, execEpoch abi.ChainEpoch) (*ExecuteTipsetResult, error) { var ( syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) vmRand = NewFixedRand() @@ -121,11 +121,10 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot messages []*types.Message results []*vm.ApplyRet - epoch = abi.ChainEpoch(tipset.Epoch) basefee = abi.NewTokenAmount(tipset.BaseFee.Int64()) ) - postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, epoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { + postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, execEpoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { messages = append(messages, msg) results = append(results, ret) return nil diff --git a/conformance/runner.go b/conformance/runner.go index d489ac288..6f9d73305 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -30,11 +30,11 @@ import ( ) // ExecuteMessageVector executes a message-class test vector. -func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { +func ExecuteMessageVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) { var ( - ctx = context.Background() - epoch = vector.Pre.Epoch - root = vector.Pre.StateTree.RootCID + ctx = context.Background() + baseEpoch = variant.Epoch + root = vector.Pre.StateTree.RootCID ) // Load the CAR into a new temporary Blockstore. @@ -53,16 +53,16 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { r.Fatalf("failed to deserialize message: %s", err) } - // add an epoch if one's set. - if m.Epoch != nil { - epoch = *m.Epoch + // add the epoch offset if one is set. + if m.EpochOffset != nil { + baseEpoch += *m.EpochOffset } // Execute the message. var ret *vm.ApplyRet ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{ Preroot: root, - Epoch: abi.ChainEpoch(epoch), + Epoch: abi.ChainEpoch(baseEpoch), Message: msg, BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), @@ -86,10 +86,10 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // ExecuteTipsetVector executes a tipset-class test vector. -func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { +func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) { var ( ctx = context.Background() - prevEpoch = vector.Pre.Epoch + baseEpoch = abi.ChainEpoch(variant.Epoch) root = vector.Pre.StateTree.RootCID tmpds = ds.NewMapDatastore() ) @@ -105,9 +105,11 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { // Apply every tipset. var receiptsIdx int + var prevEpoch = baseEpoch for i, ts := range vector.ApplyTipsets { ts := ts // capture - ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts) + execEpoch := baseEpoch + abi.ChainEpoch(ts.EpochOffset) + ret, err := driver.ExecuteTipset(bs, tmpds, root, prevEpoch, &ts, execEpoch) if err != nil { r.Fatalf("failed to apply tipset %d message: %s", i, err) } @@ -122,7 +124,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual) } - prevEpoch = ts.Epoch + prevEpoch = execEpoch root = ret.PostStateRoot } diff --git a/go.mod b/go.mod index 0118baa88..1516cba81 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/filecoin-project/specs-actors v0.9.12 github.com/filecoin-project/specs-actors/v2 v2.1.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.4 + github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 2b0859bfd..47301c784 100644 --- a/go.sum +++ b/go.sum @@ -282,8 +282,8 @@ github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= -github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 h1:qnleaW7X8Gi2e3QtqPMFdqVn/DUaNI5tCnq7wNVMnio= +github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 12f36c5bda4c8c49b42bbaf19d0964bccf448987 Mon Sep 17 00:00:00 2001 From: Whyrusleeping Date: Wed, 14 Oct 2020 09:22:25 -0500 Subject: [PATCH 052/106] Update cmd/lotus-gateway/api.go Co-authored-by: dirkmc --- cmd/lotus-gateway/api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 381dd931b..8f1d69c18 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -182,6 +182,9 @@ func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg [] } func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } return a.api.StateReadState(ctx, actor, tsk) } From ab4f051b837646cd2fe7efeef98513604e2abf82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 14 Oct 2020 18:09:31 +0200 Subject: [PATCH 053/106] sync wait --watch --- cli/sync.go | 12 +++++++++--- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus-storage-miner/init_restore.go | 2 +- cmd/lotus-storage-miner/run.go | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cli/sync.go b/cli/sync.go index ea066cbda..c3f25eb1d 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -84,6 +84,12 @@ var syncStatusCmd = &cli.Command{ var syncWaitCmd = &cli.Command{ Name: "wait", Usage: "Wait for sync to be complete", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "watch", + Usage: "don't exit after node is synced", + }, + }, Action: func(cctx *cli.Context) error { napi, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -92,7 +98,7 @@ var syncWaitCmd = &cli.Command{ defer closer() ctx := ReqContext(cctx) - return SyncWait(ctx, napi) + return SyncWait(ctx, napi, cctx.Bool("watch")) }, } @@ -234,7 +240,7 @@ var syncCheckpointCmd = &cli.Command{ }, } -func SyncWait(ctx context.Context, napi api.FullNode) error { +func SyncWait(ctx context.Context, napi api.FullNode, watch bool) error { tick := time.Second / 4 lastLines := 0 @@ -311,7 +317,7 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { _ = target // todo: maybe print? (creates a bunch of line wrapping issues with most tipsets) - if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { + if !watch && time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { fmt.Println("\nDone!") return nil } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index ccea707af..326a39e5e 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -155,7 +155,7 @@ var initCmd = &cli.Command{ log.Info("Checking full node sync status") if !cctx.Bool("genesis-miner") && !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, api); err != nil { + if err := lcli.SyncWait(ctx, api, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index bdbb99fe0..83a9ad87c 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -72,7 +72,7 @@ var initRestoreCmd = &cli.Command{ } if !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, api); err != nil { + if err := lcli.SyncWait(ctx, api, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 98a9cfaba..c043bd903 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -84,7 +84,7 @@ var runCmd = &cli.Command{ log.Info("Checking full node sync status") if !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, nodeApi); err != nil { + if err := lcli.SyncWait(ctx, nodeApi, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } From b060569fe92cb374ef79c22df2dab40e93c43477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 14 Oct 2020 19:55:36 +0200 Subject: [PATCH 054/106] Revert "Add support for /https, /http, /wss API multiaddresses." --- cli/util/apiinfo.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/cli/util/apiinfo.go b/cli/util/apiinfo.go index dec4c2e19..1f9a83769 100644 --- a/cli/util/apiinfo.go +++ b/cli/util/apiinfo.go @@ -44,22 +44,7 @@ func (a APIInfo) DialArgs() (string, error) { return "", err } - protocol := "ws" - - // If the user specifies the multiaddress as - // /something/tcp/1234/http or/something/tcp/1234/https - // or /something/tcp/1234/wss then honor that. - for _, p := range []int{ - multiaddr.P_HTTP, - multiaddr.P_HTTPS, - multiaddr.P_WSS, - } { - if _, err := ma.ValueForProtocol(p); err == nil { - protocol = multiaddr.ProtocolWithCode(p).Name - break - } - } - return protocol + "://" + addr + "/rpc/v0", nil + return "ws://" + addr + "/rpc/v0", nil } _, err = url.Parse(a.Addr) From 8df58064e3da1a19cc9ae1e869443143b1636be6 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 14 Oct 2020 13:20:18 -0700 Subject: [PATCH 055/106] feat(markets): update to 0.9.1 bugfix release --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0118baa88..c3935be19 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.9.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.9.0 + github.com/filecoin-project/go-fil-markets v0.9.1 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index 2b0859bfd..feff6abbd 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.9.0 h1:RVdMKeOdRtpHawB14+uvJhvq243/UfZ4g3yFa/6qekk= -github.com/filecoin-project/go-fil-markets v0.9.0/go.mod h1:h+bJ/IUnYjnW5HMKyt9JQSnhslqetkpuzwwugc3K8vM= +github.com/filecoin-project/go-fil-markets v0.9.1 h1:MgO+UkpreD6x8DV2Zkw2xlBogixfpw9/wf4+nBii7bU= +github.com/filecoin-project/go-fil-markets v0.9.1/go.mod h1:h+bJ/IUnYjnW5HMKyt9JQSnhslqetkpuzwwugc3K8vM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= From 470a565c349cd150675b01607ea08cd995b407ee Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Oct 2020 14:27:35 -0700 Subject: [PATCH 056/106] fix: return true when deadlines changed --- chain/actors/builtin/miner/v0.go | 2 +- chain/actors/builtin/miner/v2.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 14718d002..1b1c817b6 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -274,7 +274,7 @@ func (s *state0) DeadlinesChanged(other State) (bool, error) { return true, nil } - return s.State.Deadlines.Equals(other0.Deadlines), nil + return !s.State.Deadlines.Equals(other0.Deadlines), nil } func (s *state0) Info() (MinerInfo, error) { diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 9e599f891..7217f6b58 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -273,7 +273,7 @@ func (s *state2) DeadlinesChanged(other State) (bool, error) { return true, nil } - return s.State.Deadlines.Equals(other2.Deadlines), nil + return !s.State.Deadlines.Equals(other2.Deadlines), nil } func (s *state2) Info() (MinerInfo, error) { From ea1349205b69c548b35b774830c592ba8bdd6997 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Oct 2020 14:37:33 -0700 Subject: [PATCH 057/106] fix: return true when partitions changed --- chain/actors/builtin/miner/v0.go | 2 +- chain/actors/builtin/miner/v2.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 1b1c817b6..c79f4a2f7 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -370,7 +370,7 @@ func (d *deadline0) PartitionsChanged(other Deadline) (bool, error) { return true, nil } - return d.Deadline.Partitions.Equals(other0.Deadline.Partitions), nil + return !d.Deadline.Partitions.Equals(other0.Deadline.Partitions), nil } func (d *deadline0) PostSubmissions() (bitfield.BitField, error) { diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 7217f6b58..c686fa483 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -369,7 +369,7 @@ func (d *deadline2) PartitionsChanged(other Deadline) (bool, error) { return true, nil } - return d.Deadline.Partitions.Equals(other2.Deadline.Partitions), nil + return !d.Deadline.Partitions.Equals(other2.Deadline.Partitions), nil } func (d *deadline2) PostSubmissions() (bitfield.BitField, error) { From bbc6de94fc5ce48aa18928827ff6485f96c7f27d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 12:23:11 -0700 Subject: [PATCH 058/106] write messages to a temp blockstore when validating --- chain/sync.go | 86 ++++++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index dd83bf319..97915d8df 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -323,25 +323,35 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc) } - // Collect the CIDs of both types of messages separately: BLS and Secpk. - var bcids, scids []cid.Cid - for _, m := range fblk.BlsMessages { - bcids = append(bcids, m.Cid()) - } - - for _, m := range fblk.SecpkMessages { - scids = append(scids, m.Cid()) - } - // TODO: IMPORTANT(GARBAGE). These message puts and the msgmeta // computation need to go into the 'temporary' side of the blockstore when // we implement that - blockstore := syncer.store.Blockstore() - bs := cbor.NewCborStore(blockstore) + // We use a temporary bstore here to avoid writing intermediate pieces + // into the blockstore. + blockstore := bstore.NewTemporary() + cst := cbor.NewCborStore(blockstore) + + var bcids, scids []cid.Cid + + for _, m := range fblk.BlsMessages { + c, err := store.PutMessage(blockstore, m) + if err != nil { + return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) + } + bcids = append(bcids, c) + } + + for _, m := range fblk.SecpkMessages { + c, err := store.PutMessage(blockstore, m) + if err != nil { + return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) + } + scids = append(scids, c) + } // Compute the root CID of the combined message trie. - smroot, err := computeMsgMeta(bs, bcids, scids) + smroot, err := computeMsgMeta(cst, bcids, scids) if err != nil { return xerrors.Errorf("validating msgmeta, compute failed: %w", err) } @@ -351,21 +361,8 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { return xerrors.Errorf("messages in full block did not match msgmeta root in header (%s != %s)", fblk.Header.Messages, smroot) } - for _, m := range fblk.BlsMessages { - _, err := store.PutMessage(blockstore, m) - if err != nil { - return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) - } - } - - for _, m := range fblk.SecpkMessages { - _, err := store.PutMessage(blockstore, m) - if err != nil { - return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) - } - } - - return nil + // Finally, flush. + return vm.Copy(context.TODO(), blockstore, syncer.store.Blockstore(), smroot) } func (syncer *Syncer) LocalPeer() peer.ID { @@ -1064,8 +1061,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return err } - cst := cbor.NewCborStore(syncer.store.Blockstore()) - st, err := state.LoadStateTree(cst, stateroot) + st, err := state.LoadStateTree(syncer.store.Store(ctx), stateroot) if err != nil { return xerrors.Errorf("failed to load base state tree: %w", err) } @@ -1111,21 +1107,28 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return nil } - store := adt0.WrapStore(ctx, cst) + // Validate message arrays in a temporary blockstore. + tmpbs := bstore.NewTemporary() + tmpstore := adt0.WrapStore(ctx, cbor.NewCborStore(tmpbs)) - bmArr := adt0.MakeEmptyArray(store) + bmArr := adt0.MakeEmptyArray(tmpstore) for i, m := range b.BlsMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid bls message at index %d: %w", i, err) } - c := cbg.CborCid(m.Cid()) - if err := bmArr.Set(uint64(i), &c); err != nil { + c, err := store.PutMessage(tmpbs, m) + if err != nil { + return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err) + } + + k := cbg.CborCid(c) + if err := bmArr.Set(uint64(i), &k); err != nil { return xerrors.Errorf("failed to put bls message at index %d: %w", i, err) } } - smArr := adt0.MakeEmptyArray(store) + smArr := adt0.MakeEmptyArray(tmpstore) for i, m := range b.SecpkMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err) @@ -1142,8 +1145,12 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return xerrors.Errorf("secpk message %s has invalid signature: %w", m.Cid(), err) } - c := cbg.CborCid(m.Cid()) - if err := smArr.Set(uint64(i), &c); err != nil { + c, err := store.PutMessage(tmpbs, m) + if err != nil { + return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err) + } + k := cbg.CborCid(c) + if err := smArr.Set(uint64(i), &k); err != nil { return xerrors.Errorf("failed to put secpk message at index %d: %w", i, err) } } @@ -1158,7 +1165,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return err } - mrcid, err := cst.Put(ctx, &types.MsgMeta{ + mrcid, err := tmpstore.Put(ctx, &types.MsgMeta{ BlsMessages: bmroot, SecpkMessages: smroot, }) @@ -1170,7 +1177,8 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return fmt.Errorf("messages didnt match message root in header") } - return nil + // Finally, flush. + return vm.Copy(ctx, tmpbs, syncer.store.Blockstore(), mrcid) } func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signature, msgs []cid.Cid, pubks [][]byte) error { From e2fbbdcb15d30c79e02810a5bca1d8a5467e5c4b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 12:23:30 -0700 Subject: [PATCH 059/106] reject messages with invalid CIDs These can't possibly be valid messages, so we should just drop the block. --- chain/sub/incoming.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 68ee5e20c..99ca2fc65 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -37,6 +37,7 @@ import ( "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/node/impl/client" ) var log = logging.Logger("sub") @@ -44,6 +45,13 @@ var log = logging.Logger("sub") var ErrSoftFailure = errors.New("soft validation failure") var ErrInsufficientPower = errors.New("incoming block's miner does not have minimum power") +var msgCidPrefix = cid.Prefix{ + Version: 1, + Codec: cid.DagCBOR, + MhType: client.DefaultHashFunction, + MhLength: 32, +} + func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bs bserv.BlockService, cmgr connmgr.ConnManager) { // Timeout after (block time + propagation delay). This is useless at // this point. @@ -168,6 +176,9 @@ func fetchCids( cidIndex := make(map[cid.Cid]int) for i, c := range cids { + if c.Prefix() != msgCidPrefix { + return fmt.Errorf("invalid msg CID: %s", c) + } cidIndex[c] = i } if len(cids) != len(cidIndex) { From 4b38809c0b4cab14fef028c24502c096937c1c7b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Oct 2020 15:52:38 -0700 Subject: [PATCH 060/106] in-memory blockstore Instead of using an in-memory datastore and dealing with the overhead of computing datastore keys, creating new blocks, etc, use an in-memory blockstore. --- lib/blockstore/blockstore.go | 9 ++-- lib/blockstore/memstore.go | 80 ++++++++++++++++++++++++++++++++++++ lib/blockstore/syncstore.go | 68 ++++++++++++++++++++++++++++++ lib/bufbstore/buf_bstore.go | 4 +- 4 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 lib/blockstore/memstore.go create mode 100644 lib/blockstore/syncstore.go diff --git a/lib/blockstore/blockstore.go b/lib/blockstore/blockstore.go index 9e74f4373..f274cd708 100644 --- a/lib/blockstore/blockstore.go +++ b/lib/blockstore/blockstore.go @@ -18,19 +18,18 @@ import ( "context" ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" ) // NewTemporary returns a temporary blockstore. -func NewTemporary() blockstore.Blockstore { - return NewBlockstore(ds.NewMapDatastore()) +func NewTemporary() MemStore { + return make(MemStore) } // NewTemporarySync returns a thread-safe temporary blockstore. -func NewTemporarySync() blockstore.Blockstore { - return NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) +func NewTemporarySync() *SyncStore { + return &SyncStore{bs: make(MemStore)} } // WrapIDStore wraps the underlying blockstore in an "identity" blockstore. diff --git a/lib/blockstore/memstore.go b/lib/blockstore/memstore.go new file mode 100644 index 000000000..9745d6f03 --- /dev/null +++ b/lib/blockstore/memstore.go @@ -0,0 +1,80 @@ +package blockstore + +import ( + "context" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" +) + +type MemStore map[cid.Cid]blocks.Block + +func (m MemStore) DeleteBlock(k cid.Cid) error { + delete(m, k) + return nil +} +func (m MemStore) Has(k cid.Cid) (bool, error) { + _, ok := m[k] + return ok, nil +} +func (m MemStore) Get(k cid.Cid) (blocks.Block, error) { + b, ok := m[k] + if !ok { + return nil, blockstore.ErrNotFound + } + return b, nil +} + +// GetSize returns the CIDs mapped BlockSize +func (m MemStore) GetSize(k cid.Cid) (int, error) { + b, ok := m[k] + if !ok { + return 0, blockstore.ErrNotFound + } + return len(b.RawData()), nil +} + +// Put puts a given block to the underlying datastore +func (m MemStore) Put(b blocks.Block) error { + // Convert to a basic block for safety, but try to reuse the existing + // block if it's already a basic block. + k := b.Cid() + if _, ok := b.(*blocks.BasicBlock); !ok { + // If we already have the block, abort. + if _, ok := m[k]; ok { + return nil + } + // the error is only for debugging. + b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid()) + } + m[b.Cid()] = b + return nil +} + +// PutMany puts a slice of blocks at the same time using batching +// capabilities of the underlying datastore whenever possible. +func (m MemStore) PutMany(bs []blocks.Block) error { + for _, b := range bs { + _ = m.Put(b) // can't fail + } + return nil +} + +// AllKeysChan returns a channel from which +// the CIDs in the Blockstore can be read. It should respect +// the given context, closing the channel if it becomes Done. +func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + ch := make(chan cid.Cid, len(m)) + for k := range m { + ch <- k + } + close(ch) + return ch, nil +} + +// HashOnRead specifies if every read block should be +// rehashed to make sure it matches its CID. +func (m MemStore) HashOnRead(enabled bool) { + // no-op +} diff --git a/lib/blockstore/syncstore.go b/lib/blockstore/syncstore.go new file mode 100644 index 000000000..be9f6b5c4 --- /dev/null +++ b/lib/blockstore/syncstore.go @@ -0,0 +1,68 @@ +package blockstore + +import ( + "context" + "sync" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" +) + +type SyncStore struct { + mu sync.RWMutex + bs MemStore // specifically use a memStore to save indirection overhead. +} + +func (m *SyncStore) DeleteBlock(k cid.Cid) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.DeleteBlock(k) +} +func (m *SyncStore) Has(k cid.Cid) (bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.Has(k) +} +func (m *SyncStore) Get(k cid.Cid) (blocks.Block, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.Get(k) +} + +// GetSize returns the CIDs mapped BlockSize +func (m *SyncStore) GetSize(k cid.Cid) (int, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.GetSize(k) +} + +// Put puts a given block to the underlying datastore +func (m *SyncStore) Put(b blocks.Block) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.Put(b) +} + +// PutMany puts a slice of blocks at the same time using batching +// capabilities of the underlying datastore whenever possible. +func (m *SyncStore) PutMany(bs []blocks.Block) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.PutMany(bs) +} + +// AllKeysChan returns a channel from which +// the CIDs in the Blockstore can be read. It should respect +// the given context, closing the channel if it becomes Done. +func (m *SyncStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + m.mu.RLock() + defer m.mu.RUnlock() + // this blockstore implementation doesn't do any async work. + return m.bs.AllKeysChan(ctx) +} + +// HashOnRead specifies if every read block should be +// rehashed to make sure it matches its CID. +func (m *SyncStore) HashOnRead(enabled bool) { + // noop +} diff --git a/lib/bufbstore/buf_bstore.go b/lib/bufbstore/buf_bstore.go index a766c2b52..4ea746444 100644 --- a/lib/bufbstore/buf_bstore.go +++ b/lib/bufbstore/buf_bstore.go @@ -19,10 +19,12 @@ type BufferedBS struct { } func NewBufferedBstore(base bstore.Blockstore) *BufferedBS { - buf := bstore.NewTemporary() + var buf bstore.Blockstore if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" { log.Warn("VM BLOCKSTORE BUFFERING IS DISABLED") buf = base + } else { + buf = bstore.NewTemporary() } return &BufferedBS{ From 15fe998c68b3980cc064feabfe770eded2e77fde Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 14:02:19 -0700 Subject: [PATCH 061/106] add an timed-cache blockstore This blockstore lets us write to a temporary scratch location where blocks older than the specified cache time are automatically cleared. --- lib/timedbs/timedbs.go | 156 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 lib/timedbs/timedbs.go diff --git a/lib/timedbs/timedbs.go b/lib/timedbs/timedbs.go new file mode 100644 index 000000000..bb03a59e9 --- /dev/null +++ b/lib/timedbs/timedbs.go @@ -0,0 +1,156 @@ +package timedbs + +import ( + "context" + "fmt" + "sync" + "time" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "go.uber.org/multierr" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/lib/blockstore" +) + +// TimedCacheBS is a blockstore that keeps blocks for at least the specified +// caching interval before discarding them. Garbage collection must be started +// and stopped by calling Start/Stop. +// +// Under the covers, it's implemented with an active and an inactive blockstore +// that are rotated every cache time interval. This means all blocks will be +// stored at most 2x the cache interval. +type TimedCacheBS struct { + mu sync.RWMutex + active, inactive blockstore.MemStore + + interval time.Duration + closeCh chan struct{} +} + +func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS { + return &TimedCacheBS{ + active: blockstore.NewTemporary(), + inactive: blockstore.NewTemporary(), + interval: cacheTime, + } +} + +func (t *TimedCacheBS) Start(ctx context.Context) error { + t.mu.Lock() + defer t.mu.Unlock() + if t.closeCh != nil { + return fmt.Errorf("already started") + } + t.closeCh = make(chan struct{}) + go func() { + ticker := build.Clock.Ticker(t.interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + t.rotate() + case <-t.closeCh: + return + } + } + }() + return nil +} + +func (t *TimedCacheBS) Stop(ctx context.Context) error { + t.mu.Lock() + defer t.mu.Unlock() + if t.closeCh == nil { + return fmt.Errorf("not started started") + } + select { + case <-t.closeCh: + // already closed + default: + close(t.closeCh) + } + return nil +} + +func (t *TimedCacheBS) rotate() { + newBs := blockstore.NewTemporary() + + t.mu.Lock() + t.inactive, t.active = t.active, newBs + t.mu.Unlock() +} + +func (t *TimedCacheBS) Put(b blocks.Block) error { + // Don't check the inactive set here. We want to keep this block for at + // least one interval. + t.mu.Lock() + defer t.mu.Unlock() + return t.active.Put(b) +} + +func (t *TimedCacheBS) PutMany(bs []blocks.Block) error { + t.mu.Lock() + defer t.mu.Unlock() + return t.active.PutMany(bs) +} + +func (t *TimedCacheBS) Get(k cid.Cid) (blocks.Block, error) { + t.mu.RLock() + defer t.mu.RUnlock() + b, err := t.active.Get(k) + if err == blockstore.ErrNotFound { + b, err = t.inactive.Get(k) + } + return b, err +} + +func (t *TimedCacheBS) GetSize(k cid.Cid) (int, error) { + t.mu.RLock() + defer t.mu.RUnlock() + size, err := t.active.GetSize(k) + if err == blockstore.ErrNotFound { + size, err = t.inactive.GetSize(k) + } + return size, err +} + +func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) { + t.mu.RLock() + defer t.mu.RUnlock() + if has, err := t.active.Has(k); err != nil { + return false, err + } else if has { + return true, nil + } + return t.inactive.Has(k) +} + +func (t *TimedCacheBS) HashOnRead(_ bool) { + // no-op +} + +func (t *TimedCacheBS) DeleteBlock(k cid.Cid) error { + t.mu.Lock() + defer t.mu.Unlock() + return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k)) +} + +func (t *TimedCacheBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + t.mu.RLock() + defer t.mu.RUnlock() + + ch := make(chan cid.Cid, len(t.active)+len(t.inactive)) + for c := range t.active { + ch <- c + } + for c := range t.inactive { + if _, ok := t.active[c]; ok { + continue + } + ch <- c + } + close(ch) + return ch, nil +} From ddade32bd37654f343e13ba4ad0983d03f1f05e9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 13:49:43 -0700 Subject: [PATCH 062/106] write bitswap blocks into a temporary, in-memory block cache If they end up validating, we'll write them back to the underlying blockstore. Otherwise, there's no reason to keep them. --- node/modules/chain.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/node/modules/chain.go b/node/modules/chain.go index f563b4cdd..ce3e9f749 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "os" + "time" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" @@ -30,6 +31,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" + "github.com/filecoin-project/lotus/lib/timedbs" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" @@ -41,8 +44,15 @@ func ChainBitswap(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt r bitswapNetwork := network.NewFromIpfsHost(host, rt, network.Prefix("/chain")) bitswapOptions := []bitswap.Option{bitswap.ProvideEnabled(false)} + // Write all incoming bitswap blocks into a temporary blockstore for two + // block times. If they validate, they'll be persisted later. + cache := timedbs.NewTimedCacheBS(2 * time.Duration(build.BlockDelaySecs) * time.Second) + lc.Append(fx.Hook{OnStop: cache.Stop, OnStart: cache.Start}) + + bitswapBs := bufbstore.NewTieredBstore(bs, cache) + // Use just exch.Close(), closing the context is not needed - exch := bitswap.New(mctx, bitswapNetwork, bs, bitswapOptions...) + exch := bitswap.New(mctx, bitswapNetwork, bitswapBs, bitswapOptions...) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return exch.Close() From 28823fb8e9f7f276e915f0935e424c404287e957 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Oct 2020 14:50:59 -0700 Subject: [PATCH 063/106] fix: bad indirection of deadline diffing --- chain/actors/builtin/miner/diff_deadlines.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/chain/actors/builtin/miner/diff_deadlines.go b/chain/actors/builtin/miner/diff_deadlines.go index e1e839960..7d686ece5 100644 --- a/chain/actors/builtin/miner/diff_deadlines.go +++ b/chain/actors/builtin/miner/diff_deadlines.go @@ -7,9 +7,9 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" ) -type DeadlinesDiff map[uint64]*DeadlineDiff +type DeadlinesDiff map[uint64]DeadlineDiff -func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { +func DiffDeadlines(pre, cur State) (DeadlinesDiff, error) { changed, err := pre.DeadlinesChanged(cur) if err != nil { return nil, err @@ -18,11 +18,7 @@ func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { return nil, nil } - numDl, err := pre.NumDeadlines() - if err != nil { - return nil, err - } - dlDiff := make(DeadlinesDiff, numDl) + dlDiff := make(DeadlinesDiff) if err := pre.ForEachDeadline(func(idx uint64, preDl Deadline) error { curDl, err := cur.LoadDeadline(idx) if err != nil { @@ -39,12 +35,12 @@ func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { }); err != nil { return nil, err } - return &dlDiff, nil + return dlDiff, nil } type DeadlineDiff map[uint64]*PartitionDiff -func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { +func DiffDeadline(pre, cur Deadline) (DeadlineDiff, error) { changed, err := pre.PartitionsChanged(cur) if err != nil { return nil, err @@ -104,7 +100,7 @@ func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { return nil, err } - return &partDiff, nil + return partDiff, nil } type PartitionDiff struct { From 811f1304e673c2208a8902c761da9d3548473d3d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 16:17:55 -0700 Subject: [PATCH 064/106] test timed cache blockstore --- lib/timedbs/timedbs_test.go | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 lib/timedbs/timedbs_test.go diff --git a/lib/timedbs/timedbs_test.go b/lib/timedbs/timedbs_test.go new file mode 100644 index 000000000..75f298a1d --- /dev/null +++ b/lib/timedbs/timedbs_test.go @@ -0,0 +1,78 @@ +package timedbs_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/lib/timedbs" +) + +func TestTimedBSSimple(t *testing.T) { + tc := timedbs.NewTimedCacheBS(3 * time.Millisecond) + _ = tc.Start(context.Background()) + defer func() { + _ = tc.Stop(context.Background()) + }() + + b1 := blocks.NewBlock([]byte("foo")) + require.NoError(t, tc.Put(b1)) + + b2 := blocks.NewBlock([]byte("bar")) + require.NoError(t, tc.Put(b2)) + + b3 := blocks.NewBlock([]byte("baz")) + + b1out, err := tc.Get(b1.Cid()) + require.NoError(t, err) + require.Equal(t, b1.RawData(), b1out.RawData()) + + has, err := tc.Has(b1.Cid()) + require.NoError(t, err) + require.True(t, has) + + time.Sleep(4 * time.Millisecond) + + // We should still have everything. + has, err = tc.Has(b1.Cid()) + require.NoError(t, err) + require.True(t, has) + + has, err = tc.Has(b2.Cid()) + require.NoError(t, err) + require.True(t, has) + + // extend b2, add b3. + require.NoError(t, tc.Put(b2)) + require.NoError(t, tc.Put(b3)) + + // all keys once. + allKeys, err := tc.AllKeysChan(context.Background()) + var ks []cid.Cid + for k := range allKeys { + ks = append(ks, k) + } + require.NoError(t, err) + require.ElementsMatch(t, ks, []cid.Cid{b1.Cid(), b2.Cid(), b3.Cid()}) + + time.Sleep(4 * time.Millisecond) + + // should still have b2, and b3, but not b1 + + has, err = tc.Has(b1.Cid()) + require.NoError(t, err) + require.False(t, has) + + has, err = tc.Has(b2.Cid()) + require.NoError(t, err) + require.True(t, has) + + has, err = tc.Has(b3.Cid()) + require.NoError(t, err) + require.True(t, has) +} From 86e30729bf142a2c8db12b9b26ee63c83de279c6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 16:51:26 -0700 Subject: [PATCH 065/106] give the TimeCacheBS tests a bit more time Otherwise, they flake once in a while. --- lib/timedbs/timedbs_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/timedbs/timedbs_test.go b/lib/timedbs/timedbs_test.go index 75f298a1d..2126b9287 100644 --- a/lib/timedbs/timedbs_test.go +++ b/lib/timedbs/timedbs_test.go @@ -14,7 +14,7 @@ import ( ) func TestTimedBSSimple(t *testing.T) { - tc := timedbs.NewTimedCacheBS(3 * time.Millisecond) + tc := timedbs.NewTimedCacheBS(10 * time.Millisecond) _ = tc.Start(context.Background()) defer func() { _ = tc.Stop(context.Background()) @@ -36,7 +36,7 @@ func TestTimedBSSimple(t *testing.T) { require.NoError(t, err) require.True(t, has) - time.Sleep(4 * time.Millisecond) + time.Sleep(15 * time.Millisecond) // We should still have everything. has, err = tc.Has(b1.Cid()) @@ -60,7 +60,7 @@ func TestTimedBSSimple(t *testing.T) { require.NoError(t, err) require.ElementsMatch(t, ks, []cid.Cid{b1.Cid(), b2.Cid(), b3.Cid()}) - time.Sleep(4 * time.Millisecond) + time.Sleep(10 * time.Millisecond) // should still have b2, and b3, but not b1 From 698b49b7ef5dfcce4a6eb277b8d38ad9c094887c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 15 Oct 2020 02:46:47 +0200 Subject: [PATCH 066/106] Improve gas defaults --- chain/messagepool/messagepool.go | 4 +++- chain/types/fil.go | 9 +++++++++ node/config/def.go | 10 +++++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 83b1a7f29..88e721ef4 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -59,6 +59,8 @@ var MaxUntrustedActorPendingMessages = 10 var MaxNonceGap = uint64(4) +var DefaultMaxFee = abi.TokenAmount(types.MustParseFIL("0.007")) + var ( ErrMessageTooBig = errors.New("message too big") @@ -183,7 +185,7 @@ func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount { func CapGasFee(msg *types.Message, maxFee abi.TokenAmount) { if maxFee.Equals(big.Zero()) { - maxFee = types.NewInt(build.FilecoinPrecision / 10) + maxFee = DefaultMaxFee } gl := types.NewInt(uint64(msg.GasLimit)) diff --git a/chain/types/fil.go b/chain/types/fil.go index 7eac8ce93..0ea77660c 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -81,5 +81,14 @@ func ParseFIL(s string) (FIL, error) { return FIL{r.Num()}, nil } +func MustParseFIL(s string) FIL { + n, err := ParseFIL(s) + if err != nil { + panic(err) + } + + return n +} + var _ encoding.TextMarshaler = (*FIL)(nil) var _ encoding.TextUnmarshaler = (*FIL)(nil) diff --git a/node/config/def.go b/node/config/def.go index 7eb7e19f0..f1bb1edef 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -182,11 +182,11 @@ func DefaultStorageMiner() *StorageMiner { }, Fees: MinerFeeConfig{ - MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05 - MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), - MaxWindowPoStGasFee: types.FIL(types.FromFil(50)), - MaxPublishDealsFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(33))), // 0.03ish - MaxMarketBalanceAddFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(100))), // 0.01 + MaxPreCommitGasFee: types.MustParseFIL("0.025"), + MaxCommitGasFee: types.MustParseFIL("0.05"), + MaxWindowPoStGasFee: types.MustParseFIL("5"), + MaxPublishDealsFee: types.MustParseFIL("0.05"), + MaxMarketBalanceAddFee: types.MustParseFIL("0.007"), }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" From df6f4a572cc0d45e5695a1135c72e3963fecc23c Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 7 Oct 2020 21:00:24 +0200 Subject: [PATCH 067/106] Change default gas premium to for 6 block inclusion Signed-off-by: Jakub Sztandera --- node/impl/full/gas.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 0cb1eb084..2e5063ccb 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -277,7 +277,7 @@ func (m *GasModule) GasEstimateMessageGas(ctx context.Context, msg *types.Messag } if msg.GasPremium == types.EmptyInt || types.BigCmp(msg.GasPremium, types.NewInt(0)) == 0 { - gasPremium, err := m.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) + gasPremium, err := m.GasEstimateGasPremium(ctx, 10, msg.From, msg.GasLimit, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas price: %w", err) } From 251374eca723959cf836e2f5cc48d585b742fe6b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 14 Oct 2020 21:01:20 -0400 Subject: [PATCH 068/106] Lotus version 0.10.2 --- CHANGELOG.md | 18 ++++++++++++++++++ build/version.go | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c94e387c..701f221b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Lotus changelog +# 0.10.2 / 2020-10-14 + +This is an optional release of Lotus that updates markets to 0.9.1, which fixes an issue affecting deals that were mid-transfer when the node was upgraded to 0.9.0. This release also includes some tweaks to default gas values and minor performance improvements. + +## Changes + +- Use updated stored ask API (https://github.com/filecoin-project/lotus/pull/4384) +- tvx: trace puts to blockstore for inclusion in CAR. (https://github.com/filecoin-project/lotus/pull/4278) +- Add propose remove (https://github.com/filecoin-project/lotus/pull/4311) +- Update to 0.9.1 bugfix release (https://github.com/filecoin-project/lotus/pull/4402) +- Update drand endpoints (https://github.com/filecoin-project/lotus/pull/4125) +- fix: return true when deadlines changed (https://github.com/filecoin-project/lotus/pull/4403) +- sync wait --watch (https://github.com/filecoin-project/lotus/pull/4396) +- reduce garbage in blockstore (https://github.com/filecoin-project/lotus/pull/4406) +- give the TimeCacheBS tests a bit more time (https://github.com/filecoin-project/lotus/pull/4407) +- Improve gas defaults (https://github.com/filecoin-project/lotus/pull/4408) +- Change default gas premium to for 10 block inclusion (https://github.com/filecoin-project/lotus/pull/4222) + # 0.10.1 / 2020-10-14 This is an optional release of Lotus that updates markets to 0.9.0, which adds the ability to restart data transfers. This release also introduces Ledger support, and various UX improvements. diff --git a/build/version.go b/build/version.go index 39f9d48ff..df1fe28de 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.10.1" +const BuildVersion = "0.10.2" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From a40fc98e5635545cb70f4fd675971fdb1df5de11 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 17:22:39 -0700 Subject: [PATCH 069/106] fix code coverage ignore It looks like we need to use globs. --- .codecov.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.codecov.yml b/.codecov.yml index 1551f2276..a53081be7 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,5 +1,9 @@ comment: off ignore: - - "cbor_gen.go" + - "**/cbor_gen.go" + - "api/test/**/*" + - "api/test/*" + - "gen/**/*" + - "gen/*" github_checks: annotations: false From 3fb5334959eab2a2f8aaf5493dcef2b6ad581f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 15 Oct 2020 03:36:55 +0200 Subject: [PATCH 070/106] mpool: Don't block node startup loading messages --- chain/messagepool/messagepool.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 88e721ef4..cc26be0b5 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -370,11 +370,23 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, j journ return err }) - if err := mp.loadLocal(); err != nil { - log.Errorf("loading local messages: %+v", err) - } + mp.curTsLk.Lock() + mp.lk.Lock() - go mp.runLoop() + go func() { + err := mp.loadLocal() + + mp.lk.Unlock() + mp.curTsLk.Unlock() + + if err != nil { + log.Errorf("loading local messages: %+v", err) + } + + log.Info("mpool ready") + + mp.runLoop() + }() return mp, nil } @@ -667,9 +679,6 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { return err } - mp.curTsLk.Lock() - defer mp.curTsLk.Unlock() - curTs := mp.curTs if curTs == nil { @@ -685,9 +694,6 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { return xerrors.Errorf("minimum expected nonce is %d: %w", snonce, ErrNonceTooLow) } - mp.lk.Lock() - defer mp.lk.Unlock() - _, err = mp.verifyMsgBeforeAdd(m, curTs, true) if err != nil { return err From b29e3b5242704d4a8a82f07abcc195016417a1b3 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 14 Oct 2020 21:28:36 -0700 Subject: [PATCH 071/106] feat(markets): use build.BlockDelaySecs for start buffer instead of constant from spec-actors which uses hardcoded start epoch, use one that factors in possibility of different block time --- node/modules/storageminer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index adacf5405..cd5517a1c 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -474,7 +474,7 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline // Reject if it's more than 7 days in the future // TODO: read from cfg - maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.EpochsInDay) + maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.SecondsInDay/build.BlockDelaySecs) if deal.Proposal.StartEpoch > maxStartEpoch { return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil } From 6d771d29c7c5420d13f00de4785e689a97166ee8 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 14 Oct 2020 10:56:51 -0700 Subject: [PATCH 072/106] improve the UX of multisig approves --- cli/multisig.go | 132 ++++++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 56 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 831b1bbe4..19a80e66f 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-address" + cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -41,6 +42,13 @@ import ( var multisigCmd = &cli.Command{ Name: "msig", Usage: "Interact with a multisig wallet", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "confirmations", + Usage: "number of block confirmations to wait for", + Value: int(build.MessageConfidence), + }, + }, Subcommands: []*cli.Command{ msigCreateCmd, msigInspectCmd, @@ -146,7 +154,7 @@ var msigCreateCmd = &cli.Command{ } // wait for it to get mined into a block - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -410,7 +418,7 @@ var msigProposeCmd = &cli.Command{ fmt.Println("send proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -446,14 +454,18 @@ var msigApproveCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - if cctx.Args().Len() < 5 { - return ShowHelp(cctx, fmt.Errorf("must pass multisig address, message ID, proposer address, destination, and value")) + if cctx.Args().Len() < 2 { + return ShowHelp(cctx, fmt.Errorf("must pass at least multisig address and message ID")) } if cctx.Args().Len() > 5 && cctx.Args().Len() != 7 { return ShowHelp(cctx, fmt.Errorf("usage: msig approve [ ]")) } + if cctx.Args().Len() > 2 && cctx.Args().Len() != 5 { + return ShowHelp(cctx, fmt.Errorf("usage: msig approve ")) + } + api, closer, err := GetFullNodeAPI(cctx) if err != nil { return err @@ -471,44 +483,6 @@ var msigApproveCmd = &cli.Command{ return err } - proposer, err := address.NewFromString(cctx.Args().Get(2)) - if err != nil { - return err - } - - if proposer.Protocol() != address.ID { - proposer, err = api.StateLookupID(ctx, proposer, types.EmptyTSK) - if err != nil { - return err - } - } - - dest, err := address.NewFromString(cctx.Args().Get(3)) - if err != nil { - return err - } - - value, err := types.ParseFIL(cctx.Args().Get(4)) - if err != nil { - return err - } - - var method uint64 - var params []byte - if cctx.Args().Len() == 7 { - m, err := strconv.ParseUint(cctx.Args().Get(5), 10, 64) - if err != nil { - return err - } - method = m - - p, err := hex.DecodeString(cctx.Args().Get(6)) - if err != nil { - return err - } - params = p - } - var from address.Address if cctx.IsSet("from") { f, err := address.NewFromString(cctx.String("from")) @@ -524,14 +498,60 @@ var msigApproveCmd = &cli.Command{ from = defaddr } - msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) - if err != nil { - return err + var msgCid cid.Cid + if cctx.Args().Len() == 2 { + msgCid, err = api.MsigApprove(ctx, msig, txid, from) + if err != nil { + return err + } + } else { + proposer, err := address.NewFromString(cctx.Args().Get(2)) + if err != nil { + return err + } + + if proposer.Protocol() != address.ID { + proposer, err = api.StateLookupID(ctx, proposer, types.EmptyTSK) + if err != nil { + return err + } + } + + dest, err := address.NewFromString(cctx.Args().Get(3)) + if err != nil { + return err + } + + value, err := types.ParseFIL(cctx.Args().Get(4)) + if err != nil { + return err + } + + var method uint64 + var params []byte + if cctx.Args().Len() == 7 { + m, err := strconv.ParseUint(cctx.Args().Get(5), 10, 64) + if err != nil { + return err + } + method = m + + p, err := hex.DecodeString(cctx.Args().Get(6)) + if err != nil { + return err + } + params = p + } + + msgCid, err = api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) + if err != nil { + return err + } } fmt.Println("sent approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -602,7 +622,7 @@ var msigRemoveProposeCmd = &cli.Command{ fmt.Println("sent remove proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -680,7 +700,7 @@ var msigAddProposeCmd = &cli.Command{ fmt.Println("sent add proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -762,7 +782,7 @@ var msigAddApproveCmd = &cli.Command{ fmt.Println("sent add approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -839,7 +859,7 @@ var msigAddCancelCmd = &cli.Command{ fmt.Println("sent add cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -911,7 +931,7 @@ var msigSwapProposeCmd = &cli.Command{ fmt.Println("sent swap proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -993,7 +1013,7 @@ var msigSwapApproveCmd = &cli.Command{ fmt.Println("sent swap approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1070,7 +1090,7 @@ var msigSwapCancelCmd = &cli.Command{ fmt.Println("sent swap cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1157,7 +1177,7 @@ var msigLockProposeCmd = &cli.Command{ fmt.Println("sent lock proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1254,7 +1274,7 @@ var msigLockApproveCmd = &cli.Command{ fmt.Println("sent lock approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1346,7 +1366,7 @@ var msigLockCancelCmd = &cli.Command{ fmt.Println("sent lock cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } From 3bd2e2f7771b068f7d32e37e2813c38f58149d41 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 14 Oct 2020 22:55:23 -0700 Subject: [PATCH 073/106] fix typo --- cli/multisig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/multisig.go b/cli/multisig.go index 19a80e66f..d606d2acb 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -44,7 +44,7 @@ var multisigCmd = &cli.Command{ Usage: "Interact with a multisig wallet", Flags: []cli.Flag{ &cli.IntFlag{ - Name: "confirmations", + Name: "confidence", Usage: "number of block confirmations to wait for", Value: int(build.MessageConfidence), }, From fdb115e3692493d5e74f7224aee7a0781d540838 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 14 Oct 2020 23:10:00 -0700 Subject: [PATCH 074/106] add command to propose changing multisig threshold --- cli/multisig.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/cli/multisig.go b/cli/multisig.go index d606d2acb..73589086a 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -65,6 +65,7 @@ var multisigCmd = &cli.Command{ msigLockApproveCmd, msigLockCancelCmd, msigVestedCmd, + msigProposeThresholdCmd, }, } @@ -1440,3 +1441,78 @@ var msigVestedCmd = &cli.Command{ return nil }, } + +var msigProposeThresholdCmd = &cli.Command{ + Name: "propose-threshold", + Usage: "Propose setting a different signing threshold on the account", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "account to send the proposal from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 2 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address and new threshold value")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + newM, err := strconv.ParseUint(cctx.Args().Get(2), 10, 64) + if err != nil { + return err + } + + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr + } + + params, actErr := actors.SerializeParams(&msig0.ChangeNumApprovalsThresholdParams{ + NewThreshold: newM, + }) + + if actErr != nil { + return actErr + } + + msgCid, err := api.MsigPropose(ctx, msig, msig, types.NewInt(0), from, uint64(builtin2.MethodsMultisig.ChangeNumApprovalsThreshold), params) + if err != nil { + return fmt.Errorf("failed to propose change of threshold: %w", err) + } + + fmt.Println("sent change threshold proposal in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("change threshold proposal returned exit %d", wait.Receipt.ExitCode) + } + + return nil + }, +} From ccc780f35ac27eb7136efed3b55851e3076ed68b Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 14 Oct 2020 23:16:31 -0700 Subject: [PATCH 075/106] fix wrong index --- cli/multisig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/multisig.go b/cli/multisig.go index 73589086a..e352ed64b 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -1469,7 +1469,7 @@ var msigProposeThresholdCmd = &cli.Command{ return err } - newM, err := strconv.ParseUint(cctx.Args().Get(2), 10, 64) + newM, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) if err != nil { return err } From 9c99171cb830b811daf23be21bf64394c700bf63 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 15 Oct 2020 12:15:21 +0200 Subject: [PATCH 076/106] fix: ensure msig inspect cli works with lotus-lite --- api/api_gateway.go | 2 + api/apistruct/struct.go | 10 +++ cli/multisig.go | 25 +++--- cli/multisig_test.go | 55 +++++++++++++ cli/paych_test.go | 1 + cli/test/mockcli.go | 124 +++++++++++++++++++++++++++++ cli/test/multisig.go | 77 ++++++++++++++++++ cmd/lotus-gateway/api.go | 25 +++++- cmd/lotus-gateway/api_test.go | 8 ++ cmd/lotus-gateway/endtoend_test.go | 52 +++++++++--- cmd/lotus-gateway/main.go | 2 +- node/impl/full/chain.go | 10 ++- 12 files changed, 361 insertions(+), 30 deletions(-) create mode 100644 cli/multisig_test.go create mode 100644 cli/test/mockcli.go create mode 100644 cli/test/multisig.go diff --git a/api/api_gateway.go b/api/api_gateway.go index 95d28887d..c99ff8408 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -10,9 +10,11 @@ import ( ) type GatewayAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7723e0b1f..3854e1dd6 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -371,9 +371,11 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? + ChainHasObj func(context.Context, cid.Cid) (bool, error) ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainHead func(ctx context.Context) (*types.TipSet, error) + ChainReadObj func(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) @@ -1432,6 +1434,10 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } +func (g GatewayStruct) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { + return g.Internal.ChainHasObj(ctx, c) +} + func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { return g.Internal.ChainHead(ctx) } @@ -1444,6 +1450,10 @@ func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEp return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) } +func (g GatewayStruct) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + return g.Internal.ChainReadObj(ctx, c) +} + func (g GatewayStruct) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { return g.Internal.GasEstimateMessageGas(ctx, msg, spec, tsk) } diff --git a/cli/multisig.go b/cli/multisig.go index 831b1bbe4..97a77a75b 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "os" "reflect" "sort" "strconv" @@ -153,7 +152,7 @@ var msigCreateCmd = &cli.Command{ // check it executed successfully if wait.Receipt.ExitCode != 0 { - fmt.Println("actor creation failed!") + fmt.Fprintln(cctx.App.Writer, "actor creation failed!") return err } @@ -163,7 +162,7 @@ var msigCreateCmd = &cli.Command{ if err := execreturn.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil { return err } - fmt.Println("Created new multisig: ", execreturn.IDAddress, execreturn.RobustAddress) + fmt.Fprintln(cctx.App.Writer, "Created new multisig: ", execreturn.IDAddress, execreturn.RobustAddress) // TODO: maybe register this somewhere return nil @@ -227,25 +226,25 @@ var msigInspectCmd = &cli.Command{ return err } - fmt.Printf("Balance: %s\n", types.FIL(act.Balance)) - fmt.Printf("Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked))) + fmt.Fprintf(cctx.App.Writer, "Balance: %s\n", types.FIL(act.Balance)) + fmt.Fprintf(cctx.App.Writer, "Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked))) if cctx.Bool("vesting") { ib, err := mstate.InitialBalance() if err != nil { return err } - fmt.Printf("InitialBalance: %s\n", types.FIL(ib)) + fmt.Fprintf(cctx.App.Writer, "InitialBalance: %s\n", types.FIL(ib)) se, err := mstate.StartEpoch() if err != nil { return err } - fmt.Printf("StartEpoch: %d\n", se) + fmt.Fprintf(cctx.App.Writer, "StartEpoch: %d\n", se) ud, err := mstate.UnlockDuration() if err != nil { return err } - fmt.Printf("UnlockDuration: %d\n", ud) + fmt.Fprintf(cctx.App.Writer, "UnlockDuration: %d\n", ud) } signers, err := mstate.Signers() @@ -256,10 +255,10 @@ var msigInspectCmd = &cli.Command{ if err != nil { return err } - fmt.Printf("Threshold: %d / %d\n", threshold, len(signers)) - fmt.Println("Signers:") + fmt.Fprintf(cctx.App.Writer, "Threshold: %d / %d\n", threshold, len(signers)) + fmt.Fprintln(cctx.App.Writer, "Signers:") for _, s := range signers { - fmt.Printf("\t%s\n", s) + fmt.Fprintf(cctx.App.Writer, "\t%s\n", s) } pending := make(map[int64]multisig.Transaction) @@ -271,7 +270,7 @@ var msigInspectCmd = &cli.Command{ } decParams := cctx.Bool("decode-params") - fmt.Println("Transactions: ", len(pending)) + fmt.Fprintln(cctx.App.Writer, "Transactions: ", len(pending)) if len(pending) > 0 { var txids []int64 for txid := range pending { @@ -281,7 +280,7 @@ var msigInspectCmd = &cli.Command{ return txids[i] < txids[j] }) - w := tabwriter.NewWriter(os.Stdout, 8, 4, 2, ' ', 0) + w := tabwriter.NewWriter(cctx.App.Writer, 8, 4, 2, ' ', 0) fmt.Fprintf(w, "ID\tState\tApprovals\tTo\tValue\tMethod\tParams\n") for _, txid := range txids { tx := pending[txid] diff --git a/cli/multisig_test.go b/cli/multisig_test.go new file mode 100644 index 000000000..09cdbe056 --- /dev/null +++ b/cli/multisig_test.go @@ -0,0 +1,55 @@ +package cli + +import ( + "context" + "os" + "testing" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api/test" + clitest "github.com/filecoin-project/lotus/cli/test" + builder "github.com/filecoin-project/lotus/node/test" +) + +// TestMultisig does a basic test to exercise the multisig CLI +// commands +func TestMultisig(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + + blocktime := 5 * time.Millisecond + ctx := context.Background() + nodes, _ := startNodes(ctx, t, blocktime) + clientNode := nodes[0] + clitest.RunMultisigTest(t, Commands, clientNode) +} + +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) { + n, sn := builder.RPCMockSbBuilder(t, test.OneFull, test.OneMiner) + + full := n[0] + miner := sn[0] + + // Get everyone connected + addrs, err := full.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrs); err != nil { + t.Fatal(err) + } + + // Start mining blocks + bm := test.NewBlockMiner(ctx, t, miner, blocktime) + bm.MineBlocks() + + // Get the creator's address + creatorAddr, err := full.WalletDefaultAddress(ctx) + if err != nil { + t.Fatal(err) + } + + // Create mock CLI + return n, []address.Address{creatorAddr} +} diff --git a/cli/paych_test.go b/cli/paych_test.go index b94ba27e7..77a6e61eb 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -437,6 +437,7 @@ type mockCLI struct { out *bytes.Buffer } +// TODO: refactor to use the methods in cli/test/mockcli.go func newMockCLI(t *testing.T) *mockCLI { // Create a CLI App with an --api-url flag so that we can specify which node // the command should be executed against diff --git a/cli/test/mockcli.go b/cli/test/mockcli.go new file mode 100644 index 000000000..f5d5cfcba --- /dev/null +++ b/cli/test/mockcli.go @@ -0,0 +1,124 @@ +package test + +import ( + "bytes" + "flag" + "strings" + "testing" + + "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/require" + lcli "github.com/urfave/cli/v2" +) + +type mockCLI struct { + t *testing.T + cmds []*lcli.Command + cctx *lcli.Context + out *bytes.Buffer +} + +func newMockCLI(t *testing.T, cmds []*lcli.Command) *mockCLI { + // Create a CLI App with an --api-url flag so that we can specify which node + // the command should be executed against + app := &lcli.App{ + Flags: []lcli.Flag{ + &lcli.StringFlag{ + Name: "api-url", + Hidden: true, + }, + }, + Commands: cmds, + } + + var out bytes.Buffer + app.Writer = &out + app.Setup() + + cctx := lcli.NewContext(app, &flag.FlagSet{}, nil) + return &mockCLI{t: t, cmds: cmds, cctx: cctx, out: &out} +} + +func (c *mockCLI) client(addr multiaddr.Multiaddr) *mockCLIClient { + return &mockCLIClient{t: c.t, cmds: c.cmds, addr: addr, cctx: c.cctx, out: c.out} +} + +// mockCLIClient runs commands against a particular node +type mockCLIClient struct { + t *testing.T + cmds []*lcli.Command + addr multiaddr.Multiaddr + cctx *lcli.Context + out *bytes.Buffer +} + +func (c *mockCLIClient) run(cmd []string, params []string, args []string) string { + // Add parameter --api-url= + apiFlag := "--api-url=" + c.addr.String() + params = append([]string{apiFlag}, params...) + + err := c.cctx.App.Run(append(append(cmd, params...), args...)) + require.NoError(c.t, err) + + // Get the output + str := strings.TrimSpace(c.out.String()) + c.out.Reset() + return str +} + +func (c *mockCLIClient) runCmd(input []string) string { + cmd := c.cmdByNameSub(input[0], input[1]) + out, err := c.runCmdRaw(cmd, input[2:]) + require.NoError(c.t, err) + + return out +} + +func (c *mockCLIClient) cmdByNameSub(name string, sub string) *lcli.Command { + for _, c := range c.cmds { + if c.Name == name { + for _, s := range c.Subcommands { + if s.Name == sub { + return s + } + } + } + } + return nil +} + +func (c *mockCLIClient) runCmdRaw(cmd *lcli.Command, input []string) (string, error) { + // prepend --api-url= + apiFlag := "--api-url=" + c.addr.String() + input = append([]string{apiFlag}, input...) + + fs := c.flagSet(cmd) + err := fs.Parse(input) + require.NoError(c.t, err) + + err = cmd.Action(lcli.NewContext(c.cctx.App, fs, c.cctx)) + + // Get the output + str := strings.TrimSpace(c.out.String()) + c.out.Reset() + return str, err +} + +func (c *mockCLIClient) flagSet(cmd *lcli.Command) *flag.FlagSet { + // Apply app level flags (so we can process --api-url flag) + fs := &flag.FlagSet{} + for _, f := range c.cctx.App.Flags { + err := f.Apply(fs) + if err != nil { + c.t.Fatal(err) + } + } + // Apply command level flags + for _, f := range cmd.Flags { + err := f.Apply(fs) + if err != nil { + c.t.Fatal(err) + } + } + return fs +} diff --git a/cli/test/multisig.go b/cli/test/multisig.go new file mode 100644 index 000000000..96025aa8c --- /dev/null +++ b/cli/test/multisig.go @@ -0,0 +1,77 @@ +package test + +import ( + "context" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + lcli "github.com/urfave/cli/v2" +) + +func QuietMiningLogs() { + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") +} + +func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode) { + ctx := context.Background() + + // Create mock CLI + mockCLI := newMockCLI(t, cmds) + clientCLI := mockCLI.client(clientNode.ListenAddr) + + // Create some wallets on the node to use for testing multisig + var walletAddrs []address.Address + for i := 0; i < 4; i++ { + addr, err := clientNode.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + walletAddrs = append(walletAddrs, addr) + + test.SendFunds(ctx, t, clientNode, addr, types.NewInt(1e15)) + } + + // Create an msig with three of the addresses and threshold of two sigs + // msig create --required=2 --duration=50 --value=1000attofil + amtAtto := types.NewInt(1000) + threshold := 2 + paramDuration := "--duration=50" + paramRequired := fmt.Sprintf("--required=%d", threshold) + paramValue := fmt.Sprintf("--value=%dattofil", amtAtto) + cmd := []string{ + "msig", "create", + paramRequired, + paramDuration, + paramValue, + walletAddrs[0].String(), + walletAddrs[1].String(), + walletAddrs[2].String(), + } + out := clientCLI.runCmd(cmd) + fmt.Println(out) + + // Extract msig robust address from output + expCreateOutPrefix := "Created new multisig:" + require.Regexp(t, regexp.MustCompile(expCreateOutPrefix), out) + parts := strings.Split(strings.TrimSpace(strings.Replace(out, expCreateOutPrefix, "", -1)), " ") + require.Len(t, parts, 2) + msigRobustAddr := parts[1] + fmt.Println("msig robust address:", msigRobustAddr) + + // msig inspect + cmd = []string{"msig", "inspect", msigRobustAddr} + out = clientCLI.runCmd(cmd) + fmt.Println(out) + + require.Regexp(t, regexp.MustCompile("Balance: 0.000000000000001 FIL"), out) +} diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index d5fac0a06..6d47ff214 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -26,9 +26,11 @@ var ( // gatewayDepsAPI defines the API methods that the GatewayAPI depends on // (to make it easy to mock for tests) type gatewayDepsAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) @@ -40,7 +42,18 @@ type gatewayDepsAPI interface { } type GatewayAPI struct { - api gatewayDepsAPI + api gatewayDepsAPI + lookbackCap time.Duration +} + +// NewGatewayAPI creates a new GatewayAPI with the default lookback cap +func NewGatewayAPI(api gatewayDepsAPI) *GatewayAPI { + return newGatewayAPI(api, LookbackCap) +} + +// used by the tests +func newGatewayAPI(api gatewayDepsAPI, lookbackCap time.Duration) *GatewayAPI { + return &GatewayAPI{api: api, lookbackCap: lookbackCap} } func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { @@ -76,13 +89,17 @@ func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error } func (a *GatewayAPI) checkTimestamp(at time.Time) error { - if time.Since(at) > LookbackCap { + if time.Since(at) > a.lookbackCap { return ErrLookbackTooLong } return nil } +func (a *GatewayAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { + return a.api.ChainHasObj(ctx, c) +} + func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) @@ -112,6 +129,10 @@ func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoc return a.api.ChainGetTipSetByHeight(ctx, h, tsk) } +func (a *GatewayAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + return a.api.ChainReadObj(ctx, c) +} + func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index f34f887f5..4946e2a26 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -109,6 +109,10 @@ type mockGatewayDepsAPI struct { tipsets []*types.TipSet } +func (m *mockGatewayDepsAPI) ChainHasObj(context.Context, cid.Cid) (bool, error) { + panic("implement me") +} + func (m *mockGatewayDepsAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { m.lk.RLock() defer m.lk.RUnlock() @@ -158,6 +162,10 @@ func (m *mockGatewayDepsAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.C return m.tipsets[h], nil } +func (m *mockGatewayDepsAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + panic("implement me") +} + func (m *mockGatewayDepsAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { panic("implement me") } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index eebf56a9f..379c96053 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -4,10 +4,14 @@ import ( "bytes" "context" "fmt" + "math" "os" "testing" "time" + "github.com/filecoin-project/lotus/cli" + clitest "github.com/filecoin-project/lotus/cli/test" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -26,20 +30,22 @@ import ( builder "github.com/filecoin-project/lotus/node/test" ) +const maxLookbackCap = time.Duration(math.MaxInt64) + func init() { policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } -// TestEndToEnd tests that API calls can be made on a lite node that is -// connected through a gateway to a full API node -func TestEndToEnd(t *testing.T) { +// TestEndToEndWalletMsig tests that wallet and msig API calls can be made +// on a lite node that is connected through a gateway to a full API node +func TestEndToEndWalletMsig(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") blocktime := 5 * time.Millisecond ctx := context.Background() - full, lite, closer := startNodes(ctx, t, blocktime) + full, lite, closer := startNodes(ctx, t, blocktime, maxLookbackCap) defer closer() // The full node starts with a wallet @@ -56,11 +62,11 @@ func TestEndToEnd(t *testing.T) { require.NoError(t, err) // Send some funds from the full node to the lite node - err = sendFunds(ctx, t, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) require.NoError(t, err) // Send some funds from the lite node back to the full node - err = sendFunds(ctx, t, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) + err = sendFunds(ctx, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) require.NoError(t, err) // Sign some data with the lite node wallet address @@ -81,7 +87,7 @@ func TestEndToEnd(t *testing.T) { walletAddrs = append(walletAddrs, addr) - err = sendFunds(ctx, t, lite, liteWalletAddr, addr, types.NewInt(1e15)) + err = sendFunds(ctx, lite, liteWalletAddr, addr, types.NewInt(1e15)) require.NoError(t, err) } @@ -135,7 +141,33 @@ func TestEndToEnd(t *testing.T) { require.True(t, approveReturn.Applied) } -func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { +// TestEndToEndMsigCLI tests that msig CLI calls can be made +// on a lite node that is connected through a gateway to a full API node +func TestEndToEndMsigCLI(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + clitest.QuietMiningLogs() + + blocktime := 5 * time.Millisecond + ctx := context.Background() + full, lite, closer := startNodes(ctx, t, blocktime, maxLookbackCap) + defer closer() + + // The full node starts with a wallet + fullWalletAddr, err := full.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // Create a wallet on the lite node + liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + // Send some funds from the full node to the lite node + err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + require.NoError(t, err) + + clitest.RunMultisigTest(t, cli.Commands, lite) +} + +func sendFunds(ctx context.Context, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { msg := &types.Message{ From: fromAddr, To: toAddr, @@ -158,7 +190,7 @@ func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAd return nil } -func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration, lookbackCap time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { var closer jsonrpc.ClientCloser // Create one miner and two full nodes. @@ -175,7 +207,7 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (tes fullNode := nodes[0] // Create a gateway server in front of the full node - _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) + _, addr, err := builder.CreateRPCServer(newGatewayAPI(fullNode, lookbackCap)) require.NoError(t, err) // Create a gateway client API that connects to the gateway server diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index c19599084..34e068ffd 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -76,7 +76,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &GatewayAPI{api: api}) + rpcServer.Register("Filecoin", NewGatewayAPI(api)) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 5b4f41114..095b51ed8 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -40,9 +40,11 @@ import ( var log = logging.Logger("fullnode") type ChainModuleAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) } // ChainModule provides a default implementation of ChainModuleAPI. @@ -206,8 +208,8 @@ func (m *ChainModule) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpo return m.Chain.GetTipsetByHeight(ctx, h, ts, true) } -func (a *ChainAPI) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { - blk, err := a.Chain.Blockstore().Get(obj) +func (m *ChainModule) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { + blk, err := m.Chain.Blockstore().Get(obj) if err != nil { return nil, xerrors.Errorf("blockstore get: %w", err) } @@ -219,8 +221,8 @@ func (a *ChainAPI) ChainDeleteObj(ctx context.Context, obj cid.Cid) error { return a.Chain.Blockstore().DeleteBlock(obj) } -func (a *ChainAPI) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) { - return a.Chain.Blockstore().Has(obj) +func (m *ChainModule) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) { + return m.Chain.Blockstore().Has(obj) } func (a *ChainAPI) ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (api.ObjStat, error) { From f74852bd73ab9beed57500f879d5e07b0abf3dcf Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 15 Oct 2020 12:45:02 +0200 Subject: [PATCH 077/106] test: add more msig cli tests --- cli/multisig.go | 2 +- cli/test/multisig.go | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 97a77a75b..90b074421 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -677,7 +677,7 @@ var msigAddProposeCmd = &cli.Command{ return err } - fmt.Println("sent add proposal in message: ", msgCid) + fmt.Fprintln(cctx.App.Writer, "sent add proposal in message: ", msgCid) wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) if err != nil { diff --git a/cli/test/multisig.go b/cli/test/multisig.go index 96025aa8c..e77879346 100644 --- a/cli/test/multisig.go +++ b/cli/test/multisig.go @@ -68,10 +68,43 @@ func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNod msigRobustAddr := parts[1] fmt.Println("msig robust address:", msigRobustAddr) - // msig inspect - cmd = []string{"msig", "inspect", msigRobustAddr} + // Propose to add a new address to the msig + // msig add-propose --from= + paramFrom := fmt.Sprintf("--from=%s", walletAddrs[0]) + cmd = []string{ + "msig", "add-propose", + paramFrom, + msigRobustAddr, + walletAddrs[3].String(), + } out = clientCLI.runCmd(cmd) fmt.Println(out) + // msig inspect + cmd = []string{"msig", "inspect", "--vesting", "--decode-params", msigRobustAddr} + out = clientCLI.runCmd(cmd) + fmt.Println(out) + + // Expect correct balance require.Regexp(t, regexp.MustCompile("Balance: 0.000000000000001 FIL"), out) + // Expect 1 transaction + require.Regexp(t, regexp.MustCompile(`Transactions:\s*1`), out) + // Expect transaction to be "AddSigner" + require.Regexp(t, regexp.MustCompile(`AddSigner`), out) + + // Approve adding the new address + // msig add-approve --from= 0 false + txnID := "0" + paramFrom = fmt.Sprintf("--from=%s", walletAddrs[1]) + cmd = []string{ + "msig", "add-approve", + paramFrom, + msigRobustAddr, + walletAddrs[0].String(), + txnID, + walletAddrs[3].String(), + "false", + } + out = clientCLI.runCmd(cmd) + fmt.Println(out) } From f5d48b0e86a3771a9aa36232c21e749aca188491 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 15 Oct 2020 12:47:24 +0200 Subject: [PATCH 078/106] fix: lotus-gateway api tests --- cmd/lotus-gateway/api_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index 4946e2a26..5b8261974 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -88,7 +88,7 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { mock := &mockGatewayDepsAPI{} - a := &GatewayAPI{api: mock} + a := NewGatewayAPI(mock) // Create tipsets from genesis up to tskh and return the highest ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS) From 4e659e30c54e58af2a59be858aa5f3d14cafb6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 15 Oct 2020 12:14:16 +0100 Subject: [PATCH 079/106] fix lint errors. --- cmd/tvx/codenames_test.go | 2 +- cmd/tvx/exec.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tvx/codenames_test.go b/cmd/tvx/codenames_test.go index 0b1de9fc2..00d107707 100644 --- a/cmd/tvx/codenames_test.go +++ b/cmd/tvx/codenames_test.go @@ -18,7 +18,7 @@ func TestProtocolCodenames(t *testing.T) { t.Fatal("expected breeze codename") } - if height := abi.ChainEpoch(build.UpgradeActorsV2Height + 1); GetProtocolCodename(height) != "actorsv2" { + if height := build.UpgradeActorsV2Height + 1; GetProtocolCodename(height) != "actorsv2" { t.Fatal("expected actorsv2 codename") } diff --git a/cmd/tvx/exec.go b/cmd/tvx/exec.go index de7d2a398..89ad23913 100644 --- a/cmd/tvx/exec.go +++ b/cmd/tvx/exec.go @@ -76,7 +76,7 @@ func executeTestVector(tv schema.TestVector) error { for _, v := range tv.Pre.Variants { r := new(conformance.LogReporter) - switch class := tv.Class; class { + switch class, v := tv.Class, v; class { case "message": conformance.ExecuteMessageVector(r, &tv, &v) case "tipset": From 7e6ede7563fc859b909fa5f54660bb5e5127a4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 15 Oct 2020 12:49:23 +0100 Subject: [PATCH 080/106] update schema; test-vectors submodule. --- extern/test-vectors | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extern/test-vectors b/extern/test-vectors index a8f968ade..d9a75a787 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit a8f968adeba1995f161f7be0048188affc425079 +Subproject commit d9a75a7873aee0db28b87e3970d2ea16a2f37c6a diff --git a/go.mod b/go.mod index 1516cba81..a810e118c 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/filecoin-project/specs-actors v0.9.12 github.com/filecoin-project/specs-actors/v2 v2.1.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 + github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 47301c784..3d543fab0 100644 --- a/go.sum +++ b/go.sum @@ -282,8 +282,8 @@ github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 h1:qnleaW7X8Gi2e3QtqPMFdqVn/DUaNI5tCnq7wNVMnio= -github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= +github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 1fb9ed1c9a53fc304151f65d8409ccb4ae072d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 15 Oct 2020 12:58:43 +0100 Subject: [PATCH 081/106] remove TODO. --- cmd/tvx/codenames.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/tvx/codenames.go b/cmd/tvx/codenames.go index 1cb1b32fb..851e9d841 100644 --- a/cmd/tvx/codenames.go +++ b/cmd/tvx/codenames.go @@ -16,9 +16,6 @@ var ProtocolCodenames = []struct { name string }{ {0, "genesis"}, - // TODO there is some off-by-one trickery in GetNtwkVersion. Not sure if the - // protocol version really kicks in at the designated height, or at the - // following epoch. {build.UpgradeBreezeHeight + 1, "breeze"}, {build.UpgradeSmokeHeight + 1, "smoke"}, {build.UpgradeIgnitionHeight + 1, "ignition"}, From a931ff94e908a6467f35aaedf183f10f21dc5d9c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 08:54:36 -0700 Subject: [PATCH 082/106] fix message list api --- api/api_full.go | 7 ++++++- api/apistruct/struct.go | 4 ++-- cli/state.go | 2 +- node/impl/full/state.go | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 65182078a..89a41dede 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -327,7 +327,7 @@ type FullNode interface { // StateReadState returns the indicated actor's state. StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) // StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height. - StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) + StateListMessages(ctx context.Context, match *MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) // StateNetworkName returns the name of the network the node is synced to StateNetworkName(context.Context) (dtypes.NetworkName, error) @@ -918,3 +918,8 @@ type MsigVesting struct { StartEpoch abi.ChainEpoch UnlockDuration abi.ChainEpoch } + +type MessageMatch struct { + To address.Address + From address.Address +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3854e1dd6..ba3a39516 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -206,7 +206,7 @@ type FullNodeStruct struct { StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"` StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` - StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` + StateListMessages func(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` StateVerifierStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` @@ -948,7 +948,7 @@ func (c *FullNodeStruct) StateGetReceipt(ctx context.Context, msg cid.Cid, tsk t return c.Internal.StateGetReceipt(ctx, msg, tsk) } -func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) { +func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) { return c.Internal.StateListMessages(ctx, match, tsk, toht) } diff --git a/cli/state.go b/cli/state.go index 1e7e24ddb..a02a70505 100644 --- a/cli/state.go +++ b/cli/state.go @@ -844,7 +844,7 @@ var stateListMessagesCmd = &cli.Command{ return err } - msgs, err := api.StateListMessages(ctx, &types.Message{To: toa, From: froma}, ts.Key(), abi.ChainEpoch(toh)) + msgs, err := api.StateListMessages(ctx, &lapi.MessageMatch{To: toa, From: froma}, ts.Key(), abi.ChainEpoch(toh)) if err != nil { return err } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 08ac62f2e..81947cbe3 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -760,7 +760,7 @@ func (a *StateAPI) StateSectorPartition(ctx context.Context, maddr address.Addre return mas.FindSector(sectorNumber) } -func (a *StateAPI) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toheight abi.ChainEpoch) ([]cid.Cid, error) { +func (a *StateAPI) StateListMessages(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toheight abi.ChainEpoch) ([]cid.Cid, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) From 8c7d2efe72f864c7ebb483d047aee12138df7359 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 11:51:40 -0400 Subject: [PATCH 083/106] Make audit balances capable of printing robust addresses --- cmd/lotus-shed/balances.go | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index dfdaf9fa9..b12c069f5 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -5,6 +5,10 @@ import ( "fmt" "strconv" + "github.com/filecoin-project/lotus/chain/gen/genesis" + + _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/docker/go-units" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" @@ -136,6 +140,9 @@ var chainBalanceStateCmd = &cli.Command{ &cli.BoolFlag{ Name: "miner-info", }, + &cli.BoolFlag{ + Name: "robust-addresses", + }, }, Action: func(cctx *cli.Context) error { ctx := context.TODO() @@ -187,6 +194,33 @@ var chainBalanceStateCmd = &cli.Command{ minerInfo := cctx.Bool("miner-info") + robustMap := make(map[address.Address]address.Address) + if cctx.Bool("robust-addresses") { + iact, err := tree.GetActor(_init.Address) + if err != nil { + return xerrors.Errorf("failed to load init actor: %w", err) + } + + ist, err := _init.Load(store, iact) + if err != nil { + return xerrors.Errorf("failed to load init actor state: %w", err) + } + + err = ist.ForEachActor(func(id abi.ActorID, addr address.Address) error { + idAddr, err := address.NewIDAddress(uint64(id)) + if err != nil { + return xerrors.Errorf("failed to write to addr map: %w", err) + } + + robustMap[idAddr] = addr + + return nil + }) + if err != nil { + return xerrors.Errorf("failed to invert init address map: %w", err) + } + } + var infos []accountInfo err = tree.ForEach(func(addr address.Address, act *types.Actor) error { @@ -201,6 +235,23 @@ var chainBalanceStateCmd = &cli.Command{ VestingAmount: types.FIL(big.NewInt(0)), } + if cctx.Bool("robust-addresses") { + robust, found := robustMap[addr] + if found { + ai.Address = robust + } else { + id, err := address.IDFromAddress(addr) + if err != nil { + return xerrors.Errorf("failed to get ID address: %w", err) + } + + // TODO: This is not the correctest way to determine whether a robust address should exist + if id >= genesis.MinerStart { + return xerrors.Errorf("address doesn't have a robust address: %s", addr) + } + } + } + if minerInfo && builtin.IsStorageMinerActor(act.Code) { pow, _, _, err := stmgr.GetPowerRaw(ctx, sm, sroot, addr) if err != nil { From 0b7dc6971dbe1a1e5caae2678d5ed6606bb7dcdc Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Thu, 15 Oct 2020 09:24:48 -0700 Subject: [PATCH 084/106] Rebasing --- markets/dealfilter/cli.go | 68 ++++++++++++++++++++----------- node/builder.go | 6 ++- node/modules/dtypes/miner.go | 4 +- node/modules/storageminer.go | 78 +++++++++++++++++++++++------------- 4 files changed, 102 insertions(+), 54 deletions(-) diff --git a/markets/dealfilter/cli.go b/markets/dealfilter/cli.go index 2cb9d6c4f..af832bfa0 100644 --- a/markets/dealfilter/cli.go +++ b/markets/dealfilter/cli.go @@ -6,35 +6,57 @@ import ( "encoding/json" "os/exec" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func CliDealFilter(cmd string) dtypes.DealFilter { - // TODO: run some checks on the cmd string - +func CliStorageDealFilter(cmd string) dtypes.StorageDealFilter { return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { - j, err := json.MarshalIndent(deal, "", " ") - if err != nil { - return false, "", err + d := struct { + storagemarket.MinerDeal + DealType string + }{ + MinerDeal: deal, + DealType: "storage", } - - var out bytes.Buffer - - c := exec.Command("sh", "-c", cmd) - c.Stdin = bytes.NewReader(j) - c.Stdout = &out - c.Stderr = &out - - switch err := c.Run().(type) { - case nil: - return true, "", nil - case *exec.ExitError: - return false, out.String(), nil - default: - return false, "filter cmd run error", err - } - + return runDealFilter(ctx, cmd, d) + } +} + +func CliRetrievalDealFilter(cmd string) dtypes.RetrievalDealFilter { + return func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) { + d := struct { + retrievalmarket.ProviderDealState + DealType string + }{ + ProviderDealState: deal, + DealType: "retrieval", + } + return runDealFilter(ctx, cmd, d) + } +} + +func runDealFilter(ctx context.Context, cmd string, deal interface{}) (bool, string, error) { + j, err := json.MarshalIndent(deal, "", " ") + if err != nil { + return false, "", err + } + + var out bytes.Buffer + + c := exec.Command("sh", "-c", cmd) + c.Stdin = bytes.NewReader(j) + c.Stdout = &out + c.Stderr = &out + + switch err := c.Run().(type) { + case nil: + return true, "", nil + case *exec.ExitError: + return false, out.String(), nil + default: + return false, "filter cmd run error", err } } diff --git a/node/builder.go b/node/builder.go index f96505eec..b6b75ecc6 100644 --- a/node/builder.go +++ b/node/builder.go @@ -354,7 +354,8 @@ func Online() Option { Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), Override(new(*storedask.StoredAsk), modules.NewStorageAsk), - Override(new(dtypes.DealFilter), modules.BasicDealFilter(nil)), + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)), + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), Override(new(modules.ProviderDealFunds), modules.NewProviderDealFunds), Override(new(storagemarket.StorageProvider), modules.StorageProvider), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil)), @@ -484,7 +485,8 @@ func ConfigStorageMiner(c interface{}) Option { ConfigCommon(&cfg.Common), If(cfg.Dealmaking.Filter != "", - Override(new(dtypes.DealFilter), modules.BasicDealFilter(dealfilter.CliDealFilter(cfg.Dealmaking.Filter))), + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.Filter))), ), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees)), diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 5bb439b4d..1ef157b7e 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" @@ -71,4 +72,5 @@ type SetExpectedSealDurationFunc func(time.Duration) error // too determine how long sealing is expected to take type GetExpectedSealDurationFunc func() (time.Duration, error) -type DealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) +type StorageDealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) +type RetrievalDealFilter func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index adacf5405..74dfff5e1 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -412,16 +412,16 @@ func NewProviderDealFunds(ds dtypes.MetadataDS) (ProviderDealFunds, error) { return funds.NewDealFunds(ds, datastore.NewKey("/marketfunds/provider")) } -func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, +func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc, expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc, - spn storagemarket.StorageProviderNode) dtypes.DealFilter { + spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter { return func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc, expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc, - spn storagemarket.StorageProviderNode) dtypes.DealFilter { + spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter { return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { b, err := onlineOk() @@ -497,7 +497,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, - df dtypes.DealFilter, + df dtypes.StorageDealFilter, funds ProviderDealFunds, ) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) @@ -511,8 +511,52 @@ func StorageProvider(minerAddress dtypes.MinerAddress, return storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), store, mds, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk, funds, opt) } +func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter { + return func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter { + return func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) { + b, err := onlineOk() + if err != nil { + return false, "miner error", err + } + + if !b { + log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client") + return false, "miner is not accepting online retrieval deals", nil + } + + b, err = offlineOk() + if err != nil { + return false, "miner error", err + } + + if !b { + log.Info("offline retrieval has not been implemented yet") + } + + if userFilter != nil { + return userFilter(ctx, state) + } + + return true, "", nil + } + } +} + // RetrievalProvider creates a new retrieval provider attached to the provider blockstore -func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, mds dtypes.StagingMultiDstore, dt dtypes.ProviderDataTransfer, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) { +func RetrievalProvider(h host.Host, + miner *storage.Miner, + sealer sectorstorage.SectorManager, + full lapi.FullNode, + ds dtypes.MetadataDS, + pieceStore dtypes.ProviderPieceStore, + mds dtypes.StagingMultiDstore, + dt dtypes.ProviderDataTransfer, + onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc, + userFilter dtypes.RetrievalDealFilter, +) (retrievalmarket.RetrievalProvider, error) { adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full) maddr, err := minerAddrFromDS(ds) @@ -521,29 +565,7 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S } netwk := rmnet.NewFromLibp2pHost(h) - - opt := retrievalimpl.DealDeciderOpt(func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) { - b, err := onlineOk() - if err != nil { - return false, "miner error", err - } - - if !b { - log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client") - return false, "miner is not accepting online retrieval deals", nil - } - - b, err = offlineOk() - if err != nil { - return false, "miner error", err - } - - if !b { - log.Info("offline retrieval has not been implemented yet") - } - - return true, "", nil - }) + opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter)) return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt) } From 5bee85d57a43d45d77acc910800b1514b8e4517d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 09:38:27 -0700 Subject: [PATCH 085/106] incremental output for list-messages --- cli/state.go | 55 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/cli/state.go b/cli/state.go index a02a70505..99ace833a 100644 --- a/cli/state.go +++ b/cli/state.go @@ -837,33 +837,66 @@ var stateListMessagesCmd = &cli.Command{ froma = a } - toh := cctx.Uint64("toheight") + toh := abi.ChainEpoch(cctx.Uint64("toheight")) ts, err := LoadTipSet(ctx, cctx, api) if err != nil { return err } - msgs, err := api.StateListMessages(ctx, &lapi.MessageMatch{To: toa, From: froma}, ts.Key(), abi.ChainEpoch(toh)) - if err != nil { - return err + if ts == nil { + head, err := api.ChainHead(ctx) + if err != nil { + return err + } + ts = head } - for _, c := range msgs { - if cctx.Bool("cids") { - fmt.Println(c.String()) - continue + windowSize := abi.ChainEpoch(100) + + cur := ts + for cur.Height() > toh { + if ctx.Err() != nil { + return ctx.Err() } - m, err := api.ChainGetMessage(ctx, c) + end := toh + if cur.Height()-windowSize > end { + end = cur.Height() - windowSize + } + + msgs, err := api.StateListMessages(ctx, &lapi.MessageMatch{To: toa, From: froma}, cur.Key(), end) if err != nil { return err } - b, err := json.MarshalIndent(m, "", " ") + + for _, c := range msgs { + if cctx.Bool("cids") { + fmt.Println(c.String()) + continue + } + + m, err := api.ChainGetMessage(ctx, c) + if err != nil { + return err + } + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + } + + if end <= 0 { + break + } + + next, err := api.ChainGetTipSetByHeight(ctx, end-1, cur.Key()) if err != nil { return err } - fmt.Println(string(b)) + + cur = next } return nil From eae7d62a2b416c7e51d0884a0a875c644a72431f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 12:07:41 -0700 Subject: [PATCH 086/106] docsgen --- documentation/en/api-methods.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index a905ef2e3..4c3bd44ff 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3352,19 +3352,8 @@ Inputs: ```json [ { - "Version": 42, "To": "f01234", - "From": "f01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==", - "CID": { - "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" - } + "From": "f01234" }, [ { From 726eacbe037c02b5d5cd9648f0171e968dedf63e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 13:54:58 -0700 Subject: [PATCH 087/106] add endpoints needed by spacegap --- cmd/lotus-gateway/api.go | 57 ++++++++++++++++++++++++++++++++--- cmd/lotus-gateway/api_test.go | 3 ++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 1cdd37915..fe68018d8 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -6,10 +6,12 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" _ "github.com/filecoin-project/lotus/lib/sigs/bls" @@ -44,6 +46,12 @@ type gatewayDepsAPI interface { StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) + StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) + StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) + StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) + StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) } type GatewayAPI struct { @@ -198,10 +206,6 @@ func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence u return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } -func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { - return sigs.Verify(sig, k, msg) == nil, nil -} - func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -209,6 +213,51 @@ func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, return a.api.StateReadState(ctx, actor, tsk) } +func (a *GatewayAPI) StateMinerPower(ctx context.Context, m address.Address, tsk types.TipSetKey) (*api.MinerPower, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerPower(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerFaults(ctx context.Context, m address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return bitfield.BitField{}, err + } + return a.api.StateMinerFaults(ctx, m, tsk) +} +func (a *GatewayAPI) StateMinerRecoveries(ctx context.Context, m address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return bitfield.BitField{}, err + } + return a.api.StateMinerRecoveries(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerInfo(ctx context.Context, m address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return miner.MinerInfo{}, err + } + return a.api.StateMinerInfo(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]api.Deadline, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerDeadlines(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerAvailableBalance(ctx context.Context, m address.Address, tsk types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.BigInt{}, err + } + return a.api.StateMinerAvailableBalance(ctx, m, tsk) +} + +func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { + return sigs.Verify(sig, k, msg) == nil, nil +} + var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.GasModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index 4f10fe987..ae161945b 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -107,6 +107,9 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { type mockGatewayDepsAPI struct { lk sync.RWMutex tipsets []*types.TipSet + + gatewayDepsAPI // satisfies all interface requirements but will panic if + // methods are called. easier than filling out with panic stubs IMO } func (m *mockGatewayDepsAPI) ChainHasObj(context.Context, cid.Cid) (bool, error) { From cb801d47c7712898aa25a757ce225916f605a354 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 18:48:51 -0400 Subject: [PATCH 088/106] Add GasCost to InvocResult --- api/api_full.go | 1 + chain/stmgr/call.go | 2 ++ chain/stmgr/stmgr.go | 1 + chain/stmgr/utils.go | 13 +++++++++++++ node/impl/full/state.go | 13 +++---------- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 65182078a..130cba957 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -740,6 +740,7 @@ type InvocResult struct { MsgCid cid.Cid Msg *types.Message MsgRct *types.MessageReceipt + GasCost MsgGasCost ExecutionTrace types.ExecutionTrace Error string Duration time.Duration diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 86035140b..5eaa2e16a 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -116,6 +116,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, + GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Error: errs, Duration: ret.Duration, @@ -232,6 +233,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, + GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Error: errs, Duration: ret.Duration, diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b7d5050e1..d5cb1958d 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -203,6 +203,7 @@ func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, MsgCid: mcid, Msg: msg, MsgRct: &ret.MessageReceipt, + GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Duration: ret.Duration, } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index cbc78ca5b..de4f947df 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -701,3 +701,16 @@ func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi return sum, nil } + +func MakeMsgGasCost(msg *types.Message, ret *vm.ApplyRet) api.MsgGasCost { + return api.MsgGasCost{ + Message: msg.Cid(), + GasUsed: big.NewInt(ret.GasUsed), + BaseFeeBurn: ret.GasCosts.BaseFeeBurn, + OverEstimationBurn: ret.GasCosts.OverEstimationBurn, + MinerPenalty: ret.GasCosts.MinerPenalty, + MinerTip: ret.GasCosts.MinerTip, + Refund: ret.GasCosts.Refund, + TotalCost: big.Sub(msg.RequiredFunds(), ret.GasCosts.Refund), + } +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 08ac62f2e..9558219c7 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -365,6 +365,7 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. MsgCid: mc, Msg: m, MsgRct: &r.MessageReceipt, + GasCost: stmgr.MakeMsgGasCost(m, r), ExecutionTrace: r.ExecutionTrace, Error: errstr, Duration: r.Duration, @@ -1304,14 +1305,6 @@ func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk ty return nil, err } - return &api.MsgGasCost{ - Message: msg, - GasUsed: big.NewInt(r.GasUsed), - BaseFeeBurn: r.GasCosts.BaseFeeBurn, - OverEstimationBurn: r.GasCosts.OverEstimationBurn, - MinerPenalty: r.GasCosts.MinerPenalty, - MinerTip: r.GasCosts.MinerTip, - Refund: r.GasCosts.Refund, - TotalCost: big.Sub(m.RequiredFunds(), r.GasCosts.Refund), - }, nil + gc := stmgr.MakeMsgGasCost(m, r) + return &gc, nil } From 7826cc0babf005dfb84f5c4ab7614ab3d1fa201b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 19:14:53 -0400 Subject: [PATCH 089/106] Rename StateReplay to StateTransplant --- api/api_full.go | 4 ++-- api/apistruct/struct.go | 6 +++--- cli/state.go | 27 ++++++++------------------- lotuspond/front/src/Block.js | 2 +- node/impl/full/state.go | 2 +- 5 files changed, 15 insertions(+), 26 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 130cba957..9eaca53f9 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -320,8 +320,8 @@ type FullNode interface { // StateCall runs the given message and returns its result without any persisted changes. StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) - // StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. - StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) + // StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. + StateTransplant(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) // StateGetActor returns the indicated actor's nonce and balance. StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // StateReadState returns the indicated actor's state. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3854e1dd6..3e474f596 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -188,7 +188,7 @@ type FullNodeStruct struct { StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) `perm:"read"` StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"` StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` - StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` + StateTransplant func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` @@ -880,8 +880,8 @@ func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk return c.Internal.StateCall(ctx, msg, tsk) } -func (c *FullNodeStruct) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { - return c.Internal.StateReplay(ctx, tsk, mc) +func (c *FullNodeStruct) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { + return c.Internal.StateTransplant(ctx, tsk, mc) } func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { diff --git a/cli/state.go b/cli/state.go index 1e7e24ddb..362e29636 100644 --- a/cli/state.go +++ b/cli/state.go @@ -60,7 +60,7 @@ var stateCmd = &cli.Command{ stateSectorCmd, stateGetActorCmd, stateLookupIDCmd, - stateReplaySetCmd, + stateTransplantCmd, stateSectorSizeCmd, stateReadStateCmd, stateListMessagesCmd, @@ -384,9 +384,9 @@ var stateExecTraceCmd = &cli.Command{ }, } -var stateReplaySetCmd = &cli.Command{ - Name: "replay", - Usage: "Replay a particular message within a tipset", +var stateTransplantCmd = &cli.Command{ + Name: "transplant", + Usage: "Play a particular message within a tipset", ArgsUsage: "[tipsetKey messageCid]", Action: func(cctx *cli.Context) error { if cctx.Args().Len() < 1 { @@ -437,30 +437,19 @@ var stateReplaySetCmd = &cli.Command{ return err } } else { - var r *api.MsgLookup - r, err = fapi.StateWaitMsg(ctx, mcid, build.MessageConfidence) - if err != nil { - return xerrors.Errorf("finding message in chain: %w", err) - } - - childTs, err := fapi.ChainGetTipSet(ctx, r.TipSet) - if err != nil { - return xerrors.Errorf("loading tipset: %w", err) - } - ts, err = fapi.ChainGetTipSet(ctx, childTs.Parents()) + ts, err = fapi.ChainHead(ctx) if err != nil { return err } } - } - res, err := fapi.StateReplay(ctx, ts.Key(), mcid) + res, err := fapi.StateTransplant(ctx, ts.Key(), mcid) if err != nil { - return xerrors.Errorf("replay call failed: %w", err) + return xerrors.Errorf("transplant call failed: %w", err) } - fmt.Println("Replay receipt:") + fmt.Println("Transplant receipt:") fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) fmt.Printf("Return: %x\n", res.MsgRct.Return) fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) diff --git a/lotuspond/front/src/Block.js b/lotuspond/front/src/Block.js index 4bae57c81..cb8d25c9d 100644 --- a/lotuspond/front/src/Block.js +++ b/lotuspond/front/src/Block.js @@ -26,7 +26,7 @@ class Block extends React.Component { messages = await Promise.all(messages.map(async (msg, i) => { if (msg.receipt.ExitCode !== 0) { - let reply = await this.props.conn.call('Filecoin.StateReplay', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, msg.Cid]) + let reply = await this.props.conn.call('Filecoin.StateTransplant', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, msg.Cid]) if(!reply.Error) { reply.Error = "reply: no error" } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 9558219c7..349a4918b 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -346,7 +346,7 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. return res, err } -func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { +func (a *StateAPI) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) From 8351420404bc97e3e4eef0ad85db4e43eed11a11 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 16:27:49 -0700 Subject: [PATCH 090/106] Add more endpoints for spacegap --- cmd/lotus-gateway/api.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index fe68018d8..855dbe1e8 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -37,6 +38,7 @@ type gatewayDepsAPI interface { ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) + ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) @@ -52,6 +54,8 @@ type gatewayDepsAPI interface { StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) + StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) } type GatewayAPI struct { @@ -146,6 +150,10 @@ func (a *GatewayAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error return a.api.ChainReadObj(ctx, c) } +func (a *GatewayAPI) ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) { + return a.api.ChainGetNode(ctx, p) +} + func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -254,6 +262,21 @@ func (a *GatewayAPI) StateMinerAvailableBalance(ctx context.Context, m address.A return a.api.StateMinerAvailableBalance(ctx, m, tsk) } +func (a *GatewayAPI) StateMinerProvingDeadline(ctx context.Context, m address.Address, tsk types.TipSetKey) (*dline.Info, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerProvingDeadline(ctx, m, tsk) +} + +func (a *GatewayAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.BigInt{}, err + } + return a.api.StateCirculatingSupply(ctx, tsk) + +} + func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { return sigs.Verify(sig, k, msg) == nil, nil } From af1a422ddd0773c9de68b7b9489d56ff456846a3 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 16:36:45 -0700 Subject: [PATCH 091/106] statevminternal circ supply --- cmd/lotus-gateway/api.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 855dbe1e8..86e49e62f 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -56,6 +56,7 @@ type gatewayDepsAPI interface { StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) + StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) } type GatewayAPI struct { @@ -277,6 +278,13 @@ func (a *GatewayAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSe } +func (a *GatewayAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return api.CirculatingSupply{}, err + } + return a.api.StateVMCirculatingSupplyInternal(ctx, tsk) +} + func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { return sigs.Verify(sig, k, msg) == nil, nil } From 99c07703f8f60799e9e2637bffc30613267d14de Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 19:47:02 -0400 Subject: [PATCH 092/106] Add a new StateReplay CLI/API endpoint --- api/api_full.go | 4 +- api/apistruct/struct.go | 5 +++ cli/state.go | 91 +++++++++++++++++++++++++++++++++++++++++ node/impl/full/state.go | 40 ++++++++++++++++++ 4 files changed, 139 insertions(+), 1 deletion(-) diff --git a/api/api_full.go b/api/api_full.go index 9eaca53f9..29688a5e6 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -315,13 +315,15 @@ type FullNode interface { // MethodGroup: State // The State methods are used to query, inspect, and interact with chain state. - // All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. + // Most methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. // A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. // StateCall runs the given message and returns its result without any persisted changes. StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) // StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. StateTransplant(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) + // StateReplay searches for where the given message was executed, and replays it in that tipset. + StateReplay(context.Context, cid.Cid) (*InvocResult, error) // StateGetActor returns the indicated actor's nonce and balance. StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // StateReadState returns the indicated actor's state. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3e474f596..33670c276 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -189,6 +189,7 @@ type FullNodeStruct struct { StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"` StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` StateTransplant func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` + StateReplay func(context.Context, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` @@ -884,6 +885,10 @@ func (c *FullNodeStruct) StateTransplant(ctx context.Context, tsk types.TipSetKe return c.Internal.StateTransplant(ctx, tsk, mc) } +func (c *FullNodeStruct) StateReplay(ctx context.Context, mc cid.Cid) (*api.InvocResult, error) { + return c.Internal.StateReplay(ctx, mc) +} + func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { return c.Internal.StateGetActor(ctx, actor, tsk) } diff --git a/cli/state.go b/cli/state.go index 362e29636..bff41e268 100644 --- a/cli/state.go +++ b/cli/state.go @@ -61,6 +61,7 @@ var stateCmd = &cli.Command{ stateGetActorCmd, stateLookupIDCmd, stateTransplantCmd, + stateReplayCmd, stateSectorSizeCmd, stateReadStateCmd, stateListMessagesCmd, @@ -388,6 +389,16 @@ var stateTransplantCmd = &cli.Command{ Name: "transplant", Usage: "Play a particular message within a tipset", ArgsUsage: "[tipsetKey messageCid]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-trace", + Usage: "print out full execution trace for given message", + }, + &cli.BoolFlag{ + Name: "detailed-gas", + Usage: "print out detailed gas costs for given message", + }, + }, Action: func(cctx *cli.Context) error { if cctx.Args().Len() < 1 { fmt.Println("usage: [tipset] ") @@ -453,10 +464,90 @@ var stateTransplantCmd = &cli.Command{ fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) fmt.Printf("Return: %x\n", res.MsgRct.Return) fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) + + if cctx.Bool("detailed-gas") { + fmt.Printf("Base Fee Burn: %d\n", res.GasCost.BaseFeeBurn) + fmt.Printf("Overestimaton Burn: %d\n", res.GasCost.OverEstimationBurn) + fmt.Printf("Miner Penalty: %d\n", res.GasCost.MinerPenalty) + fmt.Printf("Miner Tip: %d\n", res.GasCost.MinerTip) + fmt.Printf("Refund: %d\n", res.GasCost.Refund) + } + fmt.Printf("Total Message Cost: %d\n", res.GasCost.TotalCost) + if res.MsgRct.ExitCode != 0 { fmt.Printf("Error message: %q\n", res.Error) } + if cctx.Bool("show-trace") { + fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", res.Msg.From, res.Msg.To, res.Msg.Value, res.Msg.Method, res.Msg.Params, res.MsgRct.ExitCode, res.MsgRct.Return) + printInternalExecutions("\t", res.ExecutionTrace.Subcalls) + } + + return nil + }, +} + +var stateReplayCmd = &cli.Command{ + Name: "replay", + Usage: "Replay a particular message", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-trace", + Usage: "print out full execution trace for given message", + }, + &cli.BoolFlag{ + Name: "detailed-gas", + Usage: "print out detailed gas costs for given message", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + fmt.Println("must provide cid of message to replay") + return nil + } + + mcid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("message cid was invalid: %s", err) + } + + fapi, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + res, err := fapi.StateReplay(ctx, mcid) + if err != nil { + return xerrors.Errorf("replay call failed: %w", err) + } + + fmt.Println("Replay receipt:") + fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) + fmt.Printf("Return: %x\n", res.MsgRct.Return) + fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) + + if cctx.Bool("detailed-gas") { + fmt.Printf("Base Fee Burn: %d\n", res.GasCost.BaseFeeBurn) + fmt.Printf("Overestimaton Burn: %d\n", res.GasCost.OverEstimationBurn) + fmt.Printf("Miner Penalty: %d\n", res.GasCost.MinerPenalty) + fmt.Printf("Miner Tip: %d\n", res.GasCost.MinerTip) + fmt.Printf("Refund: %d\n", res.GasCost.Refund) + } + fmt.Printf("Total Message Cost: %d\n", res.GasCost.TotalCost) + + if res.MsgRct.ExitCode != 0 { + fmt.Printf("Error message: %q\n", res.Error) + } + + if cctx.Bool("show-trace") { + fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", res.Msg.From, res.Msg.To, res.Msg.Value, res.Msg.Method, res.Msg.Params, res.MsgRct.ExitCode, res.MsgRct.Return) + printInternalExecutions("\t", res.ExecutionTrace.Subcalls) + } + return nil }, } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 349a4918b..959660085 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -372,6 +372,46 @@ func (a *StateAPI) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc }, nil } +func (a *StateAPI) StateReplay(ctx context.Context, mc cid.Cid) (*api.InvocResult, error) { + mlkp, err := a.StateSearchMsg(ctx, mc) + if err != nil { + return nil, xerrors.Errorf("searching for msg %s: %w", mc, err) + } + if mlkp == nil { + return nil, xerrors.Errorf("didn't find msg %s", mc) + } + + executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) + } + + ts, err := a.Chain.LoadTipSet(executionTs.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) + } + + m, r, err := a.StateManager.Replay(ctx, ts, mlkp.Message) + if err != nil { + return nil, err + } + + var errstr string + if r.ActorErr != nil { + errstr = r.ActorErr.Error() + } + + return &api.InvocResult{ + MsgCid: mlkp.Message, + Msg: m, + MsgRct: &r.MessageReceipt, + GasCost: stmgr.MakeMsgGasCost(m, r), + ExecutionTrace: r.ExecutionTrace, + Error: errstr, + Duration: r.Duration, + }, nil +} + func stateForTs(ctx context.Context, ts *types.TipSet, cstore *store.ChainStore, smgr *stmgr.StateManager) (*state.StateTree, error) { if ts == nil { ts = cstore.GetHeaviestTipSet() From a71fbee2d13e83c9fbe49a3dd59d27fa39a711c7 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 16:49:25 -0700 Subject: [PATCH 093/106] bump lookback limit to 12 hours --- cmd/lotus-gateway/api.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 86e49e62f..105f8881e 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -22,7 +22,7 @@ import ( ) const ( - LookbackCap = time.Hour + LookbackCap = time.Hour * 12 stateWaitLookbackLimit = abi.ChainEpoch(20) ) @@ -129,9 +129,19 @@ func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (* } func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.api.ChainGetTipSet(ctx, tsk) - if err != nil { - return nil, err + var ts *types.TipSet + if tsk.IsEmpty() { + head, err := a.api.ChainHead(ctx) + if err != nil { + return nil, err + } + ts = head + } else { + gts, err := a.api.ChainGetTipSet(ctx, tsk) + if err != nil { + return nil, err + } + ts = gts } // Check if the tipset key refers to a tipset that's too far in the past From 6f86b95f62bd59b38f1499ee381b93643247500b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 21:00:47 -0400 Subject: [PATCH 094/106] Remove StateMsgGasCost --- api/api_full.go | 2 - api/apistruct/struct.go | 5 - cli/state.go | 55 ----------- documentation/en/api-methods.md | 169 ++++++++++++++++++++++---------- node/impl/full/state.go | 41 -------- 5 files changed, 119 insertions(+), 153 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 29688a5e6..88bcb3cba 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -372,8 +372,6 @@ type FullNode interface { StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) // StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) - // StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip - StateMsgGasCost(context.Context, cid.Cid, types.TipSetKey) (*MsgGasCost, error) // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 33670c276..0279e8c8a 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -192,7 +192,6 @@ type FullNodeStruct struct { StateReplay func(context.Context, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` - StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` StateWaitMsgLimited func(context.Context, cid.Cid, uint64, abi.ChainEpoch) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` @@ -897,10 +896,6 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres return c.Internal.StateReadState(ctx, addr, tsk) } -func (c *FullNodeStruct) StateMsgGasCost(ctx context.Context, msgc cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { - return c.Internal.StateMsgGasCost(ctx, msgc, tsk) -} - func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) { return c.Internal.StateWaitMsg(ctx, msgc, confidence) } diff --git a/cli/state.go b/cli/state.go index bff41e268..38454ec27 100644 --- a/cli/state.go +++ b/cli/state.go @@ -70,7 +70,6 @@ var stateCmd = &cli.Command{ stateGetDealSetCmd, stateWaitMsgCmd, stateSearchMsgCmd, - stateMsgCostCmd, stateMinerInfo, stateMarketCmd, stateExecTraceCmd, @@ -1504,60 +1503,6 @@ var stateSearchMsgCmd = &cli.Command{ }, } -var stateMsgCostCmd = &cli.Command{ - Name: "msg-cost", - Usage: "Get the detailed gas costs of a message", - ArgsUsage: "[messageCid]", - Action: func(cctx *cli.Context) error { - if !cctx.Args().Present() { - return fmt.Errorf("must specify message cid to get gas costs for") - } - - api, closer, err := GetFullNodeAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := ReqContext(cctx) - - msg, err := cid.Decode(cctx.Args().First()) - if err != nil { - return err - } - - tsk := types.EmptyTSK - - ts, err := LoadTipSet(ctx, cctx, api) - if err != nil { - return err - } - - if ts != nil { - tsk = ts.Key() - } - - mgc, err := api.StateMsgGasCost(ctx, msg, tsk) - if err != nil { - return err - } - - if mgc != nil { - fmt.Printf("Message CID: %s", mgc.Message) - fmt.Printf("\nGas Used: %d", mgc.GasUsed) - fmt.Printf("\nBase Fee Burn: %d", mgc.BaseFeeBurn) - fmt.Printf("\nOverestimation Burn: %d", mgc.OverEstimationBurn) - fmt.Printf("\nMiner Tip: %d", mgc.MinerTip) - fmt.Printf("\nRefund: %d", mgc.Refund) - fmt.Printf("\nTotal Cost: %d", mgc.TotalCost) - fmt.Printf("\nMiner Penalty: %d", mgc.MinerPenalty) - } else { - fmt.Print("message was not found on chain") - } - return nil - }, -} - var stateCallCmd = &cli.Command{ Name: "call", Usage: "Invoke a method on an actor locally", diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index a905ef2e3..9be12722d 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -156,7 +156,6 @@ * [StateMinerRecoveries](#StateMinerRecoveries) * [StateMinerSectorCount](#StateMinerSectorCount) * [StateMinerSectors](#StateMinerSectors) - * [StateMsgGasCost](#StateMsgGasCost) * [StateNetworkName](#StateNetworkName) * [StateNetworkVersion](#StateNetworkVersion) * [StateReadState](#StateReadState) @@ -166,6 +165,7 @@ * [StateSectorGetInfo](#StateSectorGetInfo) * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) + * [StateTransplant](#StateTransplant) * [StateVMCirculatingSupplyInternal](#StateVMCirculatingSupplyInternal) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) @@ -2989,7 +2989,7 @@ Response: ## State The State methods are used to query, inspect, and interact with chain state. -All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. +Most methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. @@ -3100,6 +3100,18 @@ Response: "Return": "Ynl0ZSBhcnJheQ==", "GasUsed": 9 }, + "GasCost": { + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" + }, "ExecutionTrace": { "Msg": { "Version": 42, @@ -3974,45 +3986,6 @@ Inputs: Response: `null` -### StateMsgGasCost -StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "GasUsed": "0", - "BaseFeeBurn": "0", - "OverEstimationBurn": "0", - "MinerPenalty": "0", - "MinerTip": "0", - "Refund": "0", - "TotalCost": "0" -} -``` - ### StateNetworkName StateNetworkName returns the name of the network the node is synced to @@ -4075,7 +4048,7 @@ Response: ``` ### StateReplay -StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. +StateReplay searches for where the given message was executed, and replays it in that tipset. Perms: read @@ -4083,14 +4056,6 @@ Perms: read Inputs: ```json [ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" } @@ -4123,6 +4088,18 @@ Response: "Return": "Ynl0ZSBhcnJheQ==", "GasUsed": 9 }, + "GasCost": { + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" + }, "ExecutionTrace": { "Msg": { "Version": 42, @@ -4342,6 +4319,98 @@ Response: } ``` +### StateTransplant +StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "MsgCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Msg": { + "Version": 42, + "To": "f01234", + "From": "f01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "GasCost": { + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" + }, + "ExecutionTrace": { + "Msg": { + "Version": 42, + "To": "f01234", + "From": "f01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "Error": "string value", + "Duration": 60000000000, + "GasCharges": null, + "Subcalls": null + }, + "Error": "string value", + "Duration": 60000000000 +} +``` + ### StateVMCirculatingSupplyInternal StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset. This is the value reported by the runtime interface to actors code. diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 959660085..a63ac8c67 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1307,44 +1307,3 @@ func (a *StateAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) return a.StateManager.GetNtwkVersion(ctx, ts.Height()), nil } - -func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { - var msg cid.Cid - var ts *types.TipSet - var err error - if tsk != types.EmptyTSK { - msg = inputMsg - ts, err = a.Chain.LoadTipSet(tsk) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) - } - } else { - mlkp, err := a.StateSearchMsg(ctx, inputMsg) - if err != nil { - return nil, xerrors.Errorf("searching for msg %s: %w", inputMsg, err) - } - if mlkp == nil { - return nil, xerrors.Errorf("didn't find msg %s", inputMsg) - } - - executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) - } - - ts, err = a.Chain.LoadTipSet(executionTs.Parents()) - if err != nil { - return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) - } - - msg = mlkp.Message - } - - m, r, err := a.StateManager.Replay(ctx, ts, msg) - if err != nil { - return nil, err - } - - gc := stmgr.MakeMsgGasCost(m, r) - return &gc, nil -} From 89f46cb5ccd7a19b72c7432404f658eb1d7ecfb5 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 21:12:38 -0400 Subject: [PATCH 095/106] Make ApplyRet's GasOutputs a pointer, return nil when unused --- chain/stmgr/call.go | 1 - chain/stmgr/forks.go | 4 ++-- chain/stmgr/stmgr.go | 4 +++- chain/vm/vm.go | 16 ++++++++-------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 5eaa2e16a..3d7d284bc 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -116,7 +116,6 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, - GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Error: errs, Duration: ret.Duration, diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 156c68783..488f84167 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -498,7 +498,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal Subcalls: subcalls, }, Duration: 0, - GasCosts: vm.ZeroGasOutputs(), + GasCosts: nil, }); err != nil { return cid.Undef, xerrors.Errorf("recording transfers: %w", err) } @@ -799,7 +799,7 @@ func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Add Subcalls: subcalls, }, Duration: 0, - GasCosts: vm.ZeroGasOutputs(), + GasCosts: nil, }); err != nil { return xerrors.Errorf("recording transfers: %w", err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d5cb1958d..be14797d9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -203,13 +203,15 @@ func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, MsgCid: mcid, Msg: msg, MsgRct: &ret.MessageReceipt, - GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Duration: ret.Duration, } if ret.ActorErr != nil { ir.Error = ret.ActorErr.Error() } + if ret.GasCosts != nil { + ir.GasCost = MakeMsgGasCost(msg, ret) + } *trace = append(*trace, ir) return nil } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index a4efccb29..72ad731aa 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -214,7 +214,7 @@ type ApplyRet struct { ActorErr aerrors.ActorError ExecutionTrace types.ExecutionTrace Duration time.Duration - GasCosts GasOutputs + GasCosts *GasOutputs } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, @@ -361,7 +361,7 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - GasCosts: GasOutputs{}, + GasCosts: nil, Duration: time.Since(start), }, actorErr } @@ -397,7 +397,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -417,7 +417,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "actor not found: %s", msg.From), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -434,7 +434,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "send from not account actor: %s", fromActor.Code), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -450,7 +450,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor nonce invalid: msg:%d != state:%d", msg.Nonce, fromActor.Nonce), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -466,7 +466,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor balance less than needed: %s < %s", types.FIL(fromActor.Balance), types.FIL(gascost)), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -560,7 +560,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } From 2fd4a976984bb3b51e8b227b3175ed0c5eb61af3 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 22:45:11 -0400 Subject: [PATCH 096/106] Remove StateTransplant, modify StateReplay --- api/api_full.go | 7 +-- api/apistruct/struct.go | 11 +--- cli/state.go | 105 +------------------------------- documentation/en/api-methods.md | 104 +++---------------------------- lotuspond/front/src/Block.js | 2 +- node/impl/full/state.go | 74 +++++++++------------- 6 files changed, 46 insertions(+), 257 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 88bcb3cba..1946ca28f 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -320,10 +320,9 @@ type FullNode interface { // StateCall runs the given message and returns its result without any persisted changes. StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) - // StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. - StateTransplant(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) - // StateReplay searches for where the given message was executed, and replays it in that tipset. - StateReplay(context.Context, cid.Cid) (*InvocResult, error) + // StateReplay replays a given message, assuming it was included in a block in the specified tipset. + // If no tipset key is provided, the appropriate tipset is looked up. + StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) // StateGetActor returns the indicated actor's nonce and balance. StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // StateReadState returns the indicated actor's state. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 0279e8c8a..6effcf727 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -188,8 +188,7 @@ type FullNodeStruct struct { StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) `perm:"read"` StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"` StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` - StateTransplant func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` - StateReplay func(context.Context, cid.Cid) (*api.InvocResult, error) `perm:"read"` + StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` @@ -880,12 +879,8 @@ func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk return c.Internal.StateCall(ctx, msg, tsk) } -func (c *FullNodeStruct) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { - return c.Internal.StateTransplant(ctx, tsk, mc) -} - -func (c *FullNodeStruct) StateReplay(ctx context.Context, mc cid.Cid) (*api.InvocResult, error) { - return c.Internal.StateReplay(ctx, mc) +func (c *FullNodeStruct) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { + return c.Internal.StateReplay(ctx, tsk, mc) } func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { diff --git a/cli/state.go b/cli/state.go index 38454ec27..0fb3ecde6 100644 --- a/cli/state.go +++ b/cli/state.go @@ -60,7 +60,6 @@ var stateCmd = &cli.Command{ stateSectorCmd, stateGetActorCmd, stateLookupIDCmd, - stateTransplantCmd, stateReplayCmd, stateSectorSizeCmd, stateReadStateCmd, @@ -384,108 +383,6 @@ var stateExecTraceCmd = &cli.Command{ }, } -var stateTransplantCmd = &cli.Command{ - Name: "transplant", - Usage: "Play a particular message within a tipset", - ArgsUsage: "[tipsetKey messageCid]", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "show-trace", - Usage: "print out full execution trace for given message", - }, - &cli.BoolFlag{ - Name: "detailed-gas", - Usage: "print out detailed gas costs for given message", - }, - }, - Action: func(cctx *cli.Context) error { - if cctx.Args().Len() < 1 { - fmt.Println("usage: [tipset] ") - fmt.Println("The last cid passed will be used as the message CID") - fmt.Println("All preceding ones will be used as the tipset") - return nil - } - - args := cctx.Args().Slice() - mcid, err := cid.Decode(args[len(args)-1]) - if err != nil { - return fmt.Errorf("message cid was invalid: %s", err) - } - - fapi, closer, err := GetFullNodeAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := ReqContext(cctx) - - var ts *types.TipSet - { - var tscids []cid.Cid - for _, s := range args[:len(args)-1] { - c, err := cid.Decode(s) - if err != nil { - return fmt.Errorf("tipset cid was invalid: %s", err) - } - tscids = append(tscids, c) - } - - if len(tscids) > 0 { - var headers []*types.BlockHeader - for _, c := range tscids { - h, err := fapi.ChainGetBlock(ctx, c) - if err != nil { - return err - } - - headers = append(headers, h) - } - - ts, err = types.NewTipSet(headers) - if err != nil { - return err - } - } else { - ts, err = fapi.ChainHead(ctx) - if err != nil { - return err - } - } - } - - res, err := fapi.StateTransplant(ctx, ts.Key(), mcid) - if err != nil { - return xerrors.Errorf("transplant call failed: %w", err) - } - - fmt.Println("Transplant receipt:") - fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) - fmt.Printf("Return: %x\n", res.MsgRct.Return) - fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) - - if cctx.Bool("detailed-gas") { - fmt.Printf("Base Fee Burn: %d\n", res.GasCost.BaseFeeBurn) - fmt.Printf("Overestimaton Burn: %d\n", res.GasCost.OverEstimationBurn) - fmt.Printf("Miner Penalty: %d\n", res.GasCost.MinerPenalty) - fmt.Printf("Miner Tip: %d\n", res.GasCost.MinerTip) - fmt.Printf("Refund: %d\n", res.GasCost.Refund) - } - fmt.Printf("Total Message Cost: %d\n", res.GasCost.TotalCost) - - if res.MsgRct.ExitCode != 0 { - fmt.Printf("Error message: %q\n", res.Error) - } - - if cctx.Bool("show-trace") { - fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", res.Msg.From, res.Msg.To, res.Msg.Value, res.Msg.Method, res.Msg.Params, res.MsgRct.ExitCode, res.MsgRct.Return) - printInternalExecutions("\t", res.ExecutionTrace.Subcalls) - } - - return nil - }, -} - var stateReplayCmd = &cli.Command{ Name: "replay", Usage: "Replay a particular message", @@ -519,7 +416,7 @@ var stateReplayCmd = &cli.Command{ ctx := ReqContext(cctx) - res, err := fapi.StateReplay(ctx, mcid) + res, err := fapi.StateReplay(ctx, types.EmptyTSK, mcid) if err != nil { return xerrors.Errorf("replay call failed: %w", err) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 9be12722d..9c2a82cad 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -165,7 +165,6 @@ * [StateSectorGetInfo](#StateSectorGetInfo) * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) - * [StateTransplant](#StateTransplant) * [StateVMCirculatingSupplyInternal](#StateVMCirculatingSupplyInternal) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) @@ -4048,7 +4047,8 @@ Response: ``` ### StateReplay -StateReplay searches for where the given message was executed, and replays it in that tipset. +StateReplay replays a given message, assuming it was included in a block in the specified tipset. +If no tipset key is provided, the appropriate tipset is looked up. Perms: read @@ -4056,6 +4056,14 @@ Perms: read Inputs: ```json [ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" } @@ -4319,98 +4327,6 @@ Response: } ``` -### StateTransplant -StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "MsgCid": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Msg": { - "Version": 42, - "To": "f01234", - "From": "f01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==", - "CID": { - "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" - } - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "GasCost": { - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "GasUsed": "0", - "BaseFeeBurn": "0", - "OverEstimationBurn": "0", - "MinerPenalty": "0", - "MinerTip": "0", - "Refund": "0", - "TotalCost": "0" - }, - "ExecutionTrace": { - "Msg": { - "Version": 42, - "To": "f01234", - "From": "f01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==", - "CID": { - "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" - } - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "Error": "string value", - "Duration": 60000000000, - "GasCharges": null, - "Subcalls": null - }, - "Error": "string value", - "Duration": 60000000000 -} -``` - ### StateVMCirculatingSupplyInternal StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset. This is the value reported by the runtime interface to actors code. diff --git a/lotuspond/front/src/Block.js b/lotuspond/front/src/Block.js index cb8d25c9d..4bae57c81 100644 --- a/lotuspond/front/src/Block.js +++ b/lotuspond/front/src/Block.js @@ -26,7 +26,7 @@ class Block extends React.Component { messages = await Promise.all(messages.map(async (msg, i) => { if (msg.receipt.ExitCode !== 0) { - let reply = await this.props.conn.call('Filecoin.StateTransplant', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, msg.Cid]) + let reply = await this.props.conn.call('Filecoin.StateReplay', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, msg.Cid]) if(!reply.Error) { reply.Error = "reply: no error" } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index a63ac8c67..221b83e3a 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -346,12 +346,34 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. return res, err } -func (a *StateAPI) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) +func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { + msgToReplay := mc + var ts *types.TipSet + if tsk == types.EmptyTSK { + mlkp, err := a.StateSearchMsg(ctx, mc) + if err != nil { + return nil, xerrors.Errorf("searching for msg %s: %w", mc, err) + } + if mlkp == nil { + return nil, xerrors.Errorf("didn't find msg %s", mc) + } + + msgToReplay = mlkp.Message + + executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) + } + + ts, err = a.Chain.LoadTipSet(executionTs.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) + } + } else { + ts = a.Chain.GetHeaviestTipSet() } - m, r, err := a.StateManager.Replay(ctx, ts, mc) + + m, r, err := a.StateManager.Replay(ctx, ts, msgToReplay) if err != nil { return nil, err } @@ -362,47 +384,7 @@ func (a *StateAPI) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc } return &api.InvocResult{ - MsgCid: mc, - Msg: m, - MsgRct: &r.MessageReceipt, - GasCost: stmgr.MakeMsgGasCost(m, r), - ExecutionTrace: r.ExecutionTrace, - Error: errstr, - Duration: r.Duration, - }, nil -} - -func (a *StateAPI) StateReplay(ctx context.Context, mc cid.Cid) (*api.InvocResult, error) { - mlkp, err := a.StateSearchMsg(ctx, mc) - if err != nil { - return nil, xerrors.Errorf("searching for msg %s: %w", mc, err) - } - if mlkp == nil { - return nil, xerrors.Errorf("didn't find msg %s", mc) - } - - executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) - } - - ts, err := a.Chain.LoadTipSet(executionTs.Parents()) - if err != nil { - return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) - } - - m, r, err := a.StateManager.Replay(ctx, ts, mlkp.Message) - if err != nil { - return nil, err - } - - var errstr string - if r.ActorErr != nil { - errstr = r.ActorErr.Error() - } - - return &api.InvocResult{ - MsgCid: mlkp.Message, + MsgCid: msgToReplay, Msg: m, MsgRct: &r.MessageReceipt, GasCost: stmgr.MakeMsgGasCost(m, r), From d0e4150ceaf036f4af367261b8b983a3e0c51e68 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 16 Oct 2020 08:30:55 -0700 Subject: [PATCH 097/106] Use separate config for Retrieval Filter --- node/builder.go | 5 ++++- node/config/def.go | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/node/builder.go b/node/builder.go index b6b75ecc6..1dc12e80a 100644 --- a/node/builder.go +++ b/node/builder.go @@ -486,7 +486,10 @@ func ConfigStorageMiner(c interface{}) Option { If(cfg.Dealmaking.Filter != "", Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), - Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.Filter))), + ), + + If(cfg.Dealmaking.RetrievalFilter != "", + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))), ), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees)), diff --git a/node/config/def.go b/node/config/def.go index 7eb7e19f0..a1b57286c 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -45,7 +45,8 @@ type DealmakingConfig struct { PieceCidBlocklist []cid.Cid ExpectedSealDuration Duration - Filter string + Filter string + RetrievalFilter string } type SealingConfig struct { From 4de8f506424ad9664b15459f0e96a834e4727e69 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 16 Oct 2020 19:13:33 +0000 Subject: [PATCH 098/106] replace bootstrap peers --- build/bootstrap/bootstrappers.pi | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index f3f8b8a55..886ac8e99 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1,9 +1,12 @@ -/dns4/bootstrap-0.testnet.fildev.network/tcp/1347/p2p/12D3KooWJTUBUjtzWJGWU1XSiY21CwmHaCNLNYn2E7jqHEHyZaP7 -/dns4/bootstrap-1.testnet.fildev.network/tcp/1347/p2p/12D3KooW9yeKXha4hdrJKq74zEo99T8DhriQdWNoojWnnQbsgB3v -/dns4/bootstrap-2.testnet.fildev.network/tcp/1347/p2p/12D3KooWCrx8yVG9U9Kf7w8KLN3Edkj5ZKDhgCaeMqQbcQUoB6CT -/dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 -/dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T -/dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W +/dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj +/dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc +/dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4 +/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R +/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc +/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH +/dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ +/dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf +/dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR /dns4/lotus-bootstrap.forceup.cn/tcp/41778/p2p/12D3KooWFQsv3nRMUevZNWWsY1Wu6NUzUbawnWU5NcRhgKuJA37C /dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz /dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u From d24d3e420d66d4ef2a3f0cd530bbd6ffa21511a1 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 19 Oct 2020 14:27:04 -0400 Subject: [PATCH 099/106] Fix StateReplay to use provided tipset --- node/impl/full/state.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index b2e9c3e51..a6564296b 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -349,6 +349,7 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { msgToReplay := mc var ts *types.TipSet + var err error if tsk == types.EmptyTSK { mlkp, err := a.StateSearchMsg(ctx, mc) if err != nil { @@ -370,7 +371,10 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) } } else { - ts = a.Chain.GetHeaviestTipSet() + ts, err = a.Chain.LoadTipSet(tsk) + if err != nil { + return nil, xerrors.Errorf("loading specified tipset %s: %w", tsk, err) + } } m, r, err := a.StateManager.Replay(ctx, ts, msgToReplay) From 1974490ab05fd005619ea0f1b8eac9802d31ed53 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Mon, 19 Oct 2020 13:12:28 -0700 Subject: [PATCH 100/106] Don't overwrite previously-configured maxPieceSize for a persisted ask --- node/modules/storageminer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index bcc38898e..7f510aa70 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -398,8 +398,6 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat if err != nil { return nil, err } - a := storedAsk.GetAsk().Ask - err = storedAsk.SetAsk(a.Price, a.VerifiedPrice, a.Expiry-a.Timestamp) if err != nil { return storedAsk, err } From be42dd824b696a1195290348359577753ac7fb15 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 19 Oct 2020 22:32:05 +0200 Subject: [PATCH 101/106] state: optimize state snapshot address cache Signed-off-by: Jakub Sztandera --- chain/state/statetree.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index e9b76ea77..35760fdf9 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -35,7 +35,8 @@ type StateTree struct { } type stateSnaps struct { - layers []*stateSnapLayer + layers []*stateSnapLayer + lastMaybeNonEmptyResolveCache int } type stateSnapLayer struct { @@ -67,7 +68,12 @@ func (ss *stateSnaps) addLayer() { func (ss *stateSnaps) dropLayer() { ss.layers[len(ss.layers)-1] = nil // allow it to be GCed + ss.layers = ss.layers[:len(ss.layers)-1] + + if ss.lastMaybeNonEmptyResolveCache == len(ss.layers) { + ss.lastMaybeNonEmptyResolveCache = len(ss.layers) - 1 + } } func (ss *stateSnaps) mergeLastLayer() { @@ -86,7 +92,13 @@ func (ss *stateSnaps) mergeLastLayer() { } func (ss *stateSnaps) resolveAddress(addr address.Address) (address.Address, bool) { - for i := len(ss.layers) - 1; i >= 0; i-- { + for i := ss.lastMaybeNonEmptyResolveCache; i >= 0; i-- { + if len(ss.layers[i].resolveCache) == 0 { + if ss.lastMaybeNonEmptyResolveCache == i { + ss.lastMaybeNonEmptyResolveCache = i - 1 + } + continue + } resa, ok := ss.layers[i].resolveCache[addr] if ok { return resa, true @@ -97,6 +109,7 @@ func (ss *stateSnaps) resolveAddress(addr address.Address) (address.Address, boo func (ss *stateSnaps) cacheResolveAddress(addr, resa address.Address) { ss.layers[len(ss.layers)-1].resolveCache[addr] = resa + ss.lastMaybeNonEmptyResolveCache = len(ss.layers) - 1 } func (ss *stateSnaps) getActor(addr address.Address) (*types.Actor, error) { From cb0513f4c1a978f18391dc6a28e0f4bb9051bb67 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 20 Oct 2020 01:25:41 +0200 Subject: [PATCH 102/106] Add test Signed-off-by: Jakub Sztandera --- chain/state/statetree.go | 50 +++++++++++------- chain/state/statetree_test.go | 98 +++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 19 deletions(-) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 35760fdf9..7fa55b31c 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -26,10 +26,11 @@ var log = logging.Logger("statetree") // StateTree stores actors state by their ID. type StateTree struct { - root adt.Map - version types.StateTreeVersion - info cid.Cid - Store cbor.IpldStore + root adt.Map + version types.StateTreeVersion + info cid.Cid + Store cbor.IpldStore + lookupIDFun func(address.Address) (address.Address, error) snaps *stateSnaps } @@ -173,13 +174,15 @@ func NewStateTree(cst cbor.IpldStore, ver types.StateTreeVersion) (*StateTree, e return nil, err } - return &StateTree{ + s := &StateTree{ root: root, info: info, version: ver, Store: cst, snaps: newStateSnaps(), - }, nil + } + s.lookupIDFun = s.lookupIDinternal + return s, nil } func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { @@ -203,13 +206,15 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { return nil, err } - return &StateTree{ + s := &StateTree{ root: nd, info: root.Info, version: root.Version, Store: cst, snaps: newStateSnaps(), - }, nil + } + s.lookupIDFun = s.lookupIDinternal + return s, nil default: return nil, xerrors.Errorf("unsupported state tree version: %d", root.Version) } @@ -226,17 +231,7 @@ func (st *StateTree) SetActor(addr address.Address, act *types.Actor) error { return nil } -// LookupID gets the ID address of this actor's `addr` stored in the `InitActor`. -func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { - if addr.Protocol() == address.ID { - return addr, nil - } - - resa, ok := st.snaps.resolveAddress(addr) - if ok { - return resa, nil - } - +func (st *StateTree) lookupIDinternal(addr address.Address) (address.Address, error) { act, err := st.GetActor(init_.Address) if err != nil { return address.Undef, xerrors.Errorf("getting init actor: %w", err) @@ -254,6 +249,23 @@ func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { if err != nil { return address.Undef, xerrors.Errorf("resolve address %s: %w", addr, err) } + return a, err +} + +// LookupID gets the ID address of this actor's `addr` stored in the `InitActor`. +func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { + if addr.Protocol() == address.ID { + return addr, nil + } + + resa, ok := st.snaps.resolveAddress(addr) + if ok { + return resa, nil + } + a, err := st.lookupIDFun(addr) + if err != nil { + return a, err + } st.snaps.cacheResolveAddress(addr, a) diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index ed1fb1889..584d9ba7e 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -73,6 +73,104 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { } } +func TestResolveCache(t *testing.T) { + cst := cbor.NewMemCborStore() + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) + if err != nil { + t.Fatal(err) + } + nonId := address.NewForTestGetter()() + id, _ := address.NewIDAddress(1000) + + st.lookupIDFun = func(a address.Address) (address.Address, error) { + if a == nonId { + return id, nil + } else { + return address.Undef, types.ErrActorNotFound + } + } + + err = st.SetActor(nonId, &types.Actor{Nonce: 1}) + if err != nil { + t.Fatal(err) + } + + { + err = st.Snapshot(context.TODO()) + if err != nil { + t.Fatal(err) + } + act, err := st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 1 { + t.Fatalf("expected nonce 1, got %d", act.Nonce) + } + err = st.SetActor(nonId, &types.Actor{Nonce: 2}) + if err != nil { + t.Fatal(err) + } + + act, err = st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 2 { + t.Fatalf("expected nonce 2, got %d", act.Nonce) + } + + if err := st.Revert(); err != nil { + t.Fatal(err) + } + st.ClearSnapshot() + } + + act, err := st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 1 { + t.Fatalf("expected nonce 1, got %d", act.Nonce) + } + + { + err = st.Snapshot(context.TODO()) + if err != nil { + t.Fatal(err) + } + act, err := st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 1 { + t.Fatalf("expected nonce 1, got %d", act.Nonce) + } + err = st.SetActor(nonId, &types.Actor{Nonce: 2}) + if err != nil { + t.Fatal(err) + } + + act, err = st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 2 { + t.Fatalf("expected nonce 2, got %d", act.Nonce) + } + st.ClearSnapshot() + } + + act, err = st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 2 { + t.Fatalf("expected nonce 2, got %d", act.Nonce) + } + +} + func BenchmarkStateTree10kGetActor(b *testing.B) { cst := cbor.NewMemCborStore() st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) From a055540c254d13648031d5f873acb653d71c06f5 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 20 Oct 2020 01:32:51 +0200 Subject: [PATCH 103/106] Fix lint Signed-off-by: Jakub Sztandera --- chain/state/statetree_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 584d9ba7e..61e7d8c22 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -85,9 +85,8 @@ func TestResolveCache(t *testing.T) { st.lookupIDFun = func(a address.Address) (address.Address, error) { if a == nonId { return id, nil - } else { - return address.Undef, types.ErrActorNotFound } + return address.Undef, types.ErrActorNotFound } err = st.SetActor(nonId, &types.Actor{Nonce: 1}) From 1ee9fb44e237da3490115eba8aea97a7a03f4eb7 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 17 Oct 2020 19:06:12 -0400 Subject: [PATCH 104/106] Lotus version 1.0.0 --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 701f221b0..76aa51de2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Lotus changelog +# 1.0.0 / 2020-10-19 + +It's 1.0.0! This is an optional release of Lotus that introduces some UX improvements to the 0.10 series. + +This very small release is largely cosmetic, and intended to flag the code that the Filecoin mainnet was launched with. + +## API changes + +- `StateMsgGasCost` has been removed. The equivalent information can be gained by calling `StateReplay`. +- A `GasCost` field has been added to the `InvocResult` type, meaning detailed gas costs will be returned when calling `StateReplay`, `StateCompute`, and `StateCall`. +- The behaviour of `StateReplay` in response to an empty tipset key has been changed. Instead of simply using the heaviest tipset (which is almost guaranteed to be an unsuccessful replay), we search now search the chain for the tipset that included the message, and replay the message in that tipset (we fail if no such tipset is found). + +## Changes + +- Increase code coverage! (https://github.com/filecoin-project/lotus/pull/4410) +- Mpool: Don't block node startup loading messages (https://github.com/filecoin-project/lotus/pull/4411) +- Improve the UX of multisig approves (https://github.com/filecoin-project/lotus/pull/4398) +- Use build.BlockDelaySecs for deal start buffer (https://github.com/filecoin-project/lotus/pull/4415) +- Conformance: support multiple protocol versions (https://github.com/filecoin-project/lotus/pull/4393) +- Ensure msig inspect cli works with lotus-lite (https://github.com/filecoin-project/lotus/pull/4421) +- Add command to (slowly) prune lotus chain datastore (https://github.com/filecoin-project/lotus/pull/3876) +- Add WalletVerify to lotus-gateway (https://github.com/filecoin-project/lotus/pull/4373) +- Improve StateMsg APIs (https://github.com/filecoin-project/lotus/pull/4429) +- Add endpoints needed by spacegap (https://github.com/filecoin-project/lotus/pull/4426) +- Make audit balances capable of printing robust addresses (https://github.com/filecoin-project/lotus/pull/4423) +- Custom filters for retrieval deals (https://github.com/filecoin-project/lotus/pull/4424) +- Fix message list api (https://github.com/filecoin-project/lotus/pull/4422) +- Replace bootstrap peers (https://github.com/filecoin-project/lotus/pull/4447) +- Don't overwrite previously-configured maxPieceSize for a persisted ask (https://github.com/filecoin-project/lotus/pull/4480) +- State: optimize state snapshot address cache (https://github.com/filecoin-project/lotus/pull/4481) + # 0.10.2 / 2020-10-14 This is an optional release of Lotus that updates markets to 0.9.1, which fixes an issue affecting deals that were mid-transfer when the node was upgraded to 0.9.0. This release also includes some tweaks to default gas values and minor performance improvements. diff --git a/build/version.go b/build/version.go index df1fe28de..352fa9b0e 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.10.2" +const BuildVersion = "1.0.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From d8d2174104320ab3cee32fb66adebaacce32ab6c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 16 Oct 2020 20:35:46 -0400 Subject: [PATCH 105/106] Update to actors v2.2.0 at PostLiftoff epoch --- build/params_2k.go | 2 ++ build/params_mainnet.go | 3 +++ build/params_shared_vals.go | 2 +- build/params_testground.go | 1 + chain/stmgr/forks.go | 4 ++++ cmd/tvx/codenames.go | 1 + documentation/en/api-methods.md | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- 9 files changed, 19 insertions(+), 8 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index b09b60fae..5a0e8fd61 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -22,6 +22,8 @@ const UpgradeTapeHeight = -4 var UpgradeActorsV2Height = abi.ChainEpoch(10) var UpgradeLiftoffHeight = abi.ChainEpoch(-5) +const UpgradeKumquatHeight = -6 + var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, } diff --git a/build/params_mainnet.go b/build/params_mainnet.go index 54f50ac6e..9d5c48ff6 100644 --- a/build/params_mainnet.go +++ b/build/params_mainnet.go @@ -37,6 +37,9 @@ const UpgradeTapeHeight = 140760 // We still have upgrades and state changes to do, but can happen after signaling timing here. const UpgradeLiftoffHeight = 148888 +// TODO: Confirm epoch +const UpgradeKumquatHeight = 170000 + func init() { policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40)) policy.SetSupportedProofTypes( diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index ede40c0e3..b80472594 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -25,7 +25,7 @@ const UnixfsLinksPerLevel = 1024 // Consensus / Network const AllowableClockDriftSecs = uint64(1) -const NewestNetworkVersion = network.Version5 +const NewestNetworkVersion = network.Version6 const ActorUpgradeNetworkVersion = network.Version4 // Epochs diff --git a/build/params_testground.go b/build/params_testground.go index 7ef034234..beee1c727 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -86,6 +86,7 @@ var ( UpgradeTapeHeight abi.ChainEpoch = -4 UpgradeActorsV2Height abi.ChainEpoch = 10 UpgradeLiftoffHeight abi.ChainEpoch = -5 + UpgradeKumquatHeight abi.ChainEpoch = -6 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 488f84167..5d6cac51f 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -86,6 +86,10 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Height: build.UpgradeLiftoffHeight, Network: network.Version5, Migration: UpgradeLiftoff, + }, { + Height: build.UpgradeKumquatHeight, + Network: network.Version6, + Migration: nil, }} if build.UpgradeActorsV2Height == math.MaxInt64 { // disable actors upgrade diff --git a/cmd/tvx/codenames.go b/cmd/tvx/codenames.go index 851e9d841..b9f590914 100644 --- a/cmd/tvx/codenames.go +++ b/cmd/tvx/codenames.go @@ -23,6 +23,7 @@ var ProtocolCodenames = []struct { {build.UpgradeActorsV2Height + 1, "actorsv2"}, {build.UpgradeTapeHeight + 1, "tape"}, {build.UpgradeLiftoffHeight + 1, "liftoff"}, + {build.UpgradeKumquatHeight + 1, "postliftoff"}, } // GetProtocolCodename gets the protocol codename associated with a height. diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index c5bc24b04..3aeb6c096 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -4004,7 +4004,7 @@ Inputs: ] ``` -Response: `5` +Response: `6` ### StateReadState StateReadState returns the indicated actor's state. diff --git a/go.mod b/go.mod index a6a074e49..3aa30fb9c 100644 --- a/go.mod +++ b/go.mod @@ -34,12 +34,12 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b + github.com/filecoin-project/go-state-types v0.0.0-20201013222834-41ea465f274f github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.1.0 + github.com/filecoin-project/specs-actors/v2 v2.2.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 4cf7b3628..37aa63cfd 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b h1:bMUfG6Sy6YSMbsjQAO1Q2vEZldbSdsbRy/FX3OlTck0= -github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.0.0-20201013222834-41ea465f274f h1:TZDTu4MtBKSFLXWGKLy+cvC3nHfMFIrVgWLAz/+GgZQ= +github.com/filecoin-project/go-state-types v0.0.0-20201013222834-41ea465f274f/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= @@ -278,8 +278,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= -github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT+dGVWJTr5yDevU00g= -github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= +github.com/filecoin-project/specs-actors/v2 v2.2.0 h1:IyCICb0NHYeD0sdSqjVGwWydn/7r7xXuxdpvGAcRCGY= +github.com/filecoin-project/specs-actors/v2 v2.2.0/go.mod h1:rlv5Mx9wUhV8Qsz+vUezZNm+zL4tK08O0HreKKPB2Wc= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= From 143dcafdb748fa7c2ba8c9a1f46574dbb9c141e3 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 19 Oct 2020 13:26:07 -0400 Subject: [PATCH 106/106] VM: Enforce a call depth limit --- chain/actors/version.go | 2 +- chain/gen/genesis/miners.go | 2 +- chain/vm/runtime.go | 1 + chain/vm/vm.go | 42 ++++++++++++++++++++----------------- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/chain/actors/version.go b/chain/actors/version.go index 2efd903bb..fe16d521e 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -18,7 +18,7 @@ func VersionForNetwork(version network.Version) Version { switch version { case network.Version0, network.Version1, network.Version2, network.Version3: return Version0 - case network.Version4, network.Version5: + case network.Version4, network.Version5, network.Version6: return Version2 default: panic(fmt.Sprintf("unsupported network version %d", version)) diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 1023e5efa..6f0c136a3 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -405,7 +405,7 @@ func circSupply(ctx context.Context, vmi *vm.VM, maddr address.Address) abi.Toke rt := unsafeVM.MakeRuntime(ctx, &types.Message{ GasLimit: 1_000_000_000, From: maddr, - }, maddr, 0, 0, 0) + }) return rt.TotalFilCircSupply() } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 8f124247c..ed5501fe9 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -72,6 +72,7 @@ type Runtime struct { originNonce uint64 executionTrace types.ExecutionTrace + depth uint64 numActorsCreated uint64 allowInternal bool callerValidated bool diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 72ad731aa..a7aa05719 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -38,6 +38,8 @@ import ( "github.com/filecoin-project/lotus/lib/bufbstore" ) +const MaxCallDepth = 4096 + var log = logging.Logger("vm") var actorLog = logging.Logger("actors") var gasOnActorExec = newGasCharge("OnActorExec", 0, 0) @@ -97,24 +99,37 @@ func (bs *gasChargingBlocks) Put(blk block.Block) error { return nil } -func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin address.Address, originNonce uint64, usedGas int64, nac uint64) *Runtime { +func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, parent *Runtime) *Runtime { rt := &Runtime{ ctx: ctx, vm: vm, state: vm.cstate, - origin: origin, - originNonce: originNonce, + origin: msg.From, + originNonce: msg.Nonce, height: vm.blockHeight, - gasUsed: usedGas, + gasUsed: 0, gasAvailable: msg.GasLimit, - numActorsCreated: nac, + depth: 0, + numActorsCreated: 0, pricelist: PricelistByEpoch(vm.blockHeight), allowInternal: true, callerValidated: false, executionTrace: types.ExecutionTrace{Msg: msg}, } + if parent != nil { + rt.gasUsed = parent.gasUsed + rt.origin = parent.origin + rt.originNonce = parent.originNonce + rt.numActorsCreated = parent.numActorsCreated + rt.depth = parent.depth + 1 + } + + if rt.depth > MaxCallDepth && rt.NetworkVersion() >= network.Version6 { + rt.Abortf(exitcode.SysErrForbidden, "message execution exceeds call depth") + } + rt.cst = &cbor.BasicIpldStore{ Blocks: &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks}, Atlas: vm.cst.Atlas, @@ -148,8 +163,8 @@ type UnsafeVM struct { VM *VM } -func (vm *UnsafeVM) MakeRuntime(ctx context.Context, msg *types.Message, origin address.Address, originNonce uint64, usedGas int64, nac uint64) *Runtime { - return vm.VM.makeRuntime(ctx, msg, origin, originNonce, usedGas, nac) +func (vm *UnsafeVM) MakeRuntime(ctx context.Context, msg *types.Message) *Runtime { + return vm.VM.makeRuntime(ctx, msg, nil) } type CircSupplyCalculator func(context.Context, abi.ChainEpoch, *state.StateTree) (abi.TokenAmount, error) @@ -224,18 +239,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, st := vm.cstate - origin := msg.From - on := msg.Nonce - var nac uint64 = 0 - var gasUsed int64 - if parent != nil { - gasUsed = parent.gasUsed - origin = parent.origin - on = parent.originNonce - nac = parent.numActorsCreated - } - - rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) + rt := vm.makeRuntime(ctx, msg, parent) if EnableGasTracing { rt.lastGasChargeTime = start if parent != nil {