diff --git a/.gitignore b/.gitignore index 529f938c3..d841170a8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /lotus-seal-worker /lotus-seed /lotus-health +/lotus-shed /pond /townhall /fountain diff --git a/Makefile b/Makefile index 90e5828ed..c12095f53 100644 --- a/Makefile +++ b/Makefile @@ -77,8 +77,8 @@ lotus-shed: $(BUILD_DEPS) rm -f lotus-shed go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build -.PHONY: lotus-seal-worker -BINS+=lotus-seal-worker +.PHONY: lotus-shed +BINS+=lotus-shed build: lotus lotus-storage-miner lotus-seal-worker @[[ $$(type -P "lotus") ]] && echo "Caution: you have \ diff --git a/api/api_full.go b/api/api_full.go index 1dce534ac..67a73abab 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -4,6 +4,8 @@ import ( "context" "time" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/specs-actors/actors/abi" @@ -82,6 +84,7 @@ type FullNode interface { WalletBalance(context.Context, address.Address) (types.BigInt, error) WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) + WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) bool WalletDefaultAddress(context.Context) (address.Address, error) WalletSetDefault(context.Context, address.Address) error WalletExport(context.Context, address.Address) (*types.KeyInfo, error) @@ -108,8 +111,8 @@ type FullNode interface { //ClientListAsks() []Ask // if tipset is nil, we'll use heaviest - StateCall(context.Context, *types.Message, types.TipSetKey) (*MethodCall, error) - StateReplay(context.Context, types.TipSetKey, cid.Cid) (*ReplayResults, error) + StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) + StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) StateReadState(ctx context.Context, act *types.Actor, tsk types.TipSetKey) (*ActorState, error) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) @@ -300,10 +303,11 @@ type RetrievalOrder struct { MinerPeerID peer.ID } -type ReplayResults struct { - Msg *types.Message - Receipt *types.MessageReceipt - Error string +type InvocResult struct { + Msg *types.Message + MsgRct *types.MessageReceipt + InternalExecutions []*vm.ExecutionResult + Error string } type MethodCall struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 0ad84f857..758c19c6d 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -89,19 +89,20 @@ type FullNodeStruct struct { WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"` WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` + WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) bool `perm:"read"` WalletDefaultAddress func(context.Context) (address.Address, error) `perm:"write"` WalletSetDefault func(context.Context, address.Address) error `perm:"admin"` WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` - ClientImport func(ctx context.Context, ref api.FileRef) (cid.Cid, error) `perm:"admin"` + ClientImport func(ctx context.Context, ref api.FileRef) (cid.Cid, error) `perm:"admin"` ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` ClientFindData func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` - ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error `perm:"admin"` + ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"` StateMinerSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` @@ -112,8 +113,8 @@ type FullNodeStruct struct { StateMinerPostState func(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*miner.PoStState, error) `perm:"read"` StateMinerSectorSize func(context.Context, address.Address, types.TipSetKey) (abi.SectorSize, error) `perm:"read"` StateMinerFaults func(context.Context, address.Address, types.TipSetKey) ([]abi.SectorNumber, error) `perm:"read"` - StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.MethodCall, error) `perm:"read"` - StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.ReplayResults, error) `perm:"read"` + StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` + StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, *types.Actor, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StatePledgeCollateral func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"` @@ -329,6 +330,10 @@ func (c *FullNodeStruct) WalletSignMessage(ctx context.Context, k address.Addres return c.Internal.WalletSignMessage(ctx, k, msg) } +func (c *FullNodeStruct) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool { + return c.Internal.WalletVerify(ctx, k, msg, sig) +} + func (c *FullNodeStruct) WalletDefaultAddress(ctx context.Context) (address.Address, error) { return c.Internal.WalletDefaultAddress(ctx) } @@ -465,11 +470,11 @@ func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Add return c.Internal.StateMinerFaults(ctx, actor, tsk) } -func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.MethodCall, error) { +func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { return c.Internal.StateCall(ctx, msg, tsk) } -func (c *FullNodeStruct) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.ReplayResults, error) { +func (c *FullNodeStruct) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { return c.Internal.StateReplay(ctx, tsk, mc) } diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 01ec2d0db..a73a56a07 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate cid.Cid, r vm.Rand, bheight abi.ChainEpoch) (*api.MethodCall, error) { +func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate cid.Cid, r vm.Rand, bheight abi.ChainEpoch) (*api.InvocResult, error) { ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw") defer span.End() @@ -62,14 +62,17 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate errs = ret.ActorErr.Error() log.Warnf("chain call failed: %s", ret.ActorErr) } - return &api.MethodCall{ - MessageReceipt: ret.MessageReceipt, - Error: errs, + + return &api.InvocResult{ + Msg: msg, + MsgRct: &ret.MessageReceipt, + InternalExecutions: ret.InternalExecutions, + Error: errs, }, nil } -func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.MethodCall, error) { +func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { if ts == nil { ts = sm.cs.GetHeaviestTipSet() } diff --git a/chain/store/store.go b/chain/store/store.go index ed86b1727..f9935a3ec 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -15,12 +15,14 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/specs-actors/actors/util/adt" - "go.opencensus.io/trace" - "go.uber.org/multierr" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/metrics" + "go.opencensus.io/stats" + "go.opencensus.io/trace" + "go.uber.org/multierr" amt "github.com/filecoin-project/go-amt-ipld/v2" @@ -107,7 +109,15 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Sys return nil } - cs.headChangeNotifs = append(cs.headChangeNotifs, hcnf) + hcmetric := func(rev, app []*types.TipSet) error { + ctx := context.Background() + for _, r := range app { + stats.Record(ctx, metrics.ChainNodeHeight.M(int64(r.Height()))) + } + return nil + } + + cs.headChangeNotifs = append(cs.headChangeNotifs, hcnf, hcmetric) return cs } diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 6ee41aabf..03c9feaf0 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -2,6 +2,7 @@ package sub import ( "context" + "fmt" "time" lru "github.com/hashicorp/golang-lru" @@ -10,11 +11,14 @@ import ( connmgr "github.com/libp2p/go-libp2p-core/connmgr" peer "github.com/libp2p/go-libp2p-peer" pubsub "github.com/libp2p/go-libp2p-pubsub" + "go.opencensus.io/stats" + "go.opencensus.io/tag" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/metrics" ) var log = logging.Logger("sub") @@ -107,15 +111,25 @@ func (bv *BlockValidator) flagPeer(p peer.ID) { } func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) bool { + stats.Record(ctx, metrics.BlockReceived.M(1)) + ctx, _ = tag.New( + ctx, + tag.Insert(metrics.PeerID, pid.String()), + tag.Insert(metrics.ReceivedFrom, msg.ReceivedFrom.String()), + ) blk, err := types.DecodeBlockMsg(msg.GetData()) if err != nil { log.Error("got invalid block over pubsub: ", err) + ctx, _ = tag.New(ctx, tag.Insert(metrics.FailureType, "invalid")) + stats.Record(ctx, metrics.BlockValidationFailure.M(1)) bv.flagPeer(pid) return false } if len(blk.BlsMessages)+len(blk.SecpkMessages) > build.BlockMessageLimit { log.Warnf("received block with too many messages over pubsub") + ctx, _ = tag.New(ctx, tag.Insert(metrics.FailureType, "too_many_messages")) + stats.Record(ctx, metrics.BlockValidationFailure.M(1)) bv.flagPeer(pid) return false } @@ -127,6 +141,7 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub } msg.ValidatorData = blk + stats.Record(ctx, metrics.BlockValidationSuccess.M(1)) return true } @@ -153,9 +168,44 @@ func (brc *blockReceiptCache) add(bcid cid.Cid) int { return val.(int) } +type MessageValidator struct { + mpool *messagepool.MessagePool +} + +func NewMessageValidator(mp *messagepool.MessagePool) *MessageValidator { + return &MessageValidator{mp} +} + +func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) bool { + stats.Record(ctx, metrics.MessageReceived.M(1)) + ctx, _ = tag.New(ctx, tag.Insert(metrics.PeerID, pid.String())) + m, err := types.DecodeSignedMessage(msg.Message.GetData()) + if err != nil { + log.Warnf("failed to decode incoming message: %s", err) + ctx, _ = tag.New(ctx, tag.Insert(metrics.FailureType, "decode")) + stats.Record(ctx, metrics.MessageValidationFailure.M(1)) + return false + } + + if err := mv.mpool.Add(m); err != nil { + log.Warnf("failed to add message from network to message pool (From: %s, To: %s, Nonce: %d, Value: %s): %s", m.Message.From, m.Message.To, m.Message.Nonce, types.FIL(m.Message.Value), err) + ctx, _ = tag.New( + ctx, + tag.Insert(metrics.MessageFrom, m.Message.From.String()), + tag.Insert(metrics.MessageTo, m.Message.To.String()), + tag.Insert(metrics.MessageNonce, fmt.Sprint(m.Message.Nonce)), + tag.Insert(metrics.FailureType, "add"), + ) + stats.Record(ctx, metrics.MessageValidationFailure.M(1)) + return false + } + stats.Record(ctx, metrics.MessageValidationSuccess.M(1)) + return true +} + func HandleIncomingMessages(ctx context.Context, mpool *messagepool.MessagePool, msub *pubsub.Subscription) { for { - msg, err := msub.Next(ctx) + _, err := msub.Next(ctx) if err != nil { log.Warn("error from message subscription: ", err) if ctx.Err() != nil { @@ -165,15 +215,6 @@ func HandleIncomingMessages(ctx context.Context, mpool *messagepool.MessagePool, continue } - m, ok := msg.ValidatorData.(*types.SignedMessage) - if !ok { - log.Errorf("message validator func passed on wrong type: %#v", msg.ValidatorData) - continue - } - - if err := mpool.Add(m); err != nil { - log.Warnf("failed to add message from network to message pool (From: %s, To: %s, Nonce: %d, Value: %s): %s", m.Message.From, m.Message.To, m.Message.Nonce, types.FIL(m.Message.Value), err) - continue - } + // Do nothing... everything happens in validate } } diff --git a/chain/sync.go b/chain/sync.go index c52f6f4a3..8f72f7fdd 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -27,6 +27,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" "github.com/whyrusleeping/pubsub" + "go.opencensus.io/stats" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -43,6 +44,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" + "github.com/filecoin-project/lotus/metrics" ) var log = logging.Logger("chain") @@ -1076,6 +1078,7 @@ func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []* return xerrors.Errorf("message processing failed: %w", err) } + stats.Record(ctx, metrics.ChainNodeWorkerHeight.M(int64(fts.TipSet().Height()))) ss.SetHeight(fts.TipSet().Height()) return nil diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 56308ec92..e4260d91c 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -54,6 +54,12 @@ const ( outOfGasErrCode = 200 ) +type ExecutionResult struct { + Msg *types.Message + MsgRct *types.MessageReceipt + Error string +} + type VMContext struct { ctx context.Context @@ -73,6 +79,8 @@ type VMContext struct { // address that started invoke chain origin address.Address + + internalExecutions []*ExecutionResult } // Message is the message that kicked off the current invocation @@ -174,6 +182,24 @@ func (vmc *VMContext) Send(to address.Address, method abi.MethodNum, value types return nil, aerrors.Escalate(err, "failed to revert state tree after failed subcall") } } + + mr := types.MessageReceipt{ + ExitCode: exitcode.ExitCode(aerrors.RetCode(err)), + Return: ret, + GasUsed: types.EmptyInt, + } + + var es = "" + if err != nil { + es = err.Error() + } + er := ExecutionResult{ + Msg: msg, + MsgRct: &mr, + Error: es, + } + + vmc.internalExecutions = append(vmc.internalExecutions, &er) return ret, err } @@ -368,8 +394,9 @@ type Rand interface { type ApplyRet struct { types.MessageReceipt - ActorErr aerrors.ActorError - Penalty big.Int + ActorErr aerrors.ActorError + Penalty big.Int + InternalExecutions []*ExecutionResult } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *VMContext, @@ -566,7 +593,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet, Return: ret, GasUsed: gasUsed, }, - ActorErr: actorErr, + ActorErr: actorErr, + InternalExecutions: vmctx.internalExecutions, }, nil } diff --git a/cli/client.go b/cli/client.go index 2c418bd56..d8af62fd1 100644 --- a/cli/client.go +++ b/cli/client.go @@ -369,7 +369,7 @@ var clientQueryAskCmd = &cli.Command{ var pid peer.ID if pidstr := cctx.String("peerid"); pidstr != "" { - p, err := peer.IDFromString(pidstr) + p, err := peer.Decode(pidstr) if err != nil { return err } diff --git a/cli/cmd.go b/cli/cmd.go index f8eefe208..ac3811d0f 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -28,6 +28,20 @@ const ( metadataTraceConetxt = "traceContext" ) +// custom CLI error + +type ErrCmdFailed struct { + msg string +} + +func (e *ErrCmdFailed) Error() string { + return e.msg +} + +func NewCliError(s string) error { + return &ErrCmdFailed{s} +} + // ApiConnector returns API instance type ApiConnector func() api.FullNode diff --git a/cli/net.go b/cli/net.go index cf634df64..b49d9604f 100644 --- a/cli/net.go +++ b/cli/net.go @@ -2,10 +2,11 @@ package cli import ( "fmt" - "github.com/libp2p/go-libp2p-core/peer" "sort" "strings" + "github.com/libp2p/go-libp2p-core/peer" + "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/lib/addrutil" diff --git a/cli/state.go b/cli/state.go index 1ee19a42a..67e2e0f94 100644 --- a/cli/state.go +++ b/cli/state.go @@ -345,10 +345,10 @@ var stateReplaySetCmd = &cli.Command{ } fmt.Println("Replay receipt:") - fmt.Printf("Exit code: %d\n", res.Receipt.ExitCode) - fmt.Printf("Return: %x\n", res.Receipt.Return) - fmt.Printf("Gas Used: %s\n", res.Receipt.GasUsed) - if res.Receipt.ExitCode != 0 { + fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) + fmt.Printf("Return: %x\n", res.MsgRct.Return) + fmt.Printf("Gas Used: %s\n", res.MsgRct.GasUsed) + if res.MsgRct.ExitCode != 0 { fmt.Printf("Error message: %q\n", res.Error) } @@ -918,11 +918,11 @@ var stateCallCmd = &cli.Command{ return fmt.Errorf("state call failed: %s", err) } - if ret.ExitCode != 0 { - return fmt.Errorf("invocation failed (exit: %d): %s", ret.ExitCode, ret.Error) + if ret.MsgRct.ExitCode != 0 { + return fmt.Errorf("invocation failed (exit: %d): %s", ret.MsgRct.ExitCode, ret.Error) } - s, err := formatOutput(cctx.String("ret"), ret.Return) + s, err := formatOutput(cctx.String("ret"), ret.MsgRct.Return) if err != nil { return fmt.Errorf("failed to format output: %s", err) } diff --git a/cli/wallet.go b/cli/wallet.go index ed80c062c..5982e5cb8 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-address" types "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/specs-actors/actors/crypto" "gopkg.in/urfave/cli.v2" ) @@ -27,6 +28,8 @@ var walletCmd = &cli.Command{ walletImport, walletGetDefault, walletSetDefault, + walletSign, + walletVerify, }, } @@ -243,3 +246,93 @@ var walletImport = &cli.Command{ return nil }, } + +var walletSign = &cli.Command{ + Name: "sign", + Usage: "sign a message", + ArgsUsage: " ", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + if !cctx.Args().Present() || cctx.NArg() != 2 { + return fmt.Errorf("must specify signing address and message to sign") + } + + addr, err := address.NewFromString(cctx.Args().First()) + + if err != nil { + return err + } + + msg, err := hex.DecodeString(cctx.Args().Get(1)) + + if err != nil { + return err + } + + sig, err := api.WalletSign(ctx, addr, msg) + + if err != nil { + return err + } + + sigBytes := append([]byte{byte(sig.Type)}, sig.Data...) + + fmt.Println(hex.EncodeToString(sigBytes)) + return nil + }, +} + +var walletVerify = &cli.Command{ + Name: "verify", + Usage: "verify the signature of a message", + ArgsUsage: " ", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + if !cctx.Args().Present() || cctx.NArg() != 3 { + return fmt.Errorf("must specify signing address, message, and signature to verify") + } + + addr, err := address.NewFromString(cctx.Args().First()) + + if err != nil { + return err + } + + msg, err := hex.DecodeString(cctx.Args().Get(1)) + + if err != nil { + return err + } + + sigBytes, err := hex.DecodeString(cctx.Args().Get(2)) + + if err != nil { + return err + } + + var sig crypto.Signature + if err := sig.UnmarshalBinary(sigBytes); err != nil { + return err + } + + if api.WalletVerify(ctx, addr, msg, &sig) { + fmt.Println("valid") + return nil + } else { + fmt.Println("invalid") + return NewCliError("CLI Verify called with invalid signature") + } + }, +} diff --git a/cmd/lotus-chainwatch/storage.go b/cmd/lotus-chainwatch/storage.go index 374066ced..c90a653b9 100644 --- a/cmd/lotus-chainwatch/storage.go +++ b/cmd/lotus-chainwatch/storage.go @@ -233,6 +233,9 @@ create table if not exists miner_heads primary key (head, addr) ); +create index if not exists miner_heads_stateroot_index + on miner_heads (stateroot); + create or replace function miner_tips(epoch bigint) returns table (head text, addr text, diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 19eba8377..f17bb38a0 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -19,8 +19,9 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -30,6 +31,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/lib/peermgr" + "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/testing" @@ -98,7 +100,7 @@ var DaemonCmd = &cli.Command{ defer pprof.StopCPUProfile() } - ctx := context.Background() + ctx, _ := tag.New(context.Background(), tag.Insert(metrics.Version, build.BuildVersion), tag.Insert(metrics.Commit, build.CurrentCommit)) { dir, err := homedir.Expand(cctx.String("repo")) if err != nil { @@ -186,16 +188,15 @@ var DaemonCmd = &cli.Command{ } } - // Add lotus version info to prometheus metrics - var lotusInfoMetric = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "lotus_info", - Help: "Lotus version information.", - }, []string{"version"}) + // Register all metric views + if err = view.Register( + metrics.DefaultViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } - // Setting to 1 lets us multiply it with other stats to add the version labels - lotusInfoMetric.With(prometheus.Labels{ - "version": build.UserVersion, - }).Set(1) + // Set the metric to one so it is published to the exporter + stats.Record(ctx, metrics.LotusInfo.M(1)) endpoint, err := r.APIEndpoint() if err != nil { diff --git a/cmd/lotus/debug_advance.go b/cmd/lotus/debug_advance.go index 0ffc905cc..0fd7a0f78 100644 --- a/cmd/lotus/debug_advance.go +++ b/cmd/lotus/debug_advance.go @@ -53,7 +53,15 @@ func init() { return xerrors.Errorf("StateMinerWorker: %w", err) } +<<<<<<< HEAD rand, err := api.ChainGetRandomness(ctx, head.Key(), crypto.DomainSeparationTag_TicketProduction, head.Height(), addr.Bytes()) +======= + if ret.MsgRct.ExitCode != 0 { + return xerrors.Errorf("failed to get miner worker addr (exit code %d)", ret.MsgRct.ExitCode) + } + + w, err := address.NewFromBytes(ret.MsgRct.Return) +>>>>>>> master if err != nil { return xerrors.Errorf("failed to get randomness: %w", err) } diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index da003fb1c..f0d4442b4 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -73,7 +73,12 @@ func main() { Code: trace.StatusCodeFailedPrecondition, Message: err.Error(), }) - log.Warnf("%+v", err) + _, ok := err.(*lcli.ErrCmdFailed) + if ok { + log.Debugf("%+v", err) + } else { + log.Warnf("%+v", err) + } os.Exit(1) } } diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index f75b68008..5b264421a 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -23,7 +23,7 @@ import ( manet "github.com/multiformats/go-multiaddr-net" "golang.org/x/xerrors" - "github.com/prometheus/client_golang/prometheus/promhttp" + "contrib.go.opencensus.io/exporter/prometheus" ) var log = logging.Logger("main") @@ -46,7 +46,14 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr) erro http.Handle("/rest/v0/import", importAH) - http.Handle("/metrics", promhttp.Handler()) + exporter, err := prometheus.NewExporter(prometheus.Options{ + Namespace: "lotus", + }) + if err != nil { + log.Fatalf("could not create the prometheus stats exporter: %v", err) + } + + http.Handle("/debug/metrics", exporter) lst, err := manet.Listen(addr) if err != nil { diff --git a/documentation/en/.library.json b/documentation/en/.library.json index f223e4fed..e74e1934c 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -111,6 +111,13 @@ "value": null, "posts": [] }, + { + "title": "Command Line Interface", + "slug": "en+cli", + "github": "en/cli.md", + "value": null, + "posts": [] + }, { "title": "API", "slug": "en+api", diff --git a/documentation/en/cli.md b/documentation/en/cli.md new file mode 100644 index 000000000..bfb579511 --- /dev/null +++ b/documentation/en/cli.md @@ -0,0 +1,108 @@ +# Lotus Command Line Interface + +The Command Line Interface (CLI) is a convenient way to interact with +a Lotus node. You can use the CLI to operate your node, +get information about the blockchain, +manage your accounts and transfer funds, +create storage deals, and much more! + +The CLI is intended to be self-documenting, so when in doubt, simply add `--help` +to whatever command you're trying to run! This will also display all of the +input parameters that can be provided to a command. + +We highlight some of the commonly +used features of the CLI below. +All CLI commands should be run from the home directory of the Lotus project. + +## Operating a Lotus node + +### Starting up a node + +```sh +lotus daemon +``` +This command will start up your Lotus node, with its API port open at 1234. +You can pass `--api=` to use a different port. + +### Checking your sync progress + +```sh +lotus sync status +``` +This command will print your current tipset height under `Height`, and the target tipset height +under `Taregt`. + +You can also run `lotus sync wait` to get constant updates on your sync progress. + +### Getting the head tipset + +```sh +lotus chain head +``` + +### Control the logging level + +```sh +lotus log set-level +``` +This command can be used to toggle the logging levels of the different +systems of a Lotus node. In decreasing order +of logging detail, the levels are `debug`, `info`, `warn`, and `error`. + +As an example, +to set the `chain` and `blocksync` to log at the `debug` level, run +`lotus log set-level --system chain --system blocksync debug`. + +To see the various logging system, run `lotus log list`. + +### Find out what version of Lotus you're running + +```sh +lotus version +``` + +## Managing your accounts + +### Listing accounts in your wallet + +```sh +lo*tus wallet list +``` + +### Creating a new account + +```sh +lotus wallet new bls +``` +This command will create a new BLS account in your wallet; these +addresses start with the prefix `t3`. Running `lotus wallet new secp256k1` +(or just `lotus wallet new`) will create +a new Secp256k1 account, which begins with the prefix `t1`. + +### Getting an account's balance + +```sh +lotus wallet balance
+``` + +### Transferring funds + +```sh +lotus send --source= +``` +This command will transfer `amount` (in attoFIL) from `source address` to `destination address`. + +### Importing an account into your wallet + +```sh +lotus wallet import +``` +This command will import an account whose private key is saved at the specified file. + +### Exporting an account from your wallet + +```sh +lotus wallet export
+``` +This command will print out the private key of the specified address +if it is in your wallet. Always be careful with your private key! diff --git a/documentation/en/join-testnet.md b/documentation/en/join-testnet.md index 6474d81b0..21725027f 100644 --- a/documentation/en/join-testnet.md +++ b/documentation/en/join-testnet.md @@ -36,13 +36,13 @@ In order to connect to the network, you need to be connected to at least 1 peer. ## Chain sync -While the daemon is running, the next requirement is to sync the chain. Run the command below to start the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/). +While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/). ```sh lotus sync wait ``` -- This step will take anywhere between 30 minutes to a few hours. +- This step will take anywhere between a few hours to a couple of days. - You will be able to perform **Lotus Testnet** operations after it is finished. ## Create your first address diff --git a/go.mod b/go.mod index f9c3738d8..219f00dea 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( contrib.go.opencensus.io/exporter/jaeger v0.1.0 + contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/BurntSushi/toml v0.3.1 github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee @@ -74,6 +75,7 @@ require ( github.com/libp2p/go-libp2p-record v0.1.1 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 github.com/libp2p/go-libp2p-secio v0.2.1 + github.com/libp2p/go-libp2p-swarm v0.2.2 github.com/libp2p/go-libp2p-tls v0.1.0 github.com/libp2p/go-libp2p-yamux v0.2.1 github.com/libp2p/go-maddr-filter v0.0.5 @@ -86,6 +88,7 @@ require ( github.com/multiformats/go-multiaddr-net v0.1.2 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 + github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 github.com/prometheus/common v0.4.0 github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index 4cb5a6d96..6688a869d 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc= contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= +contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= +contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -684,42 +686,37 @@ github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXx github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smola/gocompat v0.2.0 h1:6b1oIMlUXIpz//VKEDzPVBK8KG7beVwmHIUEBIs/Pns= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -732,7 +729,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -740,20 +736,16 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= -github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= -github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2biEZZXdBKt9HX7DN3bYGFUqljqqy0DqgnY= github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= @@ -798,13 +790,11 @@ go.uber.org/dig v1.7.0 h1:E5/L92iQTNJTjfgJF2KgU+/JpMaiuvK2DHLBj0+kSZk= go.uber.org/dig v1.7.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg= go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY= go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw= -go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= @@ -835,7 +825,6 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -845,6 +834,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -911,7 +901,6 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200108195415-316d2f248479 h1:csuS+MHeEA2eWhyjQCMaPMq4z1+/PohkBSjJZHSIbOE= golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -934,16 +923,13 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs= @@ -955,7 +941,5 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= diff --git a/lib/jsonrpc/handler.go b/lib/jsonrpc/handler.go index 0edc25fa9..88a20ee1f 100644 --- a/lib/jsonrpc/handler.go +++ b/lib/jsonrpc/handler.go @@ -9,6 +9,9 @@ import ( "io" "reflect" + "github.com/filecoin-project/lotus/metrics" + "go.opencensus.io/stats" + "go.opencensus.io/tag" "go.opencensus.io/trace" "go.opencensus.io/trace/propagation" "golang.org/x/xerrors" @@ -151,18 +154,22 @@ func (handlers) getSpan(ctx context.Context, req request) (context.Context, *tra } func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer)), rpcError rpcErrFunc, done func(keepCtx bool), chOut chanOut) { + // Not sure if we need to sanitize the incoming req.Method or not. ctx, span := h.getSpan(ctx, req) + ctx, _ = tag.New(ctx, tag.Insert(metrics.RPCMethod, req.Method)) defer span.End() handler, ok := h[req.Method] if !ok { rpcError(w, &req, rpcMethodNotFound, fmt.Errorf("method '%s' not found", req.Method)) + stats.Record(ctx, metrics.RPCInvalidMethod.M(1)) done(false) return } if len(req.Params) != handler.nParams { rpcError(w, &req, rpcInvalidParams, fmt.Errorf("wrong param count")) + stats.Record(ctx, metrics.RPCRequestError.M(1)) done(false) return } @@ -172,6 +179,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer if chOut == nil && outCh { rpcError(w, &req, rpcMethodNotFound, fmt.Errorf("method '%s' not supported in this mode (no out channel support)", req.Method)) + stats.Record(ctx, metrics.RPCRequestError.M(1)) return } @@ -185,6 +193,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer rp := reflect.New(handler.paramReceivers[i]) if err := json.NewDecoder(bytes.NewReader(req.Params[i].data)).Decode(rp.Interface()); err != nil { rpcError(w, &req, rpcParseError, xerrors.Errorf("unmarshaling params for '%s': %w", handler.handlerFunc, err)) + stats.Record(ctx, metrics.RPCRequestError.M(1)) return } @@ -196,6 +205,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer callResult, err := doCall(req.Method, handler.handlerFunc, callParams) if err != nil { rpcError(w, &req, 0, xerrors.Errorf("fatal error calling '%s': %w", req.Method, err)) + stats.Record(ctx, metrics.RPCRequestError.M(1)) return } if req.ID == nil { @@ -213,6 +223,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer err := callResult[handler.errOut].Interface() if err != nil { log.Warnf("error in RPC call to '%s': %+v", req.Method, err) + stats.Record(ctx, metrics.RPCResponseError.M(1)) resp.Error = &respError{ Code: 1, Message: err.(error).Error(), @@ -234,6 +245,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer } log.Warnf("failed to setup channel in RPC call to '%s': %+v", req.Method, err) + stats.Record(ctx, metrics.RPCResponseError.M(1)) resp.Error = &respError{ Code: 1, Message: err.(error).Error(), @@ -243,6 +255,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer w(func(w io.Writer) { if err := json.NewEncoder(w).Encode(resp); err != nil { log.Error(err) + stats.Record(ctx, metrics.RPCResponseError.M(1)) return } }) diff --git a/lib/peermgr/peermgr.go b/lib/peermgr/peermgr.go index e1b41e14d..490f2b229 100644 --- a/lib/peermgr/peermgr.go +++ b/lib/peermgr/peermgr.go @@ -5,7 +5,9 @@ import ( "sync" "time" + "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node/modules/dtypes" + "go.opencensus.io/stats" "go.uber.org/fx" host "github.com/libp2p/go-libp2p-core/host" @@ -115,6 +117,7 @@ func (pmgr *PeerMgr) Run(ctx context.Context) { } else if pcount > pmgr.maxFilPeers { log.Debug("peer count about threshold: %d > %d", pcount, pmgr.maxFilPeers) } + stats.Record(ctx, metrics.PeerCount.M(int64(pmgr.getPeerCount()))) } } } diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 000000000..9ec9caefd --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,102 @@ +package metrics + +import ( + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +// Global Tags +var ( + Version, _ = tag.NewKey("version") + Commit, _ = tag.NewKey("commit") + RPCMethod, _ = tag.NewKey("method") + PeerID, _ = tag.NewKey("peer_id") + FailureType, _ = tag.NewKey("failure_type") + MessageFrom, _ = tag.NewKey("message_from") + MessageTo, _ = tag.NewKey("message_to") + MessageNonce, _ = tag.NewKey("message_nonce") + ReceivedFrom, _ = tag.NewKey("received_from") +) + +// Measures +var ( + LotusInfo = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless) + ChainNodeHeight = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless) + ChainNodeWorkerHeight = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless) + MessageReceived = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless) + MessageValidationFailure = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless) + MessageValidationSuccess = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless) + BlockReceived = stats.Int64("block/received", "Counter for total received blocks", stats.UnitDimensionless) + BlockValidationFailure = stats.Int64("block/failure", "Counter for block validation failures", stats.UnitDimensionless) + BlockValidationSuccess = stats.Int64("block/success", "Counter for block validation successes", stats.UnitDimensionless) + PeerCount = stats.Int64("peer/count", "Current number of FIL peers", stats.UnitDimensionless) + RPCInvalidMethod = stats.Int64("rpc/invalid_method", "Total number of invalid RPC methods called", stats.UnitDimensionless) + RPCRequestError = stats.Int64("rpc/request_error", "Total number of request errors handled", stats.UnitDimensionless) + RPCResponseError = stats.Int64("rpc/response_error", "Total number of responses errors handled", stats.UnitDimensionless) +) + +// DefaultViews is an array of Consensus views for metric gathering purposes +var DefaultViews = []*view.View{ + &view.View{ + Name: "info", + Description: "Lotus node information", + Measure: LotusInfo, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{Version, Commit}, + }, + &view.View{ + Measure: ChainNodeHeight, + Aggregation: view.LastValue(), + }, + &view.View{ + Measure: ChainNodeWorkerHeight, + Aggregation: view.LastValue(), + }, + &view.View{ + Measure: BlockReceived, + Aggregation: view.Count(), + }, + &view.View{ + Measure: BlockValidationFailure, + Aggregation: view.Count(), + TagKeys: []tag.Key{FailureType, PeerID, ReceivedFrom}, + }, + &view.View{ + Measure: BlockValidationSuccess, + Aggregation: view.Count(), + }, + &view.View{ + Measure: MessageReceived, + Aggregation: view.Count(), + }, + &view.View{ + Measure: MessageValidationFailure, + Aggregation: view.Count(), + TagKeys: []tag.Key{FailureType, MessageFrom, MessageTo, MessageNonce}, + }, + &view.View{ + Measure: MessageValidationSuccess, + Aggregation: view.Count(), + }, + &view.View{ + Measure: PeerCount, + Aggregation: view.LastValue(), + }, + // All RPC related metrics should at the very least tag the RPCMethod + &view.View{ + Measure: RPCInvalidMethod, + Aggregation: view.Count(), + TagKeys: []tag.Key{RPCMethod}, + }, + &view.View{ + Measure: RPCRequestError, + Aggregation: view.Count(), + TagKeys: []tag.Key{RPCMethod}, + }, + &view.View{ + Measure: RPCResponseError, + Aggregation: view.Count(), + TagKeys: []tag.Key{RPCMethod}, + }, +} diff --git a/node/impl/common.go b/node/impl/common.go index 1420a7b7e..359a64543 100644 --- a/node/impl/common.go +++ b/node/impl/common.go @@ -2,6 +2,7 @@ package impl import ( "context" + "github.com/filecoin-project/lotus/node/modules/lp2p" logging "github.com/ipfs/go-log/v2" @@ -10,6 +11,7 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + swarm "github.com/libp2p/go-libp2p-swarm" ma "github.com/multiformats/go-multiaddr" "go.uber.org/fx" "golang.org/x/xerrors" @@ -69,6 +71,10 @@ func (a *CommonAPI) NetPeers(context.Context) ([]peer.AddrInfo, error) { } func (a *CommonAPI) NetConnect(ctx context.Context, p peer.AddrInfo) error { + if swrm, ok := a.Host.Network().(*swarm.Swarm); ok { + swrm.Backoff().Clear(p.ID) + } + return a.Host.Connect(ctx, p) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 2f8f1dc18..61b962821 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -152,8 +152,8 @@ func (a *StateAPI) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKe return types.NewInt(0), xerrors.Errorf("failed to get miner worker addr: %w", err) } - if ret.ExitCode != 0 { - return types.NewInt(0), xerrors.Errorf("failed to get miner worker addr (exit code %d)", ret.ExitCode) + if ret.MsgRct.ExitCode != 0 { + return types.NewInt(0), xerrors.Errorf("failed to get miner worker addr (exit code %d)", ret.MsgRct.ExitCode) } return types.BigFromBytes(ret.Return), nil*/ @@ -161,7 +161,7 @@ func (a *StateAPI) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKe return big.Zero(), nil } -func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.MethodCall, error) { +func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -169,7 +169,7 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. return a.StateManager.Call(ctx, msg, ts) } -func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.ReplayResults, error) { +func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -184,10 +184,11 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. errstr = r.ActorErr.Error() } - return &api.ReplayResults{ - Msg: m, - Receipt: &r.MessageReceipt, - Error: errstr, + return &api.InvocResult{ + Msg: m, + MsgRct: &r.MessageReceipt, + InternalExecutions: r.InternalExecutions, + Error: errstr, }, nil } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index e9266c788..ddf47ccea 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -3,6 +3,8 @@ package full import ( "context" + "github.com/filecoin-project/lotus/lib/sigs" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -55,6 +57,10 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms }, nil } +func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool { + return sigs.Verify(sig, k, msg) == nil +} + func (a *WalletAPI) WalletDefaultAddress(ctx context.Context) (address.Address, error) { return a.Wallet.GetDefault() } diff --git a/node/modules/services.go b/node/modules/services.go index cac24391b..89a32b03f 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/sub" - "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/peermgr" "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -76,18 +75,9 @@ func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub panic(err) } - v := func(ctx context.Context, pid peer.ID, msg *pubsub.Message) bool { - m, err := types.DecodeSignedMessage(msg.GetData()) - if err != nil { - log.Errorf("got incorrectly formatted Message: %s", err) - return false - } + v := sub.NewMessageValidator(mpool) - msg.ValidatorData = m - return true - } - - if err := ps.RegisterTopicValidator(build.MessagesTopic, v); err != nil { + if err := ps.RegisterTopicValidator(build.MessagesTopic, v.Validate); err != nil { panic(err) } diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 42212060c..597663a94 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -250,7 +250,7 @@ func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address return false, err } - if ret.ExitCode != 0 { + if ret.MsgRct.ExitCode != 0 { return false, nil } diff --git a/storage/miner.go b/storage/miner.go index a95c685e5..1b535d786 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -45,7 +45,7 @@ type Miner struct { type storageMinerApi interface { // Call a read only method on actors (no interaction with the chain required) - StateCall(context.Context, *types.Message, types.TipSetKey) (*api.MethodCall, error) + StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) StateMinerWorker(context.Context, address.Address, types.TipSetKey) (address.Address, error) StateMinerPostState(ctx context.Context, actor address.Address, ts types.TipSetKey) (*miner.PoStState, error) StateMinerSectors(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) diff --git a/storage/sealing/checks.go b/storage/sealing/checks.go index 381be0196..c7bec0228 100644 --- a/storage/sealing/checks.go +++ b/storage/sealing/checks.go @@ -98,12 +98,12 @@ func checkSeal(ctx context.Context, maddr address.Address, si SectorInfo, api se if err != nil { return &ErrApi{xerrors.Errorf("calling ComputeDataCommitment: %w", err)} } - if r.ExitCode != 0 { - return &ErrBadCommD{xerrors.Errorf("receipt for ComputeDataCommitment had exit code %d", r.ExitCode)} + if r.MsgRct.ExitCode != 0 { + return &ErrBadCommD{xerrors.Errorf("receipt for ComputeDataCommitment had exit code %d", r.MsgRct.ExitCode)} } var c cbg.CborCid - if err := c.UnmarshalCBOR(bytes.NewReader(r.Return)); err != nil { + if err := c.UnmarshalCBOR(bytes.NewReader(r.MsgRct.Return)); err != nil { return err } diff --git a/storage/sealing/sealing.go b/storage/sealing/sealing.go index 89294eb91..b74d34db1 100644 --- a/storage/sealing/sealing.go +++ b/storage/sealing/sealing.go @@ -32,7 +32,7 @@ type TicketFn func(context.Context) (*api.SealTicket, error) type sealingApi interface { // TODO: trim down // Call a read only method on actors (no interaction with the chain required) - StateCall(context.Context, *types.Message, types.TipSetKey) (*api.MethodCall, error) + StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) StateMinerWorker(context.Context, address.Address, types.TipSetKey) (address.Address, error) StateMinerPostState(ctx context.Context, actor address.Address, ts types.TipSetKey) (*miner.PoStState, error) StateMinerSectors(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) diff --git a/tools/stats/head_buffer.go b/tools/stats/head_buffer.go new file mode 100644 index 000000000..77102e034 --- /dev/null +++ b/tools/stats/head_buffer.go @@ -0,0 +1,47 @@ +package main + +import ( + "container/list" + + "github.com/filecoin-project/lotus/chain/store" +) + +type headBuffer struct { + buffer *list.List + size int +} + +func NewHeadBuffer(size int) *headBuffer { + buffer := list.New() + buffer.Init() + + return &headBuffer{ + buffer: buffer, + size: size, + } +} + +func (h *headBuffer) Push(hc *store.HeadChange) (rethc *store.HeadChange) { + if h.buffer.Len() == h.size { + var ok bool + + el := h.buffer.Front() + rethc, ok = el.Value.(*store.HeadChange) + if !ok { + panic("Value from list is not the correct type") + } + + h.buffer.Remove(el) + } + + h.buffer.PushBack(hc) + + return +} + +func (h *headBuffer) Pop() { + el := h.buffer.Back() + if el != nil { + h.buffer.Remove(el) + } +} diff --git a/tools/stats/head_buffer_test.go b/tools/stats/head_buffer_test.go new file mode 100644 index 000000000..3ae7d749c --- /dev/null +++ b/tools/stats/head_buffer_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "testing" + + "github.com/filecoin-project/lotus/chain/store" + "github.com/stretchr/testify/require" +) + +func TestHeadBuffer(t *testing.T) { + + t.Run("Straight push through", func(t *testing.T) { + hb := NewHeadBuffer(5) + require.Nil(t, hb.Push(&store.HeadChange{Type: "1"})) + require.Nil(t, hb.Push(&store.HeadChange{Type: "2"})) + require.Nil(t, hb.Push(&store.HeadChange{Type: "3"})) + require.Nil(t, hb.Push(&store.HeadChange{Type: "4"})) + require.Nil(t, hb.Push(&store.HeadChange{Type: "5"})) + + hc := hb.Push(&store.HeadChange{Type: "6"}) + require.Equal(t, hc.Type, "1") + }) + + t.Run("Reverts", func(t *testing.T) { + hb := NewHeadBuffer(5) + require.Nil(t, hb.Push(&store.HeadChange{Type: "1"})) + require.Nil(t, hb.Push(&store.HeadChange{Type: "2"})) + require.Nil(t, hb.Push(&store.HeadChange{Type: "3"})) + hb.Pop() + require.Nil(t, hb.Push(&store.HeadChange{Type: "3a"})) + hb.Pop() + require.Nil(t, hb.Push(&store.HeadChange{Type: "3b"})) + require.Nil(t, hb.Push(&store.HeadChange{Type: "4"})) + require.Nil(t, hb.Push(&store.HeadChange{Type: "5"})) + + hc := hb.Push(&store.HeadChange{Type: "6"}) + require.Equal(t, hc.Type, "1") + hc = hb.Push(&store.HeadChange{Type: "7"}) + require.Equal(t, hc.Type, "2") + hc = hb.Push(&store.HeadChange{Type: "8"}) + require.Equal(t, hc.Type, "3b") + }) +} diff --git a/tools/stats/main.go b/tools/stats/main.go index bd84f3193..c5442fe5e 100644 --- a/tools/stats/main.go +++ b/tools/stats/main.go @@ -23,10 +23,12 @@ func main() { var database string = "lotus" var reset bool = false var height int64 = 0 + var headlag int = 3 flag.StringVar(&repo, "repo", repo, "lotus repo path") flag.StringVar(&database, "database", database, "influx database") flag.Int64Var(&height, "height", height, "block height to start syncing from (0 will resume)") + flag.IntVar(&headlag, "head-lag", headlag, "number of head events to hold to protect against small reorgs") flag.BoolVar(&reset, "reset", reset, "truncate database before starting stats gathering") flag.Parse() @@ -67,7 +69,7 @@ func main() { log.Fatal(err) } - tipsetsCh, err := GetTips(ctx, api, abi.ChainEpoch(height)) + tipsetsCh, err := GetTips(ctx, api, abi.ChainEpoch(height), headlag) if err != nil { log.Fatal(err) } @@ -76,6 +78,7 @@ func main() { defer wq.Close() for tipset := range tipsetsCh { + log.Infow("Collect stats", "height", tipset.Height()) pl := NewPointList() height := tipset.Height() diff --git a/tools/stats/rpc.go b/tools/stats/rpc.go index 03c75ced4..7f90ea169 100644 --- a/tools/stats/rpc.go +++ b/tools/stats/rpc.go @@ -121,9 +121,11 @@ sync_complete: } } -func GetTips(ctx context.Context, api api.FullNode, lastHeight abi.ChainEpoch) (<-chan *types.TipSet, error) { +func GetTips(ctx context.Context, api api.FullNode, lastHeight abi.ChainEpoch, headlag int) (<-chan *types.TipSet, error) { chmain := make(chan *types.TipSet) + hb := NewHeadBuffer(headlag) + notif, err := api.ChainNotify(ctx) if err != nil { return nil, err @@ -152,7 +154,11 @@ func GetTips(ctx context.Context, api api.FullNode, lastHeight abi.ChainEpoch) ( chmain <- tipset } case store.HCApply: - chmain <- change.Val + if out := hb.Push(change); out != nil { + chmain <- out.Val + } + case store.HCRevert: + hb.Pop() } } case <-ping: