From 2f4582147a81fe5f72b73cbfd70f47171548d1b0 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Tue, 27 Oct 2020 18:26:32 -0400 Subject: [PATCH 01/12] Add `termination-estimate` to get an estimation for how much a termination penalty will be: - Usage example: `./lotus-shed sectors termation-estimate 5 8 10`. --- cmd/lotus-shed/sectors.go | 99 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/cmd/lotus-shed/sectors.go b/cmd/lotus-shed/sectors.go index 2e78469fa..64f3faf79 100644 --- a/cmd/lotus-shed/sectors.go +++ b/cmd/lotus-shed/sectors.go @@ -25,6 +25,7 @@ var sectorsCmd = &cli.Command{ Flags: []cli.Flag{}, Subcommands: []*cli.Command{ terminateSectorCmd, + terminateSectorPenaltyEstimationCmd, }, } @@ -131,3 +132,101 @@ var terminateSectorCmd = &cli.Command{ return nil }, } + +func findPenaltyInInternalExecutions(prefix string, trace []types.ExecutionTrace) { + for _, im := range trace { + if im.Msg.To.String() == "f099" /*Burn actor*/ { + fmt.Printf("Estimated termination penalty: %s attoFIL\n", im.Msg.Value) + return + } + findPenaltyInInternalExecutions(prefix+"\t", im.Subcalls) + } +} + +var terminateSectorPenaltyEstimationCmd = &cli.Command{ + Name: "termination-estimate", + Usage: "Estimate the termination penalty", + ArgsUsage: "[sectorNum1 sectorNum2 ...]", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() < 1 { + return fmt.Errorf("at least one sector must be specified") + } + + nodeApi, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + api, acloser, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer acloser() + + ctx := lcli.ReqContext(cctx) + + maddr, err := api.ActorAddress(ctx) + if err != nil { + return err + } + + mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + terminationDeclarationParams := []miner2.TerminationDeclaration{} + + for _, sn := range cctx.Args().Slice() { + sectorNum, err := strconv.ParseUint(sn, 10, 64) + if err != nil { + return fmt.Errorf("could not parse sector number: %w", err) + } + + sectorbit := bitfield.New() + sectorbit.Set(sectorNum) + + loca, err := nodeApi.StateSectorPartition(ctx, maddr, abi.SectorNumber(sectorNum), types.EmptyTSK) + if err != nil { + return fmt.Errorf("get state sector partition %s", err) + } + + para := miner2.TerminationDeclaration{ + Deadline: loca.Deadline, + Partition: loca.Partition, + Sectors: sectorbit, + } + + terminationDeclarationParams = append(terminationDeclarationParams, para) + } + + terminateSectorParams := &miner2.TerminateSectorsParams{ + Terminations: terminationDeclarationParams, + } + + sp, err := actors.SerializeParams(terminateSectorParams) + if err != nil { + return xerrors.Errorf("serializing params: %w", err) + } + + msg := &types.Message{ + From: mi.Owner, + To: maddr, + Method: miner.Methods.TerminateSectors, + + Value: big.Zero(), + Params: sp, + } + + //TODO: 4667 add an option to give a more precise estimation with pending termination penalty excluded + + invocResult, err := nodeApi.StateCall(ctx, msg, types.TipSetKey{}) + if err != nil { + return xerrors.Errorf("fail to state call: %w", err) + } + + findPenaltyInInternalExecutions("\t", invocResult.ExecutionTrace.Subcalls) + return nil + }, +} From fe4298674f9f82e5f23dfb71388d39d4d8d18902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 3 Nov 2020 15:51:33 +0100 Subject: [PATCH 02/12] Use https for blst submodule --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 2e260f891..018b51157 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,4 +13,4 @@ url = https://github.com/filecoin-project/oni [submodule "extern/blst"] path = extern/blst - url = git@github.com:supranational/blst.git + url = https://github.com/supranational/blst.git From 97a06444f2d69d2c4a42c4d181b6885af0eda1b3 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 3 Nov 2020 20:21:55 +0100 Subject: [PATCH 03/12] Fix .gitmodules Signed-off-by: Jakub Sztandera --- .gitmodules | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 2e260f891..5d82758a2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,16 +1,15 @@ [submodule "extern/filecoin-ffi"] path = extern/filecoin-ffi url = https://github.com/filecoin-project/filecoin-ffi.git - branch = master [submodule "extern/serialization-vectors"] path = extern/serialization-vectors - url = https://github.com/filecoin-project/serialization-vectors + url = https://github.com/filecoin-project/serialization-vectors.git [submodule "extern/test-vectors"] path = extern/test-vectors url = https://github.com/filecoin-project/test-vectors.git [submodule "extern/oni"] path = extern/oni - url = https://github.com/filecoin-project/oni + url = https://github.com/filecoin-project/oni.git [submodule "extern/blst"] path = extern/blst - url = git@github.com:supranational/blst.git + url = https://github.com/supranational/blst.git From b0824ada155e3815a377c1d8ec8b4b47e5c6ec61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 3 Nov 2020 23:02:01 +0100 Subject: [PATCH 04/12] bufbs: Get from write blockstore first --- lib/bufbstore/buf_bstore.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/bufbstore/buf_bstore.go b/lib/bufbstore/buf_bstore.go index 4ea746444..d3c5dc442 100644 --- a/lib/bufbstore/buf_bstore.go +++ b/lib/bufbstore/buf_bstore.go @@ -94,7 +94,7 @@ func (bs *BufferedBS) DeleteBlock(c cid.Cid) error { } func (bs *BufferedBS) Get(c cid.Cid) (block.Block, error) { - if out, err := bs.read.Get(c); err != nil { + if out, err := bs.write.Get(c); err != nil { if err != bstore.ErrNotFound { return nil, err } @@ -102,7 +102,7 @@ func (bs *BufferedBS) Get(c cid.Cid) (block.Block, error) { return out, nil } - return bs.write.Get(c) + return bs.read.Get(c) } func (bs *BufferedBS) GetSize(c cid.Cid) (int, error) { @@ -115,7 +115,7 @@ func (bs *BufferedBS) GetSize(c cid.Cid) (int, error) { } func (bs *BufferedBS) Put(blk block.Block) error { - has, err := bs.read.Has(blk.Cid()) + has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check if err != nil { return err } @@ -128,7 +128,7 @@ func (bs *BufferedBS) Put(blk block.Block) error { } func (bs *BufferedBS) Has(c cid.Cid) (bool, error) { - has, err := bs.read.Has(c) + has, err := bs.write.Has(c) if err != nil { return false, err } @@ -136,7 +136,7 @@ func (bs *BufferedBS) Has(c cid.Cid) (bool, error) { return true, nil } - return bs.write.Has(c) + return bs.read.Has(c) } func (bs *BufferedBS) HashOnRead(hor bool) { From a1e1b03ca48960146c4f48f73f26e175bfafdff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 3 Nov 2020 23:04:44 +0100 Subject: [PATCH 05/12] Optionally allow bitswap for chainstore --- chain/gen/gen.go | 2 +- chain/gen/genesis/genesis.go | 2 +- chain/store/index_test.go | 2 +- chain/store/store.go | 17 +++--- chain/store/store_test.go | 6 +- cmd/lotus-bench/import.go | 2 +- cmd/lotus-shed/balances.go | 4 +- cmd/lotus-shed/export.go | 2 +- cmd/lotus-shed/genesis-verify.go | 2 +- cmd/lotus-shed/pruning.go | 2 +- cmd/lotus/daemon.go | 2 +- conformance/driver.go | 2 +- lib/blockstore/fallbackstore.go | 95 ++++++++++++++++++++++++++++++++ node/builder.go | 10 +++- node/modules/chain.go | 26 +++++++-- node/modules/dtypes/storage.go | 3 +- 16 files changed, 151 insertions(+), 28 deletions(-) create mode 100644 lib/blockstore/fallbackstore.go diff --git a/chain/gen/gen.go b/chain/gen/gen.go index d56f285a0..93e090ac8 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -236,7 +236,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("make genesis block failed: %w", err) } - cs := store.NewChainStore(bs, ds, sys, j) + cs := store.NewChainStore(bs, bs, ds, sys, j) genfb := &types.FullBlock{Header: genb.Genesis} gents := store.NewFullTipSet([]*types.FullBlock{genfb}) diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index 6a1090784..e441af7ae 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -482,7 +482,7 @@ func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blocksto } // temp chainstore - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys, j) + cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), sys, j) // Verify PreSealed Data stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs) diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 5283d10dc..11ff4371f 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -31,7 +31,7 @@ func TestIndexSeeks(t *testing.T) { ctx := context.TODO() nbs := blockstore.NewTemporarySync() - cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil) + cs := store.NewChainStore(nbs, nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil) _, err = cs.Import(bytes.NewReader(gencar)) if err != nil { diff --git a/chain/store/store.go b/chain/store/store.go index 00a78500e..f9df20d4f 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -104,8 +104,9 @@ type HeadChangeEvt struct { // 1. a tipset cache // 2. a block => messages references cache. type ChainStore struct { - bs bstore.Blockstore - ds dstore.Batching + bs bstore.Blockstore + localbs bstore.Blockstore + ds dstore.Batching heaviestLk sync.Mutex heaviest *types.TipSet @@ -130,7 +131,8 @@ type ChainStore struct { journal journal.Journal } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore { +// localbs is guaranteed to fail Get* if requested block isn't stored locally +func NewChainStore(bs bstore.Blockstore, localbs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore { c, _ := lru.NewARC(DefaultMsgMetaCacheSize) tsc, _ := lru.NewARC(DefaultTipSetCacheSize) if j == nil { @@ -138,6 +140,7 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallB } cs := &ChainStore{ bs: bs, + localbs: localbs, ds: ds, bestTips: pubsub.New(64), tipsets: make(map[abi.ChainEpoch][]cid.Cid), @@ -522,7 +525,7 @@ func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) { // GetBlock fetches a BlockHeader with the supplied CID. It returns // blockstore.ErrNotFound if the block was not found in the BlockStore. func (cs *ChainStore) GetBlock(c cid.Cid) (*types.BlockHeader, error) { - sb, err := cs.bs.Get(c) + sb, err := cs.localbs.Get(c) if err != nil { return nil, err } @@ -793,7 +796,7 @@ func (cs *ChainStore) GetCMessage(c cid.Cid) (types.ChainMsg, error) { } func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) { - sb, err := cs.bs.Get(c) + sb, err := cs.localbs.Get(c) if err != nil { log.Errorf("get message get failed: %s: %s", c, err) return nil, err @@ -803,7 +806,7 @@ func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) { } func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error) { - sb, err := cs.bs.Get(c) + sb, err := cs.localbs.Get(c) if err != nil { log.Errorf("get message get failed: %s: %s", c, err) return nil, err @@ -939,7 +942,7 @@ func (cs *ChainStore) ReadMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) return mmcids.bls, mmcids.secpk, nil } - cst := cbor.NewCborStore(cs.bs) + cst := cbor.NewCborStore(cs.localbs) var msgmeta types.MsgMeta if err := cst.Get(context.TODO(), mmc, &msgmeta); err != nil { return nil, nil, xerrors.Errorf("failed to load msgmeta (%s): %w", mmc, err) diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 160527104..61ff98620 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -63,7 +63,7 @@ func BenchmarkGetRandomness(b *testing.B) { bs := blockstore.NewBlockstore(bds) - cs := store.NewChainStore(bs, mds, nil, nil) + cs := store.NewChainStore(bs, bs, mds, nil, nil) b.ResetTimer() @@ -97,7 +97,7 @@ func TestChainExportImport(t *testing.T) { } nbs := blockstore.NewTemporary() - cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, nil) + cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil) root, err := cs.Import(buf) if err != nil { @@ -131,7 +131,7 @@ func TestChainExportImportFull(t *testing.T) { } nbs := blockstore.NewTemporary() - cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, nil) + cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil) root, err := cs.Import(buf) if err != nil { t.Fatal(err) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index acbf9ebdc..bb7baf2b8 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -193,7 +193,7 @@ var importBenchCmd = &cli.Command{ return nil } - cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier), nil) + cs := store.NewChainStore(bs, bs, ds, vm.Syscalls(verifier), nil) stm := stmgr.NewStateManager(cs) if cctx.Bool("global-profile") { diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index b12c069f5..8f3c9574e 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -180,7 +180,7 @@ var chainBalanceStateCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) + cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) @@ -394,7 +394,7 @@ var chainPledgeCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) + cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index 3be49f0e0..dcf45e9a8 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -83,7 +83,7 @@ var exportChainCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, nil, nil) + cs := store.NewChainStore(bs, bs, mds, nil, nil) if err := cs.Load(); err != nil { return err } diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index 4b197c58f..e15a42374 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -52,7 +52,7 @@ var genesisVerifyCmd = &cli.Command{ } bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil, nil) + cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), nil, nil) cf := cctx.Args().Get(0) f, err := os.Open(cf) diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index 6cf4f8c6f..f61c8d8ea 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -162,7 +162,7 @@ var stateTreePruneCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) + cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) if err := cs.Load(); err != nil { return fmt.Errorf("loading chainstore: %w", err) } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 7d078407a..42fee736b 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -422,7 +422,7 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { if err != nil { return xerrors.Errorf("failed to open journal: %w", err) } - cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), j) + cst := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), j) log.Infof("importing chain from %s...", fname) diff --git a/conformance/driver.go b/conformance/driver.go index 95b6f2659..91f461722 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -87,7 +87,7 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) vmRand = NewFixedRand() - cs = store.NewChainStore(bs, ds, syscalls, nil) + cs = store.NewChainStore(bs, bs, ds, syscalls, nil) sm = stmgr.NewStateManager(cs) ) diff --git a/lib/blockstore/fallbackstore.go b/lib/blockstore/fallbackstore.go new file mode 100644 index 000000000..0ce397d44 --- /dev/null +++ b/lib/blockstore/fallbackstore.go @@ -0,0 +1,95 @@ +package blockstore + +import ( + "context" + "sync" + "time" + + "golang.org/x/xerrors" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + logging "github.com/ipfs/go-log" +) + +var log = logging.Logger("blockstore") + +type FallbackStore struct { + blockstore.Blockstore + + fallbackGetBlock func(context.Context, cid.Cid) (blocks.Block, error) + lk sync.RWMutex +} + +func (fbs *FallbackStore) SetFallback(fg func(context.Context, cid.Cid) (blocks.Block, error)) { + fbs.lk.Lock() + defer fbs.lk.Unlock() + + fbs.fallbackGetBlock = fg +} + +func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) { + log.Errorw("fallbackstore: Block not found locally, fetching from the network", "cid", c) + fbs.lk.RLock() + defer fbs.lk.RUnlock() + + if fbs.fallbackGetBlock == nil { + // FallbackStore wasn't configured yet (chainstore/bitswap aren't up yet) + // Wait for a bit and retry + fbs.lk.RUnlock() + time.Sleep(5 * time.Second) + fbs.lk.RLock() + + if fbs.fallbackGetBlock == nil { + log.Errorw("fallbackstore: fallbackGetBlock not configured yet") + return nil, blockstore.ErrNotFound + } + } + + ctx, cancel := context.WithTimeout(context.TODO(), 120*time.Second) + defer cancel() + + b, err := fbs.fallbackGetBlock(ctx, c) + if err != nil { + return nil, err + } + + // chain bitswap puts blocks in temp blockstore which is cleaned up + // every few min (to drop any messages we fetched but don't want) + // in this case we want to keep this block around + if err := fbs.Put(b); err != nil { + return nil, xerrors.Errorf("persisting fallback-fetched block: %w", err) + } + return b, nil +} + +func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) { + b, err := fbs.Blockstore.Get(c) + switch err { + case nil: + return b, nil + case blockstore.ErrNotFound: + return fbs.getFallback(c) + default: + return b, err + } +} + +func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) { + sz, err := fbs.Blockstore.GetSize(c) + switch err { + case nil: + return sz, nil + case blockstore.ErrNotFound: + b, err := fbs.getFallback(c) + if err != nil { + return 0, err + } + return len(b.RawData()), nil + default: + return sz, err + } +} + +var _ blockstore.Blockstore = &FallbackStore{} diff --git a/node/builder.go b/node/builder.go index d797ebb2f..1ab27f486 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,6 +3,7 @@ package node import ( "context" "errors" + "os" "time" metricsi "github.com/ipfs/go-metrics-interface" @@ -138,6 +139,7 @@ const ( HeadMetricsKey SettlePaymentChannelsKey RunPeerTaggerKey + SetupFallbackBlockstoreKey SetApiEndpointKey @@ -521,7 +523,13 @@ func Repo(r repo.Repo) Option { Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing Override(new(dtypes.MetadataDS), modules.Datastore), - Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore), + Override(new(dtypes.ChainRawBlockstore), modules.ChainRawBlockstore), + Override(new(dtypes.ChainBlockstore), From(new(dtypes.ChainRawBlockstore))), + + If(os.Getenv("LOTUS_ENABLE_CHAINSTORE_FALLBACK") == "1", + Override(new(dtypes.ChainBlockstore), modules.FallbackChainBlockstore), + Override(SetupFallbackBlockstoreKey, modules.SetupFallbackBlockstore), + ), Override(new(dtypes.ClientImportMgr), modules.ClientImportMgr), Override(new(dtypes.ClientMultiDstore), modules.ClientMultiDatastore), diff --git a/node/modules/chain.go b/node/modules/chain.go index d1414b307..e5a0f7412 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -76,7 +76,7 @@ func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds return mp, nil } -func ChainBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.ChainBlockstore, error) { +func ChainRawBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.ChainRawBlockstore, error) { blocks, err := r.Datastore("/chain") if err != nil { return nil, err @@ -91,16 +91,32 @@ func ChainBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo return cbs, nil } -func ChainGCBlockstore(bs dtypes.ChainBlockstore, gcl dtypes.ChainGCLocker) dtypes.ChainGCBlockstore { +func ChainGCBlockstore(bs dtypes.ChainRawBlockstore, gcl dtypes.ChainGCLocker) dtypes.ChainGCBlockstore { return blockstore.NewGCBlockstore(bs, gcl) } -func ChainBlockService(bs dtypes.ChainBlockstore, rem dtypes.ChainBitswap) dtypes.ChainBlockService { +func ChainBlockService(bs dtypes.ChainRawBlockstore, rem dtypes.ChainBitswap) dtypes.ChainBlockService { return blockservice.New(bs, rem) } -func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore { - chain := store.NewChainStore(bs, ds, syscalls, j) +func FallbackChainBlockstore(rbs dtypes.ChainRawBlockstore) dtypes.ChainBlockstore { + return &blockstore.FallbackStore{ + Blockstore: rbs, + } +} + +func SetupFallbackBlockstore(cbs dtypes.ChainBlockstore, rem dtypes.ChainBitswap) error { + fbs, ok := cbs.(*blockstore.FallbackStore) + if !ok { + return xerrors.Errorf("expected a FallbackStore") + } + + fbs.SetFallback(rem.GetBlock) + return nil +} + +func ChainStore(bs dtypes.ChainBlockstore, lbs dtypes.ChainRawBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore { + chain := store.NewChainStore(bs, lbs, ds, syscalls, j) if err := chain.Load(); err != nil { log.Warnf("loading chain state from disk: %s", err) diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index 13defda8d..9d7364577 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -23,7 +23,8 @@ import ( // dy default it's namespaced under /metadata in main repo datastore type MetadataDS datastore.Batching -type ChainBlockstore blockstore.Blockstore +type ChainRawBlockstore blockstore.Blockstore +type ChainBlockstore blockstore.Blockstore // optionally bitswap backed type ChainGCLocker blockstore.GCLocker type ChainGCBlockstore blockstore.GCBlockstore From 7d540fc0d82798d5c79a72a02578443568087455 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 3 Nov 2020 17:25:53 -0800 Subject: [PATCH 06/12] polish: add msig pendingtxn diffing and comp --- chain/actors/builtin/multisig/diff.go | 134 ++++++++++++++++++++++++ chain/actors/builtin/multisig/state.go | 5 + chain/actors/builtin/multisig/state0.go | 24 +++++ chain/actors/builtin/multisig/state2.go | 23 ++++ 4 files changed, 186 insertions(+) create mode 100644 chain/actors/builtin/multisig/diff.go diff --git a/chain/actors/builtin/multisig/diff.go b/chain/actors/builtin/multisig/diff.go new file mode 100644 index 000000000..680d0870a --- /dev/null +++ b/chain/actors/builtin/multisig/diff.go @@ -0,0 +1,134 @@ +package multisig + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lotus/chain/actors/adt" +) + +type PendingTransactionChanges struct { + Added []TransactionChange + Modified []TransactionModification + Removed []TransactionChange +} + +type TransactionChange struct { + TxID int64 + Tx Transaction +} + +type TransactionModification struct { + TxID int64 + From Transaction + To Transaction +} + +func DiffPendingTransactions(pre, cur State) (*PendingTransactionChanges, error) { + results := new(PendingTransactionChanges) + if changed, err := pre.PendingTxnChanged(cur); err != nil { + return nil, err + } else if !changed { // if nothing has changed then return an empty result and bail. + return results, nil + } + + pret, err := pre.transactions() + if err != nil { + return nil, err + } + + curt, err := cur.transactions() + if err != nil { + return nil, err + } + + if err := adt.DiffAdtMap(pret, curt, &transactionDiffer{results, pre, cur}); err != nil { + return nil, err + } + return results, nil +} + +type transactionDiffer struct { + Results *PendingTransactionChanges + pre, after State +} + +func (t *transactionDiffer) AsKey(key string) (abi.Keyer, error) { + txID, err := abi.ParseIntKey(key) + if err != nil { + return nil, err + } + return abi.IntKey(txID), nil +} + +func (t *transactionDiffer) Add(key string, val *cbg.Deferred) error { + txID, err := abi.ParseIntKey(key) + if err != nil { + return err + } + tx, err := t.after.decodeTransaction(val) + if err != nil { + return err + } + t.Results.Added = append(t.Results.Added, TransactionChange{ + TxID: txID, + Tx: tx, + }) + return nil +} + +func (t *transactionDiffer) Modify(key string, from, to *cbg.Deferred) error { + txID, err := abi.ParseIntKey(key) + if err != nil { + return err + } + + txFrom, err := t.pre.decodeTransaction(from) + if err != nil { + return err + } + + txTo, err := t.after.decodeTransaction(to) + if err != nil { + return err + } + + if approvalsChanged(txFrom.Approved, txTo.Approved) { + t.Results.Modified = append(t.Results.Modified, TransactionModification{ + TxID: txID, + From: txFrom, + To: txTo, + }) + } + + return nil +} + +func approvalsChanged(from, to []address.Address) bool { + if len(from) != len(to) { + return true + } + for idx := range from { + if from[idx] != to[idx] { + return true + } + } + return false +} + +func (t *transactionDiffer) Remove(key string, val *cbg.Deferred) error { + txID, err := abi.ParseIntKey(key) + if err != nil { + return err + } + tx, err := t.pre.decodeTransaction(val) + if err != nil { + return err + } + t.Results.Removed = append(t.Results.Removed, TransactionChange{ + TxID: txID, + Tx: tx, + }) + return nil +} diff --git a/chain/actors/builtin/multisig/state.go b/chain/actors/builtin/multisig/state.go index 89a7eedad..fea42ba5f 100644 --- a/chain/actors/builtin/multisig/state.go +++ b/chain/actors/builtin/multisig/state.go @@ -1,6 +1,7 @@ package multisig import ( + cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -47,6 +48,10 @@ type State interface { Signers() ([]address.Address, error) ForEachPendingTxn(func(id int64, txn Transaction) error) error + PendingTxnChanged(State) (bool, error) + + transactions() (adt.Map, error) + decodeTransaction(val *cbg.Deferred) (Transaction, error) } type Transaction = msig0.Transaction diff --git a/chain/actors/builtin/multisig/state0.go b/chain/actors/builtin/multisig/state0.go index c934343e7..e6f9a9c36 100644 --- a/chain/actors/builtin/multisig/state0.go +++ b/chain/actors/builtin/multisig/state0.go @@ -1,17 +1,20 @@ package multisig import ( + "bytes" "encoding/binary" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + multisig0 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" ) var _ State = (*state0)(nil) @@ -68,3 +71,24 @@ func (s *state0) ForEachPendingTxn(cb func(id int64, txn Transaction) error) err return cb(txid, (Transaction)(out)) }) } + +func (s *state0) PendingTxnChanged(other State) (bool, error) { + other0, ok := other.(*state0) + if !ok { + // treat an upgrade as a change, always + return true, nil + } + return !s.State.PendingTxns.Equals(other0.PendingTxns), nil +} + +func (s *state0) transactions() (adt.Map, error) { + return adt0.AsMap(s.store, s.PendingTxns) +} + +func (s *state0) decodeTransaction(val *cbg.Deferred) (Transaction, error) { + var tx multisig0.Transaction + if err := tx.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { + return Transaction{}, err + } + return tx, nil +} diff --git a/chain/actors/builtin/multisig/state2.go b/chain/actors/builtin/multisig/state2.go index a78b07d55..628da3f2c 100644 --- a/chain/actors/builtin/multisig/state2.go +++ b/chain/actors/builtin/multisig/state2.go @@ -1,11 +1,13 @@ package multisig import ( + "bytes" "encoding/binary" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -68,3 +70,24 @@ func (s *state2) ForEachPendingTxn(cb func(id int64, txn Transaction) error) err return cb(txid, (Transaction)(out)) }) } + +func (s *state2) PendingTxnChanged(other State) (bool, error) { + other2, ok := other.(*state2) + if !ok { + // treat an upgrade as a change, always + return true, nil + } + return !s.State.PendingTxns.Equals(other2.PendingTxns), nil +} + +func (s *state2) transactions() (adt.Map, error) { + return adt2.AsMap(s.store, s.PendingTxns) +} + +func (s *state2) decodeTransaction(val *cbg.Deferred) (Transaction, error) { + var tx msig2.Transaction + if err := tx.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { + return Transaction{}, err + } + return tx, nil +} From bd4322f56fd699b92dd4baf27335dd3b8c18ba7f Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 3 Nov 2020 18:13:19 +0000 Subject: [PATCH 07/12] Update lotus-stats with a richer cli This updates lotus stats to use urfave instead of the golang flags package. This brings with it some common features from other lotus tools such as the use of the `FULLNODE_API_INFO` env and other parts of the lotus cli package. This also includes the latest dashboard. --- Makefile | 2 +- cmd/lotus-stats/chain.dashboard.json | 1165 +++++++++++++++++++++++--- cmd/lotus-stats/docker-compose.yml | 6 +- cmd/lotus-stats/env.stats | 2 +- cmd/lotus-stats/main.go | 193 +++-- cmd/lotus-stats/setup.bash | 4 +- 6 files changed, 1207 insertions(+), 165 deletions(-) diff --git a/Makefile b/Makefile index 093f62ef6..535873614 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,7 @@ BINS+=lotus-bench lotus-stats: rm -f lotus-stats - go build -o lotus-stats ./cmd/lotus-stats + go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats go run github.com/GeertJohan/go.rice/rice append --exec lotus-stats -i ./build .PHONY: lotus-stats BINS+=lotus-stats diff --git a/cmd/lotus-stats/chain.dashboard.json b/cmd/lotus-stats/chain.dashboard.json index 5ff7654d0..8083c96b1 100644 --- a/cmd/lotus-stats/chain.dashboard.json +++ b/cmd/lotus-stats/chain.dashboard.json @@ -1,20 +1,11 @@ { - "__inputs": [ - { - "name": "DS_INFLUXDB", - "label": "InfluxDB", - "description": "", - "type": "datasource", - "pluginId": "influxdb", - "pluginName": "InfluxDB" - } - ], + "__inputs": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "6.5.0-pre" + "version": "7.3.0" }, { "type": "panel", @@ -36,8 +27,8 @@ }, { "type": "panel", - "id": "table", - "name": "Table", + "id": "table-old", + "name": "Table (old)", "version": "" } ], @@ -58,6 +49,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, + "iteration": 1604018016916, "links": [], "panels": [ { @@ -65,8 +57,15 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 3, "fillGradient": 0, "gridPos": { @@ -75,6 +74,7 @@ "x": 0, "y": 0 }, + "hiddenSeries": false, "hideTimeOverride": false, "id": 38, "interval": "", @@ -93,15 +93,25 @@ }, "lines": false, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.0", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "all", + "bars": false, + "color": "rgb(99, 99, 99)", + "fill": 1, + "lines": true, + "stack": false + } + ], "spaceLength": 10, "stack": true, "steppedLine": false, @@ -128,10 +138,11 @@ "type": "fill" } ], + "hide": false, "measurement": "chain.election", "orderByTime": "ASC", "policy": "default", - "query": "SELECT count(\"value\") FROM \"chain.election\" WHERE $timeFilter -10m GROUP BY time($__interval), \"miner\" fill(null)", + "query": "SELECT sum(\"value\") FROM \"chain.election\" WHERE $timeFilter GROUP BY time($blockInterval), \"miner\" fill(null)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -156,13 +167,52 @@ ] ], "tags": [] + }, + { + "alias": "all", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "orderByTime": "ASC", + "policy": "defult", + "query": "SELECT TRIPLE_EXPONENTIAL_MOVING_AVERAGE(sum(\"value\"), 40) FROM \"chain.election\" WHERE $timeFilter -$blockInterval*40 AND time < now() - $blockInterval*3 GROUP BY time($blockInterval) fill(0)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blocks Won", + "title": "Blocks and Win Counts", "tooltip": { "shared": true, "sort": 2, @@ -207,7 +257,14 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -216,6 +273,7 @@ "x": 0, "y": 9 }, + "hiddenSeries": false, "id": 22, "interval": "", "legend": { @@ -232,9 +290,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -318,7 +377,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "s", "gauge": { "maxValue": 100, @@ -350,7 +415,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -422,7 +486,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "bytes", "gauge": { "maxValue": 100, @@ -454,7 +524,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -493,7 +562,7 @@ ], "orderByTime": "ASC", "policy": "default", - "query": "SELECT sum(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY time(45s)", + "query": "SELECT sum(\"value\") FROM \"chain.power\" WHERE $timeFilter GROUP BY time(25s)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -538,7 +607,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "none", "gauge": { "maxValue": 100, @@ -570,7 +645,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -596,7 +670,7 @@ "groupBy": [ { "params": [ - "$interval" + "$blockInterval" ], "type": "time" } @@ -616,7 +690,7 @@ }, { "params": [], - "type": "sum" + "type": "count" } ] ], @@ -648,7 +722,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "none", "gauge": { "maxValue": 100, @@ -680,7 +760,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -746,7 +825,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "s", "gauge": { "maxValue": 100, @@ -778,7 +863,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -848,7 +932,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "none", "gauge": { "maxValue": 100, @@ -880,7 +970,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -906,7 +995,7 @@ "groupBy": [ { "params": [ - "$__interval" + "$blockInterval" ], "type": "time" }, @@ -917,7 +1006,7 @@ "type": "fill" } ], - "measurement": "chain.message_gasprice", + "measurement": "chain.message_gaspremium", "orderByTime": "ASC", "policy": "default", "refId": "A", @@ -932,7 +1021,7 @@ }, { "params": [], - "type": "mean" + "type": "median" } ] ], @@ -942,7 +1031,7 @@ "thresholds": "", "timeFrom": null, "timeShift": null, - "title": "Avg Gas Price", + "title": "Avg Gas Premium", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -963,7 +1052,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "decbytes", "gauge": { "maxValue": 100, @@ -995,7 +1090,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -1021,7 +1115,7 @@ "groupBy": [ { "params": [ - "$__interval" + "$blockInterval" ], "type": "time" }, @@ -1078,7 +1172,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "bytes", "gauge": { "maxValue": 100, @@ -1110,7 +1210,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -1136,7 +1235,7 @@ "groupBy": [ { "params": [ - "$__interval" + "$blockInterval" ], "type": "time" }, @@ -1193,7 +1292,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "none", "gauge": { "maxValue": 100, @@ -1225,7 +1330,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "pluginVersion": "6.4.2", "postfix": "", "postfixFontSize": "50%", @@ -1252,7 +1356,7 @@ "groupBy": [ { "params": [ - "$__interval" + "$blockInterval" ], "type": "time" }, @@ -1311,8 +1415,14 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "dateTimeFromNow", "gauge": { "maxValue": 100, @@ -1344,7 +1454,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -1413,7 +1522,14 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1422,6 +1538,7 @@ "x": 4, "y": 16 }, + "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, @@ -1441,9 +1558,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1569,7 +1687,13 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "none", "gauge": { "maxValue": 100, @@ -1601,7 +1725,6 @@ "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, - "options": {}, "postfix": "FIL", "postfixFontSize": "50%", "prefix": "", @@ -1660,7 +1783,13 @@ }, { "columns": [], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fontSize": "100%", "gridPos": { "h": 21, @@ -1669,7 +1798,6 @@ "y": 19 }, "id": 28, - "options": {}, "pageSize": null, "showHeader": true, "sort": { @@ -1679,12 +1807,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "hidden" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -1701,6 +1831,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -1741,7 +1872,7 @@ "timeShift": null, "title": "Top Power Table", "transform": "table", - "type": "table" + "type": "table-old" }, { "aliasColors": {}, @@ -1749,7 +1880,14 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 5, "fillGradient": 0, "gridPos": { @@ -1758,8 +1896,9 @@ "x": 4, "y": 19 }, + "hiddenSeries": false, "id": 40, - "interval": "", + "interval": "300s", "legend": { "alignAsTable": true, "avg": false, @@ -1778,11 +1917,12 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": true, + "pluginVersion": "7.3.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1817,7 +1957,7 @@ "measurement": "chain.miner_power", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY time($__interval), \"miner\" fill(previous)", + "query": "SELECT mean(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY time($__interval), \"miner\" fill(null)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -1885,7 +2025,13 @@ }, { "columns": [], - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fontSize": "100%", "gridPos": { "h": 21, @@ -1894,7 +2040,6 @@ "y": 19 }, "id": 18, - "options": {}, "pageSize": null, "showHeader": true, "sort": { @@ -1904,6 +2049,7 @@ "styles": [ { "alias": "Height", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "link": false, "mappingType": 1, @@ -1914,6 +2060,7 @@ }, { "alias": "Tipset", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -1930,6 +2077,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -1973,74 +2121,77 @@ "timeShift": null, "title": "Chain Table", "transform": "timeseries_to_columns", - "type": "table" + "type": "table-old" }, { "aliasColors": {}, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 7, "w": 12, "x": 4, "y": 27 }, - "id": 24, + "hiddenSeries": false, + "id": 50, "legend": { - "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.0", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*/", - "color": "rgb(31, 120, 193)" - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Total GasLimit", "groupBy": [ { "params": [ - "$__interval" + "$blockInterval" ], "type": "time" }, { "params": [ - "previous" + "null" ], "type": "fill" } ], - "measurement": "chain.pledge_collateral", + "measurement": "chain.gas_limit_total", "orderByTime": "ASC", "policy": "default", + "query": "SELECT max(\"value\") FROM \"chain.gas_limit_total\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": false, "refId": "A", "resultFormat": "time_series", "select": [ @@ -2053,18 +2204,107 @@ }, { "params": [], - "type": "mean" + "type": "max" + } + ] + ], + "tags": [] + }, + { + "alias": "Total GasUsed", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.gas_used_total", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM \"chain.gas_used_total\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + }, + { + "alias": "Total Unique GasLimit", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.gas_limit_uniq_total", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM \"chain.gas_limit_total\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" } ] ], "tags": [] } ], - "thresholds": [], + "thresholds": [ + { + "colorMode": "custom", + "fill": false, + "fillColor": "rgba(50, 116, 217, 0.2)", + "line": true, + "lineColor": "rgba(31, 96, 196, 0.6)", + "op": "gt", + "value": 25000000000, + "yaxis": "left" + } + ], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pledge Collateral", + "title": "Network Gas", "tooltip": { "shared": true, "sort": 0, @@ -2081,7 +2321,7 @@ "yaxes": [ { "format": "short", - "label": "FIL", + "label": null, "logBase": 1, "max": null, "min": null, @@ -2107,15 +2347,23 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 4, - "y": 33 + "y": 34 }, + "hiddenSeries": false, "id": 44, "legend": { "avg": false, @@ -2131,9 +2379,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2146,7 +2395,7 @@ "groupBy": [ { "params": [ - "$__interval" + "$blockInterval" ], "type": "time" }, @@ -2228,7 +2477,14 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2237,6 +2493,7 @@ "x": 0, "y": 40 }, + "hiddenSeries": false, "id": 34, "legend": { "alignAsTable": true, @@ -2251,11 +2508,12 @@ }, "lines": false, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2269,7 +2527,7 @@ "groupBy": [ { "params": [ - "$__interval" + "$blockInterval" ], "type": "time" }, @@ -2360,7 +2618,14 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_INFLUXDB}", + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2369,6 +2634,7 @@ "x": 12, "y": 40 }, + "hiddenSeries": false, "id": 36, "legend": { "alignAsTable": true, @@ -2387,11 +2653,12 @@ }, "lines": false, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2437,7 +2704,7 @@ "measurement": "chain.message_count", "orderByTime": "ASC", "policy": "default", - "query": "SELECT sum(\"value\") FROM \"chain.message_count\" WHERE $timeFilter GROUP BY time($__interval), \"method\", \"exitcode\", \"actor\" fill(null)", + "query": "SELECT sum(\"value\") FROM \"chain.message_count\" WHERE $timeFilter GROUP BY time($blockInterval), \"method\", \"exitcode\", \"actor\" fill(null)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -2498,14 +2765,701 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Transfer Fee", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.basefee", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\")*1000000 FROM \"chain.basefee\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cost of simple transfer [FIL]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "sci", + "label": "", + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 49 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Transfer Fee", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.basefee", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"chain.basefee\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Base Fee[FIL]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$network", + "decimals": null, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Precommit Transfer Fee", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.basefee", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\")*24000000 FROM \"chain.basefee\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "Commit Transfer Fee", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.basefee", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\")*56000000 FROM \"chain.basefee\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Message Gas fees [FIL]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 52, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "10 PIB PoSt Fee", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.basefee", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\")*940000000 FROM \"chain.basefee\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "750TiB miner PoSt Fee", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.basefee", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\")*580000000 FROM \"chain.basefee\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "10TiB miner PoSt Fee", + "groupBy": [ + { + "params": [ + "$blockInterval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.basefee", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\")*380000000 FROM \"chain.basefee\" WHERE $timeFilter GROUP BY time($blockInterval) fill(null)", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Message Gas fees [FIL]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "refresh": "45s", - "schemaVersion": 20, + "refresh": false, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "current": { + "selected": false, + "text": "filecoin-ntwk-testnet", + "value": "filecoin-ntwk-testnet" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Network", + "multi": false, + "name": "network", + "options": [], + "query": "influxdb", + "queryValue": "", + "refresh": 1, + "regex": "/^filecoin-ntwk-/", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "30s", + "value": "30s" + }, + "error": null, + "hide": 2, + "label": null, + "name": "blockInterval", + "options": [ + { + "selected": true, + "text": "30s", + "value": "30s" + } + ], + "query": "30s", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] }, "time": { "from": "now-30m", @@ -2515,6 +3469,7 @@ "refresh_intervals": [ "5s", "10s", + "25s", "30s", "45s", "1m", @@ -2527,7 +3482,7 @@ ] }, "timezone": "", - "title": "Chain", + "title": "Filecoin Chain Stats", "uid": "z6FtI92Zz", - "version": 9 + "version": 4 } diff --git a/cmd/lotus-stats/docker-compose.yml b/cmd/lotus-stats/docker-compose.yml index 03d573b94..b08a2157e 100644 --- a/cmd/lotus-stats/docker-compose.yml +++ b/cmd/lotus-stats/docker-compose.yml @@ -4,10 +4,10 @@ services: influxdb: image: influxdb:latest container_name: influxdb + ports: + - "18086:8086" environment: - INFLUXDB_DB=lotus - ports: - - "8086:8086" volumes: - influxdb:/var/lib/influxdb @@ -15,7 +15,7 @@ services: image: grafana/grafana:latest container_name: grafana ports: - - "3000:3000" + - "13000:3000" links: - influxdb volumes: diff --git a/cmd/lotus-stats/env.stats b/cmd/lotus-stats/env.stats index a76e7554a..ad5ec1619 100644 --- a/cmd/lotus-stats/env.stats +++ b/cmd/lotus-stats/env.stats @@ -1,3 +1,3 @@ -export INFLUX_ADDR="http://localhost:8086" +export INFLUX_ADDR="http://localhost:18086" export INFLUX_USER="" export INFLUX_PASS="" diff --git a/cmd/lotus-stats/main.go b/cmd/lotus-stats/main.go index 3ca139b7d..b3fce79bf 100644 --- a/cmd/lotus-stats/main.go +++ b/cmd/lotus-stats/main.go @@ -2,71 +2,158 @@ package main import ( "context" - "flag" "os" + "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/tools/stats" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" ) var log = logging.Logger("stats") -const ( - influxAddrEnvVar = "INFLUX_ADDR" - influxUserEnvVar = "INFLUX_USER" - influxPassEnvVar = "INFLUX_PASS" -) - func main() { - var repo string = "~/.lotus" - var database string = "lotus" - var reset bool = false - var nosync bool = false - var height int64 = 0 - var headlag int = 3 - - flag.StringVar(&repo, "repo", repo, "lotus repo path") - flag.StringVar(&database, "database", database, "influx database") - flag.Int64Var(&height, "height", height, "block height to start syncing from (0 will resume)") - flag.IntVar(&headlag, "head-lag", headlag, "number of head events to hold to protect against small reorgs") - flag.BoolVar(&reset, "reset", reset, "truncate database before starting stats gathering") - flag.BoolVar(&nosync, "nosync", nosync, "skip waiting for sync") - - flag.Parse() - - ctx := context.Background() - - influx, err := stats.InfluxClient(os.Getenv(influxAddrEnvVar), os.Getenv(influxUserEnvVar), os.Getenv(influxPassEnvVar)) - if err != nil { - log.Fatal(err) + local := []*cli.Command{ + runCmd, + versionCmd, } - if reset { - if err := stats.ResetDatabase(influx, database); err != nil { - log.Fatal(err) - } + app := &cli.App{ + Name: "lotus-stats", + Usage: "Collect basic information about a filecoin network using lotus", + Version: build.UserVersion(), + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "lotus-path", + EnvVars: []string{"LOTUS_PATH"}, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + &cli.StringFlag{ + Name: "log-level", + EnvVars: []string{"LOTUS_STATS_LOG_LEVEL"}, + Value: "info", + }, + }, + Before: func(cctx *cli.Context) error { + return logging.SetLogLevel("stats", cctx.String("log-level")) + }, + Commands: local, } - if !reset && height == 0 { - h, err := stats.GetLastRecordedHeight(influx, database) - if err != nil { - log.Info(err) - } - - height = h + if err := app.Run(os.Args); err != nil { + log.Errorw("exit in error", "err", err) + os.Exit(1) + return } - - api, closer, err := stats.GetFullNodeAPI(ctx, repo) - if err != nil { - log.Fatal(err) - } - defer closer() - - if !nosync { - if err := stats.WaitForSyncComplete(ctx, api); err != nil { - log.Fatal(err) - } - } - - stats.Collect(ctx, api, influx, database, height, headlag) +} + +var versionCmd = &cli.Command{ + Name: "version", + Usage: "Print version", + Action: func(cctx *cli.Context) error { + cli.VersionPrinter(cctx) + return nil + }, +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "influx-database", + EnvVars: []string{"LOTUS_STATS_INFLUX_DATABASE"}, + Usage: "influx database", + Value: "", + }, + &cli.StringFlag{ + Name: "influx-hostname", + EnvVars: []string{"LOTUS_STATS_INFLUX_HOSTNAME"}, + Value: "http://localhost:8086", + Usage: "influx hostname", + }, + &cli.StringFlag{ + Name: "influx-username", + EnvVars: []string{"LOTUS_STATS_INFLUX_USERNAME"}, + Usage: "influx username", + Value: "", + }, + &cli.StringFlag{ + Name: "influx-password", + EnvVars: []string{"LOTUS_STATS_INFLUX_PASSWORD"}, + Usage: "influx password", + Value: "", + }, + &cli.IntFlag{ + Name: "height", + EnvVars: []string{"LOTUS_STATS_HEIGHT"}, + Usage: "tipset height to start processing from", + Value: 0, + }, + &cli.IntFlag{ + Name: "head-lag", + EnvVars: []string{"LOTUS_STATS_HEAD_LAG"}, + Usage: "the number of tipsets to delay processing on to smooth chain reorgs", + Value: int(build.MessageConfidence), + }, + &cli.BoolFlag{ + Name: "no-sync", + EnvVars: []string{"LOTUS_STATS_NO_SYNC"}, + Usage: "do not wait for chain sync to complete", + Value: false, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + + resetFlag := cctx.Bool("reset") + noSyncFlag := cctx.Bool("no-sync") + heightFlag := cctx.Int("height") + headLagFlag := cctx.Int("head-lag") + + influxAddrFlag := cctx.String("influx-addr") + influxUserFlag := cctx.String("influx-user") + influxPassFlag := cctx.String("influx-pass") + influxDatabaseFlag := cctx.String("influx-database") + + influx, err := stats.InfluxClient(influxAddrFlag, influxUserFlag, influxPassFlag) + if err != nil { + log.Fatal(err) + } + + if resetFlag { + if err := stats.ResetDatabase(influx, influxDatabaseFlag); err != nil { + log.Fatal(err) + } + } + + height := int64(heightFlag) + + if !resetFlag && height == 0 { + h, err := stats.GetLastRecordedHeight(influx, influxDatabaseFlag) + if err != nil { + log.Info(err) + } + + height = h + } + + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + if !noSyncFlag { + if err := stats.WaitForSyncComplete(ctx, api); err != nil { + log.Fatal(err) + } + } + + stats.Collect(ctx, api, influx, influxDatabaseFlag, height, headLagFlag) + + return nil + }, } diff --git a/cmd/lotus-stats/setup.bash b/cmd/lotus-stats/setup.bash index e2812b93a..6510c2fc6 100755 --- a/cmd/lotus-stats/setup.bash +++ b/cmd/lotus-stats/setup.bash @@ -1,10 +1,10 @@ #!/usr/bin/env bash -GRAFANA_HOST="localhost:3000" +GRAFANA_HOST="http://localhost:13000" curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF { - "name":"InfluxDB", + "name":"filecoin-ntwk-localstats", "type":"influxdb", "database":"lotus", "url": "http://influxdb:8086", From fe95d19e2903dcf486a3f96a28f395083935dce7 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 4 Nov 2020 20:08:42 +0100 Subject: [PATCH 08/12] Move gas multiplier as property of pricelist Signed-off-by: Jakub Sztandera --- chain/vm/gas.go | 5 ++++- chain/vm/gas_v0.go | 12 +++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/chain/vm/gas.go b/chain/vm/gas.go index 95551f153..731f0c4d5 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -30,7 +30,7 @@ type GasCharge struct { } func (g GasCharge) Total() int64 { - return g.ComputeGas*GasComputeMulti + g.StorageGas*GasStorageMulti + return g.ComputeGas + g.StorageGas } func (g GasCharge) WithVirtual(compute, storage int64) GasCharge { out := g @@ -85,6 +85,9 @@ type Pricelist interface { var prices = map[abi.ChainEpoch]Pricelist{ abi.ChainEpoch(0): &pricelistV0{ + computeGasMulti: 1, + storageGasMulti: 1000, + onChainMessageComputeBase: 38863, onChainMessageStorageBase: 36, onChainMessageStoragePerByte: 1, diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index e4028039b..df3709058 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -18,6 +18,8 @@ type scalingCost struct { } type pricelistV0 struct { + computeGasMulti int64 + storageGasMulti int64 /////////////////////////////////////////////////////////////////////////// // System operations /////////////////////////////////////////////////////////////////////////// @@ -99,12 +101,12 @@ var _ Pricelist = (*pricelistV0)(nil) // OnChainMessage returns the gas used for storing a message of a given size in the chain. func (pl *pricelistV0) OnChainMessage(msgSize int) GasCharge { return newGasCharge("OnChainMessage", pl.onChainMessageComputeBase, - pl.onChainMessageStorageBase+pl.onChainMessageStoragePerByte*int64(msgSize)) + (pl.onChainMessageStorageBase+pl.onChainMessageStoragePerByte*int64(msgSize))*pl.storageGasMulti) } // OnChainReturnValue returns the gas used for storing the response of a message in the chain. func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge { - return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte) + return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte*pl.storageGasMulti) } // OnMethodInvocation returns the gas used when invoking a method. @@ -136,18 +138,18 @@ func (pl *pricelistV0) OnIpldGet() GasCharge { // OnIpldPut returns the gas used for storing an object func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge { - return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte). + return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte*pl.storageGasMulti). WithExtra(dataSize) } // OnCreateActor returns the gas used for creating an actor func (pl *pricelistV0) OnCreateActor() GasCharge { - return newGasCharge("OnCreateActor", pl.createActorCompute, pl.createActorStorage) + return newGasCharge("OnCreateActor", pl.createActorCompute, pl.createActorStorage*pl.storageGasMulti) } // OnDeleteActor returns the gas used for deleting an actor func (pl *pricelistV0) OnDeleteActor() GasCharge { - return newGasCharge("OnDeleteActor", 0, pl.deleteActor) + return newGasCharge("OnDeleteActor", 0, pl.deleteActor*pl.storageGasMulti) } // OnVerifySignature From 105f0fd49af6689b4babf537289105eda8695ddc Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 5 Nov 2020 04:21:03 -0500 Subject: [PATCH 09/12] Run kumquat upgrade on devnets --- build/params_2k.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/params_2k.go b/build/params_2k.go index 5a0e8fd61..6d1f3d46c 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -22,7 +22,7 @@ const UpgradeTapeHeight = -4 var UpgradeActorsV2Height = abi.ChainEpoch(10) var UpgradeLiftoffHeight = abi.ChainEpoch(-5) -const UpgradeKumquatHeight = -6 +const UpgradeKumquatHeight = 15 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, From 94438bf3cdc99d3290cde3fad41af825b753c85f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 5 Nov 2020 21:14:38 +0100 Subject: [PATCH 10/12] Use TSK passed to GasEstimateGasLimit Signed-off-by: Jakub Sztandera --- node/impl/full/gas.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 5d21121ee..13c344599 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -192,11 +192,19 @@ func gasEstimateGasPremium(cstore *store.ChainStore, nblocksincl uint64) (types. return premium, nil } -func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) { - return gasEstimateGasLimit(ctx, a.Chain, a.Stmgr, a.Mpool, msgIn) +func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, tsk types.TipSetKey) (int64, error) { + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return -1, xerrors.Errorf("getting tipset: %w", err) + } + return gasEstimateGasLimit(ctx, a.Chain, a.Stmgr, a.Mpool, msgIn, ts) } -func (m *GasModule) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) { - return gasEstimateGasLimit(ctx, m.Chain, m.Stmgr, m.Mpool, msgIn) +func (m *GasModule) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, tsk types.TipSetKey) (int64, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) + if err != nil { + return -1, xerrors.Errorf("getting tipset: %w", err) + } + return gasEstimateGasLimit(ctx, m.Chain, m.Stmgr, m.Mpool, msgIn, ts) } func gasEstimateGasLimit( ctx context.Context, @@ -204,13 +212,13 @@ func gasEstimateGasLimit( smgr *stmgr.StateManager, mpool *messagepool.MessagePool, msgIn *types.Message, + currTs *types.TipSet, ) (int64, error) { msg := *msgIn msg.GasLimit = build.BlockGasLimit msg.GasFeeCap = types.NewInt(uint64(build.MinimumBaseFee) + 1) msg.GasPremium = types.NewInt(1) - currTs := cstore.GetHeaviestTipSet() fromA, err := smgr.ResolveToKeyAddress(ctx, msgIn.From, currTs) if err != nil { return -1, xerrors.Errorf("getting key address: %w", err) From a3f5d0f5903f7be7d5a0247a0277822dbac5d3a8 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 6 Nov 2020 11:06:57 +0800 Subject: [PATCH 11/12] match data type for reward state api --- chain/actors/builtin/reward/v2.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/actors/builtin/reward/v2.go b/chain/actors/builtin/reward/v2.go index b7cb49102..c9a591532 100644 --- a/chain/actors/builtin/reward/v2.go +++ b/chain/actors/builtin/reward/v2.go @@ -28,7 +28,7 @@ type state2 struct { store adt.Store } -func (s *state2) ThisEpochReward() (abi.StoragePower, error) { +func (s *state2) ThisEpochReward() (abi.TokenAmount, error) { return s.State.ThisEpochReward, nil } @@ -55,11 +55,11 @@ func (s *state2) EffectiveNetworkTime() (abi.ChainEpoch, error) { return s.State.EffectiveNetworkTime, nil } -func (s *state2) CumsumBaseline() (abi.StoragePower, error) { +func (s *state2) CumsumBaseline() (reward2.Spacetime, error) { return s.State.CumsumBaseline, nil } -func (s *state2) CumsumRealized() (abi.StoragePower, error) { +func (s *state2) CumsumRealized() (reward2.Spacetime, error) { return s.State.CumsumRealized, nil } From ee9df632211e89a422faa4c37831157ab7e8af31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Nov 2020 16:20:57 +0100 Subject: [PATCH 12/12] shed: Util for creating ID CIDs --- cmd/lotus-shed/cid.go | 48 ++++++++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + 2 files changed, 49 insertions(+) create mode 100644 cmd/lotus-shed/cid.go diff --git a/cmd/lotus-shed/cid.go b/cmd/lotus-shed/cid.go new file mode 100644 index 000000000..7839ec601 --- /dev/null +++ b/cmd/lotus-shed/cid.go @@ -0,0 +1,48 @@ +package main + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + + "github.com/urfave/cli/v2" + + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +var cidCmd = &cli.Command{ + Name: "cid", + Subcommands: cli.Commands{ + cidIdCmd, + }, +} + +var cidIdCmd = &cli.Command{ + Name: "id", + Usage: "create identity CID from hex or base64 data", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return fmt.Errorf("must specify data") + } + + dec, err := hex.DecodeString(cctx.Args().First()) + if err != nil { + dec, err = base64.StdEncoding.DecodeString(cctx.Args().First()) + if err != nil { + return err + } + + } + + builder := cid.V1Builder{Codec: cid.Raw, MhType: mh.IDENTITY} + + c, err := builder.Sum(dec) + if err != nil { + return err + } + + fmt.Println(c) + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 33c7a159c..5da5c0188 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -48,6 +48,7 @@ func main() { msgCmd, electionCmd, rpcCmd, + cidCmd, } app := &cli.App{