From fb37ee98814ac11497b55a73fbd7af91e233e18e Mon Sep 17 00:00:00 2001 From: i-norden Date: Tue, 19 Oct 2021 16:40:56 -0500 Subject: [PATCH] cli and toml bindings; fix unit tests; remove unused code --- cmd/env.go | 14 +++ cmd/root.go | 4 + cmd/serve.go | 14 ++- cmd/write.go | 24 ++++-- environments/example.toml | 5 ++ pkg/builder_test.go | 3 +- pkg/reader.go | 6 +- pkg/testhelpers/helpers.go | 1 + pkg/testhelpers/mocks/blockchain.go | 128 ---------------------------- pkg/testhelpers/mocks/builder.go | 65 -------------- pkg/testhelpers/test_data.go | 6 -- 11 files changed, 59 insertions(+), 211 deletions(-) delete mode 100644 pkg/testhelpers/mocks/blockchain.go delete mode 100644 pkg/testhelpers/mocks/builder.go diff --git a/cmd/env.go b/cmd/env.go index eb5f7aa..8557727 100644 --- a/cmd/env.go +++ b/cmd/env.go @@ -28,6 +28,13 @@ const ( ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK" ETH_NETWORK_ID = "ETH_NETWORK_ID" ETH_CHAIN_ID = "ETH_CHAIN_ID" + + DB_CACHE_SIZE_MB = "DB_CACHE_SIZE_MB" + TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB" + + WRITE_SERVER = "WRITE_SERVER" + + STATEDIFF_WORKERS = "STATEDIFF_WORKERS" ) // Bind env vars for eth node and DB configuration @@ -46,4 +53,11 @@ func init() { viper.BindEnv("database.maxIdle", pg.DATABASE_MAX_IDLE_CONNECTIONS) viper.BindEnv("database.maxOpen", pg.DATABASE_MAX_OPEN_CONNECTIONS) viper.BindEnv("database.maxLifetime", pg.DATABASE_MAX_CONN_LIFETIME) + + viper.BindEnv("cache.database", DB_CACHE_SIZE_MB) + viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB) + + viper.BindEnv("write.serve", WRITE_SERVER) + + viper.BindEnv("statediff.workers", STATEDIFF_WORKERS) } diff --git a/cmd/root.go b/cmd/root.go index c2c6811..ebb92d6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -106,6 +106,8 @@ func init() { "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash") rootCmd.PersistentFlags().String("eth-network-id", "1", "eth network id") rootCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id") + rootCmd.PersistentFlags().Int("cache-db", 1024, "megabytes of memory allocated to database cache") + rootCmd.PersistentFlags().Int("cache-trie", 1024, "Megabytes of memory allocated to trie cache") viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file")) viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) @@ -122,6 +124,8 @@ func init() { viper.BindPFlag("ethereum.genesisBlock", rootCmd.PersistentFlags().Lookup("eth-genesis-block")) viper.BindPFlag("ethereum.networkID", rootCmd.PersistentFlags().Lookup("eth-network-id")) viper.BindPFlag("ethereum.chainID", rootCmd.PersistentFlags().Lookup("eth-chain-id")) + viper.BindPFlag("cache.database", rootCmd.PersistentFlags().Lookup("cache-db")) + viper.BindPFlag("cache.trie", rootCmd.PersistentFlags().Lookup("cache-trie")) } func initConfig() { diff --git a/cmd/serve.go b/cmd/serve.go index 9add5b8..ba7baf5 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -66,7 +67,18 @@ func serve() { // create leveldb reader logWithCommand.Info("Creating leveldb reader") - lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config) + conf := sd.ReaderConfig{ + TrieConfig: &trie.Config{ + Cache: viper.GetInt("cache.trie"), + Journal: "", + Preimages: false, + }, + ChainConfig: config, + Path: path, + AncientPath: ancientPath, + DBCacheSize: viper.GetInt("cache.database"), + } + lvlDBReader, err := sd.NewLvlDBReader(conf) if err != nil { logWithCommand.Fatal(err) } diff --git a/cmd/write.go b/cmd/write.go index 58d8686..f4d32e6 100644 --- a/cmd/write.go +++ b/cmd/write.go @@ -25,6 +25,7 @@ import ( gethsd "github.com/ethereum/go-ethereum/statediff" ind "github.com/ethereum/go-ethereum/statediff/indexer" "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/trie" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -41,9 +42,7 @@ var writeCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { subCommand = cmd.CalledAs() logWithCommand = *logrus.WithField("SubCommand", subCommand) - - addr, _ := cmd.Flags().GetString("serve") - write(addr) + write() }, } @@ -51,10 +50,11 @@ type blockRange [2]uint64 func init() { rootCmd.AddCommand(writeCmd) - writeCmd.Flags().String("serve", ":8888", "starts a server which handles write request through endpoints") + writeCmd.Flags().String("serve", "", "starts a server which handles write request through endpoints") + viper.BindPFlag("write.serve", writeCmd.PersistentFlags().Lookup("serve")) } -func write(addr string) { +func write() { logWithCommand.Info("Starting statediff writer") // load params @@ -72,7 +72,18 @@ func write(addr string) { // create leveldb reader logWithCommand.Info("Creating leveldb reader") - lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config) + conf := sd.ReaderConfig{ + TrieConfig: &trie.Config{ + Cache: viper.GetInt("cache.trie"), + Journal: "", + Preimages: false, + }, + ChainConfig: config, + Path: path, + AncientPath: ancientPath, + DBCacheSize: viper.GetInt("cache.database"), + } + lvlDBReader, err := sd.NewLvlDBReader(conf) if err != nil { logWithCommand.Fatal(err) } @@ -108,6 +119,7 @@ func write(addr string) { viper.UnmarshalKey("write.params", &diffParams) blockRangesCh := make(chan blockRange, 100) + addr := viper.GetString("write.serve") go func() { for _, r := range blockRanges { blockRangesCh <- r diff --git a/environments/example.toml b/environments/example.toml index 4ece698..940a48a 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -13,6 +13,7 @@ ranges = [ [0, 0] ] + serve = ":8888" [log] file = "" @@ -27,3 +28,7 @@ port = 5432 user = "vulcanize" password = "..." + +[cache] + database = 1024 + trie = 1024 diff --git a/pkg/builder_test.go b/pkg/builder_test.go index 003abce..6111122 100644 --- a/pkg/builder_test.go +++ b/pkg/builder_test.go @@ -22,12 +22,11 @@ import ( "sort" "testing" - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" sdtypes "github.com/ethereum/go-ethereum/statediff/types" pkg "github.com/vulcanize/eth-statediff-service/pkg" diff --git a/pkg/reader.go b/pkg/reader.go index eb58aff..bc88660 100644 --- a/pkg/reader.go +++ b/pkg/reader.go @@ -37,10 +37,10 @@ type LvlDBReader struct { // ReaderConfig struct for initializing a LvlDBReader type ReaderConfig struct { - TrieConfig *trie.Config - ChainConfig *params.ChainConfig + TrieConfig *trie.Config + ChainConfig *params.ChainConfig Path, AncientPath string - DBCacheSize int + DBCacheSize int } // NewLvlDBReader creates a new LvlDBReader diff --git a/pkg/testhelpers/helpers.go b/pkg/testhelpers/helpers.go index 7fd320b..02ac018 100644 --- a/pkg/testhelpers/helpers.go +++ b/pkg/testhelpers/helpers.go @@ -32,6 +32,7 @@ import ( // the returned hash chain is ordered head->parent. func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) { config := params.TestChainConfig + config.LondonBlock.Set(big.NewInt(10)) blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen) chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) return blocks, chain diff --git a/pkg/testhelpers/mocks/blockchain.go b/pkg/testhelpers/mocks/blockchain.go deleted file mode 100644 index a995e47..0000000 --- a/pkg/testhelpers/mocks/blockchain.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mocks - -import ( - "errors" - "math/big" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" -) - -// BlockChain is a mock blockchain for testing -type BlockChain struct { - HashesLookedUp []common.Hash - blocksToReturnByHash map[common.Hash]*types.Block - blocksToReturnByNumber map[uint64]*types.Block - callCount int - ChainEvents []core.ChainEvent - Receipts map[common.Hash]types.Receipts - TDByHash map[common.Hash]*big.Int -} - -// SetBlocksForHashes mock method -func (blockChain *BlockChain) SetBlocksForHashes(blocks map[common.Hash]*types.Block) { - if blockChain.blocksToReturnByHash == nil { - blockChain.blocksToReturnByHash = make(map[common.Hash]*types.Block) - } - blockChain.blocksToReturnByHash = blocks -} - -// GetBlockByHash mock method -func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { - blockChain.HashesLookedUp = append(blockChain.HashesLookedUp, hash) - - var block *types.Block - if len(blockChain.blocksToReturnByHash) > 0 { - block = blockChain.blocksToReturnByHash[hash] - } - - return block -} - -// SetChainEvents mock method -func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) { - blockChain.ChainEvents = chainEvents -} - -// SubscribeChainEvent mock method -func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { - subErr := errors.New("subscription error") - - var eventCounter int - subscription := event.NewSubscription(func(quit <-chan struct{}) error { - for _, chainEvent := range blockChain.ChainEvents { - if eventCounter > 1 { - time.Sleep(250 * time.Millisecond) - return subErr - } - select { - case ch <- chainEvent: - case <-quit: - return nil - } - eventCounter++ - } - return nil - }) - - return subscription -} - -// SetReceiptsForHash test method -func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) { - if blockChain.Receipts == nil { - blockChain.Receipts = make(map[common.Hash]types.Receipts) - } - blockChain.Receipts[hash] = receipts -} - -// GetReceiptsByHash mock method -func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { - return blockChain.Receipts[hash] -} - -// SetBlockForNumber test method -func (blockChain *BlockChain) SetBlockForNumber(block *types.Block, number uint64) { - if blockChain.blocksToReturnByNumber == nil { - blockChain.blocksToReturnByNumber = make(map[uint64]*types.Block) - } - blockChain.blocksToReturnByNumber[number] = block -} - -// GetBlockByNumber mock method -func (blockChain *BlockChain) GetBlockByNumber(number uint64) *types.Block { - return blockChain.blocksToReturnByNumber[number] -} - -// GetTdByHash mock method -func (blockChain *BlockChain) GetTdByHash(hash common.Hash) *big.Int { - return blockChain.TDByHash[hash] -} - -func (blockChain *BlockChain) SetTdByHash(hash common.Hash, td *big.Int) { - if blockChain.TDByHash == nil { - blockChain.TDByHash = make(map[common.Hash]*big.Int) - } - blockChain.TDByHash[hash] = td -} - -func (blockChain *BlockChain) UnlockTrie(root common.Hash) {} diff --git a/pkg/testhelpers/mocks/builder.go b/pkg/testhelpers/mocks/builder.go deleted file mode 100644 index 55833a0..0000000 --- a/pkg/testhelpers/mocks/builder.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mocks - -import ( - "github.com/ethereum/go-ethereum/core/types" - sd "github.com/ethereum/go-ethereum/statediff/types" -) - -// Builder is a mock state diff builder -type Builder struct { - Args sd.Args - Params sd.Params - stateDiff sd.StateObject - block *types.Block - stateTrie sd.StateObject - builderError error -} - -// BuildStateDiffObject mock method -func (builder *Builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sd.StateObject, error) { - builder.Args = args - builder.Params = params - - return builder.stateDiff, builder.builderError -} - -// BuildStateDiffObject mock method -func (builder *Builder) WriteStateDiffObject(args sd.StateRoots, params sd.Params, output sd.StateNodeSink, codeOutput sd.CodeSink) error { - builder.StateRoots = args - builder.Params = params - - return builder.builderError -} - -// BuildStateTrieObject mock method -func (builder *Builder) BuildStateTrieObject(block *types.Block) (sd.StateObject, error) { - builder.block = block - - return builder.stateTrie, builder.builderError -} - -// SetStateDiffToBuild mock method -func (builder *Builder) SetStateDiffToBuild(stateDiff sd.StateObject) { - builder.stateDiff = stateDiff -} - -// SetBuilderError mock method -func (builder *Builder) SetBuilderError(err error) { - builder.builderError = err -} diff --git a/pkg/testhelpers/test_data.go b/pkg/testhelpers/test_data.go index cc5374d..044c401 100644 --- a/pkg/testhelpers/test_data.go +++ b/pkg/testhelpers/test_data.go @@ -18,7 +18,6 @@ package testhelpers import ( "math/big" - "math/rand" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -42,12 +41,7 @@ func AddressToEncodedPath(address common.Address) []byte { // Test variables var ( EvenLeafFlag = []byte{byte(2) << 4} - BlockNumber = big.NewInt(rand.Int63()) - BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" NullCodeHash = crypto.Keccak256Hash([]byte{}) - StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes() - StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() - StorageValue = common.Hex2Bytes("0x03") NullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") Testdb = rawdb.NewMemoryDatabase()