cli and toml bindings; fix unit tests; remove unused code

cmd/env.go

@@ -28,6 +28,13 @@ const (
 	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
 	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
 	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+
+	DB_CACHE_SIZE_MB   = "DB_CACHE_SIZE_MB"
+	TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
+
+	WRITE_SERVER = "WRITE_SERVER"
+
+	STATEDIFF_WORKERS = "STATEDIFF_WORKERS"
 )
 
 // Bind env vars for eth node and DB configuration
@@ -46,4 +53,11 @@ func init() {
 	viper.BindEnv("database.maxIdle", pg.DATABASE_MAX_IDLE_CONNECTIONS)
 	viper.BindEnv("database.maxOpen", pg.DATABASE_MAX_OPEN_CONNECTIONS)
 	viper.BindEnv("database.maxLifetime", pg.DATABASE_MAX_CONN_LIFETIME)
+
+	viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
+	viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
+
+	viper.BindEnv("write.serve", WRITE_SERVER)
+
+	viper.BindEnv("statediff.workers", STATEDIFF_WORKERS)
 }

cmd/root.go

@@ -106,6 +106,8 @@ func init() {
 		"0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
 	rootCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
 	rootCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
+	rootCmd.PersistentFlags().Int("cache-db", 1024, "megabytes of memory allocated to database cache")
+	rootCmd.PersistentFlags().Int("cache-trie", 1024, "Megabytes of memory allocated to trie cache")
 
 	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
@@ -122,6 +124,8 @@ func init() {
 	viper.BindPFlag("ethereum.genesisBlock", rootCmd.PersistentFlags().Lookup("eth-genesis-block"))
 	viper.BindPFlag("ethereum.networkID", rootCmd.PersistentFlags().Lookup("eth-network-id"))
 	viper.BindPFlag("ethereum.chainID", rootCmd.PersistentFlags().Lookup("eth-chain-id"))
+	viper.BindPFlag("cache.database", rootCmd.PersistentFlags().Lookup("cache-db"))
+	viper.BindPFlag("cache.trie", rootCmd.PersistentFlags().Lookup("cache-trie"))
 }
 
 func initConfig() {

cmd/serve.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/trie"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -66,7 +67,18 @@ func serve() {
 
 	// create leveldb reader
 	logWithCommand.Info("Creating leveldb reader")
-	lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config)
+	conf := sd.ReaderConfig{
+		TrieConfig: &trie.Config{
+			Cache:     viper.GetInt("cache.trie"),
+			Journal:   "",
+			Preimages: false,
+		},
+		ChainConfig: config,
+		Path:        path,
+		AncientPath: ancientPath,
+		DBCacheSize: viper.GetInt("cache.database"),
+	}
+	lvlDBReader, err := sd.NewLvlDBReader(conf)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}

cmd/write.go

@@ -25,6 +25,7 @@ import (
 	gethsd "github.com/ethereum/go-ethereum/statediff"
 	ind "github.com/ethereum/go-ethereum/statediff/indexer"
 	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+	"github.com/ethereum/go-ethereum/trie"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -41,9 +42,7 @@ var writeCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *logrus.WithField("SubCommand", subCommand)
-
+		write()
-		addr, _ := cmd.Flags().GetString("serve")
-		write(addr)
 	},
 }
 
@@ -51,10 +50,11 @@ type blockRange [2]uint64
 
 func init() {
 	rootCmd.AddCommand(writeCmd)
-	writeCmd.Flags().String("serve", ":8888", "starts a server which handles write request through endpoints")
+	writeCmd.Flags().String("serve", "", "starts a server which handles write request through endpoints")
+	viper.BindPFlag("write.serve", writeCmd.PersistentFlags().Lookup("serve"))
 }
 
-func write(addr string) {
+func write() {
 	logWithCommand.Info("Starting statediff writer")
 
 	// load params
@@ -72,7 +72,18 @@ func write(addr string) {
 
 	// create leveldb reader
 	logWithCommand.Info("Creating leveldb reader")
-	lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config)
+	conf := sd.ReaderConfig{
+		TrieConfig: &trie.Config{
+			Cache:     viper.GetInt("cache.trie"),
+			Journal:   "",
+			Preimages: false,
+		},
+		ChainConfig: config,
+		Path:        path,
+		AncientPath: ancientPath,
+		DBCacheSize: viper.GetInt("cache.database"),
+	}
+	lvlDBReader, err := sd.NewLvlDBReader(conf)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -108,6 +119,7 @@ func write(addr string) {
 	viper.UnmarshalKey("write.params", &diffParams)
 
 	blockRangesCh := make(chan blockRange, 100)
+	addr := viper.GetString("write.serve")
 	go func() {
 		for _, r := range blockRanges {
 			blockRangesCh <- r

environments/example.toml

@@ -13,6 +13,7 @@
     ranges = [
         [0, 0]
     ]
+    serve = ":8888"
 
 [log]
     file = ""
@@ -27,3 +28,7 @@
     port     = 5432
     user     = "vulcanize"
     password = "..."
+
+[cache]
+    database = 1024
+    trie = 1024

pkg/builder_test.go

@@ -22,12 +22,11 @@ import (
 	"sort"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/statediff"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	pkg "github.com/vulcanize/eth-statediff-service/pkg"

pkg/reader.go

@@ -37,10 +37,10 @@ type LvlDBReader struct {
 
 // ReaderConfig struct for initializing a LvlDBReader
 type ReaderConfig struct {
-	TrieConfig *trie.Config
+	TrieConfig        *trie.Config
-	ChainConfig *params.ChainConfig
+	ChainConfig       *params.ChainConfig
 	Path, AncientPath string
-	DBCacheSize int
+	DBCacheSize       int
 }
 
 // NewLvlDBReader creates a new LvlDBReader

pkg/testhelpers/helpers.go

@@ -32,6 +32,7 @@ import (
 // the returned hash chain is ordered head->parent.
 func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) {
 	config := params.TestChainConfig
+	config.LondonBlock.Set(big.NewInt(10))
 	blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
 	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
 	return blocks, chain

pkg/testhelpers/mocks/blockchain.go

@@ -1,128 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package mocks
-
-import (
-	"errors"
-	"math/big"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/event"
-)
-
-// BlockChain is a mock blockchain for testing
-type BlockChain struct {
-	HashesLookedUp         []common.Hash
-	blocksToReturnByHash   map[common.Hash]*types.Block
-	blocksToReturnByNumber map[uint64]*types.Block
-	callCount              int
-	ChainEvents            []core.ChainEvent
-	Receipts               map[common.Hash]types.Receipts
-	TDByHash               map[common.Hash]*big.Int
-}
-
-// SetBlocksForHashes mock method
-func (blockChain *BlockChain) SetBlocksForHashes(blocks map[common.Hash]*types.Block) {
-	if blockChain.blocksToReturnByHash == nil {
-		blockChain.blocksToReturnByHash = make(map[common.Hash]*types.Block)
-	}
-	blockChain.blocksToReturnByHash = blocks
-}
-
-// GetBlockByHash mock method
-func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
-	blockChain.HashesLookedUp = append(blockChain.HashesLookedUp, hash)
-
-	var block *types.Block
-	if len(blockChain.blocksToReturnByHash) > 0 {
-		block = blockChain.blocksToReturnByHash[hash]
-	}
-
-	return block
-}
-
-// SetChainEvents mock method
-func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) {
-	blockChain.ChainEvents = chainEvents
-}
-
-// SubscribeChainEvent mock method
-func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
-	subErr := errors.New("subscription error")
-
-	var eventCounter int
-	subscription := event.NewSubscription(func(quit <-chan struct{}) error {
-		for _, chainEvent := range blockChain.ChainEvents {
-			if eventCounter > 1 {
-				time.Sleep(250 * time.Millisecond)
-				return subErr
-			}
-			select {
-			case ch <- chainEvent:
-			case <-quit:
-				return nil
-			}
-			eventCounter++
-		}
-		return nil
-	})
-
-	return subscription
-}
-
-// SetReceiptsForHash test method
-func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) {
-	if blockChain.Receipts == nil {
-		blockChain.Receipts = make(map[common.Hash]types.Receipts)
-	}
-	blockChain.Receipts[hash] = receipts
-}
-
-// GetReceiptsByHash mock method
-func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
-	return blockChain.Receipts[hash]
-}
-
-// SetBlockForNumber test method
-func (blockChain *BlockChain) SetBlockForNumber(block *types.Block, number uint64) {
-	if blockChain.blocksToReturnByNumber == nil {
-		blockChain.blocksToReturnByNumber = make(map[uint64]*types.Block)
-	}
-	blockChain.blocksToReturnByNumber[number] = block
-}
-
-// GetBlockByNumber mock method
-func (blockChain *BlockChain) GetBlockByNumber(number uint64) *types.Block {
-	return blockChain.blocksToReturnByNumber[number]
-}
-
-// GetTdByHash mock method
-func (blockChain *BlockChain) GetTdByHash(hash common.Hash) *big.Int {
-	return blockChain.TDByHash[hash]
-}
-
-func (blockChain *BlockChain) SetTdByHash(hash common.Hash, td *big.Int) {
-	if blockChain.TDByHash == nil {
-		blockChain.TDByHash = make(map[common.Hash]*big.Int)
-	}
-	blockChain.TDByHash[hash] = td
-}
-
-func (blockChain *BlockChain) UnlockTrie(root common.Hash) {}

pkg/testhelpers/mocks/builder.go

@@ -1,65 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package mocks
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-	sd "github.com/ethereum/go-ethereum/statediff/types"
-)
-
-// Builder is a mock state diff builder
-type Builder struct {
-	Args         sd.Args
-	Params       sd.Params
-	stateDiff    sd.StateObject
-	block        *types.Block
-	stateTrie    sd.StateObject
-	builderError error
-}
-
-// BuildStateDiffObject mock method
-func (builder *Builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sd.StateObject, error) {
-	builder.Args = args
-	builder.Params = params
-
-	return builder.stateDiff, builder.builderError
-}
-
-// BuildStateDiffObject mock method
-func (builder *Builder) WriteStateDiffObject(args sd.StateRoots, params sd.Params, output sd.StateNodeSink, codeOutput sd.CodeSink) error {
-	builder.StateRoots = args
-	builder.Params = params
-
-	return builder.builderError
-}
-
-// BuildStateTrieObject mock method
-func (builder *Builder) BuildStateTrieObject(block *types.Block) (sd.StateObject, error) {
-	builder.block = block
-
-	return builder.stateTrie, builder.builderError
-}
-
-// SetStateDiffToBuild mock method
-func (builder *Builder) SetStateDiffToBuild(stateDiff sd.StateObject) {
-	builder.stateDiff = stateDiff
-}
-
-// SetBuilderError mock method
-func (builder *Builder) SetBuilderError(err error) {
-	builder.builderError = err
-}

pkg/testhelpers/test_data.go

@@ -18,7 +18,6 @@ package testhelpers
 
 import (
 	"math/big"
-	"math/rand"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -42,12 +41,7 @@ func AddressToEncodedPath(address common.Address) []byte {
 // Test variables
 var (
 	EvenLeafFlag = []byte{byte(2) << 4}
-	BlockNumber  = big.NewInt(rand.Int63())
-	BlockHash    = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
 	NullCodeHash = crypto.Keccak256Hash([]byte{})
-	StoragePath  = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
-	StorageKey   = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
-	StorageValue = common.Hex2Bytes("0x03")
 	NullHash     = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
 
 	Testdb          = rawdb.NewMemoryDatabase()