diff --git a/cmd/env.go b/cmd/env.go
index eb5f7aa..8557727 100644
--- a/cmd/env.go
+++ b/cmd/env.go
@@ -28,6 +28,13 @@ const (
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
+
+ DB_CACHE_SIZE_MB = "DB_CACHE_SIZE_MB"
+ TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
+
+ WRITE_SERVER = "WRITE_SERVER"
+
+ STATEDIFF_WORKERS = "STATEDIFF_WORKERS"
)
// Bind env vars for eth node and DB configuration
@@ -46,4 +53,11 @@ func init() {
viper.BindEnv("database.maxIdle", pg.DATABASE_MAX_IDLE_CONNECTIONS)
viper.BindEnv("database.maxOpen", pg.DATABASE_MAX_OPEN_CONNECTIONS)
viper.BindEnv("database.maxLifetime", pg.DATABASE_MAX_CONN_LIFETIME)
+
+ viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
+ viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
+
+ viper.BindEnv("write.serve", WRITE_SERVER)
+
+ viper.BindEnv("statediff.workers", STATEDIFF_WORKERS)
}
diff --git a/cmd/root.go b/cmd/root.go
index c2c6811..ebb92d6 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -106,6 +106,8 @@ func init() {
"0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
rootCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
rootCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
+ rootCmd.PersistentFlags().Int("cache-db", 1024, "megabytes of memory allocated to database cache")
+ rootCmd.PersistentFlags().Int("cache-trie", 1024, "Megabytes of memory allocated to trie cache")
viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
@@ -122,6 +124,8 @@ func init() {
viper.BindPFlag("ethereum.genesisBlock", rootCmd.PersistentFlags().Lookup("eth-genesis-block"))
viper.BindPFlag("ethereum.networkID", rootCmd.PersistentFlags().Lookup("eth-network-id"))
viper.BindPFlag("ethereum.chainID", rootCmd.PersistentFlags().Lookup("eth-chain-id"))
+ viper.BindPFlag("cache.database", rootCmd.PersistentFlags().Lookup("cache-db"))
+ viper.BindPFlag("cache.trie", rootCmd.PersistentFlags().Lookup("cache-trie"))
}
func initConfig() {
diff --git a/cmd/serve.go b/cmd/serve.go
index 9add5b8..ba7baf5 100644
--- a/cmd/serve.go
+++ b/cmd/serve.go
@@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
+ "github.com/ethereum/go-ethereum/trie"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -66,7 +67,18 @@ func serve() {
// create leveldb reader
logWithCommand.Info("Creating leveldb reader")
- lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config)
+ conf := sd.ReaderConfig{
+ TrieConfig: &trie.Config{
+ Cache: viper.GetInt("cache.trie"),
+ Journal: "",
+ Preimages: false,
+ },
+ ChainConfig: config,
+ Path: path,
+ AncientPath: ancientPath,
+ DBCacheSize: viper.GetInt("cache.database"),
+ }
+ lvlDBReader, err := sd.NewLvlDBReader(conf)
if err != nil {
logWithCommand.Fatal(err)
}
diff --git a/cmd/write.go b/cmd/write.go
index 58d8686..f4d32e6 100644
--- a/cmd/write.go
+++ b/cmd/write.go
@@ -25,6 +25,7 @@ import (
gethsd "github.com/ethereum/go-ethereum/statediff"
ind "github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+ "github.com/ethereum/go-ethereum/trie"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -41,9 +42,7 @@ var writeCmd = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *logrus.WithField("SubCommand", subCommand)
-
- addr, _ := cmd.Flags().GetString("serve")
- write(addr)
+ write()
},
}
@@ -51,10 +50,11 @@ type blockRange [2]uint64
func init() {
rootCmd.AddCommand(writeCmd)
- writeCmd.Flags().String("serve", ":8888", "starts a server which handles write request through endpoints")
+ writeCmd.Flags().String("serve", "", "starts a server which handles write request through endpoints")
+ viper.BindPFlag("write.serve", writeCmd.PersistentFlags().Lookup("serve"))
}
-func write(addr string) {
+func write() {
logWithCommand.Info("Starting statediff writer")
// load params
@@ -72,7 +72,18 @@ func write(addr string) {
// create leveldb reader
logWithCommand.Info("Creating leveldb reader")
- lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config)
+ conf := sd.ReaderConfig{
+ TrieConfig: &trie.Config{
+ Cache: viper.GetInt("cache.trie"),
+ Journal: "",
+ Preimages: false,
+ },
+ ChainConfig: config,
+ Path: path,
+ AncientPath: ancientPath,
+ DBCacheSize: viper.GetInt("cache.database"),
+ }
+ lvlDBReader, err := sd.NewLvlDBReader(conf)
if err != nil {
logWithCommand.Fatal(err)
}
@@ -108,6 +119,7 @@ func write(addr string) {
viper.UnmarshalKey("write.params", &diffParams)
blockRangesCh := make(chan blockRange, 100)
+ addr := viper.GetString("write.serve")
go func() {
for _, r := range blockRanges {
blockRangesCh <- r
diff --git a/environments/example.toml b/environments/example.toml
index 4ece698..940a48a 100644
--- a/environments/example.toml
+++ b/environments/example.toml
@@ -13,6 +13,7 @@
ranges = [
[0, 0]
]
+ serve = ":8888"
[log]
file = ""
@@ -27,3 +28,7 @@
port = 5432
user = "vulcanize"
password = "..."
+
+[cache]
+ database = 1024
+ trie = 1024
diff --git a/pkg/builder_test.go b/pkg/builder_test.go
index 003abce..6111122 100644
--- a/pkg/builder_test.go
+++ b/pkg/builder_test.go
@@ -22,12 +22,11 @@ import (
"sort"
"testing"
- "github.com/ethereum/go-ethereum/statediff"
-
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/statediff"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
pkg "github.com/vulcanize/eth-statediff-service/pkg"
diff --git a/pkg/reader.go b/pkg/reader.go
index 6987eb4..bc88660 100644
--- a/pkg/reader.go
+++ b/pkg/reader.go
@@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/trie"
)
// LvlDBReader exposes the necessary read functions on lvldb
@@ -34,16 +35,24 @@ type LvlDBReader struct {
chainConfig *params.ChainConfig
}
+// ReaderConfig struct for initializing a LvlDBReader
+type ReaderConfig struct {
+ TrieConfig *trie.Config
+ ChainConfig *params.ChainConfig
+ Path, AncientPath string
+ DBCacheSize int
+}
+
// NewLvlDBReader creates a new LvlDBReader
-func NewLvlDBReader(path, ancient string, chainConfig *params.ChainConfig) (*LvlDBReader, error) {
- edb, err := rawdb.NewLevelDBDatabaseWithFreezer(path, 1024, 256, ancient, "eth-statediff-service", true)
+func NewLvlDBReader(conf ReaderConfig) (*LvlDBReader, error) {
+ edb, err := rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
if err != nil {
return nil, err
}
return &LvlDBReader{
ethDB: edb,
- stateDB: state.NewDatabase(edb),
- chainConfig: chainConfig,
+ stateDB: state.NewDatabaseWithConfig(edb, conf.TrieConfig),
+ chainConfig: conf.ChainConfig,
}, nil
}
diff --git a/pkg/testhelpers/helpers.go b/pkg/testhelpers/helpers.go
index 7fd320b..02ac018 100644
--- a/pkg/testhelpers/helpers.go
+++ b/pkg/testhelpers/helpers.go
@@ -32,6 +32,7 @@ import (
// the returned hash chain is ordered head->parent.
func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) {
config := params.TestChainConfig
+ config.LondonBlock.Set(big.NewInt(10))
blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
return blocks, chain
diff --git a/pkg/testhelpers/mocks/blockchain.go b/pkg/testhelpers/mocks/blockchain.go
deleted file mode 100644
index a995e47..0000000
--- a/pkg/testhelpers/mocks/blockchain.go
+++ /dev/null
@@ -1,128 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package mocks
-
-import (
- "errors"
- "math/big"
- "time"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/core"
- "github.com/ethereum/go-ethereum/core/types"
- "github.com/ethereum/go-ethereum/event"
-)
-
-// BlockChain is a mock blockchain for testing
-type BlockChain struct {
- HashesLookedUp []common.Hash
- blocksToReturnByHash map[common.Hash]*types.Block
- blocksToReturnByNumber map[uint64]*types.Block
- callCount int
- ChainEvents []core.ChainEvent
- Receipts map[common.Hash]types.Receipts
- TDByHash map[common.Hash]*big.Int
-}
-
-// SetBlocksForHashes mock method
-func (blockChain *BlockChain) SetBlocksForHashes(blocks map[common.Hash]*types.Block) {
- if blockChain.blocksToReturnByHash == nil {
- blockChain.blocksToReturnByHash = make(map[common.Hash]*types.Block)
- }
- blockChain.blocksToReturnByHash = blocks
-}
-
-// GetBlockByHash mock method
-func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
- blockChain.HashesLookedUp = append(blockChain.HashesLookedUp, hash)
-
- var block *types.Block
- if len(blockChain.blocksToReturnByHash) > 0 {
- block = blockChain.blocksToReturnByHash[hash]
- }
-
- return block
-}
-
-// SetChainEvents mock method
-func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) {
- blockChain.ChainEvents = chainEvents
-}
-
-// SubscribeChainEvent mock method
-func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
- subErr := errors.New("subscription error")
-
- var eventCounter int
- subscription := event.NewSubscription(func(quit <-chan struct{}) error {
- for _, chainEvent := range blockChain.ChainEvents {
- if eventCounter > 1 {
- time.Sleep(250 * time.Millisecond)
- return subErr
- }
- select {
- case ch <- chainEvent:
- case <-quit:
- return nil
- }
- eventCounter++
- }
- return nil
- })
-
- return subscription
-}
-
-// SetReceiptsForHash test method
-func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) {
- if blockChain.Receipts == nil {
- blockChain.Receipts = make(map[common.Hash]types.Receipts)
- }
- blockChain.Receipts[hash] = receipts
-}
-
-// GetReceiptsByHash mock method
-func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
- return blockChain.Receipts[hash]
-}
-
-// SetBlockForNumber test method
-func (blockChain *BlockChain) SetBlockForNumber(block *types.Block, number uint64) {
- if blockChain.blocksToReturnByNumber == nil {
- blockChain.blocksToReturnByNumber = make(map[uint64]*types.Block)
- }
- blockChain.blocksToReturnByNumber[number] = block
-}
-
-// GetBlockByNumber mock method
-func (blockChain *BlockChain) GetBlockByNumber(number uint64) *types.Block {
- return blockChain.blocksToReturnByNumber[number]
-}
-
-// GetTdByHash mock method
-func (blockChain *BlockChain) GetTdByHash(hash common.Hash) *big.Int {
- return blockChain.TDByHash[hash]
-}
-
-func (blockChain *BlockChain) SetTdByHash(hash common.Hash, td *big.Int) {
- if blockChain.TDByHash == nil {
- blockChain.TDByHash = make(map[common.Hash]*big.Int)
- }
- blockChain.TDByHash[hash] = td
-}
-
-func (blockChain *BlockChain) UnlockTrie(root common.Hash) {}
diff --git a/pkg/testhelpers/mocks/builder.go b/pkg/testhelpers/mocks/builder.go
deleted file mode 100644
index 55833a0..0000000
--- a/pkg/testhelpers/mocks/builder.go
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package mocks
-
-import (
- "github.com/ethereum/go-ethereum/core/types"
- sd "github.com/ethereum/go-ethereum/statediff/types"
-)
-
-// Builder is a mock state diff builder
-type Builder struct {
- Args sd.Args
- Params sd.Params
- stateDiff sd.StateObject
- block *types.Block
- stateTrie sd.StateObject
- builderError error
-}
-
-// BuildStateDiffObject mock method
-func (builder *Builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sd.StateObject, error) {
- builder.Args = args
- builder.Params = params
-
- return builder.stateDiff, builder.builderError
-}
-
-// BuildStateDiffObject mock method
-func (builder *Builder) WriteStateDiffObject(args sd.StateRoots, params sd.Params, output sd.StateNodeSink, codeOutput sd.CodeSink) error {
- builder.StateRoots = args
- builder.Params = params
-
- return builder.builderError
-}
-
-// BuildStateTrieObject mock method
-func (builder *Builder) BuildStateTrieObject(block *types.Block) (sd.StateObject, error) {
- builder.block = block
-
- return builder.stateTrie, builder.builderError
-}
-
-// SetStateDiffToBuild mock method
-func (builder *Builder) SetStateDiffToBuild(stateDiff sd.StateObject) {
- builder.stateDiff = stateDiff
-}
-
-// SetBuilderError mock method
-func (builder *Builder) SetBuilderError(err error) {
- builder.builderError = err
-}
diff --git a/pkg/testhelpers/test_data.go b/pkg/testhelpers/test_data.go
index cc5374d..044c401 100644
--- a/pkg/testhelpers/test_data.go
+++ b/pkg/testhelpers/test_data.go
@@ -18,7 +18,6 @@ package testhelpers
import (
"math/big"
- "math/rand"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
@@ -42,12 +41,7 @@ func AddressToEncodedPath(address common.Address) []byte {
// Test variables
var (
EvenLeafFlag = []byte{byte(2) << 4}
- BlockNumber = big.NewInt(rand.Int63())
- BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
NullCodeHash = crypto.Keccak256Hash([]byte{})
- StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
- StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
- StorageValue = common.Hex2Bytes("0x03")
NullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
Testdb = rawdb.NewMemoryDatabase()