cerc-io/eth-statediff-service
15
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #30 from vulcanize/updates

Browse Source 
Updates
This commit is contained in:
Ian Norden 2021-10-19 17:25:50 -05:00 committed by GitHub
commit 4cbaa630c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 69 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/env.go
View File

@ -28,6 +28,13 @@ const (
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
DB_CACHE_SIZE_MB = "DB_CACHE_SIZE_MB"
TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
WRITE_SERVER = "WRITE_SERVER"
STATEDIFF_WORKERS = "STATEDIFF_WORKERS"
)
// Bind env vars for eth node and DB configuration
@ -46,4 +53,11 @@ func init() {
viper.BindEnv("database.maxIdle", pg.DATABASE_MAX_IDLE_CONNECTIONS)
viper.BindEnv("database.maxOpen", pg.DATABASE_MAX_OPEN_CONNECTIONS)
viper.BindEnv("database.maxLifetime", pg.DATABASE_MAX_CONN_LIFETIME)
viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
viper.BindEnv("write.serve", WRITE_SERVER)
viper.BindEnv("statediff.workers", STATEDIFF_WORKERS)
}

4
cmd/root.go
View File

@ -106,6 +106,8 @@ func init() {
"0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
rootCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
rootCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
rootCmd.PersistentFlags().Int("cache-db", 1024, "megabytes of memory allocated to database cache")
rootCmd.PersistentFlags().Int("cache-trie", 1024, "Megabytes of memory allocated to trie cache")
viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
@ -122,6 +124,8 @@ func init() {
viper.BindPFlag("ethereum.genesisBlock", rootCmd.PersistentFlags().Lookup("eth-genesis-block"))
viper.BindPFlag("ethereum.networkID", rootCmd.PersistentFlags().Lookup("eth-network-id"))
viper.BindPFlag("ethereum.chainID", rootCmd.PersistentFlags().Lookup("eth-chain-id"))
viper.BindPFlag("cache.database", rootCmd.PersistentFlags().Lookup("cache-db"))
viper.BindPFlag("cache.trie", rootCmd.PersistentFlags().Lookup("cache-trie"))
}
func initConfig() {

14
cmd/serve.go
View File

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@ -66,7 +67,18 @@ func serve() {
// create leveldb reader
logWithCommand.Info("Creating leveldb reader")
lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config)
conf := sd.ReaderConfig{
TrieConfig: &trie.Config{
Cache: viper.GetInt("cache.trie"),
Journal: "",
Preimages: false,
},
ChainConfig: config,
Path: path,
AncientPath: ancientPath,
DBCacheSize: viper.GetInt("cache.database"),
}
lvlDBReader, err := sd.NewLvlDBReader(conf)
if err != nil {
logWithCommand.Fatal(err)
}

24
cmd/write.go
View File

@ -25,6 +25,7 @@ import (
gethsd "github.com/ethereum/go-ethereum/statediff"
ind "github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/trie"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@ -41,9 +42,7 @@ var writeCmd = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *logrus.WithField("SubCommand", subCommand)
addr, _ := cmd.Flags().GetString("serve")
write(addr)
write()
},
}
@ -51,10 +50,11 @@ type blockRange [2]uint64
func init() {
rootCmd.AddCommand(writeCmd)
writeCmd.Flags().String("serve", ":8888", "starts a server which handles write request through endpoints")
writeCmd.Flags().String("serve", "", "starts a server which handles write request through endpoints")
viper.BindPFlag("write.serve", writeCmd.PersistentFlags().Lookup("serve"))
}
func write(addr string) {
func write() {
logWithCommand.Info("Starting statediff writer")
// load params
@ -72,7 +72,18 @@ func write(addr string) {
// create leveldb reader
logWithCommand.Info("Creating leveldb reader")
lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config)
conf := sd.ReaderConfig{
TrieConfig: &trie.Config{
Cache: viper.GetInt("cache.trie"),
Journal: "",
Preimages: false,
},
ChainConfig: config,
Path: path,
AncientPath: ancientPath,
DBCacheSize: viper.GetInt("cache.database"),
}
lvlDBReader, err := sd.NewLvlDBReader(conf)
if err != nil {
logWithCommand.Fatal(err)
}
@ -108,6 +119,7 @@ func write(addr string) {
viper.UnmarshalKey("write.params", &diffParams)
blockRangesCh := make(chan blockRange, 100)
addr := viper.GetString("write.serve")
go func() {
for _, r := range blockRanges {
blockRangesCh <- r

5
environments/example.toml
View File

@ -13,6 +13,7 @@
ranges = [
[0, 0]
]
serve = ":8888"
[log]
file = ""
@ -27,3 +28,7 @@
port = 5432
user = "vulcanize"
password = "..."
[cache]
database = 1024
trie = 1024

3
pkg/builder_test.go
View File

@ -22,12 +22,11 @@ import (
"sort"
"testing"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
pkg "github.com/vulcanize/eth-statediff-service/pkg"

17
pkg/reader.go
View File

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
// LvlDBReader exposes the necessary read functions on lvldb
@ -34,16 +35,24 @@ type LvlDBReader struct {
chainConfig *params.ChainConfig
}
// ReaderConfig struct for initializing a LvlDBReader
type ReaderConfig struct {
TrieConfig *trie.Config
ChainConfig *params.ChainConfig
Path, AncientPath string
DBCacheSize int
}
// NewLvlDBReader creates a new LvlDBReader
func NewLvlDBReader(path, ancient string, chainConfig *params.ChainConfig) (*LvlDBReader, error) {
edb, err := rawdb.NewLevelDBDatabaseWithFreezer(path, 1024, 256, ancient, "eth-statediff-service", true)
func NewLvlDBReader(conf ReaderConfig) (*LvlDBReader, error) {
edb, err := rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
if err != nil {
return nil, err
}
return &LvlDBReader{
ethDB: edb,
stateDB: state.NewDatabase(edb),
chainConfig: chainConfig,
stateDB: state.NewDatabaseWithConfig(edb, conf.TrieConfig),
chainConfig: conf.ChainConfig,
}, nil
}

1
pkg/testhelpers/helpers.go
View File

@ -32,6 +32,7 @@ import (
// the returned hash chain is ordered head->parent.
func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) {
config := params.TestChainConfig
config.LondonBlock.Set(big.NewInt(10))
blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
return blocks, chain

128
pkg/testhelpers/mocks/blockchain.go
View File

@ -1,128 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mocks
import (
"errors"
"math/big"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/event"
)
// BlockChain is a mock blockchain for testing
type BlockChain struct {
HashesLookedUp []common.Hash
blocksToReturnByHash map[common.Hash]*types.Block
blocksToReturnByNumber map[uint64]*types.Block
callCount int
ChainEvents []core.ChainEvent
Receipts map[common.Hash]types.Receipts
TDByHash map[common.Hash]*big.Int
}
// SetBlocksForHashes mock method
func (blockChain *BlockChain) SetBlocksForHashes(blocks map[common.Hash]*types.Block) {
if blockChain.blocksToReturnByHash == nil {
blockChain.blocksToReturnByHash = make(map[common.Hash]*types.Block)
}
blockChain.blocksToReturnByHash = blocks
}
// GetBlockByHash mock method
func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
blockChain.HashesLookedUp = append(blockChain.HashesLookedUp, hash)
var block *types.Block
if len(blockChain.blocksToReturnByHash) > 0 {
block = blockChain.blocksToReturnByHash[hash]
}
return block
}
// SetChainEvents mock method
func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) {
blockChain.ChainEvents = chainEvents
}
// SubscribeChainEvent mock method
func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
subErr := errors.New("subscription error")
var eventCounter int
subscription := event.NewSubscription(func(quit <-chan struct{}) error {
for _, chainEvent := range blockChain.ChainEvents {
if eventCounter > 1 {
time.Sleep(250 * time.Millisecond)
return subErr
}
select {
case ch <- chainEvent:
case <-quit:
return nil
}
eventCounter++
}
return nil
})
return subscription
}
// SetReceiptsForHash test method
func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) {
if blockChain.Receipts == nil {
blockChain.Receipts = make(map[common.Hash]types.Receipts)
}
blockChain.Receipts[hash] = receipts
}
// GetReceiptsByHash mock method
func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
return blockChain.Receipts[hash]
}
// SetBlockForNumber test method
func (blockChain *BlockChain) SetBlockForNumber(block *types.Block, number uint64) {
if blockChain.blocksToReturnByNumber == nil {
blockChain.blocksToReturnByNumber = make(map[uint64]*types.Block)
}
blockChain.blocksToReturnByNumber[number] = block
}
// GetBlockByNumber mock method
func (blockChain *BlockChain) GetBlockByNumber(number uint64) *types.Block {
return blockChain.blocksToReturnByNumber[number]
}
// GetTdByHash mock method
func (blockChain *BlockChain) GetTdByHash(hash common.Hash) *big.Int {
return blockChain.TDByHash[hash]
}
func (blockChain *BlockChain) SetTdByHash(hash common.Hash, td *big.Int) {
if blockChain.TDByHash == nil {
blockChain.TDByHash = make(map[common.Hash]*big.Int)
}
blockChain.TDByHash[hash] = td
}
func (blockChain *BlockChain) UnlockTrie(root common.Hash) {}

65
pkg/testhelpers/mocks/builder.go
View File

@ -1,65 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mocks
import (
"github.com/ethereum/go-ethereum/core/types"
sd "github.com/ethereum/go-ethereum/statediff/types"
)
// Builder is a mock state diff builder
type Builder struct {
Args sd.Args
Params sd.Params
stateDiff sd.StateObject
block *types.Block
stateTrie sd.StateObject
builderError error
}
// BuildStateDiffObject mock method
func (builder *Builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sd.StateObject, error) {
builder.Args = args
builder.Params = params
return builder.stateDiff, builder.builderError
}
// BuildStateDiffObject mock method
func (builder *Builder) WriteStateDiffObject(args sd.StateRoots, params sd.Params, output sd.StateNodeSink, codeOutput sd.CodeSink) error {
builder.StateRoots = args
builder.Params = params
return builder.builderError
}
// BuildStateTrieObject mock method
func (builder *Builder) BuildStateTrieObject(block *types.Block) (sd.StateObject, error) {
builder.block = block
return builder.stateTrie, builder.builderError
}
// SetStateDiffToBuild mock method
func (builder *Builder) SetStateDiffToBuild(stateDiff sd.StateObject) {
builder.stateDiff = stateDiff
}
// SetBuilderError mock method
func (builder *Builder) SetBuilderError(err error) {
builder.builderError = err
}

6
pkg/testhelpers/test_data.go
View File

@ -18,7 +18,6 @@ package testhelpers
import (
"math/big"
"math/rand"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
@ -42,12 +41,7 @@ func AddressToEncodedPath(address common.Address) []byte {
// Test variables
var (
EvenLeafFlag = []byte{byte(2) << 4}
BlockNumber = big.NewInt(rand.Int63())
BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
NullCodeHash = crypto.Keccak256Hash([]byte{})
StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
StorageValue = common.Hex2Bytes("0x03")
NullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
Testdb = rawdb.NewMemoryDatabase()