update to use new version of indexer; support file writing mode

This commit is contained in:
i-norden 2021-11-18 18:43:16 -06:00
parent d9eb68b2d2
commit 108a6f89f1
8 changed files with 61 additions and 46 deletions

View File

@ -62,6 +62,7 @@ const (
DATABASE_TYPE = "DATABASE_TYPE"
DATABASE_DRIVER_TYPE = "DATABASE_DRIVER_TYPE"
DATABASE_DUMP_DST = "DATABASE_DUMP_DST"
DATABASE_FILE_PATH = "DATABASE_FILE_PATH"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
@ -98,6 +99,7 @@ func init() {
viper.BindEnv("database.type", DATABASE_TYPE)
viper.BindEnv("database.driver", DATABASE_DRIVER_TYPE)
viper.BindEnv("database.dumpDestination", DATABASE_DUMP_DST)
viper.BindEnv("database.filePath", DATABASE_FILE_PATH)
viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)

View File

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
@ -137,6 +138,7 @@ func init() {
rootCmd.PersistentFlags().String("database-type", "postgres", "database type (currently supported: postgres, dump)")
rootCmd.PersistentFlags().String("database-driver", "sqlx", "database driver type (currently supported: sqlx, pgx)")
rootCmd.PersistentFlags().String("database-dump-dst", "stdout", "dump destination (for database-type=dump; options: stdout, stderr, discard)")
rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-type=file)")
rootCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
rootCmd.PersistentFlags().String("eth-client-name", "eth-statediff-service", "eth client name")
@ -280,7 +282,15 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
logWithCommand.Infof("configuring service for database type: %s", dbType)
var indexerConfig interfaces.Config
switch dbType {
case shared.FILE:
logWithCommand.Info("starting in sql file writing mode")
filePathStr := viper.GetString("database.filePath")
if filePathStr == "" {
logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
}
indexerConfig = file.Config{FilePath: filePathStr}
case shared.DUMP:
logWithCommand.Info("starting in data dump mode")
dumpDstStr := viper.GetString("database.dumpDestination")
dumpDst, err := dump.ResolveDumpType(dumpDstStr)
if err != nil {
@ -297,6 +307,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
}
case shared.POSTGRES:
logWithCommand.Info("starting in postgres mode")
driverTypeStr := viper.GetString("database.driver")
driverType, err := postgres.ResolveDriverType(driverTypeStr)
if err != nil {

View File

@ -41,6 +41,7 @@
type = "postgres"
driver = "sqlx"
dumpDestination = ""
filePath = ""
[cache]
database = 1024

View File

@ -43,6 +43,7 @@
type = "postgres"
driver = "sqlx"
dumpDestination = ""
filePath = ""
[cache]
database = 1024

2
go.mod
View File

@ -12,4 +12,4 @@ require (
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191
)
replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b
replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c

4
go.sum
View File

@ -653,8 +653,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191 h1:+xSwb0xOLCAhejoai2BQZPTJW5Z+tU9xkTaOBwGnPoU=
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191/go.mod h1:FBMQp69PTnRg3kLx9qeWGfKYfVq4fpfbr69X8lEvvfs=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b h1:TM6qCbWQCwBFTm2ZhZDKRtyjGPRU5ccSbDRp9tWHulY=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c h1:EnKLxZU0y1he4s/kmfbnLH5kr+QOaCUOf+yrBVfsq4Y=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=

View File

@ -479,7 +479,7 @@ func TestBuilder(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testEmptyDiff",
@ -489,7 +489,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -504,7 +504,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: []sdtypes.StateNode{
@ -527,7 +527,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -566,7 +566,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -637,7 +637,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{
@ -716,7 +716,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testEmptyDiff",
@ -726,7 +726,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -741,7 +741,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: []sdtypes.StateNode{
@ -764,7 +764,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -809,7 +809,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -891,7 +891,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{
@ -993,7 +993,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testEmptyDiff",
@ -1003,7 +1003,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -1018,7 +1018,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -1033,7 +1033,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -1057,7 +1057,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -1107,7 +1107,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{
@ -1171,7 +1171,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testEmptyDiff",
@ -1181,7 +1181,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -1196,7 +1196,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -1211,7 +1211,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -1235,7 +1235,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -1279,7 +1279,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{
@ -1336,7 +1336,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
// blocks 0-3 are the same as in TestBuilderWithIntermediateNodes
{
@ -1347,7 +1347,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
BlockNumber: block4.Number(),
BlockHash: block4.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block4.Number(),
BlockHash: block4.Hash(),
Nodes: []sdtypes.StateNode{
@ -1413,7 +1413,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
BlockNumber: block5.Number(),
BlockHash: block5.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block5.Number(),
BlockHash: block5.Hash(),
Nodes: []sdtypes.StateNode{
@ -1474,7 +1474,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
BlockNumber: block6.Number(),
BlockHash: block6.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block6.Number(),
BlockHash: block6.Hash(),
Nodes: []sdtypes.StateNode{
@ -1550,7 +1550,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
// blocks 0-3 are the same as in TestBuilderWithIntermediateNodes
{
@ -1561,7 +1561,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.
BlockNumber: block4.Number(),
BlockHash: block4.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block4.Number(),
BlockHash: block4.Hash(),
Nodes: []sdtypes.StateNode{
@ -1616,7 +1616,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.
BlockNumber: block5.Number(),
BlockHash: block5.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block5.Number(),
BlockHash: block5.Hash(),
Nodes: []sdtypes.StateNode{
@ -1671,7 +1671,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.
BlockNumber: block6.Number(),
BlockHash: block6.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block6.Number(),
BlockHash: block6.Hash(),
Nodes: []sdtypes.StateNode{
@ -1821,7 +1821,7 @@ func TestBuilderWithMovedAccount(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testBlock1",
@ -1831,7 +1831,7 @@ func TestBuilderWithMovedAccount(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -1890,7 +1890,7 @@ func TestBuilderWithMovedAccount(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -1958,7 +1958,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testBlock1",
@ -1968,7 +1968,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -2016,7 +2016,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -2080,12 +2080,12 @@ func TestBuildStateTrie(t *testing.T) {
var tests = []struct {
name string
block *types.Block
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testBlock1",
block1,
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -2122,7 +2122,7 @@ func TestBuildStateTrie(t *testing.T) {
{
"testBlock2",
block2,
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -2197,7 +2197,7 @@ func TestBuildStateTrie(t *testing.T) {
{
"testBlock3",
block3,
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{

View File

@ -379,7 +379,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
}
// defer handling of commit/rollback for any return case
output := func(node sdtypes.StateNode) error {
return sds.indexer.PushStateNode(tx, node)
return sds.indexer.PushStateNode(tx, node, block.Hash().String())
}
codeOutput := func(c sdtypes.CodeAndCodeHash) error {
return sds.indexer.PushCodeAndCodeHash(tx, c)