From 108a6f89f126e0cae603eae119f2ce8910b087b7 Mon Sep 17 00:00:00 2001 From: i-norden Date: Thu, 18 Nov 2021 18:43:16 -0600 Subject: [PATCH] update to use new version of indexer; support file writing mode --- cmd/env.go | 2 + cmd/root.go | 11 +++++ environments/config.toml | 1 + environments/example.toml | 1 + go.mod | 2 +- go.sum | 4 +- pkg/builder_test.go | 84 +++++++++++++++++++-------------------- pkg/service.go | 2 +- 8 files changed, 61 insertions(+), 46 deletions(-) diff --git a/cmd/env.go b/cmd/env.go index 8c500fe..c2f8b8a 100644 --- a/cmd/env.go +++ b/cmd/env.go @@ -62,6 +62,7 @@ const ( DATABASE_TYPE = "DATABASE_TYPE" DATABASE_DRIVER_TYPE = "DATABASE_DRIVER_TYPE" DATABASE_DUMP_DST = "DATABASE_DUMP_DST" + DATABASE_FILE_PATH = "DATABASE_FILE_PATH" DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS" DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS" @@ -98,6 +99,7 @@ func init() { viper.BindEnv("database.type", DATABASE_TYPE) viper.BindEnv("database.driver", DATABASE_DRIVER_TYPE) viper.BindEnv("database.dumpDestination", DATABASE_DUMP_DST) + viper.BindEnv("database.filePath", DATABASE_FILE_PATH) viper.BindEnv("cache.database", DB_CACHE_SIZE_MB) viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB) diff --git a/cmd/root.go b/cmd/root.go index 5ea2dc2..cb3a163 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/statediff/indexer/database/dump" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/node" @@ -137,6 +138,7 @@ func init() { rootCmd.PersistentFlags().String("database-type", "postgres", "database type (currently supported: postgres, dump)") rootCmd.PersistentFlags().String("database-driver", "sqlx", "database driver type (currently supported: sqlx, pgx)") rootCmd.PersistentFlags().String("database-dump-dst", "stdout", "dump destination (for database-type=dump; options: stdout, stderr, discard)") + rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-type=file)") rootCmd.PersistentFlags().String("eth-node-id", "", "eth node id") rootCmd.PersistentFlags().String("eth-client-name", "eth-statediff-service", "eth client name") @@ -280,7 +282,15 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) { logWithCommand.Infof("configuring service for database type: %s", dbType) var indexerConfig interfaces.Config switch dbType { + case shared.FILE: + logWithCommand.Info("starting in sql file writing mode") + filePathStr := viper.GetString("database.filePath") + if filePathStr == "" { + logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided") + } + indexerConfig = file.Config{FilePath: filePathStr} case shared.DUMP: + logWithCommand.Info("starting in data dump mode") dumpDstStr := viper.GetString("database.dumpDestination") dumpDst, err := dump.ResolveDumpType(dumpDstStr) if err != nil { @@ -297,6 +307,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) { return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst) } case shared.POSTGRES: + logWithCommand.Info("starting in postgres mode") driverTypeStr := viper.GetString("database.driver") driverType, err := postgres.ResolveDriverType(driverTypeStr) if err != nil { diff --git a/environments/config.toml b/environments/config.toml index 99ae78e..903bf43 100644 --- a/environments/config.toml +++ b/environments/config.toml @@ -41,6 +41,7 @@ type = "postgres" driver = "sqlx" dumpDestination = "" + filePath = "" [cache] database = 1024 diff --git a/environments/example.toml b/environments/example.toml index 299e0f1..de087e7 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -43,6 +43,7 @@ type = "postgres" driver = "sqlx" dumpDestination = "" + filePath = "" [cache] database = 1024 diff --git a/go.mod b/go.mod index d632725..1778cfa 100644 --- a/go.mod +++ b/go.mod @@ -12,4 +12,4 @@ require ( github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191 ) -replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b +replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c diff --git a/go.sum b/go.sum index 618e972..91d7524 100644 --- a/go.sum +++ b/go.sum @@ -653,8 +653,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191 h1:+xSwb0xOLCAhejoai2BQZPTJW5Z+tU9xkTaOBwGnPoU= github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191/go.mod h1:FBMQp69PTnRg3kLx9qeWGfKYfVq4fpfbr69X8lEvvfs= -github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b h1:TM6qCbWQCwBFTm2ZhZDKRtyjGPRU5ccSbDRp9tWHulY= -github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo= +github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c h1:EnKLxZU0y1he4s/kmfbnLH5kr+QOaCUOf+yrBVfsq4Y= +github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= diff --git a/pkg/builder_test.go b/pkg/builder_test.go index 6111122..dbcdf18 100644 --- a/pkg/builder_test.go +++ b/pkg/builder_test.go @@ -479,7 +479,7 @@ func TestBuilder(t *testing.T) { var tests = []struct { name string startingArguments statediff.Args - expected *statediff.StateObject + expected *sdtypes.StateObject }{ { "testEmptyDiff", @@ -489,7 +489,7 @@ func TestBuilder(t *testing.T) { BlockNumber: block0.Number(), BlockHash: block0.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), Nodes: emptyDiffs, @@ -504,7 +504,7 @@ func TestBuilder(t *testing.T) { BlockNumber: block0.Number(), BlockHash: block0.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), Nodes: []sdtypes.StateNode{ @@ -527,7 +527,7 @@ func TestBuilder(t *testing.T) { BlockNumber: block1.Number(), BlockHash: block1.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), Nodes: []sdtypes.StateNode{ @@ -566,7 +566,7 @@ func TestBuilder(t *testing.T) { BlockNumber: block2.Number(), BlockHash: block2.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), Nodes: []sdtypes.StateNode{ @@ -637,7 +637,7 @@ func TestBuilder(t *testing.T) { BlockNumber: block3.Number(), BlockHash: block3.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), Nodes: []sdtypes.StateNode{ @@ -716,7 +716,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { var tests = []struct { name string startingArguments statediff.Args - expected *statediff.StateObject + expected *sdtypes.StateObject }{ { "testEmptyDiff", @@ -726,7 +726,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { BlockNumber: block0.Number(), BlockHash: block0.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), Nodes: emptyDiffs, @@ -741,7 +741,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { BlockNumber: block0.Number(), BlockHash: block0.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), Nodes: []sdtypes.StateNode{ @@ -764,7 +764,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { BlockNumber: block1.Number(), BlockHash: block1.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), Nodes: []sdtypes.StateNode{ @@ -809,7 +809,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { BlockNumber: block2.Number(), BlockHash: block2.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), Nodes: []sdtypes.StateNode{ @@ -891,7 +891,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { BlockNumber: block3.Number(), BlockHash: block3.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), Nodes: []sdtypes.StateNode{ @@ -993,7 +993,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { var tests = []struct { name string startingArguments statediff.Args - expected *statediff.StateObject + expected *sdtypes.StateObject }{ { "testEmptyDiff", @@ -1003,7 +1003,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { BlockNumber: block0.Number(), BlockHash: block0.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), Nodes: emptyDiffs, @@ -1018,7 +1018,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { BlockNumber: block0.Number(), BlockHash: block0.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), Nodes: emptyDiffs, @@ -1033,7 +1033,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { BlockNumber: block1.Number(), BlockHash: block1.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), Nodes: []sdtypes.StateNode{ @@ -1057,7 +1057,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { BlockNumber: block2.Number(), BlockHash: block2.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), Nodes: []sdtypes.StateNode{ @@ -1107,7 +1107,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { BlockNumber: block3.Number(), BlockHash: block3.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), Nodes: []sdtypes.StateNode{ @@ -1171,7 +1171,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { var tests = []struct { name string startingArguments statediff.Args - expected *statediff.StateObject + expected *sdtypes.StateObject }{ { "testEmptyDiff", @@ -1181,7 +1181,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { BlockNumber: block0.Number(), BlockHash: block0.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), Nodes: emptyDiffs, @@ -1196,7 +1196,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { BlockNumber: block0.Number(), BlockHash: block0.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), Nodes: emptyDiffs, @@ -1211,7 +1211,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { BlockNumber: block1.Number(), BlockHash: block1.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), Nodes: []sdtypes.StateNode{ @@ -1235,7 +1235,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { BlockNumber: block2.Number(), BlockHash: block2.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), Nodes: []sdtypes.StateNode{ @@ -1279,7 +1279,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { BlockNumber: block3.Number(), BlockHash: block3.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), Nodes: []sdtypes.StateNode{ @@ -1336,7 +1336,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { var tests = []struct { name string startingArguments statediff.Args - expected *statediff.StateObject + expected *sdtypes.StateObject }{ // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes { @@ -1347,7 +1347,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { BlockNumber: block4.Number(), BlockHash: block4.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block4.Number(), BlockHash: block4.Hash(), Nodes: []sdtypes.StateNode{ @@ -1413,7 +1413,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { BlockNumber: block5.Number(), BlockHash: block5.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block5.Number(), BlockHash: block5.Hash(), Nodes: []sdtypes.StateNode{ @@ -1474,7 +1474,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { BlockNumber: block6.Number(), BlockHash: block6.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block6.Number(), BlockHash: block6.Hash(), Nodes: []sdtypes.StateNode{ @@ -1550,7 +1550,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. var tests = []struct { name string startingArguments statediff.Args - expected *statediff.StateObject + expected *sdtypes.StateObject }{ // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes { @@ -1561,7 +1561,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. BlockNumber: block4.Number(), BlockHash: block4.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block4.Number(), BlockHash: block4.Hash(), Nodes: []sdtypes.StateNode{ @@ -1616,7 +1616,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. BlockNumber: block5.Number(), BlockHash: block5.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block5.Number(), BlockHash: block5.Hash(), Nodes: []sdtypes.StateNode{ @@ -1671,7 +1671,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. BlockNumber: block6.Number(), BlockHash: block6.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block6.Number(), BlockHash: block6.Hash(), Nodes: []sdtypes.StateNode{ @@ -1821,7 +1821,7 @@ func TestBuilderWithMovedAccount(t *testing.T) { var tests = []struct { name string startingArguments statediff.Args - expected *statediff.StateObject + expected *sdtypes.StateObject }{ { "testBlock1", @@ -1831,7 +1831,7 @@ func TestBuilderWithMovedAccount(t *testing.T) { BlockNumber: block1.Number(), BlockHash: block1.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), Nodes: []sdtypes.StateNode{ @@ -1890,7 +1890,7 @@ func TestBuilderWithMovedAccount(t *testing.T) { BlockNumber: block2.Number(), BlockHash: block2.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), Nodes: []sdtypes.StateNode{ @@ -1958,7 +1958,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { var tests = []struct { name string startingArguments statediff.Args - expected *statediff.StateObject + expected *sdtypes.StateObject }{ { "testBlock1", @@ -1968,7 +1968,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { BlockNumber: block1.Number(), BlockHash: block1.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), Nodes: []sdtypes.StateNode{ @@ -2016,7 +2016,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { BlockNumber: block2.Number(), BlockHash: block2.Hash(), }, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), Nodes: []sdtypes.StateNode{ @@ -2080,12 +2080,12 @@ func TestBuildStateTrie(t *testing.T) { var tests = []struct { name string block *types.Block - expected *statediff.StateObject + expected *sdtypes.StateObject }{ { "testBlock1", block1, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), Nodes: []sdtypes.StateNode{ @@ -2122,7 +2122,7 @@ func TestBuildStateTrie(t *testing.T) { { "testBlock2", block2, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), Nodes: []sdtypes.StateNode{ @@ -2197,7 +2197,7 @@ func TestBuildStateTrie(t *testing.T) { { "testBlock3", block3, - &statediff.StateObject{ + &sdtypes.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), Nodes: []sdtypes.StateNode{ diff --git a/pkg/service.go b/pkg/service.go index 9935462..f806ac1 100644 --- a/pkg/service.go +++ b/pkg/service.go @@ -379,7 +379,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p } // defer handling of commit/rollback for any return case output := func(node sdtypes.StateNode) error { - return sds.indexer.PushStateNode(tx, node) + return sds.indexer.PushStateNode(tx, node, block.Hash().String()) } codeOutput := func(c sdtypes.CodeAndCodeHash) error { return sds.indexer.PushCodeAndCodeHash(tx, c)