cerc-io/eth-statediff-service
16
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #59 from vulcanize/schema_updates

Browse Source 
reflect changes in integrated service
This commit is contained in:
Ian Norden 2021-11-18 19:25:36 -06:00 committed by GitHub
commit ddbb690ee6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 108 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
README.md
View File

@ -23,14 +23,13 @@ Available RPC methods are:
* `statediff_streamCodeAndCodeHash()`
* `statediff_stateDiffAt()`
* `statediff_writeStateDiffAt()`
* `statediff_writeStateDiffsInRange()`
### `write`
e.g. `curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"`
To write state diffs directly to a database:
`eth-statediff-service write --config=<config path>`
This depends on the `database` settings being properly configured.
The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranged and params in the
`prerun` section of the config as shown below.
## Configuration
@ -42,27 +41,56 @@ An example config file:
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
[server]
ipcPath = "~/.vulcanize/vulcanize.ipc"
ipcPath = ".ipc"
httpPath = "127.0.0.1:8545"
[write]
ranges = [[1, 2], [3, 4]]
[write.params]
IntermediateStateNodes = true
IntermediateStorageNodes = false
IncludeBlock = true
IncludeReceipts = true
IncludeTD = true
IncludeCode = false
[statediff]
workers = 4
prerun = true
serviceWorkers = 1
workerQueueSize = 1024
trieWorkers = 4
[prerun]
only = false
ranges = [
[0, 1000]
]
[prerun.params]
intermediateStateNodes = true
intermediateStorageNodes = true
includeBlock = true
includeReceipts = true
includeTD = true
includeCode = true
watchedAddresses = []
watchedStorageKeys = []
[log]
file = "~/.vulcanize/statediff.log"
file = ""
level = "info"
[eth]
chainID = 1
[database]
name = "vulcanize_test"
hostname = "localhost"
port = 5432
user = "vulcanize"
password = "..."
type = "postgres"
driver = "sqlx"
dumpDestination = ""
filePath = ""
[cache]
database = 1024
trie = 1024
[prom]
dbStats = false
metrics = true
http = true
httpAddr = "localhost"
httpPort = "8889"
```

2
cmd/env.go
View File

@ -62,6 +62,7 @@ const (
DATABASE_TYPE = "DATABASE_TYPE"
DATABASE_DRIVER_TYPE = "DATABASE_DRIVER_TYPE"
DATABASE_DUMP_DST = "DATABASE_DUMP_DST"
DATABASE_FILE_PATH = "DATABASE_FILE_PATH"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
@ -98,6 +99,7 @@ func init() {
viper.BindEnv("database.type", DATABASE_TYPE)
viper.BindEnv("database.driver", DATABASE_DRIVER_TYPE)
viper.BindEnv("database.dumpDestination", DATABASE_DUMP_DST)
viper.BindEnv("database.filePath", DATABASE_FILE_PATH)
viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)

11
cmd/root.go
View File

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
@ -137,6 +138,7 @@ func init() {
rootCmd.PersistentFlags().String("database-type", "postgres", "database type (currently supported: postgres, dump)")
rootCmd.PersistentFlags().String("database-driver", "sqlx", "database driver type (currently supported: sqlx, pgx)")
rootCmd.PersistentFlags().String("database-dump-dst", "stdout", "dump destination (for database-type=dump; options: stdout, stderr, discard)")
rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-type=file)")
rootCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
rootCmd.PersistentFlags().String("eth-client-name", "eth-statediff-service", "eth client name")
@ -280,7 +282,15 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
logWithCommand.Infof("configuring service for database type: %s", dbType)
var indexerConfig interfaces.Config
switch dbType {
case shared.FILE:
logWithCommand.Info("starting in sql file writing mode")
filePathStr := viper.GetString("database.filePath")
if filePathStr == "" {
logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
}
indexerConfig = file.Config{FilePath: filePathStr}
case shared.DUMP:
logWithCommand.Info("starting in data dump mode")
dumpDstStr := viper.GetString("database.dumpDestination")
dumpDst, err := dump.ResolveDumpType(dumpDstStr)
if err != nil {
@ -297,6 +307,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
}
case shared.POSTGRES:
logWithCommand.Info("starting in postgres mode")
driverTypeStr := viper.GetString("database.driver")
driverType, err := postgres.ResolveDriverType(driverTypeStr)
if err != nil {

1
environments/config.toml
View File

@ -41,6 +41,7 @@
type = "postgres"
driver = "sqlx"
dumpDestination = ""
filePath = ""
[cache]
database = 1024

1
environments/example.toml
View File

@ -43,6 +43,7 @@
type = "postgres"
driver = "sqlx"
dumpDestination = ""
filePath = ""
[cache]
database = 1024

2
go.mod
View File

@ -12,4 +12,4 @@ require (
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191
)
replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b
replace github.com/ethereum/go-ethereum v1.10.9 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c

4
go.sum
View File

@ -653,8 +653,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191 h1:+xSwb0xOLCAhejoai2BQZPTJW5Z+tU9xkTaOBwGnPoU=
github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191/go.mod h1:FBMQp69PTnRg3kLx9qeWGfKYfVq4fpfbr69X8lEvvfs=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b h1:TM6qCbWQCwBFTm2ZhZDKRtyjGPRU5ccSbDRp9tWHulY=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27b/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c h1:EnKLxZU0y1he4s/kmfbnLH5kr+QOaCUOf+yrBVfsq4Y=
github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27c/go.mod h1:XO9WLkNXfwoJN05BZj0//xgOWHJyUrUPdnudbQfKlUo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=

84
pkg/builder_test.go
View File

@ -479,7 +479,7 @@ func TestBuilder(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testEmptyDiff",
@ -489,7 +489,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -504,7 +504,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: []sdtypes.StateNode{
@ -527,7 +527,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -566,7 +566,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -637,7 +637,7 @@ func TestBuilder(t *testing.T) {
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{
@ -716,7 +716,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testEmptyDiff",
@ -726,7 +726,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -741,7 +741,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: []sdtypes.StateNode{
@ -764,7 +764,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -809,7 +809,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -891,7 +891,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) {
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{
@ -993,7 +993,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testEmptyDiff",
@ -1003,7 +1003,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -1018,7 +1018,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -1033,7 +1033,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -1057,7 +1057,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -1107,7 +1107,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{
@ -1171,7 +1171,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testEmptyDiff",
@ -1181,7 +1181,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -1196,7 +1196,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block0.Number(),
BlockHash: block0.Hash(),
Nodes: emptyDiffs,
@ -1211,7 +1211,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -1235,7 +1235,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -1279,7 +1279,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) {
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{
@ -1336,7 +1336,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
// blocks 0-3 are the same as in TestBuilderWithIntermediateNodes
{
@ -1347,7 +1347,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
BlockNumber: block4.Number(),
BlockHash: block4.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block4.Number(),
BlockHash: block4.Hash(),
Nodes: []sdtypes.StateNode{
@ -1413,7 +1413,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
BlockNumber: block5.Number(),
BlockHash: block5.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block5.Number(),
BlockHash: block5.Hash(),
Nodes: []sdtypes.StateNode{
@ -1474,7 +1474,7 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) {
BlockNumber: block6.Number(),
BlockHash: block6.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block6.Number(),
BlockHash: block6.Hash(),
Nodes: []sdtypes.StateNode{
@ -1550,7 +1550,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
// blocks 0-3 are the same as in TestBuilderWithIntermediateNodes
{
@ -1561,7 +1561,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.
BlockNumber: block4.Number(),
BlockHash: block4.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block4.Number(),
BlockHash: block4.Hash(),
Nodes: []sdtypes.StateNode{
@ -1616,7 +1616,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.
BlockNumber: block5.Number(),
BlockHash: block5.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block5.Number(),
BlockHash: block5.Hash(),
Nodes: []sdtypes.StateNode{
@ -1671,7 +1671,7 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.
BlockNumber: block6.Number(),
BlockHash: block6.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block6.Number(),
BlockHash: block6.Hash(),
Nodes: []sdtypes.StateNode{
@ -1821,7 +1821,7 @@ func TestBuilderWithMovedAccount(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testBlock1",
@ -1831,7 +1831,7 @@ func TestBuilderWithMovedAccount(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -1890,7 +1890,7 @@ func TestBuilderWithMovedAccount(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -1958,7 +1958,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) {
var tests = []struct {
name string
startingArguments statediff.Args
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testBlock1",
@ -1968,7 +1968,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) {
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -2016,7 +2016,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) {
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
},
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -2080,12 +2080,12 @@ func TestBuildStateTrie(t *testing.T) {
var tests = []struct {
name string
block *types.Block
expected *statediff.StateObject
expected *sdtypes.StateObject
}{
{
"testBlock1",
block1,
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block1.Number(),
BlockHash: block1.Hash(),
Nodes: []sdtypes.StateNode{
@ -2122,7 +2122,7 @@ func TestBuildStateTrie(t *testing.T) {
{
"testBlock2",
block2,
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block2.Number(),
BlockHash: block2.Hash(),
Nodes: []sdtypes.StateNode{
@ -2197,7 +2197,7 @@ func TestBuildStateTrie(t *testing.T) {
{
"testBlock3",
block3,
&statediff.StateObject{
&sdtypes.StateObject{
BlockNumber: block3.Number(),
BlockHash: block3.Hash(),
Nodes: []sdtypes.StateNode{

2
pkg/service.go
View File

@ -379,7 +379,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
}
// defer handling of commit/rollback for any return case
output := func(node sdtypes.StateNode) error {
return sds.indexer.PushStateNode(tx, node)
return sds.indexer.PushStateNode(tx, node, block.Hash().String())
}
codeOutput := func(c sdtypes.CodeAndCodeHash) error {
return sds.indexer.PushCodeAndCodeHash(tx, c)