Upgrade geth after schema changes (#97)

* Include intermediate nodes in builder tests for watched addresses

* Upgrade geth

README.md

@@ -161,6 +161,8 @@ An example config file:
     type = "postgres"
     driver = "sqlx"
     dumpDestination = ""
+    fileMode = "csv"
+    fileCsvDir = ""
     filePath = ""
 
 [cache]

cmd/env.go

@@ -68,10 +68,12 @@ const (
 	DATABASE_USER     = "DATABASE_USER"
 	DATABASE_PASSWORD = "DATABASE_PASSWORD"
 
-	DATABASE_TYPE        = "DATABASE_TYPE"
-	DATABASE_DRIVER_TYPE = "DATABASE_DRIVER_TYPE"
-	DATABASE_DUMP_DST    = "DATABASE_DUMP_DST"
-	DATABASE_FILE_PATH   = "DATABASE_FILE_PATH"
+	DATABASE_TYPE         = "DATABASE_TYPE"
+	DATABASE_DRIVER_TYPE  = "DATABASE_DRIVER_TYPE"
+	DATABASE_DUMP_DST     = "DATABASE_DUMP_DST"
+	DATABASE_FILE_PATH    = "DATABASE_FILE_PATH"
+	DATABASE_FILE_MODE    = "DATABASE_FILE_MODE"
+	DATABASE_FILE_CSV_DIR = "DATABASE_FILE_CSV_DIR"
 
 	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
 	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
@@ -109,7 +111,9 @@ func init() {
 	viper.BindEnv("database.type", DATABASE_TYPE)
 	viper.BindEnv("database.driver", DATABASE_DRIVER_TYPE)
 	viper.BindEnv("database.dumpDestination", DATABASE_DUMP_DST)
+	viper.BindEnv("database.fileMode", DATABASE_FILE_MODE)
 	viper.BindEnv("database.filePath", DATABASE_FILE_PATH)
+	viper.BindEnv("database.fileCsvDir", DATABASE_FILE_CSV_DIR)
 
 	viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
 	viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)

cmd/root.go

@@ -140,7 +140,9 @@ func init() {
 	rootCmd.PersistentFlags().String("database-type", "postgres", "database type (currently supported: postgres, dump)")
 	rootCmd.PersistentFlags().String("database-driver", "sqlx", "database driver type (currently supported: sqlx, pgx)")
 	rootCmd.PersistentFlags().String("database-dump-dst", "stdout", "dump destination (for database-type=dump; options: stdout, stderr, discard)")
-	rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-type=file)")
+	rootCmd.PersistentFlags().String("database-file-mode", "csv", "mode for writing file (for database-type=file; options: csv, sql)")
+	rootCmd.PersistentFlags().String("database-file-csv-dir", "", "full directory path (for database-file-mode=csv)")
+	rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-file-mode=sql)")
 
 	rootCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
 	rootCmd.PersistentFlags().String("eth-client-name", "eth-statediff-service", "eth client name")
@@ -198,6 +200,8 @@ func init() {
 	viper.BindPFlag("database.type", rootCmd.PersistentFlags().Lookup("database-type"))
 	viper.BindPFlag("database.driver", rootCmd.PersistentFlags().Lookup("database-driver"))
 	viper.BindPFlag("database.dumpDestination", rootCmd.PersistentFlags().Lookup("database-dump-dst"))
+	viper.BindPFlag("database.fileMode", rootCmd.PersistentFlags().Lookup("database-file-mode"))
+	viper.BindPFlag("database.fileCsvDir", rootCmd.PersistentFlags().Lookup("database-file-csv-dir"))
 	viper.BindPFlag("database.filePath", rootCmd.PersistentFlags().Lookup("database-file-path"))
 
 	viper.BindPFlag("ethereum.nodeID", rootCmd.PersistentFlags().Lookup("eth-node-id"))
@@ -302,11 +306,28 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 	switch dbType {
 	case shared.FILE:
 		logWithCommand.Info("starting in sql file writing mode")
+
+		fileModeStr := viper.GetString("database.fileMode")
+		fileMode, err := file.ResolveFileMode(fileModeStr)
+		if err != nil {
+			utils.Fatalf("%v", err)
+		}
+
 		filePathStr := viper.GetString("database.filePath")
-		if filePathStr == "" {
+		if fileMode == file.SQL && filePathStr == "" {
 			logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
 		}
-		indexerConfig = file.Config{FilePath: filePathStr}
+
+		fileCsvDirStr := viper.GetString("database.fileCsvDir")
+		if fileMode == file.CSV && fileCsvDirStr == "" {
+			logWithCommand.Fatal("when operating in csv file writing mode a directory path must be provided")
+		}
+
+		indexerConfig = file.Config{
+			Mode:      fileMode,
+			OutputDir: fileCsvDirStr,
+			FilePath:  filePathStr,
+		}
 	case shared.DUMP:
 		logWithCommand.Info("starting in data dump mode")
 		dumpDstStr := viper.GetString("database.dumpDestination")

environments/config.toml

@@ -39,6 +39,8 @@
     type = "postgres"
     driver = "sqlx"
     dumpDestination = ""
+    fileMode = "csv"
+    fileCsvDir = ""
     filePath = ""
 
 [cache]

go.mod

@@ -128,4 +128,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.3-alpha
+replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha

go.sum

@@ -727,8 +727,8 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/vulcanize/go-eth-state-node-iterator v1.1.1 h1:/9zKsUzF1zlQDriAljdspmp1ixcIpOcbetiyHCPhbU0=
 github.com/vulcanize/go-eth-state-node-iterator v1.1.1/go.mod h1:pL9jDztI5Yv9OOCKpCyo08/v+27othmbQlggoqH84Zo=
-github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.3-alpha h1:Pr6mSkWM7bobRJ/GzeztH+EOQR5IVLScO57G+N/q20g=
-github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.3-alpha/go.mod h1:5tMN+CDbK/qI2UlfN307HJykDmVIOCB1FM5RcHK9Kp8=
+github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha h1:8ge2ban6t/e53XDwe6s28jcCevT7Ggo51lNJ0Eo1PgA=
+github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha/go.mod h1:5tMN+CDbK/qI2UlfN307HJykDmVIOCB1FM5RcHK9Kp8=
 github.com/vulcanize/leveldb-ethdb-rpc v0.1.3 h1:R0+2GNMQKPXsKVWjTc6h5khD/aZzSarxxVsPEtBHiHU=
 github.com/vulcanize/leveldb-ethdb-rpc v0.1.3/go.mod h1:A3aI/tDglAHGpB9/vmtwS/mWOvfGl1UFJ1jCbhIPMAM=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=

pkg/builder.go

@@ -114,8 +114,7 @@ func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Para
 			go func(worker uint) {
 				defer wg.Done()
 				var err error
-				if !params.IntermediateStateNodes || len(params.WatchedAddresses) > 0 {
-					// if we are watching only specific accounts then we are only diffing leaf nodes
+				if !params.IntermediateStateNodes {
 					err = sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
 				} else {
 					err = sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)

pkg/builder_test.go

@@ -999,9 +999,11 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
 	block2 = blocks[1]
 	block3 = blocks[2]
 	params := sd.Params{
-		WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr},
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		WatchedAddresses:         []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr},
 	}
-	params.ComputeWatchedAddressesLeafKeys()
+	params.ComputeWatchedAddressesLeafPaths()
 
 	var tests = []struct {
 		name              string
@@ -1050,6 +1052,12 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
 				BlockNumber: block1.Number(),
 				BlockHash:   block1.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block1BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:         []byte{'\x0e'},
 						NodeType:     sdtypes.Leaf,
@@ -1074,12 +1082,23 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
 				BlockNumber: block2.Number(),
 				BlockHash:   block2.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block2BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:      []byte{'\x06'},
 						NodeType:  sdtypes.Leaf,
 						LeafKey:   contractLeafKey,
 						NodeValue: contractAccountAtBlock2LeafNode,
 						StorageNodes: []sdtypes.StorageNode{
+							{
+								Path:      []byte{},
+								NodeType:  sdtypes.Branch,
+								NodeValue: block2StorageBranchRootNode,
+							},
 							{
 								Path:      []byte{'\x02'},
 								NodeType:  sdtypes.Leaf,
@@ -1124,12 +1143,23 @@ func TestBuilderWithWatchedAddressList(t *testing.T) {
 				BlockNumber: block3.Number(),
 				BlockHash:   block3.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block3BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:      []byte{'\x06'},
 						NodeType:  sdtypes.Leaf,
 						LeafKey:   contractLeafKey,
 						NodeValue: contractAccountAtBlock3LeafNode,
 						StorageNodes: []sdtypes.StorageNode{
+							{
+								Path:      []byte{},
+								NodeType:  sdtypes.Branch,
+								NodeValue: block3StorageBranchRootNode,
+							},
 							{
 								Path:      []byte{'\x0c'},
 								NodeType:  sdtypes.Leaf,
@@ -1611,9 +1641,11 @@ func TestBuilderWithRemovedNonWatchedAccount(t *testing.T) {
 	block5 = blocks[4]
 	block6 = blocks[5]
 	params := sd.Params{
-		WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.Account2Addr},
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		WatchedAddresses:         []common.Address{test_helpers.Account1Addr, test_helpers.Account2Addr},
 	}
-	params.ComputeWatchedAddressesLeafKeys()
+	params.ComputeWatchedAddressesLeafPaths()
 
 	var tests = []struct {
 		name              string
@@ -1632,6 +1664,12 @@ func TestBuilderWithRemovedNonWatchedAccount(t *testing.T) {
 				BlockNumber: block4.Number(),
 				BlockHash:   block4.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block4BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:         []byte{'\x0c'},
 						NodeType:     sdtypes.Leaf,
@@ -1654,6 +1692,12 @@ func TestBuilderWithRemovedNonWatchedAccount(t *testing.T) {
 				BlockNumber: block5.Number(),
 				BlockHash:   block5.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block5BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:         []byte{'\x0e'},
 						NodeType:     sdtypes.Leaf,
@@ -1676,6 +1720,12 @@ func TestBuilderWithRemovedNonWatchedAccount(t *testing.T) {
 				BlockNumber: block6.Number(),
 				BlockHash:   block6.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block6BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:         []byte{'\x0c'},
 						NodeType:     sdtypes.Leaf,
@@ -1729,9 +1779,11 @@ func TestBuilderWithRemovedWatchedAccount(t *testing.T) {
 	block5 = blocks[4]
 	block6 = blocks[5]
 	params := sd.Params{
-		WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr},
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		WatchedAddresses:         []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr},
 	}
-	params.ComputeWatchedAddressesLeafKeys()
+	params.ComputeWatchedAddressesLeafPaths()
 
 	var tests = []struct {
 		name              string
@@ -1750,12 +1802,23 @@ func TestBuilderWithRemovedWatchedAccount(t *testing.T) {
 				BlockNumber: block4.Number(),
 				BlockHash:   block4.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block4BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:      []byte{'\x06'},
 						NodeType:  sdtypes.Leaf,
 						LeafKey:   contractLeafKey,
 						NodeValue: contractAccountAtBlock4LeafNode,
 						StorageNodes: []sdtypes.StorageNode{
+							{
+								Path:      []byte{},
+								NodeType:  sdtypes.Branch,
+								NodeValue: block4StorageBranchRootNode,
+							},
 							{
 								Path:      []byte{'\x04'},
 								NodeType:  sdtypes.Leaf,
@@ -1791,12 +1854,23 @@ func TestBuilderWithRemovedWatchedAccount(t *testing.T) {
 				BlockNumber: block5.Number(),
 				BlockHash:   block5.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block5BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:      []byte{'\x06'},
 						NodeType:  sdtypes.Leaf,
 						LeafKey:   contractLeafKey,
 						NodeValue: contractAccountAtBlock5LeafNode,
 						StorageNodes: []sdtypes.StorageNode{
+							{
+								Path:      []byte{},
+								NodeType:  sdtypes.Branch,
+								NodeValue: block5StorageBranchRootNode,
+							},
 							{
 								Path:      []byte{'\x0c'},
 								NodeType:  sdtypes.Leaf,
@@ -1833,12 +1907,23 @@ func TestBuilderWithRemovedWatchedAccount(t *testing.T) {
 				BlockNumber: block6.Number(),
 				BlockHash:   block6.Hash(),
 				Nodes: []sdtypes.StateNode{
+					{
+						Path:         []byte{},
+						NodeType:     sdtypes.Branch,
+						NodeValue:    block6BranchRootNode,
+						StorageNodes: emptyStorage,
+					},
 					{
 						Path:      []byte{'\x06'},
 						NodeType:  sdtypes.Removed,
 						LeafKey:   contractLeafKey,
 						NodeValue: []byte{},
 						StorageNodes: []sdtypes.StorageNode{
+							{
+								Path:      []byte{},
+								NodeType:  sdtypes.Removed,
+								NodeValue: []byte{},
+							},
 							{
 								Path:      []byte{'\x02'},
 								NodeType:  sdtypes.Removed,

pkg/service.go

@@ -195,8 +195,8 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 	}
 	logrus.Infof("sending state diff at block %d", blockNumber)
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	if blockNumber == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
@@ -217,8 +217,8 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
 	}
 	logrus.Infof("sending state diff at block %s", blockHash.Hex())
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	if currentBlock.NumberU64() == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
@@ -290,8 +290,8 @@ func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 	}
 	logrus.Infof("sending state trie at block %d", blockNumber)
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	return sds.processStateTrie(currentBlock, params)
 }
@@ -333,8 +333,8 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 		return err
 	}
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	parentRoot := common.Hash{}
 	if blockNumber != 0 {
@@ -358,8 +358,8 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) e
 		return err
 	}
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	parentRoot := common.Hash{}
 	if currentBlock.NumberU64() != 0 {