Merge pull request #84 from deep-stack/ng-update-geth

Update vulcanize geth version to 1.10.17

.github/workflows/publish.yaml

@@ -7,19 +7,23 @@ jobs:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v2
       - name: Get the version
-        id: vars
         run: |
-          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
           echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Docker Login to Github Registry
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Pull
-        run: docker pull docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+          echo ::set-output name=sha::$(echo ${GITHUB_SHA})
+        id: vars
       - name: Docker Login to Docker Registry
-        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
-      - name: Tag docker image
-        run: docker tag docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} vulcanize/eth-statediff-service:${{steps.vars.outputs.tag}}
-      - name: Docker Push to Docker Hub
-        run: docker push vulcanize/eth-statediff-service:${{steps.vars.outputs.tag}}
+        uses: docker/login-action@v1
+        with:
+          username: vulcanizejenkins
+          password: ${{ secrets.VULCANIZEJENKINS_PAT }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: vulcanize/eth-statediff-service:${{ steps.vars.outputs.tag }}
+          build-args: |
+            GIT_COMMIT=${{ steps.vars.outputs.sha }}
 

Makefile

@@ -1,4 +1,12 @@
 ## Build docker image
 .PHONY: docker-build
 docker-build:
-	docker build -t vulcanize/eth-statediff-service .
+	docker build -t vulcanize/eth-statediff-service .
+
+.PHONY: test
+test: | $(GOOSE)
+	go test -p 1 ./pkg/... -v
+
+build:
+	go fmt ./...
+	go build

README.md

@@ -10,6 +10,69 @@ Stand up a statediffing service directly on top of a go-ethereum leveldb instanc
 This service can serve historical state data over the same rpc interface as
 [statediffing geth](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.5) without needing to run a full node
 
+## Setup
+
+Build the binary:
+
+```bash
+make build
+```
+
+### Local Setup
+
+* Create a chain config file `chain.json` according to chain config in genesis json file used by local geth.
+
+  Example:
+  ```json
+  {
+    "chainId": 41337,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "clique": {
+      "period": 5,
+      "epoch": 30000
+    }
+  }
+  ```
+
+* Change the following in [config file](./environments/config.toml)
+
+    ```toml
+    [leveldb]
+        # Path to geth leveldb data
+        path = "/path-to-local-geth-data/chaindata"
+        ancient = "/path-to-local-geth-data/chaindata/ancient"
+
+    [ethereum]
+        chainConfig = "./chain.json" # Path to custom chain  config file
+        chainID = 41337 # Same chain ID as in chain.json
+
+    [database]
+        # Update database config
+        name     = "vulcanize_testing"
+        hostname = "localhost"
+        port     = 5432
+        user     = "postgres"
+        password = "postgres"
+        type = "postgres"
+    ```
+
+* To write statediff for a range of block make changes in [config file](./environments/config.toml)
+    ```toml
+    [prerun]
+        only = false
+        ranges = [
+            [8, 15] # Block number range for which to write statediff.
+        ]
+    ```
+
 ## Usage
 
 ### `serve`
@@ -18,6 +81,12 @@ To serve state diffs over RPC:
 
 `eth-statediff-service serve --config=<config path>`
 
+Example:
+
+```bash
+./eth-statediff-service serve --config environments/config.toml
+```
+
 Available RPC methods are:
   * `statediff_stateTrieAt()`
   * `statediff_streamCodeAndCodeHash()`
@@ -63,7 +132,6 @@ An example config file:
         includeTD = true
         includeCode = true
         watchedAddresses = []
-        watchedStorageKeys = []
 
 [log]
     file = ""

cmd/env.go

@@ -26,6 +26,7 @@ const (
 	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
 	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
 	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+	ETH_CHAIN_CONFIG  = "ETH_CHAIN_CONFIG"
 
 	DB_CACHE_SIZE_MB   = "DB_CACHE_SIZE_MB"
 	TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
@@ -46,15 +47,15 @@ const (
 	PROM_HTTP_PORT = "PROM_HTTP_PORT"
 	PROM_DB_STATS  = "PROM_DB_STATS"
 
-	PRERUN_ONLY        = "PRERUN_ONLY"
-	PRERUN_RANGE_START = "PRERUN_RANGE_START"
-	PRERUN_RANGE_STOP  = "PRERUN_RANGE_STOP"
-	PRERUN_INTERMEDIATE_STATE_NODES = "PRERUN_INTERMEDIATE_STATE_NODES"
+	PRERUN_ONLY                       = "PRERUN_ONLY"
+	PRERUN_RANGE_START                = "PRERUN_RANGE_START"
+	PRERUN_RANGE_STOP                 = "PRERUN_RANGE_STOP"
+	PRERUN_INTERMEDIATE_STATE_NODES   = "PRERUN_INTERMEDIATE_STATE_NODES"
 	PRERUN_INTERMEDIATE_STORAGE_NODES = "PRERUN_INTERMEDIATE_STORAGE_NODES"
-	PRERUN_INCLUDE_BLOCK = "PRERUN_INCLUDE_BLOCK"
-	PRERUN_INCLUDE_RECEIPTS = "PRERUN_INCLUDE_RECEIPTS"
-	PRERUN_INCLUDE_TD = "PRERUN_INCLUDE_TD"
-	PRERUN_INCLUDE_CODE = "PRERUN_INCLUDE_CODE"
+	PRERUN_INCLUDE_BLOCK              = "PRERUN_INCLUDE_BLOCK"
+	PRERUN_INCLUDE_RECEIPTS           = "PRERUN_INCLUDE_RECEIPTS"
+	PRERUN_INCLUDE_TD                 = "PRERUN_INCLUDE_TD"
+	PRERUN_INCLUDE_CODE               = "PRERUN_INCLUDE_CODE"
 
 	LOG_LEVEL     = "LOG_LEVEL"
 	LOG_FILE_PATH = "LOG_FILE_PATH"
@@ -88,6 +89,7 @@ func init() {
 	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
 	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
 	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
 
 	viper.BindEnv("database.name", DATABASE_NAME)
 	viper.BindEnv("database.hostname", DATABASE_HOSTNAME)

cmd/root.go

@@ -166,7 +166,6 @@ func init() {
 	rootCmd.PersistentFlags().Bool("prerun-include-td", true, "include td in the statediff payload")
 	rootCmd.PersistentFlags().Bool("prerun-include-code", true, "include code and codehash mappings in statediff payload")
 
-
 	viper.BindPFlag("server.httpPath", rootCmd.PersistentFlags().Lookup("http-path"))
 	viper.BindPFlag("server.ipcPath", rootCmd.PersistentFlags().Lookup("ipc-path"))
 
@@ -202,6 +201,7 @@ func init() {
 	viper.BindPFlag("ethereum.genesisBlock", rootCmd.PersistentFlags().Lookup("eth-genesis-block"))
 	viper.BindPFlag("ethereum.networkID", rootCmd.PersistentFlags().Lookup("eth-network-id"))
 	viper.BindPFlag("ethereum.chainID", rootCmd.PersistentFlags().Lookup("eth-chain-id"))
+	viper.BindPFlag("ethereum.chainConfig", rootCmd.PersistentFlags().Lookup("eth-chain-config"))
 
 	viper.BindPFlag("cache.database", rootCmd.PersistentFlags().Lookup("cache-db"))
 	viper.BindPFlag("cache.trie", rootCmd.PersistentFlags().Lookup("cache-trie"))

cmd/util.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff"
 	gethsd "github.com/ethereum/go-ethereum/statediff"
 	ind "github.com/ethereum/go-ethereum/statediff/indexer"
 	"github.com/ethereum/go-ethereum/trie"
@@ -26,7 +26,17 @@ func createStateDiffService() (sd.StateDiffService, error) {
 	}
 
 	nodeInfo := getEthNodeInfo()
-	chainConf, err := chainConfig(nodeInfo.ChainID)
+
+	var chainConf *params.ChainConfig
+	var err error
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+
+	if chainConfigPath != "" {
+		chainConf, err = statediff.LoadConfig(chainConfigPath)
+	} else {
+		chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
+	}
+
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -56,7 +66,7 @@ func createStateDiffService() (sd.StateDiffService, error) {
 		logWithCommand.Fatal(err)
 	}
 	logWithCommand.Info("Creating statediff indexer")
-	indexer, err := ind.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
+	_, indexer, err := ind.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -89,13 +99,6 @@ func setupPreRunRanges() []sd.RangeRequest {
 		addrs[i] = common.HexToAddress(addrStr)
 	}
 	preRunParams.WatchedAddresses = addrs
-	var storageKeyStrs []string
-	viper.UnmarshalKey("prerun.params.watchedStorageKeys", &storageKeyStrs)
-	keys := make([]common.Hash, len(storageKeyStrs))
-	for i, keyStr := range storageKeyStrs {
-		keys[i] = common.HexToHash(keyStr)
-	}
-	preRunParams.WatchedStorageSlots = keys
 	var rawRanges []blockRange
 	viper.UnmarshalKey("prerun.ranges", &rawRanges)
 	blockRanges := make([]sd.RangeRequest, len(rawRanges))
@@ -118,18 +121,3 @@ func setupPreRunRanges() []sd.RangeRequest {
 
 	return blockRanges
 }
-
-func chainConfig(chainID uint64) (*params.ChainConfig, error) {
-	switch chainID {
-	case 1:
-		return params.MainnetChainConfig, nil
-	case 3:
-		return params.RopstenChainConfig, nil // Ropsten
-	case 4:
-		return params.RinkebyChainConfig, nil
-	case 5:
-		return params.GoerliChainConfig, nil
-	default:
-		return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
-	}
-}

cmd/version.go

@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	v "github.com/vulcanize/eth-statediff-service/version"
+)
+
+// versionCmd represents the version command
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Prints the version of eth-statediff-service",
+	Long: `Use this command to fetch the version of eth-statediff-service
+
+Usage: ./eth-statediff-service  version`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		logWithCommand.Infof("eth-statediff-service version: %s", v.Version)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}

environments/config.toml

@@ -23,7 +23,6 @@
         includeTD = true
         includeCode = true
         watchedAddresses = []
-        watchedStorageKeys = []
 
 [log]
     file = ""
@@ -52,6 +51,7 @@
     httpPort = 9100
 
 [ethereum]
+    chainConfig = ""
     nodeID = ""
     clientName = "eth-statediff-service"
     genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"

environments/example.toml

@@ -25,7 +25,6 @@
         includeTD = true
         includeCode = true
         watchedAddresses = []
-        watchedStorageKeys = []
 
 [log]
     file = ""
@@ -54,6 +53,7 @@
     httpPort = "8889"
 
 [ethereum]
+    chainConfig = ""
     nodeID = ""
     clientName = "eth-statediff-service"
     genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"

go.mod

@@ -3,13 +3,14 @@ module github.com/vulcanize/eth-statediff-service
 go 1.16
 
 require (
-	github.com/ethereum/go-ethereum v1.10.11
+	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
+	github.com/ethereum/go-ethereum v1.10.17
 	github.com/jmoiron/sqlx v1.2.0
-	github.com/prometheus/client_golang v1.0.0
+	github.com/prometheus/client_golang v1.4.0
 	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.1.1
-	github.com/spf13/viper v1.7.1
-	github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191
+	github.com/spf13/cobra v1.3.0
+	github.com/spf13/viper v1.10.1
+	github.com/vulcanize/go-eth-state-node-iterator v1.0.1
 )
 
-replace github.com/ethereum/go-ethereum v1.10.11 => github.com/vulcanize/go-ethereum v1.10.11-statediff-0.0.27g
+replace github.com/ethereum/go-ethereum v1.10.17 => github.com/vulcanize/go-ethereum v1.10.17-statediff-3.2.1

pkg/builder.go

@@ -42,7 +42,7 @@ import (
 
 var (
 	nullHashBytes     = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
-	emptyNode, _      = rlp.EncodeToBytes([]byte{})
+	emptyNode, _      = rlp.EncodeToBytes(&[]byte{})
 	emptyContractRoot = crypto.Keccak256Hash(emptyNode)
 	nullCodeHash      = crypto.Keccak256Hash([]byte{}).Bytes()
 )
@@ -163,12 +163,12 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, [
 			node.LeafKey = leafKey
 			if !bytes.Equal(account.CodeHash, nullCodeHash) {
 				var storageNodes []sdtypes.StorageNode
-				err := sdb.buildStorageNodesEventual(account.Root, nil, true, storageNodeAppender(&storageNodes))
+				err := sdb.buildStorageNodesEventual(account.Root, true, storageNodeAppender(&storageNodes))
 				if err != nil {
 					return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err)
 				}
 				node.StorageNodes = storageNodes
-				// emit codehash => code mappings for cod
+				// emit codehash => code mappings for code
 				codeHash := common.BytesToHash(account.CodeHash)
 				code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
 				if err != nil {
@@ -218,16 +218,18 @@ func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Para
 	// Load tries for old and new states
 	oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
 	if err != nil {
-		return fmt.Errorf("error creating trie for old state root: %v", err)
+		return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
 	}
 	newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
 	if err != nil {
-		return fmt.Errorf("error creating trie for new state root: %v", err)
+		return fmt.Errorf("error creating trie for newStateRoot: %v", err)
 	}
 
 	// Split old and new tries into corresponding subtrie iterators
-	oldIterFac := iter.NewSubtrieIteratorFactory(oldTrie, sdb.numWorkers)
-	newIterFac := iter.NewSubtrieIteratorFactory(newTrie, sdb.numWorkers)
+	oldIters1 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
+	oldIters2 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
+	newIters1 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
+	newIters2 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
 
 	// Create iterators ahead of time to avoid race condition in state.Trie access
 	// We do two state iterations per subtrie: one for new/updated nodes,
@@ -235,8 +237,8 @@ func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Para
 	var iterPairs [][]iterPair
 	for i := uint(0); i < sdb.numWorkers; i++ {
 		iterPairs = append(iterPairs, []iterPair{
-			{older: oldIterFac.IteratorAt(i), newer: newIterFac.IteratorAt(i)},
-			{older: oldIterFac.IteratorAt(i), newer: newIterFac.IteratorAt(i)},
+			{older: oldIters1[i], newer: newIters1[i]},
+			{older: oldIters2[i], newer: newIters2[i]},
 		})
 	}
 
@@ -297,7 +299,7 @@ func (sdb *builder) buildStateDiff(args []iterPair, params sd.Params, output sdt
 	if params.IntermediateStateNodes {
 		diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedStateWithIntermediateNodes(args[0], output)
 	} else {
-		diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedState(args[0], params.WatchedAddresses)
+		diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedState(args[0], params.WatchedAddressesLeafKeys())
 	}
 
 	if err != nil {
@@ -306,12 +308,12 @@ func (sdb *builder) buildStateDiff(args []iterPair, params sd.Params, output sdt
 
 	// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
 	// a map of their leafkey to all the accounts that were touched and exist at A
-	diffAccountsAtA, err := sdb.deletedOrUpdatedState(args[1], diffPathsAtB, output)
+	diffAccountsAtA, err := sdb.deletedOrUpdatedState(args[1], diffAccountsAtB, diffPathsAtB, params.WatchedAddressesLeafKeys(), params.IntermediateStorageNodes, output)
 	if err != nil {
 		return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
 	}
 
-	// collect and sort the leafkey keys for both account mappings into a slice
+	// collect and sort the leafkeys for both account mappings into a slice
 	createKeys := sortKeys(diffAccountsAtB)
 	deleteKeys := sortKeys(diffAccountsAtA)
 
@@ -323,13 +325,12 @@ func (sdb *builder) buildStateDiff(args []iterPair, params sd.Params, output sdt
 
 	// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
 	err = sdb.buildAccountUpdates(
-		diffAccountsAtB, diffAccountsAtA, updatedKeys,
-		params.WatchedStorageSlots, params.IntermediateStorageNodes, output)
+		diffAccountsAtB, diffAccountsAtA, updatedKeys, params.IntermediateStorageNodes, output)
 	if err != nil {
 		return fmt.Errorf("error building diff for updated accounts: %v", err)
 	}
 	// build the diff nodes for created accounts
-	err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput)
+	err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput)
 	if err != nil {
 		return fmt.Errorf("error building diff for created accounts: %v", err)
 	}
@@ -339,7 +340,7 @@ func (sdb *builder) buildStateDiff(args []iterPair, params sd.Params, output sdt
 // createdAndUpdatedState returns
 // a mapping of their leafkeys to all the accounts that exist in a different state at B than A
 // and a slice of the paths for all of the nodes included in both
-func (sdb *builder) createdAndUpdatedState(iters iterPair, watchedAddresses []common.Address) (AccountMap, map[string]bool, error) {
+func (sdb *builder) createdAndUpdatedState(iters iterPair, watchedAddressesLeafKeys map[common.Hash]struct{}) (AccountMap, map[string]bool, error) {
 	diffPathsAtB := make(map[string]bool)
 	diffAcountsAtB := make(AccountMap)
 	it, _ := trie.NewDifferenceIterator(iters.older, iters.newer)
@@ -347,7 +348,7 @@ func (sdb *builder) createdAndUpdatedState(iters iterPair, watchedAddresses []co
 		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
 			continue
 		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
 		if err != nil {
 			return nil, nil, err
 		}
@@ -363,7 +364,7 @@ func (sdb *builder) createdAndUpdatedState(iters iterPair, watchedAddresses []co
 			valueNodePath := append(node.Path, partialPath...)
 			encodedPath := trie.HexToCompact(valueNodePath)
 			leafKey := encodedPath[1:]
-			if isWatchedAddress(watchedAddresses, leafKey) {
+			if isWatchedAddress(watchedAddressesLeafKeys, leafKey) {
 				diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
 					NodeType:  node.NodeType,
 					Path:      node.Path,
@@ -435,14 +436,15 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(iters iterPair,
 
 // deletedOrUpdatedState returns a slice of all the paths that are emptied at B
 // and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
-func (sdb *builder) deletedOrUpdatedState(iters iterPair, diffPathsAtB map[string]bool, output sdtypes.StateNodeSink) (AccountMap, error) {
+func (sdb *builder) deletedOrUpdatedState(iters iterPair, diffAccountsAtB AccountMap, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, intermediateStorageNodes bool, output sdtypes.StateNodeSink) (AccountMap, error) {
 	diffAccountAtA := make(AccountMap)
 	it, _ := trie.NewDifferenceIterator(iters.newer, iters.older)
 	for it.Next(true) {
+		// skip value nodes
 		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
 			continue
 		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
 		if err != nil {
 			return nil, err
 		}
@@ -457,24 +459,48 @@ func (sdb *builder) deletedOrUpdatedState(iters iterPair, diffPathsAtB map[strin
 			valueNodePath := append(node.Path, partialPath...)
 			encodedPath := trie.HexToCompact(valueNodePath)
 			leafKey := encodedPath[1:]
-			diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{
-				NodeType:  node.NodeType,
-				Path:      node.Path,
-				NodeValue: node.NodeValue,
-				LeafKey:   leafKey,
-				Account:   &account,
-			}
-			// if this node's path did not show up in diffPathsAtB
-			// that means the node at this path was deleted (or moved) in B
-			// emit an empty "removed" diff to signify as such
-			if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
-				if err := output(sdtypes.StateNode{
+			if isWatchedAddress(watchedAddressesLeafKeys, leafKey) {
+				diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{
+					NodeType:  node.NodeType,
 					Path:      node.Path,
-					NodeValue: []byte{},
-					NodeType:  sdtypes.Removed,
+					NodeValue: node.NodeValue,
 					LeafKey:   leafKey,
-				}); err != nil {
-					return nil, err
+					Account:   &account,
+				}
+				// if this node's path did not show up in diffPathsAtB
+				// that means the node at this path was deleted (or moved) in B
+				if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
+					var diff sdtypes.StateNode
+					// if this node's leaf key also did not show up in diffAccountsAtB
+					// that means the node was deleted
+					// in that case, emit an empty "removed" diff state node
+					// include empty "removed" diff storage nodes for all the storage slots
+					if _, ok := diffAccountsAtB[common.Bytes2Hex(leafKey)]; !ok {
+						diff = sdtypes.StateNode{
+							NodeType:  sdtypes.Removed,
+							Path:      node.Path,
+							LeafKey:   leafKey,
+							NodeValue: []byte{},
+						}
+
+						var storageDiffs []sdtypes.StorageNode
+						err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
+						if err != nil {
+							return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err)
+						}
+						diff.StorageNodes = storageDiffs
+					} else {
+						// emit an empty "removed" diff with empty leaf key if the account was moved
+						diff = sdtypes.StateNode{
+							NodeType:  sdtypes.Removed,
+							Path:      node.Path,
+							NodeValue: []byte{},
+						}
+					}
+
+					if err := output(diff); err != nil {
+						return nil, err
+					}
 				}
 			}
 		case sdtypes.Extension, sdtypes.Branch:
@@ -502,7 +528,7 @@ func (sdb *builder) deletedOrUpdatedState(iters iterPair, diffPathsAtB map[strin
 // to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
 // needs to be called before building account creations and deletions as this mutates
 // those account maps to remove the accounts which were updated
-func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output sdtypes.StateNodeSink) error {
+func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, intermediateStorageNodes bool, output sdtypes.StateNodeSink) error {
 	var err error
 	for _, key := range updatedKeys {
 		createdAcc := creations[key]
@@ -511,7 +537,7 @@ func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updated
 		if deletedAcc.Account != nil && createdAcc.Account != nil {
 			oldSR := deletedAcc.Account.Root
 			newSR := createdAcc.Account.Root
-			err = sdb.buildStorageNodesIncremental(oldSR, newSR, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
+			err = sdb.buildStorageNodesIncremental(oldSR, newSR, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
 			if err != nil {
 				return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
 			}
@@ -534,7 +560,7 @@ func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updated
 
 // buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
 // it also returns the code and codehash for created contract accounts
-func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
+func (sdb *builder) buildAccountCreations(accounts AccountMap, intermediateStorageNodes bool, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
 	for _, val := range accounts {
 		diff := sdtypes.StateNode{
 			NodeType:  val.NodeType,
@@ -545,7 +571,7 @@ func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKey
 		if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
 			// For contract creations, any storage node contained is a diff
 			var storageDiffs []sdtypes.StorageNode
-			err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
+			err := sdb.buildStorageNodesEventual(val.Account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
 			if err != nil {
 				return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err)
 			}
@@ -573,7 +599,7 @@ func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKey
 
 // buildStorageNodesEventual builds the storage diff node objects for a created account
 // i.e. it returns all the storage nodes at this state, since there is no previous state
-func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
+func (sdb *builder) buildStorageNodesEventual(sr common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
 	if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
 		return nil
 	}
@@ -584,7 +610,7 @@ func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys
 		return err
 	}
 	it := sTrie.NodeIterator(make([]byte, 0))
-	err = sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes, output)
+	err = sdb.buildStorageNodesFromTrie(it, intermediateNodes, output)
 	if err != nil {
 		return err
 	}
@@ -594,7 +620,7 @@ func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys
 // buildStorageNodesFromTrie returns all the storage diff node objects in the provided node iterator
 // if any storage keys are provided it will only return those leaf nodes
 // including intermediate nodes can be turned on or off
-func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
+func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
 	for it.Next(true) {
 		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
 			continue
@@ -609,15 +635,13 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStora
 			valueNodePath := append(node.Path, partialPath...)
 			encodedPath := trie.HexToCompact(valueNodePath)
 			leafKey := encodedPath[1:]
-			if isWatchedStorageKey(watchedStorageKeys, leafKey) {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-					LeafKey:   leafKey,
-				}); err != nil {
-					return err
-				}
+			if err := output(sdtypes.StorageNode{
+				NodeType:  node.NodeType,
+				Path:      node.Path,
+				NodeValue: node.NodeValue,
+				LeafKey:   leafKey,
+			}); err != nil {
+				return err
 			}
 		case sdtypes.Extension, sdtypes.Branch:
 			if intermediateNodes {
@@ -636,8 +660,70 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStora
 	return it.Error()
 }
 
+// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account
+func (sdb *builder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
+	if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
+		return nil
+	}
+	log.Debug("Storage Root For Removed Diffs", "root", sr.Hex())
+	sTrie, err := sdb.stateCache.OpenTrie(sr)
+	if err != nil {
+		log.Info("error in build removed account storage diffs", "error", err)
+		return err
+	}
+	it := sTrie.NodeIterator(make([]byte, 0))
+	err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator
+// including intermediate nodes can be turned on or off
+func (sdb *builder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
+	for it.Next(true) {
+		// skip value nodes
+		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+			continue
+		}
+		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		if err != nil {
+			return err
+		}
+		switch node.NodeType {
+		case sdtypes.Leaf:
+			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+			valueNodePath := append(node.Path, partialPath...)
+			encodedPath := trie.HexToCompact(valueNodePath)
+			leafKey := encodedPath[1:]
+			if err := output(sdtypes.StorageNode{
+				NodeType:  sdtypes.Removed,
+				Path:      node.Path,
+				NodeValue: []byte{},
+				LeafKey:   leafKey,
+			}); err != nil {
+				return err
+			}
+		case sdtypes.Extension, sdtypes.Branch:
+			if intermediateNodes {
+				if err := output(sdtypes.StorageNode{
+					NodeType:  sdtypes.Removed,
+					Path:      node.Path,
+					NodeValue: []byte{},
+				}); err != nil {
+					return err
+				}
+			}
+		default:
+			return fmt.Errorf("unexpected node type %s", node.NodeType)
+		}
+	}
+	return it.Error()
+}
+
 // buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
-func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
+func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
 	if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
 		return nil
 	}
@@ -651,19 +737,20 @@ func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common
 		return err
 	}
 
-	diffPathsAtB, err := sdb.createdAndUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), watchedStorageKeys, intermediateNodes, output)
+	diffSlotsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), intermediateNodes, output)
 	if err != nil {
 		return err
 	}
-	err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffPathsAtB, watchedStorageKeys, intermediateNodes, output)
+	err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffSlotsAtB, diffPathsAtB, intermediateNodes, output)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) (map[string]bool, error) {
+func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) (map[string]bool, map[string]bool, error) {
 	diffPathsAtB := make(map[string]bool)
+	diffSlotsAtB := make(map[string]bool)
 	it, _ := trie.NewDifferenceIterator(a, b)
 	for it.Next(true) {
 		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
@@ -671,7 +758,7 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys
 		}
 		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		switch node.NodeType {
 		case sdtypes.Leaf:
@@ -679,15 +766,14 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys
 			valueNodePath := append(node.Path, partialPath...)
 			encodedPath := trie.HexToCompact(valueNodePath)
 			leafKey := encodedPath[1:]
-			if isWatchedStorageKey(watchedKeys, leafKey) {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-					LeafKey:   leafKey,
-				}); err != nil {
-					return nil, err
-				}
+			diffSlotsAtB[common.Bytes2Hex(leafKey)] = true
+			if err := output(sdtypes.StorageNode{
+				NodeType:  node.NodeType,
+				Path:      node.Path,
+				NodeValue: node.NodeValue,
+				LeafKey:   leafKey,
+			}); err != nil {
+				return nil, nil, err
 			}
 		case sdtypes.Extension, sdtypes.Branch:
 			if intermediateNodes {
@@ -696,57 +782,75 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys
 					Path:      node.Path,
 					NodeValue: node.NodeValue,
 				}); err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 			}
 		default:
-			return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+			return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
 		}
 		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
 	}
-	return diffPathsAtB, it.Error()
+	return diffSlotsAtB, diffPathsAtB, it.Error()
 }
 
-func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
+func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB, diffPathsAtB map[string]bool, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
 	it, _ := trie.NewDifferenceIterator(b, a)
 	for it.Next(true) {
+		// skip value nodes
 		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
 			continue
 		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
 		if err != nil {
 			return err
 		}
-		// if this node path showed up in diffPathsAtB
-		// that means this node was updated at B and we already have the updated diff for it
-		// otherwise that means this node was deleted in B and we need to add a "removed" diff to represent that event
-		if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; ok {
-			continue
-		}
+
 		switch node.NodeType {
 		case sdtypes.Leaf:
 			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
 			valueNodePath := append(node.Path, partialPath...)
 			encodedPath := trie.HexToCompact(valueNodePath)
 			leafKey := encodedPath[1:]
-			if isWatchedStorageKey(watchedKeys, leafKey) {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  sdtypes.Removed,
-					Path:      node.Path,
-					NodeValue: []byte{},
-					LeafKey:   leafKey,
-				}); err != nil {
-					return err
+
+			// if this node's path did not show up in diffPathsAtB
+			// that means the node at this path was deleted (or moved) in B
+			if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
+				// if this node's leaf key also did not show up in diffSlotsAtB
+				// that means the node was deleted
+				// in that case, emit an empty "removed" diff storage node
+				if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok {
+					if err := output(sdtypes.StorageNode{
+						NodeType:  sdtypes.Removed,
+						Path:      node.Path,
+						NodeValue: []byte{},
+						LeafKey:   leafKey,
+					}); err != nil {
+						return err
+					}
+				} else {
+					// emit an empty "removed" diff with empty leaf key if the account was moved
+					if err := output(sdtypes.StorageNode{
+						NodeType:  sdtypes.Removed,
+						Path:      node.Path,
+						NodeValue: []byte{},
+					}); err != nil {
+						return err
+					}
 				}
 			}
 		case sdtypes.Extension, sdtypes.Branch:
-			if intermediateNodes {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  sdtypes.Removed,
-					Path:      node.Path,
-					NodeValue: []byte{},
-				}); err != nil {
-					return err
+			// if this node's path did not show up in diffPathsAtB
+			// that means the node at this path was deleted in B
+			// in that case, emit an empty "removed" diff storage node
+			if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
+				if intermediateNodes {
+					if err := output(sdtypes.StorageNode{
+						NodeType:  sdtypes.Removed,
+						Path:      node.Path,
+						NodeValue: []byte{},
+					}); err != nil {
+						return err
+					}
 				}
 			}
 		default:
@@ -757,18 +861,14 @@ func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB
 }
 
 // isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch
-func isWatchedAddress(watchedAddresses []common.Address, stateLeafKey []byte) bool {
+func isWatchedAddress(watchedAddressesLeafKeys map[common.Hash]struct{}, stateLeafKey []byte) bool {
 	// If we aren't watching any specific addresses, we are watching everything
-	if len(watchedAddresses) == 0 {
+	if len(watchedAddressesLeafKeys) == 0 {
 		return true
 	}
-	for _, addr := range watchedAddresses {
-		addrHashKey := crypto.Keccak256(addr.Bytes())
-		if bytes.Equal(addrHashKey, stateLeafKey) {
-			return true
-		}
-	}
-	return false
+
+	_, ok := watchedAddressesLeafKeys[common.BytesToHash(stateLeafKey)]
+	return ok
 }
 
 // isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch

pkg/rpc/http.go

@@ -35,7 +35,7 @@ func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors [
 	if err != nil {
 		utils.Fatalf("Could not register HTTP API: %w", err)
 	}
-	handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
+	handler := node.NewHTTPHandlerStack(srv, cors, vhosts, nil)
 
 	// start http server
 	_, addr, err := node.StartHTTPEndpoint(endpoint, rpc.DefaultHTTPTimeouts, prom.HTTPMiddleware(handler))

pkg/service.go

@@ -194,6 +194,10 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 		return nil, err
 	}
 	logrus.Infof("sending state diff at block %d", blockNumber)
+
+	// compute leaf keys of watched addresses in the params
+	params.ComputeWatchedAddressesLeafKeys()
+
 	if blockNumber == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
@@ -212,6 +216,10 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
 		return nil, err
 	}
 	logrus.Infof("sending state diff at block %s", blockHash.Hex())
+
+	// compute leaf keys of watched addresses in the params
+	params.ComputeWatchedAddressesLeafKeys()
+
 	if currentBlock.NumberU64() == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
@@ -233,7 +241,7 @@ func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot commo
 	if err != nil {
 		return nil, err
 	}
-	stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+	stateDiffRlp, err := rlp.EncodeToBytes(&stateDiff)
 	if err != nil {
 		return nil, err
 	}
@@ -281,6 +289,10 @@ func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 		return nil, err
 	}
 	logrus.Infof("sending state trie at block %d", blockNumber)
+
+	// compute leaf keys of watched addresses in the params
+	params.ComputeWatchedAddressesLeafKeys()
+
 	return sds.processStateTrie(currentBlock, params)
 }
 
@@ -289,7 +301,7 @@ func (sds *Service) processStateTrie(block *types.Block, params sd.Params) (*sd.
 	if err != nil {
 		return nil, err
 	}
-	stateTrieRlp, err := rlp.EncodeToBytes(stateNodes)
+	stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes)
 	if err != nil {
 		return nil, err
 	}
@@ -320,6 +332,10 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 	if err != nil {
 		return err
 	}
+
+	// compute leaf keys of watched addresses in the params
+	params.ComputeWatchedAddressesLeafKeys()
+
 	parentRoot := common.Hash{}
 	if blockNumber != 0 {
 		parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
@@ -341,6 +357,10 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) e
 	if err != nil {
 		return err
 	}
+
+	// compute leaf keys of watched addresses in the params
+	params.ComputeWatchedAddressesLeafKeys()
+
 	parentRoot := common.Hash{}
 	if currentBlock.NumberU64() != 0 {
 		parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())

pkg/testhelpers/helpers.go

@@ -1,125 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package testhelpers
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/consensus/ethash"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/params"
-)
-
-// MakeChain creates a chain of n blocks starting at and including parent.
-// the returned hash chain is ordered head->parent.
-func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) {
-	config := params.TestChainConfig
-	config.LondonBlock.Set(big.NewInt(10))
-	blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
-	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
-	return blocks, chain
-}
-
-func TestSelfDestructChainGen(i int, block *core.BlockGen) {
-	signer := types.HomesteadSigner{}
-	switch i {
-	case 0:
-		// Block 1 is mined by Account1Addr
-		// Account1Addr creates a new contract
-		block.SetCoinbase(TestBankAddress)
-		tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, TestBankKey)
-		ContractAddr = crypto.CreateAddress(TestBankAddress, 0)
-		block.AddTx(tx)
-	case 1:
-		// Block 2 is mined by Account1Addr
-		// Account1Addr self-destructs the contract
-		block.SetCoinbase(TestBankAddress)
-		data := common.Hex2Bytes("43D726D6")
-		tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
-		block.AddTx(tx)
-	}
-}
-
-func TestChainGen(i int, block *core.BlockGen) {
-	signer := types.HomesteadSigner{}
-	switch i {
-	case 0:
-		// In block 1, the test bank sends account #1 some ether.
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
-		block.AddTx(tx)
-	case 1:
-		// In block 2, the test bank sends some more ether to account #1.
-		// Account1Addr passes it on to account #2.
-		// Account1Addr creates a test contract.
-		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
-		nonce := block.TxNonce(Account1Addr)
-		tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
-		nonce++
-		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key)
-		ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
-		block.AddTx(tx1)
-		block.AddTx(tx2)
-		block.AddTx(tx3)
-	case 2:
-		// Block 3 has a single tx from the bankAccount to the contract, that transfers no value
-		// Block 3 is mined by Account2Addr
-		block.SetCoinbase(Account2Addr)
-		//put function: c16431b9
-		//close function: 43d726d6
-		data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003")
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
-		block.AddTx(tx)
-	case 3:
-		// Block 4 has three txs from bankAccount to the contract, that transfer no value
-		// Two set the two original slot positions to 0 and one sets another position to a new value
-		// Block 4 is mined by Account2Addr
-		block.SetCoinbase(Account2Addr)
-		data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
-		data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000")
-		data3 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000009")
-
-		nonce := block.TxNonce(TestBankAddress)
-		tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data1), signer, TestBankKey)
-		nonce++
-		tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data2), signer, TestBankKey)
-		nonce++
-		tx3, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data3), signer, TestBankKey)
-		block.AddTx(tx1)
-		block.AddTx(tx2)
-		block.AddTx(tx3)
-	case 4:
-		// Block 5 has one tx from bankAccount to the contract, that transfers no value
-		// It sets the remaining storage value to zero
-		// Block 5 is mined by Account1Addr
-		block.SetCoinbase(Account1Addr)
-		data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000")
-		nonce := block.TxNonce(TestBankAddress)
-		tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
-		block.AddTx(tx)
-	case 5:
-		// Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value
-		// Block 6 is mined by Account2Addr
-		block.SetCoinbase(Account2Addr)
-		data := common.Hex2Bytes("43D726D6")
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(Account1Addr), ContractAddr, big.NewInt(0), 100000, nil, data), signer, Account1Key)
-		block.AddTx(tx)
-	}
-}

pkg/testhelpers/test_data.go

@@ -1,67 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package testhelpers
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/rlp"
-)
-
-// AddressToLeafKey hashes an returns an address
-func AddressToLeafKey(address common.Address) []byte {
-	return crypto.Keccak256(address[:])
-}
-
-// AddressToEncodedPath hashes an address and appends the even-number leaf flag to it
-func AddressToEncodedPath(address common.Address) []byte {
-	addrHash := crypto.Keccak256(address[:])
-	decodedPath := append(EvenLeafFlag, addrHash...)
-	return decodedPath
-}
-
-// Test variables
-var (
-	EvenLeafFlag = []byte{byte(2) << 4}
-	NullCodeHash = crypto.Keccak256Hash([]byte{})
-	NullHash     = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
-
-	Testdb          = rawdb.NewMemoryDatabase()
-	TestBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
-	TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
-	BankLeafKey     = AddressToLeafKey(TestBankAddress)
-	TestBankFunds   = big.NewInt(100000000)
-	Genesis         = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
-
-	Account1Key, _          = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
-	Account2Key, _          = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
-	Account1Addr            = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
-	Account2Addr            = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
-	Account1LeafKey         = AddressToLeafKey(Account1Addr)
-	Account2LeafKey         = AddressToLeafKey(Account2Addr)
-	ContractCode            = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506040518060200160405280600160ff16815250600190600161007492919061007a565b506100e4565b82606481019282156100ae579160200282015b828111156100ad578251829060ff1690559160200191906001019061008d565b5b5090506100bb91906100bf565b5090565b6100e191905b808211156100dd5760008160009055506001016100c5565b5090565b90565b6101ca806100f36000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032")
-	ByteCodeAfterDeployment = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032")
-	CodeHash                = common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127")
-	ContractAddr            common.Address
-
-	EmptyRootNode, _  = rlp.EncodeToBytes([]byte{})
-	EmptyContractRoot = crypto.Keccak256Hash(EmptyRootNode)
-)

version/version.go

@@ -0,0 +1,30 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package version
+
+import "fmt"
+
+const (
+	major = 3 // major version component of the current release
+	minor = 0 // minor version component of the current release
+	patch = 0 // patch version component of the current release
+)
+
+// Version holds the textual version string.
+var Version = func() string {
+	return fmt.Sprintf("%d.%d.%d", major, minor, patch)
+}()