Merge pull request #140 from vulcanize/release-v2.0.0

dependabot security update

.github/workflows/on-pr.yaml

@@ -16,7 +16,7 @@ jobs:
       GOPATH: /tmp/go
     strategy:
       matrix:
-        go-version: [1.14.x, 1.15.x]
+        go-version: [1.16.x, 1.17.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -28,19 +28,23 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - uses: actions/checkout@v2
       - name: Run database
-        run: docker-compose up -d db
+        run: docker-compose up -d ipld-eth-db
       - name: Test
         run: |
           sleep 10
-          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 make test
+          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test
 
   integrationtest:
     name: Run integration tests
     env:
       GOPATH: /tmp/go
+      DB_WRITE: true
+      ETH_FORWARD_ETH_CALLS: false
+      ETH_PROXY_ON_ERROR: false
+      ETH_HTTP_PATH: ""
     strategy:
       matrix:
-        go-version: [1.15.x]
+        go-version: [1.16.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -52,7 +56,36 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - uses: actions/checkout@v2
       - name: Run database
-        run: docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d db dapptools contract eth-server
+        run: docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d ipld-eth-db dapptools contract eth-server
+      - name: Test
+        run: |
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
+          make integrationtest
+
+  integrationtest_forwardethcalls:
+    name: Run integration tests for direct proxy fall-through of eth_calls
+    env:
+      GOPATH: /tmp/go
+      DB_WRITE: false
+      ETH_FORWARD_ETH_CALLS: true
+      ETH_PROXY_ON_ERROR: false
+      ETH_HTTP_PATH: "dapptools:8545"
+    strategy:
+      matrix:
+        go-version: [ 1.16.x ]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/checkout@v2
+      - name: Run database
+        run: docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d ipld-eth-db dapptools contract eth-server
       - name: Test
         run: |
           while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.16-alpine as builder
 
 RUN apk --update --no-cache add make git g++ linux-headers
 # DEBUG
@@ -44,7 +44,6 @@ COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-serv
 # keep binaries immutable
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/ipld-eth-server ipld-eth-server
 COPY --from=builder /goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/environments environments
 
 ENTRYPOINT ["/app/entrypoint.sh"]

Makefile

@@ -57,10 +57,6 @@ TEST_CONNECT_STRING_LOCAL = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)
 test: | $(GINKGO) $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	export PGPASSWORD=$(DATABASE_PASSWORD)
-	dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
-	createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
-	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
 	$(GINKGO) -r --skipPackage=test
 
 .PHONY: integrationtest
@@ -79,7 +75,8 @@ test_local: | $(GINKGO) $(GOOSE)
 integrationtest_local: | $(GINKGO) $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	./scripts/run_intregration_test.sh
+	./scripts/run_integration_test.sh
+	./scripts/run_integration_test_forward_eth_calls.sh
 
 build:
 	go fmt ./...
@@ -148,4 +145,4 @@ import:
 ## Build docker image
 .PHONY: docker-build
 docker-build:
-	docker build -t vulcanize/ipld-eth-server .
+	docker build -t vulcanize/ipld-eth-server .

cmd/common.go

@@ -0,0 +1,37 @@
+// Copyright © 2021 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func addDatabaseFlags(command *cobra.Command) {
+	// database flags
+	command.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	command.PersistentFlags().Int("database-port", 5432, "database port")
+	command.PersistentFlags().String("database-hostname", "localhost", "database hostname")
+	command.PersistentFlags().String("database-user", "", "database user")
+	command.PersistentFlags().String("database-password", "", "database password")
+
+	// database flag bindings
+	viper.BindPFlag("database.name", command.PersistentFlags().Lookup("database-name"))
+	viper.BindPFlag("database.port", command.PersistentFlags().Lookup("database-port"))
+	viper.BindPFlag("database.hostname", command.PersistentFlags().Lookup("database-hostname"))
+	viper.BindPFlag("database.user", command.PersistentFlags().Lookup("database-user"))
+	viper.BindPFlag("database.password", command.PersistentFlags().Lookup("database-password"))
+}

cmd/root.go

@@ -19,16 +19,20 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
+	"github.com/joho/godotenv"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
 	"github.com/vulcanize/ipld-eth-server/pkg/prom"
 )
 
 var (
 	cfgFile        string
+	envFile        string
 	subCommand     string
 	logWithCommand log.Entry
 )
@@ -99,6 +103,7 @@ func init() {
 	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
+	rootCmd.PersistentFlags().StringVar(&envFile, "env", "", "environment file location")
 
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
@@ -121,14 +126,32 @@ func init() {
 }
 
 func initConfig() {
+	if cfgFile == "" && envFile == "" {
+		log.Fatal("No configuration file specified, use --config , --env flag to provide configuration")
+	}
+
 	if cfgFile != "" {
-		viper.SetConfigFile(cfgFile)
+		if filepath.Ext(cfgFile) != ".toml" {
-		if err := viper.ReadInConfig(); err == nil {
+			log.Fatal("Provide .toml file for --config flag")
-			log.Printf("Using config file: %s", viper.ConfigFileUsed())
-		} else {
-			log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
 		}
-	} else {
+
-		log.Warn("No config file passed with --config flag")
+		viper.SetConfigFile(cfgFile)
+		if err := viper.ReadInConfig(); err != nil {
+			log.Fatalf("Couldn't read config file: %s", err.Error())
+		}
+
+		log.Infof("Using config file: %s", viper.ConfigFileUsed())
+	}
+
+	if envFile != "" {
+		if filepath.Ext(envFile) != ".env" {
+			log.Fatal("Provide .env file for --env flag")
+		}
+
+		if err := godotenv.Load(envFile); err != nil {
+			log.Fatalf("Failed to set environment variable from env file: %s", err.Error())
+		}
+
+		log.Infof("Using env file: %s", envFile)
 	}
 }

cmd/serve.go

@@ -23,14 +23,16 @@ import (
 	"os/signal"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/mailgun/groupcache/v2"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-
 	"github.com/vulcanize/gap-filler/pkg/mux"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/graphql"
 	srpc "github.com/vulcanize/ipld-eth-server/pkg/rpc"
 	s "github.com/vulcanize/ipld-eth-server/pkg/serve"
@@ -86,7 +88,19 @@ func serve() {
 		logWithCommand.Fatal(err)
 	}
 
-	shutdown := make(chan os.Signal)
+	err = startGroupCacheService(serverConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	if serverConfig.StateValidationEnabled {
+		go startStateTrieValidator(serverConfig, server)
+		logWithCommand.Info("state validator enabled")
+	} else {
+		logWithCommand.Info("state validator disabled")
+	}
+
+	shutdown := make(chan os.Signal, 1)
 	signal.Notify(shutdown, os.Interrupt)
 	<-shutdown
 	if graphQL != nil {
@@ -200,6 +214,83 @@ func startIpldGraphQL(settings *s.Config) error {
 	return nil
 }
 
+func startGroupCacheService(settings *s.Config) error {
+	gcc := settings.GroupCache
+
+	if gcc.Pool.Enabled {
+		logWithCommand.Info("starting up groupcache pool HTTTP server")
+
+		pool := groupcache.NewHTTPPoolOpts(gcc.Pool.HttpEndpoint, &groupcache.HTTPPoolOptions{})
+		pool.Set(gcc.Pool.PeerHttpEndpoints...)
+
+		httpURL, err := url.Parse(gcc.Pool.HttpEndpoint)
+		if err != nil {
+			return err
+		}
+
+		server := http.Server{
+			Addr:    httpURL.Host,
+			Handler: pool,
+		}
+
+		// Start a HTTP server to listen for peer requests from the groupcache
+		go server.ListenAndServe()
+
+		logWithCommand.Infof("groupcache pool endpoint opened for url %s", httpURL)
+	} else {
+		logWithCommand.Info("Groupcache pool is disabled")
+	}
+
+	return nil
+}
+
+func startStateTrieValidator(config *s.Config, server s.Server) {
+	validateEveryNthBlock := config.StateValidationEveryNthBlock
+
+	var lastBlockNumber uint64
+	backend := server.Backend()
+
+	for {
+		time.Sleep(5 * time.Second)
+
+		block, err := backend.CurrentBlock()
+		if err != nil {
+			log.Errorln("Error fetching current block for state trie validator")
+			continue
+		}
+
+		stateRoot := block.Root()
+		blockNumber := block.NumberU64()
+		blockHash := block.Hash()
+
+		if validateEveryNthBlock <= 0 || // Used for static replicas where block number doesn't progress.
+			(blockNumber > lastBlockNumber) && (blockNumber%validateEveryNthBlock == 0) {
+
+			// The validate trie call will take a long time on mainnet, e.g. a few hours.
+			if err = backend.ValidateTrie(stateRoot); err != nil {
+				log.Fatalf("Error validating trie for block number %d hash %s state root %s",
+					blockNumber,
+					blockHash,
+					stateRoot,
+				)
+			}
+
+			log.Infof("Successfully validated trie for block number %d hash %s state root %s",
+				blockNumber,
+				blockHash,
+				stateRoot,
+			)
+
+			if validateEveryNthBlock <= 0 {
+				// Static replica, sleep a long-ish time (1/2 of cache expiry time) since we only need to keep the cache warm.
+				time.Sleep((time.Minute * time.Duration(config.GroupCache.StateDB.CacheExpiryInMins)) / 2)
+			}
+
+			lastBlockNumber = blockNumber
+		}
+	}
+}
+
 func parseRpcAddresses(value string) ([]*rpc.Client, error) {
 	rpcAddresses := strings.Split(value, ",")
 	rpcClients := make([]*rpc.Client, 0, len(rpcAddresses))
@@ -224,12 +315,7 @@ func parseRpcAddresses(value string) ([]*rpc.Client, error) {
 func init() {
 	rootCmd.AddCommand(serveCmd)
 
-	// database credentials
+	addDatabaseFlags(serveCmd)
-	serveCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
-	serveCmd.PersistentFlags().Int("database-port", 5432, "database port")
-	serveCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
-	serveCmd.PersistentFlags().String("database-user", "", "database user")
-	serveCmd.PersistentFlags().String("database-password", "", "database password")
 
 	// flags for all config variables
 	// eth graphql and json-rpc parameters
@@ -258,16 +344,23 @@ func init() {
 	serveCmd.PersistentFlags().String("eth-default-sender", "", "default sender address")
 	serveCmd.PersistentFlags().String("eth-rpc-gas-cap", "", "rpc gas cap (for eth_Call execution)")
 	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
-	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether or not the proxy ethereum client supports statediffing endpoints")
+	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether the proxy ethereum client supports statediffing endpoints")
+	serveCmd.PersistentFlags().Bool("eth-forward-eth-calls", false, "whether to immediately forward eth_calls to proxy client")
+	serveCmd.PersistentFlags().Bool("eth-proxy-on-error", true, "whether to forward all failed calls to proxy client")
+
+	// groupcache flags
+	serveCmd.PersistentFlags().Bool("gcache-pool-enabled", false, "turn on the groupcache pool")
+	serveCmd.PersistentFlags().String("gcache-pool-http-path", "", "http url for groupcache node")
+	serveCmd.PersistentFlags().StringArray("gcache-pool-http-peers", []string{}, "http urls for groupcache peers")
+	serveCmd.PersistentFlags().Int("gcache-statedb-cache-size", 16, "state DB cache size in MB")
+	serveCmd.PersistentFlags().Int("gcache-statedb-cache-expiry", 60, "state DB cache expiry time in mins")
+	serveCmd.PersistentFlags().Int("gcache-statedb-log-stats-interval", 60, "state DB cache stats log interval in secs")
+
+	// state validator flags
+	serveCmd.PersistentFlags().Bool("validator-enabled", false, "turn on the state validator")
+	serveCmd.PersistentFlags().Uint("validator-every-nth-block", 1500, "only validate every Nth block")
 
 	// and their bindings
-	// database
-	viper.BindPFlag("database.name", serveCmd.PersistentFlags().Lookup("database-name"))
-	viper.BindPFlag("database.port", serveCmd.PersistentFlags().Lookup("database-port"))
-	viper.BindPFlag("database.hostname", serveCmd.PersistentFlags().Lookup("database-hostname"))
-	viper.BindPFlag("database.user", serveCmd.PersistentFlags().Lookup("database-user"))
-	viper.BindPFlag("database.password", serveCmd.PersistentFlags().Lookup("database-password"))
-
 	// eth graphql server
 	viper.BindPFlag("eth.server.graphql", serveCmd.PersistentFlags().Lookup("eth-server-graphql"))
 	viper.BindPFlag("eth.server.graphqlPath", serveCmd.PersistentFlags().Lookup("eth-server-graphql-path"))
@@ -301,4 +394,18 @@ func init() {
 	viper.BindPFlag("ethereum.rpcGasCap", serveCmd.PersistentFlags().Lookup("eth-rpc-gas-cap"))
 	viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config"))
 	viper.BindPFlag("ethereum.supportsStateDiff", serveCmd.PersistentFlags().Lookup("eth-supports-state-diff"))
+	viper.BindPFlag("ethereum.forwardEthCalls", serveCmd.PersistentFlags().Lookup("eth-forward-eth-calls"))
+	viper.BindPFlag("ethereum.proxyOnError", serveCmd.PersistentFlags().Lookup("eth-proxy-on-error"))
+
+	// groupcache flags
+	viper.BindPFlag("groupcache.pool.enabled", serveCmd.PersistentFlags().Lookup("gcache-pool-enabled"))
+	viper.BindPFlag("groupcache.pool.httpEndpoint", serveCmd.PersistentFlags().Lookup("gcache-pool-http-path"))
+	viper.BindPFlag("groupcache.pool.peerHttpEndpoints", serveCmd.PersistentFlags().Lookup("gcache-pool-http-peers"))
+	viper.BindPFlag("groupcache.statedb.cacheSizeInMB", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-size"))
+	viper.BindPFlag("groupcache.statedb.cacheExpiryInMins", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-expiry"))
+	viper.BindPFlag("groupcache.statedb.logStatsIntervalInSecs", serveCmd.PersistentFlags().Lookup("gcache-statedb-log-stats-interval"))
+
+	// state validator flags
+	viper.BindPFlag("validator.enabled", serveCmd.PersistentFlags().Lookup("validator-enabled"))
+	viper.BindPFlag("validator.everyNthBlock", serveCmd.PersistentFlags().Lookup("validator-every-nth-block"))
 }

cmd/subscribe.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -129,7 +128,7 @@ func subscribe() {
 			}
 			// This assumes leafs only
 			for _, stateNode := range ethData.StateNodes {
-				var acct state.Account
+				var acct types.StateAccount
 				err = rlp.DecodeBytes(stateNode.IPLD.Data, &acct)
 				if err != nil {
 					logWithCommand.Error(err)

cmd/validate.go

@@ -0,0 +1,87 @@
+// Copyright © 2021 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	validator "github.com/vulcanize/eth-ipfs-state-validator/pkg"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
+
+	s "github.com/vulcanize/ipld-eth-server/pkg/serve"
+)
+
+const GroupName = "statedb-validate"
+const CacheExpiryInMins = 8 * 60 // 8 hours
+const CacheSizeInMB = 16         // 16 MB
+
+var validateCmd = &cobra.Command{
+	Use:   "validate",
+	Short: "valdiate state",
+	Long:  `This command validates the trie for the given state root`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		validate()
+	},
+}
+
+func validate() {
+	config, err := s.NewConfig()
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	stateRootStr := viper.GetString("stateRoot")
+	if stateRootStr == "" {
+		logWithCommand.Fatal("must provide a state root for state validation")
+	}
+
+	stateRoot := common.HexToHash(stateRootStr)
+	cacheSize := viper.GetInt("cacheSize")
+
+	ethDB := ipfsethdb.NewDatabase(config.DB.DB, ipfsethdb.CacheConfig{
+		Name:           GroupName,
+		Size:           cacheSize * 1024 * 1024,
+		ExpiryDuration: time.Minute * time.Duration(CacheExpiryInMins),
+	})
+
+	val := validator.NewValidator(nil, ethDB)
+	if err = val.ValidateTrie(stateRoot); err != nil {
+		log.Fatalln("Error validating state root")
+	}
+
+	stats := ethDB.(*ipfsethdb.Database).GetCacheStats()
+	log.Debugf("groupcache stats %+v", stats)
+
+	log.Infoln("Successfully validated state root")
+}
+
+func init() {
+	rootCmd.AddCommand(validateCmd)
+
+	addDatabaseFlags(validateCmd)
+
+	validateCmd.PersistentFlags().String("state-root", "", "root of the state trie we wish to validate")
+	viper.BindPFlag("stateRoot", validateCmd.PersistentFlags().Lookup("state-root"))
+
+	validateCmd.PersistentFlags().Int("cache-size", CacheSizeInMB, "cache size in MB")
+	viper.BindPFlag("cacheSize", validateCmd.PersistentFlags().Lookup("cache-size"))
+}

db/migrations/00001_create_ipfs_blocks_table.sql

@@ -1,8 +0,0 @@
--- +goose Up
-CREATE TABLE IF NOT EXISTS public.blocks (
-  key TEXT UNIQUE NOT NULL,
-  data BYTEA NOT NULL
-);
-
--- +goose Down
-DROP TABLE public.blocks;

db/migrations/00002_create_nodes_table.sql

@@ -1,13 +0,0 @@
--- +goose Up
-CREATE TABLE nodes (
-  id            SERIAL PRIMARY KEY,
-  client_name   VARCHAR,
-  genesis_block VARCHAR(66),
-  network_id    VARCHAR,
-  node_id       VARCHAR(128),
-  chain_id      INTEGER DEFAULT 1,
-  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id, chain_id)
-);
-
--- +goose Down
-DROP TABLE nodes;

db/migrations/00003_create_eth_schema.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE SCHEMA eth;
-
--- +goose Down
-DROP SCHEMA eth;

db/migrations/00004_create_eth_header_cids_table.sql

@@ -1,23 +0,0 @@
--- +goose Up
-CREATE TABLE eth.header_cids (
-  id                    SERIAL PRIMARY KEY,
-  block_number          BIGINT NOT NULL,
-  block_hash            VARCHAR(66) NOT NULL,
-  parent_hash           VARCHAR(66) NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  td                    NUMERIC NOT NULL,
-  node_id               INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
-  reward                NUMERIC NOT NULL,
-  state_root            VARCHAR(66) NOT NULL,
-  tx_root               VARCHAR(66) NOT NULL,
-  receipt_root          VARCHAR(66) NOT NULL,
-  uncle_root            VARCHAR(66) NOT NULL,
-  bloom                 BYTEA NOT NULL,
-  timestamp             NUMERIC NOT NULL,
-  times_validated       INTEGER NOT NULL DEFAULT 1,
-  UNIQUE (block_number, block_hash)
-);
-
--- +goose Down
-DROP TABLE eth.header_cids;

db/migrations/00005_create_eth_uncle_cids_table.sql

@@ -1,14 +0,0 @@
--- +goose Up
-CREATE TABLE eth.uncle_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  block_hash            VARCHAR(66) NOT NULL,
-  parent_hash           VARCHAR(66) NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  reward                NUMERIC NOT NULL,
-  UNIQUE (header_id, block_hash)
-);
-
--- +goose Down
-DROP TABLE eth.uncle_cids;

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -1,16 +0,0 @@
--- +goose Up
-CREATE TABLE eth.transaction_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  tx_hash               VARCHAR(66) NOT NULL,
-  index                 INTEGER NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  dst                   VARCHAR(66) NOT NULL,
-  src                   VARCHAR(66) NOT NULL,
-  tx_data               BYTEA,
-  UNIQUE (header_id, tx_hash)
-);
-
--- +goose Down
-DROP TABLE eth.transaction_cids;

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -1,20 +0,0 @@
--- +goose Up
-CREATE TABLE eth.receipt_cids (
-  id                    SERIAL PRIMARY KEY,
-  tx_id                 INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  contract              VARCHAR(66),
-  contract_hash         VARCHAR(66),
-  topic0s               VARCHAR(66)[],
-  topic1s               VARCHAR(66)[],
-  topic2s               VARCHAR(66)[],
-  topic3s               VARCHAR(66)[],
-  log_contracts         VARCHAR(66)[],
-  post_state            VARCHAR(66),
-  post_status           INTEGER,
-  UNIQUE (tx_id)
-);
-
--- +goose Down
-DROP TABLE eth.receipt_cids;

db/migrations/00008_create_eth_state_cids_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.state_cids (
-  id                    BIGSERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  state_leaf_key        VARCHAR(66),
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  state_path            BYTEA,
-  node_type             INTEGER NOT NULL,
-  diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE (header_id, state_path)
-);
-
--- +goose Down
-DROP TABLE eth.state_cids;

db/migrations/00009_create_eth_storage_cids_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.storage_cids (
-  id                    BIGSERIAL PRIMARY KEY,
-  state_id              BIGINT NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  storage_leaf_key      VARCHAR(66),
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  storage_path          BYTEA,
-  node_type             INTEGER NOT NULL,
-  diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE (state_id, storage_path)
-);
-
--- +goose Down
-DROP TABLE eth.storage_cids;

db/migrations/00010_create_eth_state_accouts_table.sql

@@ -1,13 +0,0 @@
--- +goose Up
-CREATE TABLE eth.state_accounts (
-  id                    SERIAL PRIMARY KEY,
-  state_id              BIGINT NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  balance               NUMERIC NOT NULL,
-  nonce                 INTEGER NOT NULL,
-  code_hash             BYTEA NOT NULL,
-  storage_root          VARCHAR(66) NOT NULL,
-  UNIQUE (state_id)
-);
-
--- +goose Down
-DROP TABLE eth.state_accounts;

db/migrations/00011_create_postgraphile_comments.sql

@@ -1,6 +0,0 @@
--- +goose Up
-COMMENT ON TABLE public.nodes IS E'@name NodeInfo';
-COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
-COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
-COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
-COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';

db/migrations/00012_potgraphile_triggers.sql

@@ -1,69 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-CREATE FUNCTION eth.graphql_subscription() returns TRIGGER as $$
-declare
-    table_name text = TG_ARGV[0];
-    attribute text = TG_ARGV[1];
-    id text;
-begin
-    execute 'select $1.' || quote_ident(attribute)
-        using new
-        into id;
-    perform pg_notify('postgraphile:' || table_name,
-                      json_build_object(
-                              '__node__', json_build_array(
-                              table_name,
-                              id
-                          )
-                          )::text
-        );
-    return new;
-end;
-$$ language plpgsql;
--- +goose StatementEnd
-
-CREATE TRIGGER header_cids_ai
-    after INSERT ON eth.header_cids
-    for each row
-    execute procedure eth.graphql_subscription('header_cids', 'id');
-
-CREATE TRIGGER receipt_cids_ai
-    after INSERT ON eth.receipt_cids
-    for each row
-    execute procedure eth.graphql_subscription('receipt_cids', 'id');
-
-CREATE TRIGGER state_accounts_ai
-    after INSERT ON eth.state_accounts
-    for each row
-    execute procedure eth.graphql_subscription('state_accounts', 'id');
-
-CREATE TRIGGER state_cids_ai
-    after INSERT ON eth.state_cids
-    for each row
-    execute procedure eth.graphql_subscription('state_cids', 'id');
-
-CREATE TRIGGER storage_cids_ai
-    after INSERT ON eth.storage_cids
-    for each row
-    execute procedure eth.graphql_subscription('storage_cids', 'id');
-
-CREATE TRIGGER transaction_cids_ai
-    after INSERT ON eth.transaction_cids
-    for each row
-    execute procedure eth.graphql_subscription('transaction_cids', 'id');
-
-CREATE TRIGGER uncle_cids_ai
-    after INSERT ON eth.uncle_cids
-    for each row
-    execute procedure eth.graphql_subscription('uncle_cids', 'id');
-
--- +goose Down
-DROP TRIGGER uncle_cids_ai ON eth.uncle_cids;
-DROP TRIGGER transaction_cids_ai ON eth.transaction_cids;
-DROP TRIGGER storage_cids_ai ON eth.storage_cids;
-DROP TRIGGER state_cids_ai ON eth.state_cids;
-DROP TRIGGER state_accounts_ai ON eth.state_accounts;
-DROP TRIGGER receipt_cids_ai ON eth.receipt_cids;
-DROP TRIGGER header_cids_ai ON eth.header_cids;
-
-DROP FUNCTION eth.graphql_subscription();

db/migrations/00013_create_cid_indexes.sql

@@ -1,121 +0,0 @@
--- +goose Up
--- header indexes
-CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-
-CREATE INDEX block_hash_index ON eth.header_cids USING btree (block_hash);
-
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
-
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
-
-CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
-
-CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
-
--- transaction indexes
-CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-
-CREATE INDEX tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
-
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
-
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
-
-CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
-
-CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
-
--- receipt indexes
-CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
-
-CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
-
-CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
-
-CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
-
-CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash);
-
-CREATE INDEX rct_topic0_index ON eth.receipt_cids USING gin (topic0s);
-
-CREATE INDEX rct_topic1_index ON eth.receipt_cids USING gin (topic1s);
-
-CREATE INDEX rct_topic2_index ON eth.receipt_cids USING gin (topic2s);
-
-CREATE INDEX rct_topic3_index ON eth.receipt_cids USING gin (topic3s);
-
-CREATE INDEX rct_log_contract_index ON eth.receipt_cids USING gin (log_contracts);
-
--- state node indexes
-CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id);
-
-CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key);
-
-CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid);
-
-CREATE INDEX state_mh_index ON eth.state_cids USING btree (mh_key);
-
-CREATE INDEX state_path_index ON eth.state_cids USING btree (state_path);
-
--- storage node indexes
-CREATE INDEX storage_state_id_index ON eth.storage_cids USING btree (state_id);
-
-CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key);
-
-CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid);
-
-CREATE INDEX storage_mh_index ON eth.storage_cids USING btree (mh_key);
-
-CREATE INDEX storage_path_index ON eth.storage_cids USING btree (storage_path);
-
--- state accounts indexes
-CREATE INDEX account_state_id_index ON eth.state_accounts USING btree (state_id);
-
-CREATE INDEX storage_root_index ON eth.state_accounts USING btree (storage_root);
-
--- +goose Down
--- state account indexes
-DROP INDEX eth.storage_root_index;
-DROP INDEX eth.account_state_id_index;
-
--- storage node indexes
-DROP INDEX eth.storage_path_index;
-DROP INDEX eth.storage_mh_index;
-DROP INDEX eth.storage_cid_index;
-DROP INDEX eth.storage_leaf_key_index;
-DROP INDEX eth.storage_state_id_index;
-
--- state node indexes
-DROP INDEX eth.state_path_index;
-DROP INDEX eth.state_mh_index;
-DROP INDEX eth.state_cid_index;
-DROP INDEX eth.state_leaf_key_index;
-DROP INDEX eth.state_header_id_index;
-
--- receipt indexes
-DROP INDEX eth.rct_log_contract_index;
-DROP INDEX eth.rct_topic3_index;
-DROP INDEX eth.rct_topic2_index;
-DROP INDEX eth.rct_topic1_index;
-DROP INDEX eth.rct_topic0_index;
-DROP INDEX eth.rct_contract_hash_index;
-DROP INDEX eth.rct_contract_index;
-DROP INDEX eth.rct_mh_index;
-DROP INDEX eth.rct_cid_index;
-DROP INDEX eth.rct_tx_id_index;
-
--- transaction indexes
-DROP INDEX eth.tx_src_index;
-DROP INDEX eth.tx_dst_index;
-DROP INDEX eth.tx_mh_index;
-DROP INDEX eth.tx_cid_index;
-DROP INDEX eth.tx_hash_index;
-DROP INDEX eth.tx_header_id_index;
-
--- header indexes
-DROP INDEX eth.timestamp_index;
-DROP INDEX eth.state_root_index;
-DROP INDEX eth.header_mh_index;
-DROP INDEX eth.header_cid_index;
-DROP INDEX eth.block_hash_index;
-DROP INDEX eth.block_number_index;

db/migrations/00014_create_stored_functions.sql

@@ -1,158 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns if a storage node at the provided path was removed in the range >= the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-SELECT exists(SELECT 1
-              FROM eth.storage_cids
-                       INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-                       INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-              WHERE storage_path = path
-                AND block_number >= height
-                AND block_number <= (SELECT block_number
-                                     FROM eth.header_cids
-                                     WHERE block_hash = hash)
-                AND storage_cids.node_type = 3
-              LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-SELECT exists(SELECT 1
-              FROM eth.state_cids
-                       INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-              WHERE state_path = path
-                AND block_number > height
-                AND block_number <= (SELECT block_number
-                                     FROM eth.header_cids
-                                     WHERE block_hash = hash)
-                AND state_cids.node_type = 3
-              LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE TYPE child_result AS (
-    has_child BOOLEAN,
-    children eth.header_cids[]
-);
-
-CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66), height BIGINT) RETURNS child_result AS
-$BODY$
-DECLARE
-child_height INT;
-  temp_child eth.header_cids;
-  new_child_result child_result;
-BEGIN
-  child_height = height + 1;
-  -- short circuit if there are no children
-SELECT exists(SELECT 1
-              FROM eth.header_cids
-              WHERE parent_hash = hash
-                AND block_number = child_height
-              LIMIT 1)
-INTO new_child_result.has_child;
--- collect all the children for this header
-IF new_child_result.has_child THEN
-    FOR temp_child IN
-SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
-    LOOP
-      new_child_result.children = array_append(new_child_result.children, temp_child);
-END LOOP;
-END IF;
-RETURN new_child_result;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE OR REPLACE FUNCTION canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids AS
-$BODY$
-DECLARE
-canonical_header eth.header_cids;
-  canonical_child eth.header_cids;
-  header eth.header_cids;
-  current_child_result child_result;
-  child_headers eth.header_cids[];
-  current_header_with_child eth.header_cids;
-  has_children_count INT DEFAULT 0;
-BEGIN
-  -- for each header in the provided set
-  FOREACH header IN ARRAY headers
-  LOOP
-    -- check if it has any children
-    current_child_result = has_child(header.block_hash, header.block_number);
-    IF current_child_result.has_child THEN
-      -- if it does, take note
-      has_children_count = has_children_count + 1;
-      current_header_with_child = header;
-      -- and add the children to the growing set of child headers
-      child_headers = array_cat(child_headers, current_child_result.children);
-END IF;
-END LOOP;
-  -- if none of the headers had children, none is more canonical than the other
-  IF has_children_count = 0 THEN
-    -- return the first one selected
-SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
--- if only one header had children, it can be considered the heaviest/canonical header of the set
-ELSIF has_children_count = 1 THEN
-    -- return the only header with a child
-    canonical_header = current_header_with_child;
-  -- if there are multiple headers with children
-ELSE
-    -- find the canonical header from the child set
-    canonical_child = canonical_header_from_array(child_headers);
-    -- the header that is parent to this header, is the canonical header at this level
-SELECT * INTO canonical_header FROM unnest(headers)
-WHERE block_hash = canonical_child.parent_hash;
-END IF;
-RETURN canonical_header;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE OR REPLACE FUNCTION canonical_header_id(height BIGINT) RETURNS INTEGER AS
-$BODY$
-DECLARE
-canonical_header eth.header_cids;
-  headers eth.header_cids[];
-  header_count INT;
-  temp_header eth.header_cids;
-BEGIN
-  -- collect all headers at this height
-FOR temp_header IN
-SELECT * FROM eth.header_cids WHERE block_number = height
-    LOOP
-    headers = array_append(headers, temp_header);
-END LOOP;
-  -- count the number of headers collected
-  header_count = array_length(headers, 1);
-  -- if we have less than 1 header, return NULL
-  IF header_count IS NULL OR header_count < 1 THEN
-    RETURN NULL;
-  -- if we have one header, return its id
-  ELSIF header_count = 1 THEN
-    RETURN headers[1].id;
-  -- if we have multiple headers we need to determine which one is canonical
-ELSE
-    canonical_header = canonical_header_from_array(headers);
-RETURN canonical_header.id;
-END IF;
-END;
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION was_storage_removed;
-DROP FUNCTION was_state_removed;
-DROP FUNCTION canonical_header_id;
-DROP FUNCTION canonical_header_from_array;
-DROP FUNCTION has_child;
-DROP TYPE child_result;

docker-compose.yml

@@ -4,46 +4,36 @@ services:
   dapptools:
     restart: unless-stopped
     depends_on:
-      - statediff-migrations
+      - ipld-eth-db
-    image: vulcanize/dapptools:v0.29.0-v1.10.2-statediff-0.0.19
+    image: vulcanize/dapptools:v0.30.0-v1.10.14-statediff-0.0.29
     environment:
       DB_USER: vdbm
-      DB_NAME: vulcanize_public
+      DB_NAME: vulcanize_testing
-      DB_HOST: db
+      DB_HOST: ipld-eth-db
       DB_PORT: 5432
       DB_PASSWORD: password
+      DB_WRITE: $DB_WRITE
     ports:
       - "127.0.0.1:8545:8545"
       - "127.0.0.1:8546:8546"
 
-  statediff-migrations:
+  ipld-eth-db:
-    restart: on-failure
-    depends_on:
-      - db
-    image: vulcanize/statediff-migrations:v0.4.0
-    environment:
-      DATABASE_USER: vdbm
-      DATABASE_NAME: vulcanize_public
-      DATABASE_HOSTNAME: db
-      DATABASE_PORT: 5432
-      DATABASE_PASSWORD: password
-
-  db:
     restart: always
-    image: postgres:10.12-alpine
+    image: vulcanize/ipld-eth-db:v0.2.0
     environment:
       POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
     volumes:
      - vdb_db_eth_server:/var/lib/postgresql/data
     ports:
      - "127.0.0.1:8077:5432"
+    command: ["postgres", "-c", "log_statement=all"]
 
   eth-server:
     restart: unless-stopped
     depends_on:
-      - db
+      - ipld-eth-db
     build:
       context: ./
       cache_from:
@@ -55,12 +45,15 @@ services:
       ETH_SERVER_HTTPPATH: 0.0.0.0:8081
       VDB_COMMAND: "serve"
       ETH_CHAIN_CONFIG: "/tmp/chain.json"
-      DATABASE_NAME: "vulcanize_public"
+      DATABASE_NAME: "vulcanize_testing"
-      DATABASE_HOSTNAME: "db"
+      DATABASE_HOSTNAME: "ipld-eth-db"
       DATABASE_PORT: 5432
       DATABASE_USER: "vdbm"
       DATABASE_PASSWORD: "password"
       ETH_CHAIN_ID: 4
+      ETH_FORWARD_ETH_CALLS: $ETH_FORWARD_ETH_CALLS
+      ETH_PROXY_ON_ERROR: $ETH_PROXY_ON_ERROR
+      ETH_HTTP_PATH: $ETH_HTTP_PATH
     volumes:
     - type: bind
       source: ./chain.json
@@ -71,7 +64,7 @@ services:
   graphql:
     restart: unless-stopped
     depends_on:
-      - db
+      - ipld-eth-db
     image: vulcanize/postgraphile:v1.0.1
     environment:
       - PG_HOST=db

entrypoint.sh

@@ -1,20 +1,4 @@
 #!/bin/sh
-# Runs the db migrations and starts the watcher services
-
-# Construct the connection string for postgres
-VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
-
-# Run the DB migrations
-echo "Connecting with: $VDB_PG_CONNECT"
-echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
-rv=$?
-
-if [ $rv != 0 ]; then
-  echo "Could not run migrations. Are the database details correct?"
-  exit 1
-fi
-
 
 echo "Beginning the ipld-eth-server process"
 

environments/example.toml

@@ -21,7 +21,9 @@
     rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
     httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
     supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
+    forwardEthCalls = false # $ETH_FORWARD_ETH_CALLS
+    proxyOnError = true # $ETH_PROXY_ON_ERROR
     nodeID = "arch1" # $ETH_NODE_ID
     clientName = "Geth" # $ETH_CLIENT_NAME
     genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
-    networkID = "1" # $ETH_NETWORK_ID
+    networkID = "1" # $ETH_NETWORK_ID

go.mod

@@ -1,32 +1,39 @@
 module github.com/vulcanize/ipld-eth-server
 
-go 1.13
+go 1.15
 
 require (
-	github.com/ethereum/go-ethereum v1.9.25
+	github.com/ethereum/go-ethereum v1.10.15
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29
-	github.com/ipfs/go-block-format v0.0.2
+	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/jmoiron/sqlx v1.2.0
-	github.com/lib/pq v1.10.2
+	github.com/joho/godotenv v1.4.0
+	github.com/lib/pq v1.10.4
 	github.com/machinebox/graphql v0.2.2
-	github.com/multiformats/go-multihash v0.0.14
+	github.com/mailgun/groupcache/v2 v2.2.1
-	github.com/onsi/ginkgo v1.16.2
+	github.com/matryer/is v1.4.0 // indirect
-	github.com/onsi/gomega v1.10.1
+	github.com/mattn/go-sqlite3 v1.14.9 // indirect
-	github.com/prometheus/client_golang v1.5.1
+	github.com/multiformats/go-multihash v0.0.15
+	github.com/onsi/ginkgo v1.16.5
+	github.com/onsi/gomega v1.13.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/shirou/gopsutil v3.21.5+incompatible // indirect
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.0
 	github.com/tklauser/go-sysconf v0.3.6 // indirect
+	github.com/vulcanize/eth-ipfs-state-validator v0.1.0
 	github.com/vulcanize/gap-filler v0.3.1
-	github.com/vulcanize/ipfs-ethdb v0.0.2-alpha
+	github.com/vulcanize/ipfs-ethdb v0.0.6
-	github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha.0.20210805022537-b4692fa49849
+	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e // indirect
+	golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect
+	golang.org/x/tools v0.1.8 // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.10.4-statediff-0.0.25
+replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0
-
-replace github.com/vulcanize/ipfs-ethdb v0.0.2-alpha => github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha

pkg/eth/api.go

@@ -24,12 +24,15 @@ import (
 	"fmt"
 	"io"
 	"math/big"
+	"strconv"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/filters"
@@ -54,13 +57,21 @@ type PublicEthAPI struct {
 	B *Backend
 
 	// Proxy node for forwarding cache misses
-	supportsStateDiff bool // Whether or not the remote node supports the statediff_writeStateDiffAt endpoint, if it does we can fill the local cache when we hit a miss
+	supportsStateDiff bool // Whether the remote node supports the statediff_writeStateDiffAt endpoint, if it does we can fill the local cache when we hit a miss
 	rpc               *rpc.Client
 	ethClient         *ethclient.Client
+	forwardEthCalls   bool // if true, forward eth_call calls directly to the configured proxy node
+	proxyOnError      bool // turn on regular proxy fall-through on errors; needed to test difference between direct and indirect fall-through
 }
 
 // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend
-func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff bool) *PublicEthAPI {
+func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff, forwardEthCalls, proxyOnError bool) (*PublicEthAPI, error) {
+	if forwardEthCalls && client == nil {
+		return nil, errors.New("ipld-eth-server is configured to forward eth_calls to proxy node but no proxy node is configured")
+	}
+	if proxyOnError && client == nil {
+		return nil, errors.New("ipld-eth-server is configured to forward all calls to proxy node on errors but no proxy node is configured")
+	}
 	var ethClient *ethclient.Client
 	if client != nil {
 		ethClient = ethclient.NewClient(client)
@@ -70,7 +81,9 @@ func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff bool) *Pu
 		supportsStateDiff: supportsStateDiff,
 		rpc:               client,
 		ethClient:         ethClient,
-	}
+		forwardEthCalls:   forwardEthCalls,
+		proxyOnError:      proxyOnError,
+	}, nil
 }
 
 /*
@@ -87,12 +100,13 @@ func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.Block
 	if header != nil && err == nil {
 		return pea.rpcMarshalHeader(header)
 	}
-	if pea.ethClient != nil {
+	if pea.proxyOnError {
 		if header, err := pea.ethClient.HeaderByNumber(ctx, big.NewInt(number.Int64())); header != nil && err == nil {
 			go pea.writeStateDiffAt(number.Int64())
 			return pea.rpcMarshalHeader(header)
 		}
 	}
+
 	return nil, err
 }
 
@@ -104,7 +118,8 @@ func (pea *PublicEthAPI) GetHeaderByHash(ctx context.Context, hash common.Hash)
 			return res
 		}
 	}
-	if pea.ethClient != nil {
+
+	if pea.proxyOnError {
 		if header, err := pea.ethClient.HeaderByHash(ctx, hash); header != nil && err == nil {
 			go pea.writeStateDiffFor(hash)
 			if res, err := pea.rpcMarshalHeader(header); err != nil {
@@ -112,6 +127,7 @@ func (pea *PublicEthAPI) GetHeaderByHash(ctx context.Context, hash common.Hash)
 			}
 		}
 	}
+
 	return nil
 }
 
@@ -122,7 +138,9 @@ func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]inte
 	if err != nil {
 		return nil, err
 	}
+
 	fields["totalDifficulty"] = (*hexutil.Big)(td)
+
 	return fields, nil
 }
 
@@ -142,12 +160,14 @@ func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockN
 	if block != nil && err == nil {
 		return pea.rpcMarshalBlock(block, true, fullTx)
 	}
-	if pea.ethClient != nil {
+
+	if pea.proxyOnError {
 		if block, err := pea.ethClient.BlockByNumber(ctx, big.NewInt(number.Int64())); block != nil && err == nil {
 			go pea.writeStateDiffAt(number.Int64())
 			return pea.rpcMarshalBlock(block, true, fullTx)
 		}
 	}
+
 	return nil, err
 }
 
@@ -158,28 +178,33 @@ func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, f
 	if block != nil && err == nil {
 		return pea.rpcMarshalBlock(block, true, fullTx)
 	}
-	if pea.ethClient != nil {
+
+	if pea.proxyOnError {
 		if block, err := pea.ethClient.BlockByHash(ctx, hash); block != nil && err == nil {
 			go pea.writeStateDiffFor(hash)
 			return pea.rpcMarshalBlock(block, true, fullTx)
 		}
 	}
+
 	return nil, err
 }
 
 // ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config.
-func (pea *PublicEthAPI) ChainId() hexutil.Uint64 {
+func (pea *PublicEthAPI) ChainId() (*hexutil.Big, error) {
-	chainID := new(big.Int)
 	block, err := pea.B.CurrentBlock()
 	if err != nil {
-		logrus.Errorf("ChainId failed with err %s", err.Error())
+		if pea.proxyOnError {
+			if id, err := pea.ethClient.ChainID(context.Background()); err == nil {
+				return (*hexutil.Big)(id), nil
+			}
+		}
+		return nil, err
+	}
 
-		return 0
-	}
 	if config := pea.B.Config.ChainConfig; config.IsEIP155(block.Number()) {
-		chainID = config.ChainID
+		return (*hexutil.Big)(config.ChainID), nil
 	}
-	return (hexutil.Uint64)(chainID.Uint64())
+	return nil, fmt.Errorf("chain not synced beyond EIP-155 replay-protection fork block")
 }
 
 /*
@@ -201,12 +226,14 @@ func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, bloc
 		block = types.NewBlockWithHeader(uncles[index])
 		return pea.rpcMarshalBlock(block, false, false)
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockNumberAndIndex", blockNr, index); uncle != nil && err == nil {
 			go pea.writeStateDiffAt(blockNr.Int64())
 			return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
 		}
 	}
+
 	return nil, err
 }
 
@@ -223,12 +250,14 @@ func (pea *PublicEthAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockH
 		block = types.NewBlockWithHeader(uncles[index])
 		return pea.rpcMarshalBlock(block, false, false)
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockHashAndIndex", blockHash, index); uncle != nil && err == nil {
 			go pea.writeStateDiffFor(blockHash)
 			return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
 		}
 	}
+
 	return nil, err
 }
 
@@ -238,13 +267,15 @@ func (pea *PublicEthAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr
 		n := hexutil.Uint(len(block.Uncles()))
 		return &n
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockNumber", blockNr); num != nil && err == nil {
 			go pea.writeStateDiffAt(blockNr.Int64())
 			return num
 		}
 	}
+
 	return nil
 }
 
@@ -254,13 +285,15 @@ func (pea *PublicEthAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash
 		n := hexutil.Uint(len(block.Uncles()))
 		return &n
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockHash", blockHash); num != nil && err == nil {
 			go pea.writeStateDiffFor(blockHash)
 			return num
 		}
 	}
+
 	return nil
 }
 
@@ -276,13 +309,15 @@ func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common
 	if count != nil && err == nil {
 		return count, nil
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		var num *hexutil.Uint64
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getTransactionCount", address, blockNrOrHash); num != nil && err == nil {
 			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return num, nil
 		}
 	}
+
 	return nil, err
 }
 
@@ -291,6 +326,7 @@ func (pea *PublicEthAPI) localGetTransactionCount(ctx context.Context, address c
 	if err != nil {
 		return nil, err
 	}
+
 	nonce := hexutil.Uint64(account.Nonce)
 	return &nonce, nil
 }
@@ -301,13 +337,15 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, b
 		n := hexutil.Uint(len(block.Transactions()))
 		return &n
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByNumber", blockNr); num != nil && err == nil {
 			go pea.writeStateDiffAt(blockNr.Int64())
 			return num
 		}
 	}
+
 	return nil
 }
 
@@ -317,13 +355,15 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByHash(ctx context.Context, blo
 		n := hexutil.Uint(len(block.Transactions()))
 		return &n
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByHash", blockHash); num != nil && err == nil {
 			go pea.writeStateDiffFor(blockHash)
 			return num
 		}
 	}
+
 	return nil
 }
 
@@ -332,13 +372,15 @@ func (pea *PublicEthAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context
 	if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil {
 		return newRPCTransactionFromBlockIndex(block, uint64(index))
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
 			go pea.writeStateDiffAt(blockNr.Int64())
 			return tx
 		}
 	}
+
 	return nil
 }
 
@@ -347,13 +389,15 @@ func (pea *PublicEthAPI) GetTransactionByBlockHashAndIndex(ctx context.Context,
 	if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil {
 		return newRPCTransactionFromBlockIndex(block, uint64(index))
 	}
-	if pea.rpc != nil {
+
+	if pea.proxyOnError {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
 			go pea.writeStateDiffFor(blockHash)
 			return tx
 		}
 	}
+
 	return nil
 }
 
@@ -362,7 +406,7 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Cont
 	if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil {
 		return newRPCRawTransactionFromBlockIndex(block, uint64(index))
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
 			go pea.writeStateDiffAt(blockNr.Int64())
@@ -377,7 +421,7 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Contex
 	if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil {
 		return newRPCRawTransactionFromBlockIndex(block, uint64(index))
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
 			go pea.writeStateDiffFor(blockHash)
@@ -399,7 +443,7 @@ func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.H
 
 		return NewRPCTransaction(tx, blockHash, blockNumber, index, header.BaseFee), nil
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); tx != nil && err == nil {
 			go pea.writeStateDiffFor(hash)
@@ -416,7 +460,7 @@ func (pea *PublicEthAPI) GetRawTransactionByHash(ctx context.Context, hash commo
 	if tx != nil && err == nil {
 		return rlp.EncodeToBytes(tx)
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); tx != nil && err == nil {
 			go pea.writeStateDiffFor(hash)
@@ -438,7 +482,7 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.
 	if receipt != nil && err == nil {
 		return receipt, nil
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		if receipt := pea.remoteGetTransactionReceipt(ctx, hash); receipt != nil {
 			go pea.writeStateDiffFor(hash)
 			return receipt, nil
@@ -536,7 +580,7 @@ func (pea *PublicEthAPI) remoteGetTransactionReceipt(ctx context.Context, hash c
 // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs
 func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit filters.FilterCriteria) ([]*types.Log, error) {
 	logs, err := pea.localGetLogs(crit)
-	if err != nil && pea.rpc != nil {
+	if err != nil && pea.proxyOnError {
 		var res []*types.Log
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getLogs", crit); err == nil {
 			go pea.writeStateDiffWithCriteria(crit)
@@ -586,24 +630,14 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log
 		}
 	}()
 
-	// If we have a blockhash to filter on, fire off single retrieval query
+	// If we have a blockHash to filter on, fire off single retrieval query
 	if crit.BlockHash != nil {
-		rctCIDs, err := pea.B.Retriever.RetrieveRctCIDs(tx, filter, 0, crit.BlockHash, nil)
+		filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, 0, crit.BlockHash)
 		if err != nil {
 			return nil, err
 		}
-		rctIPLDs, err := pea.B.Fetcher.FetchRcts(tx, rctCIDs)
+
-		if err != nil {
+		return decomposeLogs(filteredLogs)
-			return nil, err
-		}
-		if err := tx.Commit(); err != nil {
-			return nil, err
-		}
-		block, err := pea.B.BlockByHash(context.Background(), *crit.BlockHash)
-		if err != nil {
-			return nil, err
-		}
-		return extractLogsOfInterest(pea.B.Config.ChainConfig, *crit.BlockHash, block.NumberU64(), block.Transactions(), rctIPLDs, filter)
 	}
 
 	// Otherwise, create block range from criteria
@@ -626,27 +660,17 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log
 	end := endingBlock.Int64()
 	var logs []*types.Log
 	for i := start; i <= end; i++ {
-		rctCIDs, err := pea.B.Retriever.RetrieveRctCIDs(tx, filter, i, nil, nil)
+		filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, i, nil)
 		if err != nil {
 			return nil, err
 		}
 
-		block, err := pea.B.BlockByNumber(context.Background(), rpc.BlockNumber(i))
+		logCIDs, err := decomposeLogs(filteredLogs)
 		if err != nil {
 			return nil, err
 		}
 
-		rctIPLDs, err := pea.B.Fetcher.FetchRcts(tx, rctCIDs)
+		logs = append(logs, logCIDs...)
-		if err != nil {
-			return nil, err
-		}
-
-		log, err := extractLogsOfInterest(pea.B.Config.ChainConfig, block.Hash(), uint64(i), block.Transactions(), rctIPLDs, filter)
-		if err != nil {
-			return nil, err
-		}
-
-		logs = append(logs, log...)
 	}
 
 	if err := tx.Commit(); err != nil {
@@ -670,7 +694,7 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address,
 	if bal != nil && err == nil {
 		return bal, nil
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		var res *hexutil.Big
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getBalance", address, blockNrOrHash); res != nil && err == nil {
 			go pea.writeStateDiffAtOrFor(blockNrOrHash)
@@ -710,7 +734,7 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres
 
 		return value[:], nil
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		var res hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil {
 			go pea.writeStateDiffAtOrFor(blockNrOrHash)
@@ -729,7 +753,7 @@ func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, bl
 	if code != nil && err == nil {
 		return code, nil
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		var res hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil {
 			go pea.writeStateDiffAtOrFor(blockNrOrHash)
@@ -749,7 +773,7 @@ func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, s
 	if proof != nil && err == nil {
 		return proof, nil
 	}
-	if pea.rpc != nil {
+	if pea.proxyOnError {
 		var res *AccountResult
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getProof", address, storageKeys, blockNrOrHash); res != nil && err == nil {
 			go pea.writeStateDiffAtOrFor(blockNrOrHash)
@@ -808,50 +832,59 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre
 	}, state.Error()
 }
 
-// Call executes the given transaction on the state for the given block number.
+// revertError is an API error that encompassas an EVM revertal with JSON error
-//
+// code and a binary data blob.
-// Additionally, the caller can specify a batch of contract for fields overriding.
+type revertError struct {
-//
+	error
-// Note, this function doesn't make and changes in the state/blockchain and is
+	reason string // revert reason hex encoded
-// useful to execute and retrieve values.
-func (pea *PublicEthAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *map[common.Address]account) (hexutil.Bytes, error) {
-	var accounts map[common.Address]account
-	if overrides != nil {
-		accounts = *overrides
-	}
-	res, _, failed, err := DoCall(ctx, pea.B, args, blockNrOrHash, accounts, 5*time.Second, pea.B.Config.RPCGasCap)
-	if (failed || err != nil) && pea.rpc != nil {
-		var hex hexutil.Bytes
-		if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil {
-			go pea.writeStateDiffAtOrFor(blockNrOrHash)
-			return hex, nil
-		}
-	}
-	if failed && err == nil {
-		return nil, errors.New("eth_call failed without error")
-	}
-	return (hexutil.Bytes)(res), err
 }
 
-func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides map[common.Address]account, timeout time.Duration, globalGasCap *big.Int) ([]byte, uint64, bool, error) {
+// ErrorCode returns the JSON error code for a revertal.
-	defer func(start time.Time) {
+// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal
-		logrus.Debugf("Executing EVM call finished %s runtime %s", time.Now().String(), time.Since(start).String())
+func (e *revertError) ErrorCode() int {
-	}(time.Now())
+	return 3
-	state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash)
+}
-	if state == nil || err != nil {
+
-		return nil, 0, false, err
+// ErrorData returns the hex encoded revert reason.
+func (e *revertError) ErrorData() interface{} {
+	return e.reason
+}
+
+func newRevertError(result *core.ExecutionResult) *revertError {
+	reason, errUnpack := abi.UnpackRevert(result.Revert())
+	err := errors.New("execution reverted")
+	if errUnpack == nil {
+		err = fmt.Errorf("execution reverted: %v", reason)
 	}
-	// Set sender address or use a default if none specified
+	return &revertError{
-	var addr common.Address
+		error:  err,
-	if args.From == nil {
+		reason: hexutil.Encode(result.Revert()),
-		if b.Config.DefaultSender != nil {
-			addr = *b.Config.DefaultSender
-		}
-	} else {
-		addr = *args.From
 	}
-	// Override the fields of specified contracts before execution.
+}
-	for addr, account := range overrides {
+
+// OverrideAccount indicates the overriding fields of account during the execution
+// of a message call.
+// Note, state and stateDiff can't be specified at the same time. If state is
+// set, message execution will only use the data in the given state. Otherwise
+// if statDiff is set, all diff will be applied first and then execute the call
+// message.
+type OverrideAccount struct {
+	Nonce     *hexutil.Uint64              `json:"nonce"`
+	Code      *hexutil.Bytes               `json:"code"`
+	Balance   **hexutil.Big                `json:"balance"`
+	State     *map[common.Hash]common.Hash `json:"state"`
+	StateDiff *map[common.Hash]common.Hash `json:"stateDiff"`
+}
+
+// StateOverride is the collection of overridden accounts.
+type StateOverride map[common.Address]OverrideAccount
+
+// Apply overrides the fields of specified accounts into the given state.
+func (diff *StateOverride) Apply(state *state.StateDB) error {
+	if diff == nil {
+		return nil
+	}
+	for addr, account := range *diff {
 		// Override account nonce.
 		if account.Nonce != nil {
 			state.SetNonce(addr, uint64(*account.Nonce))
@@ -865,7 +898,7 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 			state.SetBalance(addr, (*big.Int)(*account.Balance))
 		}
 		if account.State != nil && account.StateDiff != nil {
-			return nil, 0, false, fmt.Errorf("account %s has both 'state' and 'stateDiff'", addr.Hex())
+			return fmt.Errorf("account %s has both 'state' and 'stateDiff'", addr.Hex())
 		}
 		// Replace entire state if caller requires.
 		if account.State != nil {
@@ -878,74 +911,57 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 			}
 		}
 	}
-	// Set default gas & gas price if none were set
+	return nil
-	gas := uint64(math.MaxUint64 / 2)
+}
-	if args.Gas != nil {
+
-		gas = uint64(*args.Gas)
+// Call executes the given transaction on the state for the given block number.
+//
+// Additionally, the caller can specify a batch of contract for fields overriding.
+//
+// Note, this function doesn't make and changes in the state/blockchain and is
+// useful to execute and retrieve values.
+func (pea *PublicEthAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) {
+	if pea.forwardEthCalls {
+		var hex hexutil.Bytes
+		err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides)
+		return hex, err
 	}
 
-	if globalGasCap != nil && globalGasCap.Uint64() < gas {
+	result, err := DoCall(ctx, pea.B, args, blockNrOrHash, overrides, 5*time.Second, pea.B.Config.RPCGasCap.Uint64())
-		logrus.Warnf("Caller gas above allowance, capping; requested: %d, cap: %d", gas, globalGasCap)
-		gas = globalGasCap.Uint64()
-	}
 
-	var (
+	// If the result contains a revert reason, try to unpack and return it.
-		gasPrice  *big.Int
+	if err == nil {
-		gasFeeCap *big.Int
+		if len(result.Revert()) > 0 {
-		gasTipCap *big.Int
+			err = newRevertError(result)
-	)
+		} else if result.Err != nil {
-
+			err = result.Err
-	if header.BaseFee == nil {
-		// If there's no basefee, then it must be a non-1559 execution
-		gasPrice = new(big.Int)
-		if args.GasPrice != nil {
-			gasPrice = args.GasPrice.ToInt()
-		}
-		gasFeeCap, gasTipCap = gasPrice, gasPrice
-	} else {
-		// A basefee is provided, necessitating 1559-type execution
-		if args.GasPrice != nil {
-			// User specified the legacy gas field, convert to 1559 gas typing
-			gasPrice = args.GasPrice.ToInt()
-			gasFeeCap, gasTipCap = gasPrice, gasPrice
-		} else {
-			// User specified 1559 gas feilds (or none), use those
-			gasFeeCap = new(big.Int)
-			if args.MaxFeePerGas != nil {
-				gasFeeCap = args.MaxFeePerGas.ToInt()
-			}
-			gasTipCap = new(big.Int)
-			if args.MaxPriorityFeePerGas != nil {
-				gasTipCap = args.MaxPriorityFeePerGas.ToInt()
-			}
-			// Backfill the legacy gasPrice for EVM execution, unless we're all zeroes
-			gasPrice = new(big.Int)
-			if gasFeeCap.BitLen() > 0 || gasTipCap.BitLen() > 0 {
-				gasPrice = math.BigMin(new(big.Int).Add(gasTipCap, header.BaseFee), gasFeeCap)
-			}
 		}
 	}
 
-	value := new(big.Int)
+	if err != nil && pea.proxyOnError {
-	if args.Value != nil {
+		var hex hexutil.Bytes
-		value = args.Value.ToInt()
+		if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
+			return hex, nil
+		}
+	}
+	return result.Return(), err
+}
+
+func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) {
+	defer func(start time.Time) {
+		logrus.Debugf("Executing EVM call finished %s runtime %s", time.Now().String(), time.Since(start).String())
+	}(time.Now())
+
+	state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash)
+	if state == nil || err != nil {
+		return nil, err
 	}
 
-	var data []byte
+	if err := overrides.Apply(state); err != nil {
-	if args.Input != nil {
+		return nil, err
-		data = *args.Input
-	} else if args.Data != nil {
-		data = *args.Data
 	}
 
-	var accessList types.AccessList
-	if args.AccessList != nil {
-		accessList = *args.AccessList
-	}
-
-	// Create new call message
-	msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, gasFeeCap, gasTipCap, data, accessList, false)
-
 	// Setup context so it may be cancelled the call has completed
 	// or, in case of unmetered gas, setup a context with a timeout.
 	var cancel context.CancelFunc
@@ -959,10 +975,16 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 	defer cancel()
 
 	// Get a new instance of the EVM.
-	evm, err := b.GetEVM(ctx, msg, state, header)
+	msg, err := args.ToMessage(globalGasCap, header.BaseFee)
 	if err != nil {
-		return nil, 0, false, err
+		return nil, err
 	}
+
+	evm, vmError, err := b.GetEVM(ctx, msg, state, header)
+	if err != nil {
+		return nil, err
+	}
+
 	// Wait for the context to be done and cancel the evm. Even if the
 	// EVM has finished, cancelling may be done (repeatedly)
 	go func() {
@@ -970,18 +992,21 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 		evm.Cancel()
 	}()
 
-	// Setup the gas pool (also for unmetered requests)
+	// Execute the message.
-	// and apply the message.
 	gp := new(core.GasPool).AddGas(math.MaxUint64)
 	result, err := core.ApplyMessage(evm, msg, gp)
-	if err != nil {
+	if err := vmError(); err != nil {
-		return nil, 0, false, fmt.Errorf("execution failed: %v", err)
+		return nil, err
 	}
+
 	// If the timer caused an abort, return an appropriate error message
 	if evm.Cancelled() {
-		return nil, 0, false, fmt.Errorf("execution aborted (timeout = %v)", timeout)
+		return nil, fmt.Errorf("execution aborted (timeout = %v)", timeout)
 	}
-	return result.Return(), result.UsedGas, result.Failed(), err
+	if err != nil {
+		return result, fmt.Errorf("err: %w (supplied gas %d)", err, msg.Gas())
+	}
+	return result, nil
 }
 
 // writeStateDiffAtOrFor calls out to the proxy statediffing geth client to fill in a gap in the index
@@ -1105,3 +1130,42 @@ func toHexSlice(b [][]byte) []string {
 	}
 	return r
 }
+
+// decomposeLogs return logs from LogResult.
+func decomposeLogs(logCIDs []LogResult) ([]*types.Log, error) {
+	logs := make([]*types.Log, len(logCIDs))
+	for i, l := range logCIDs {
+		topics := make([]common.Hash, 0)
+		if l.Topic0 != "" {
+			topics = append(topics, common.HexToHash(l.Topic0))
+		}
+		if l.Topic1 != "" {
+			topics = append(topics, common.HexToHash(l.Topic1))
+		}
+		if l.Topic2 != "" {
+			topics = append(topics, common.HexToHash(l.Topic2))
+		}
+		if l.Topic3 != "" {
+			topics = append(topics, common.HexToHash(l.Topic3))
+		}
+
+		// TODO: should we convert string to uint ?
+		blockNum, err := strconv.ParseUint(l.BlockNumber, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+
+		logs[i] = &types.Log{
+			Address:     common.HexToAddress(l.Address),
+			Topics:      topics,
+			Data:        l.Data,
+			BlockNumber: blockNum,
+			TxHash:      common.HexToHash(l.TxHash),
+			TxIndex:     uint(l.TxnIndex),
+			BlockHash:   common.HexToHash(l.BlockHash),
+			Index:       uint(l.Index),
+		}
+	}
+
+	return logs, nil
+}

pkg/eth/api_test.go

@@ -19,6 +19,7 @@ package eth_test
 import (
 	"context"
 	"math/big"
+	"os"
 	"strconv"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -30,28 +31,28 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 var (
-	randomAddr      = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
+	randomAddr     = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
-	randomHash      = crypto.Keccak256Hash(randomAddr.Bytes())
+	randomHash     = crypto.Keccak256Hash(randomAddr.Bytes())
-	number          = rpc.BlockNumber(test_helpers.BlockNumber.Int64())
+	number         = rpc.BlockNumber(test_helpers.BlockNumber.Int64())
-	londonBlockNum  = rpc.BlockNumber(test_helpers.LondonBlockNum.Int64())
+	londonBlockNum = rpc.BlockNumber(test_helpers.LondonBlockNum.Int64())
-	wrongNumber     = rpc.BlockNumber(number + 1)
+	wrongNumber    = number + 1
-	blockHash       = test_helpers.MockBlock.Header().Hash()
+	blockHash      = test_helpers.MockBlock.Header().Hash()
-	londonBlockHash = test_helpers.MockLondonBlock.Header().Hash()
+	baseFee        = test_helpers.MockLondonBlock.BaseFee()
-	baseFee         = test_helpers.MockLondonBlock.BaseFee()
+	ctx            = context.Background()
-	ctx             = context.Background()
+	expectedBlock  = map[string]interface{}{
-	expectedBlock   = map[string]interface{}{
 		"number":           (*hexutil.Big)(test_helpers.MockBlock.Number()),
 		"hash":             test_helpers.MockBlock.Hash(),
 		"parentHash":       test_helpers.MockBlock.ParentHash(),
@@ -150,7 +151,7 @@ var (
 		"contractAddress":   nil,
 		"logs":              test_helpers.MockReceipts[0].Logs,
 		"logsBloom":         test_helpers.MockReceipts[0].Bloom,
-		"root":              hexutil.Bytes(test_helpers.MockReceipts[0].PostState),
+		"status":            hexutil.Uint(test_helpers.MockReceipts[0].Status),
 	}
 	expectedReceipt2 = map[string]interface{}{
 		"blockHash":         blockHash,
@@ -182,6 +183,19 @@ var (
 	}
 )
 
+// SetupDB is use to setup a db for watcher tests
+func SetupDB() (*postgres.DB, error) {
+	port, _ := strconv.Atoi(os.Getenv("DATABASE_PORT"))
+	uri := postgres.DbConnectionString(postgres.ConnectionParams{
+		User:     os.Getenv("DATABASE_USER"),
+		Password: os.Getenv("DATABASE_PASSWORD"),
+		Hostname: os.Getenv("DATABASE_HOSTNAME"),
+		Name:     os.Getenv("DATABASE_NAME"),
+		Port:     port,
+	})
+	return postgres.NewDB(uri, postgres.ConnectionConfig{}, node.Info{})
+}
+
 var _ = Describe("API", func() {
 	var (
 		db          *postgres.DB
@@ -191,21 +205,50 @@ var _ = Describe("API", func() {
 	// Test db setup, rather than using BeforeEach we only need to setup once since the tests do not mutate the database
 	// Note: if you focus one of the tests be sure to focus this and the defered It()
 	It("test init", func() {
-		var err error
+		var (
-		db, err = shared.SetupDB()
+			err error
+			tx  *indexer.BlockTx
+		)
+
+		db, err = SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+
+		indexAndPublisher, err := indexer.NewStateDiffIndexer(chainConfig, db)
 		Expect(err).ToNot(HaveOccurred())
-		indexAndPublisher := eth2.NewIPLDPublisher(db)
 		backend, err := eth.NewEthBackend(db, &eth.Config{
 			ChainConfig: chainConfig,
-			VmConfig:    vm.Config{},
+			VMConfig:    vm.Config{},
 			RPCGasCap:   big.NewInt(10000000000), // Max gas capacity for a rpc call.
+			GroupCacheConfig: &ethServerShared.GroupCacheConfig{
+				StateDB: ethServerShared.GroupConfig{
+					Name:                   "api_test",
+					CacheSizeInMB:          8,
+					CacheExpiryInMins:      60,
+					LogStatsIntervalInSecs: 0,
+				},
+			},
 		})
 		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend, nil, false)
+		api, _ = eth.NewPublicEthAPI(backend, nil, false, false, false)
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
+		tx, err = indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
 		Expect(err).ToNot(HaveOccurred())
-		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
+
+		for _, node := range test_helpers.MockStateNodes {
+			err = indexAndPublisher.PushStateNode(tx, node)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		ccHash := sdtypes.CodeAndCodeHash{
+			Hash: test_helpers.ContractCodeHash,
+			Code: test_helpers.ContractCode,
+		}
+
+		err = indexAndPublisher.PushCodeAndCodeHash(tx, ccHash)
 		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Close(err)
+		Expect(err).ToNot(HaveOccurred())
+
 		uncles := test_helpers.MockBlock.Uncles()
 		uncleHashes := make([]common.Hash, len(uncles))
 		for i, uncle := range uncles {
@@ -213,7 +256,15 @@ var _ = Describe("API", func() {
 		}
 		expectedBlock["uncles"] = uncleHashes
 
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedLondonPayload)
+		// setting chain config to for london block
+		chainConfig.LondonBlock = big.NewInt(2)
+		indexAndPublisher, err = indexer.NewStateDiffIndexer(chainConfig, db)
+		Expect(err).ToNot(HaveOccurred())
+
+		tx, err = indexAndPublisher.PushBlock(test_helpers.MockLondonBlock, test_helpers.MockLondonReceipts, test_helpers.MockLondonBlock.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Close(err)
 		Expect(err).ToNot(HaveOccurred())
 	})
 
@@ -339,8 +390,7 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			_, ok := block["baseFee"]
 			Expect(ok).To(Equal(false))
-
+			block, err = api.GetBlockByHash(ctx, test_helpers.MockLondonBlock.Hash(), false)
-			block, err = api.GetBlockByHash(ctx, londonBlockHash, false)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(block["baseFee"].(*big.Int)).To(Equal(baseFee))
 		})
@@ -440,14 +490,14 @@ var _ = Describe("API", func() {
 	Describe("eth_getBlockTransactionCountByNumber", func() {
 		It("Retrieves the number of transactions in the canonical block with the provided number", func() {
 			count := api.GetBlockTransactionCountByNumber(ctx, number)
-			Expect(uint64(*count)).To(Equal(uint64(3)))
+			Expect(uint64(*count)).To(Equal(uint64(4)))
 		})
 	})
 
 	Describe("eth_getBlockTransactionCountByHash", func() {
 		It("Retrieves the number of transactions in the block with the provided hash ", func() {
 			count := api.GetBlockTransactionCountByHash(ctx, blockHash)
-			Expect(uint64(*count)).To(Equal(uint64(3)))
+			Expect(uint64(*count)).To(Equal(uint64(4)))
 		})
 	})
 
@@ -490,7 +540,7 @@ var _ = Describe("API", func() {
 		})
 
 		It("Retrieves the GasFeeCap and GasTipCap for dynamic transaction from the london block hash", func() {
-			tx := api.GetTransactionByBlockHashAndIndex(ctx, londonBlockHash, 0)
+			tx := api.GetTransactionByBlockHashAndIndex(ctx, test_helpers.MockLondonBlock.Hash(), 0)
 			Expect(tx).ToNot(BeNil())
 			Expect(tx.GasFeeCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasFeeCap())))
 			Expect(tx.GasTipCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasTipCap())))
@@ -610,7 +660,13 @@ var _ = Describe("API", func() {
 			crit := filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
-						common.HexToHash("0x04"),
+						common.HexToHash("0x0c"),
+					},
+					{
+						common.HexToHash("0x0a"),
+					},
+					{
+						common.HexToHash("0x0b"),
 					},
 				},
 				FromBlock: test_helpers.MockBlock.Number(),
@@ -618,6 +674,58 @@ var _ = Describe("API", func() {
 			}
 			logs, err := api.GetLogs(ctx, crit)
 			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(0))
+
+			crit = filters.FilterCriteria{
+				Topics: [][]common.Hash{
+					{
+						common.HexToHash("0x08"),
+					},
+					{
+						common.HexToHash("0x0a"),
+					},
+					{
+						common.HexToHash("0x0c"),
+					},
+				},
+				FromBlock: test_helpers.MockBlock.Number(),
+				ToBlock:   test_helpers.MockBlock.Number(),
+			}
+			logs, err = api.GetLogs(ctx, crit)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(0))
+
+			crit = filters.FilterCriteria{
+				Topics: [][]common.Hash{
+					{
+						common.HexToHash("0x09"),
+					},
+					{
+						common.HexToHash("0x0a"),
+					},
+					{
+						common.HexToHash("0x0b"),
+					},
+				},
+				FromBlock: test_helpers.MockBlock.Number(),
+				ToBlock:   test_helpers.MockBlock.Number(),
+			}
+			logs, err = api.GetLogs(ctx, crit)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(1))
+			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog4}))
+
+			crit = filters.FilterCriteria{
+				Topics: [][]common.Hash{
+					{
+						common.HexToHash("0x04"),
+					},
+				},
+				FromBlock: test_helpers.MockBlock.Number(),
+				ToBlock:   test_helpers.MockBlock.Number(),
+			}
+			logs, err = api.GetLogs(ctx, crit)
+			Expect(err).ToNot(HaveOccurred())
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1}))
 
@@ -775,8 +883,8 @@ var _ = Describe("API", func() {
 			}
 			logs, err = api.GetLogs(ctx, crit)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
+			Expect(len(logs)).To(Equal(6))
-			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
+			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2, test_helpers.MockLog3, test_helpers.MockLog4, test_helpers.MockLog5, test_helpers.MockLog6}))
 		})
 
 		It("Uses the provided blockhash if one is provided", func() {
@@ -911,8 +1019,8 @@ var _ = Describe("API", func() {
 			}
 			logs, err = api.GetLogs(ctx, crit)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
+			Expect(len(logs)).To(Equal(6))
-			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
+			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2, test_helpers.MockLog3, test_helpers.MockLog4, test_helpers.MockLog5, test_helpers.MockLog6}))
 		})
 
 		It("Filters on contract address if any are provided", func() {
@@ -1034,20 +1142,3 @@ var _ = Describe("API", func() {
 		})
 	})
 })
-
-func publishCode(db *postgres.DB, codeHash common.Hash, code []byte) error {
-	tx, err := db.Beginx()
-	if err != nil {
-		return err
-	}
-	mhKey, err := shared.MultihashKeyFromKeccak256(codeHash)
-	if err != nil {
-		tx.Rollback()
-		return err
-	}
-	if err := shared.PublishDirect(tx, mhKey, code); err != nil {
-		tx.Rollback()
-		return err
-	}
-	return tx.Commit()
-}

pkg/eth/backend.go

@@ -22,10 +22,10 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
@@ -39,12 +39,13 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+	ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/trie"
-
+	log "github.com/sirupsen/logrus"
-	ipfsethdb "github.com/vulcanize/ipfs-ethdb"
+	validator "github.com/vulcanize/eth-ipfs-state-validator/pkg"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-	shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
@@ -54,6 +55,10 @@ var (
 	errNegativeBlockNumber = errors.New("negative block number not supported")
 	errHeaderHashNotFound  = errors.New("header for hash not found")
 	errHeaderNotFound      = errors.New("header not found")
+
+	// errMissingSignature is returned if a block's extra-data section doesn't seem
+	// to contain a 65 byte secp256k1 signature.
+	errMissingSignature = errors.New("extra-data 65 byte signature suffix missing")
 )
 
 const (
@@ -82,6 +87,10 @@ const (
 	RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1`
 )
 
+const (
+	StateDBGroupCacheName = "statedb"
+)
+
 type Backend struct {
 	// underlying postgres db
 	DB *postgres.DB
@@ -99,15 +108,30 @@ type Backend struct {
 }
 
 type Config struct {
-	ChainConfig   *params.ChainConfig
+	ChainConfig      *params.ChainConfig
-	VmConfig      vm.Config
+	VMConfig         vm.Config
-	DefaultSender *common.Address
+	DefaultSender    *common.Address
-	RPCGasCap     *big.Int
+	RPCGasCap        *big.Int
+	GroupCacheConfig *shared.GroupCacheConfig
 }
 
 func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) {
+	gcc := c.GroupCacheConfig
+
+	groupName := gcc.StateDB.Name
+	if groupName == "" {
+		groupName = StateDBGroupCacheName
+	}
+
 	r := NewCIDRetriever(db)
-	ethDB := ipfsethdb.NewDatabase(db.DB)
+	ethDB := ipfsethdb.NewDatabase(db.DB, ipfsethdb.CacheConfig{
+		Name:           groupName,
+		Size:           gcc.StateDB.CacheSizeInMB * 1024 * 1024,
+		ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins),
+	})
+
+	logStateDBStatsOnTimer(ethDB.(*ipfsethdb.Database), gcc)
+
 	return &Backend{
 		DB:            db,
 		Retriever:     r,
@@ -289,10 +313,10 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	}
 	defer func() {
 		if p := recover(); p != nil {
-			shared.Rollback(tx)
+			ethServerShared.Rollback(tx)
 			panic(p)
 		} else if err != nil {
-			shared.Rollback(tx)
+			ethServerShared.Rollback(tx)
 		} else {
 			err = tx.Commit()
 		}
@@ -351,7 +375,12 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	var receipts []*types.Receipt
 	for _, rctIPLD := range rctIPLDs {
 		var receipt types.Receipt
-		err = receipt.UnmarshalBinary(rctIPLD.Data)
+		nodeVal, err := DecodeLeafNode(rctIPLD.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		err = receipt.UnmarshalBinary(nodeVal)
 		if err != nil {
 			return nil, err
 		}
@@ -380,10 +409,10 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	}
 	defer func() {
 		if p := recover(); p != nil {
-			shared.Rollback(tx)
+			ethServerShared.Rollback(tx)
 			panic(p)
 		} else if err != nil {
-			shared.Rollback(tx)
+			ethServerShared.Rollback(tx)
 		} else {
 			err = tx.Commit()
 		}
@@ -451,7 +480,12 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	var receipts []*types.Receipt
 	for _, rctIPLD := range rctIPLDs {
 		var receipt types.Receipt
-		err = receipt.UnmarshalBinary(rctIPLD.Data)
+		nodeVal, err := DecodeLeafNode(rctIPLD.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		err = receipt.UnmarshalBinary(nodeVal)
 		if err != nil {
 			return nil, err
 		}
@@ -585,15 +619,15 @@ func (b *Backend) GetCanonicalHeader(number uint64) (string, []byte, error) {
 }
 
 // GetEVM constructs and returns a vm.EVM
-func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, error) {
+func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) {
-	state.SetBalance(msg.From(), math.MaxBig256)
+	vmError := func() error { return nil }
-	vmctx := core.NewEVMBlockContext(header, b, nil)
 	txContext := core.NewEVMTxContext(msg)
-	return vm.NewEVM(vmctx, txContext, state, b.Config.ChainConfig, b.Config.VmConfig), nil
+	context := core.NewEVMBlockContext(header, b, nil)
+	return vm.NewEVM(context, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil
 }
 
 // GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash
-func (b *Backend) GetAccountByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*state.Account, error) {
+func (b *Backend) GetAccountByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*types.StateAccount, error) {
 	if blockNr, ok := blockNrOrHash.Number(); ok {
 		return b.GetAccountByNumber(ctx, address, blockNr)
 	}
@@ -604,7 +638,7 @@ func (b *Backend) GetAccountByNumberOrHash(ctx context.Context, address common.A
 }
 
 // GetAccountByNumber returns the account object for the provided address at the canonical block at the provided height
-func (b *Backend) GetAccountByNumber(ctx context.Context, address common.Address, blockNumber rpc.BlockNumber) (*state.Account, error) {
+func (b *Backend) GetAccountByNumber(ctx context.Context, address common.Address, blockNumber rpc.BlockNumber) (*types.StateAccount, error) {
 	var err error
 	number := blockNumber.Int64()
 	if blockNumber == rpc.LatestBlockNumber {
@@ -633,7 +667,7 @@ func (b *Backend) GetAccountByNumber(ctx context.Context, address common.Address
 }
 
 // GetAccountByHash returns the account object for the provided address at the block with the provided hash
-func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, hash common.Hash) (*state.Account, error) {
+func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, hash common.Hash) (*types.StateAccount, error) {
 	_, err := b.HeaderByHash(context.Background(), hash)
 	if err == sql.ErrNoRows {
 		return nil, errHeaderHashNotFound
@@ -646,7 +680,7 @@ func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address,
 		return nil, err
 	}
 
-	acct := new(state.Account)
+	acct := new(types.StateAccount)
 	return acct, rlp.DecodeBytes(accountRlp, acct)
 }
 
@@ -701,10 +735,10 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has
 	}
 	defer func() {
 		if p := recover(); p != nil {
-			shared.Rollback(tx)
+			ethServerShared.Rollback(tx)
 			panic(p)
 		} else if err != nil {
-			shared.Rollback(tx)
+			ethServerShared.Rollback(tx)
 		} else {
 			err = tx.Commit()
 		}
@@ -714,7 +748,7 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has
 		return nil, err
 	}
 	var mhKey string
-	mhKey, err = shared2.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
+	mhKey, err = ethServerShared.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
 	if err != nil {
 		return nil, err
 	}
@@ -791,6 +825,11 @@ func (b *Backend) GetHeader(hash common.Hash, height uint64) *types.Header {
 	return header
 }
 
+// ValidateTrie validates the trie for the given stateRoot
+func (b *Backend) ValidateTrie(stateRoot common.Hash) error {
+	return validator.NewValidator(nil, b.EthDB).ValidateTrie(stateRoot)
+}
+
 // RPCGasCap returns the configured gas cap for the rpc server
 func (b *Backend) RPCGasCap() *big.Int {
 	return b.Config.RPCGasCap
@@ -823,3 +862,18 @@ func (b *Backend) BloomStatus() (uint64, uint64) {
 func (b *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
 	panic("implement me")
 }
+
+func logStateDBStatsOnTimer(ethDB *ipfsethdb.Database, gcc *shared.GroupCacheConfig) {
+	// No stats logging if interval isn't a positive integer.
+	if gcc.StateDB.LogStatsIntervalInSecs <= 0 {
+		return
+	}
+
+	ticker := time.NewTicker(time.Duration(gcc.StateDB.LogStatsIntervalInSecs) * time.Second)
+
+	go func() {
+		for range ticker.C {
+			log.Infof("%s groupcache stats: %+v", StateDBGroupCacheName, ethDB.GetCacheStats())
+		}
+	}()
+}

pkg/eth/backend_utils.go

@@ -27,11 +27,8 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
 )
 
 // RPCMarshalHeader converts the given header to the RPC output.
@@ -123,8 +120,8 @@ func RPCMarshalBlockWithUncleHashes(block *types.Block, uncleHashes []common.Has
 		}
 		fields["transactions"] = transactions
 	}
-	fields["uncles"] = uncleHashes
 
+	fields["uncles"] = uncleHashes
 	return fields, nil
 }
 
@@ -271,122 +268,6 @@ func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransacti
 	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index, b.BaseFee())
 }
 
-// extractLogsOfInterest returns logs from the receipt IPLD
-func extractLogsOfInterest(config *params.ChainConfig, blockHash common.Hash, blockNumber uint64,
-	txs types.Transactions, rctIPLDs []ipfs.BlockModel, filter ReceiptFilter) ([]*types.Log, error) {
-	receipts := make(types.Receipts, len(rctIPLDs))
-
-	for i, rctBytes := range rctIPLDs {
-		rct := new(types.Receipt)
-		if err := rct.UnmarshalBinary(rctBytes.Data); err != nil {
-			return nil, err
-		}
-		receipts[i] = rct
-	}
-
-	err := receipts.DeriveFields(config, blockHash, blockNumber, txs)
-	if err != nil {
-		return nil, err
-	}
-
-	var unfilteredLogs []*types.Log
-	for _, receipt := range receipts {
-		unfilteredLogs = append(unfilteredLogs, receipt.Logs...)
-	}
-
-	adders := make([]common.Address, len(filter.LogAddresses))
-	for i, addr := range filter.LogAddresses {
-		adders[i] = common.HexToAddress(addr)
-	}
-
-	topics := make([][]common.Hash, len(filter.Topics))
-	for i, v := range filter.Topics {
-		topics[i] = make([]common.Hash, len(v))
-		for j, topic := range v {
-			topics[i][j] = common.HexToHash(topic)
-		}
-	}
-
-	logs := filterLogs(unfilteredLogs, nil, nil, adders, topics)
-	return logs, nil
-}
-
-func includes(addresses []common.Address, a common.Address) bool {
-	for _, addr := range addresses {
-		if addr == a {
-			return true
-		}
-	}
-
-	return false
-}
-
-// filterLogs creates a slice of logs matching the given criteria.
-func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []common.Address, topics [][]common.Hash) []*types.Log {
-	var ret []*types.Log
-Logs:
-	for _, log := range logs {
-		if fromBlock != nil && fromBlock.Int64() >= 0 && fromBlock.Uint64() > log.BlockNumber {
-			continue
-		}
-		if toBlock != nil && toBlock.Int64() >= 0 && toBlock.Uint64() < log.BlockNumber {
-			continue
-		}
-
-		if len(addresses) > 0 && !includes(addresses, log.Address) {
-			continue
-		}
-		// If the to filtered topics is greater than the amount of topics in logs, skip.
-		if len(topics) > len(log.Topics) {
-			continue
-		}
-		for i, sub := range topics {
-			match := len(sub) == 0 // empty rule set == wildcard
-			for _, topic := range sub {
-				if log.Topics[i] == topic {
-					match = true
-					break
-				}
-			}
-			if !match {
-				continue Logs
-			}
-		}
-		ret = append(ret, log)
-	}
-	return ret
-}
-
-// returns true if the log matches on the filter
-func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
-	// actualTopics will always have length <= 4
-	// wantedTopics will always have length 4
-	matches := 0
-	for i, actualTopic := range actualTopics {
-		// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
-		if len(wantedTopics[i]) > 0 {
-			matches += sliceContainsHash(wantedTopics[i], actualTopic)
-		} else {
-			// Filter slot is empty, not matching any topics at this slot => counts as a match
-			matches++
-		}
-	}
-	if matches == len(actualTopics) {
-		return true
-	}
-	return false
-}
-
-// returns 1 if the slice contains the hash, 0 if it does not
-func sliceContainsHash(slice []string, hash common.Hash) int {
-	for _, str := range slice {
-		if str == hash.String() {
-			return 1
-		}
-	}
-	return 0
-}
-
 func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
 	arg := map[string]interface{}{
 		"address": q.Addresses,

pkg/eth/cid_retriever.go

@@ -22,13 +22,12 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
@@ -86,10 +85,10 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 	}()
 
 	// Retrieve cached header CIDs at this block height
-	var headers []eth.HeaderModel
+	var headers []models.HeaderModel
 	headers, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
 	if err != nil {
-		log.Error("header cid retrieval error")
+		log.Error("header cid retrieval error", err)
 		return nil, true, err
 	}
 	cws := make([]CIDWrapper, len(headers))
@@ -102,7 +101,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 			empty = false
 			if filter.HeaderFilter.Uncles {
 				// Retrieve uncle cids for this header id
-				var uncleCIDs []eth.UncleModel
+				var uncleCIDs []models.UncleModel
 				uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, header.ID)
 				if err != nil {
 					log.Error("uncle cid retrieval error")
@@ -166,18 +165,18 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 }
 
 // RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
-func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]eth.HeaderModel, error) {
+func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]models.HeaderModel, error) {
 	log.Debug("retrieving header cids for block ", blockNumber)
-	headers := make([]eth.HeaderModel, 0)
+	headers := make([]models.HeaderModel, 0)
 	pgStr := `SELECT * FROM eth.header_cids
 				WHERE block_number = $1`
 	return headers, tx.Select(&headers, pgStr, blockNumber)
 }
 
 // RetrieveUncleCIDsByHeaderID retrieves and returns all of the uncle cids for the provided header
-func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.UncleModel, error) {
+func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]models.UncleModel, error) {
 	log.Debug("retrieving uncle cids for block id ", headerID)
-	headers := make([]eth.UncleModel, 0)
+	headers := make([]models.UncleModel, 0)
 	pgStr := `SELECT * FROM eth.uncle_cids
 				WHERE header_id = $1`
 	return headers, tx.Select(&headers, pgStr, headerID)
@@ -185,10 +184,10 @@ func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64
 
 // RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
 // also returns the ids for the returned transaction cids
-func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]eth.TxModel, error) {
+func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]models.TxModel, error) {
 	log.Debug("retrieving transaction cids for header id ", headerID)
 	args := make([]interface{}, 0, 3)
-	results := make([]eth.TxModel, 0)
+	results := make([]models.TxModel, 0)
 	id := 1
 	pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
  			transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.mh_key,
@@ -210,94 +209,178 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID
 	return results, tx.Select(&results, pgStr, args...)
 }
 
-// RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
+func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) {
-// filter parameters and correspond to the provided tx ids
+	for i, topicSet := range topics {
-func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]eth.ReceiptModel, error) {
+		if len(topicSet) == 0 {
-	log.Debug("retrieving receipt cids for header id ", headerID)
+			continue
-	args := make([]interface{}, 0, 4)
+		}
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
+
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
+		if !first {
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
+			pgStr += " AND"
- 			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+		} else {
-			WHERE receipt_cids.tx_id = transaction_cids.id 
+			first = false
-			AND transaction_cids.header_id = header_cids.id
+		}
-			AND header_cids.id = $1`
+		pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id)
-	id := 2
+		args = append(args, pq.Array(topicSet))
-	args = append(args, headerID)
+		*id++
+	}
+	return pgStr, args
+}
+
+func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) {
+	if len(rctFilter.LogAddresses) > 0 {
+		pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id)
+		args = append(args, pq.Array(rctFilter.LogAddresses))
+		*id++
+	}
+
+	// Filter on topics if there are any
+	if hasTopics(rctFilter.Topics) {
+		pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
+	}
+
+	return pgStr, args
+}
+
+func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, trxIds []int64) (string, []interface{}) {
+	rctCond := " AND (receipt_cids.id = ANY ( "
+	logQuery := "SELECT receipt_id FROM eth.log_cids WHERE"
 	if len(rctFilter.LogAddresses) > 0 {
 		// Filter on log contract addresses if there are any
-		pgStr += fmt.Sprintf(` AND ((receipt_cids.log_contracts && $%d::VARCHAR(66)[]`, id)
+		pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id)
 		args = append(args, pq.Array(rctFilter.LogAddresses))
-		id++
+		*id++
+
 		// Filter on topics if there are any
 		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND ("
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
 		}
+
 		pgStr += ")"
+
 		// Filter on txIDs if there are any and we are matching txs
 		if rctFilter.MatchTxs && len(trxIds) > 0 {
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
+			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, *id)
 			args = append(args, pq.Array(trxIds))
 		}
 		pgStr += ")"
 	} else { // If there are no contract addresses to filter on
 		// Filter on topics if there are any
 		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND (("
+			pgStr += rctCond + logQuery
-			first := true
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true)
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
 			pgStr += ")"
 			// Filter on txIDs if there are any and we are matching txs
 			if rctFilter.MatchTxs && len(trxIds) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
+				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, *id)
 				args = append(args, pq.Array(trxIds))
 			}
 			pgStr += ")"
 		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
 			// If there are no contract addresses or topics to filter on,
 			// Filter on txIDs if there are any and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
+			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, *id)
 			args = append(args, pq.Array(trxIds))
 		}
 	}
+
+	return pgStr, args
+}
+
+// RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
+// filter parameters and correspond to the provided tx ids
+func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]models.ReceiptModel, error) {
+	log.Debug("retrieving receipt cids for header id ", headerID)
+	args := make([]interface{}, 0, 4)
+	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
+ 			receipt_cids.contract, receipt_cids.contract_hash
+ 			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+			WHERE receipt_cids.tx_id = transaction_cids.id 
+			AND transaction_cids.header_id = header_cids.id
+			AND header_cids.id = $1`
+	id := 2
+	args = append(args, headerID)
+
+	pgStr, args = receiptFilterConditions(&id, pgStr, args, rctFilter, trxIds)
+
 	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth.ReceiptModel, 0)
+	receiptCids := make([]models.ReceiptModel, 0)
 	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
 }
 
+// RetrieveFilteredGQLLogs retrieves and returns all the log cIDs provided blockHash that conform to the provided
+// filter parameters.
+func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids")
+	args := make([]interface{}, 0, 4)
+	id := 1
+	pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id,  
+       			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, 
+       			eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status
+				FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
+				WHERE eth.log_cids.receipt_id = receipt_cids.id
+				AND receipt_cids.tx_id = transaction_cids.id
+ 				AND transaction_cids.header_id = header_cids.id
+ 				AND log_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1`
+
+	args = append(args, blockHash.String())
+	id++
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY log_cids.index`
+
+	logCIDs := make([]LogResult, 0)
+	err := tx.Select(&logCIDs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return logCIDs, nil
+}
+
+// RetrieveFilteredLog retrieves and returns all the log cIDs provided blockHeight or blockHash that conform to the provided
+// filter parameters.
+func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids")
+	args := make([]interface{}, 0, 4)
+	pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id,  
+       			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, 
+       			eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, 
+       			header_cids.block_hash, header_cids.block_number
+				FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids
+				WHERE eth.log_cids.receipt_id = receipt_cids.id
+				AND receipt_cids.tx_id = transaction_cids.id
+ 				AND transaction_cids.header_id = header_cids.id`
+	id := 1
+	if blockNumber > 0 {
+		pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
+		args = append(args, blockNumber)
+		id++
+	}
+	if blockHash != nil {
+		pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id)
+		args = append(args, blockHash.String())
+		id++
+	}
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY log_cids.index`
+
+	logCIDs := make([]LogResult, 0)
+	err := tx.Select(&logCIDs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return logCIDs, nil
+}
+
 // RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided
 // filter parameters and correspond to the provided tx ids
-func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]eth.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]models.ReceiptModel, error) {
 	log.Debug("retrieving receipt cids for block ", blockNumber)
 	args := make([]interface{}, 0, 5)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
+	pgStr := `SELECT receipt_cids.id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
  			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
 			WHERE receipt_cids.tx_id = transaction_cids.id 
 			AND transaction_cids.header_id = header_cids.id`
@@ -313,77 +396,10 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
 		id++
 	}
 
-	// TODO: Add the below filters when we have log index in DB.
+	pgStr, args = receiptFilterConditions(&id, pgStr, args, rctFilter, trxIds)
-	if true {
-		pgStr += ` ORDER BY transaction_cids.index`
-		receiptCids := make([]eth.ReceiptModel, 0)
-		return receiptCids, tx.Select(&receiptCids, pgStr, args...)
-	}
-
-	if len(rctFilter.LogAddresses) > 0 {
-		// Filter on log contract addresses if there are any
-		pgStr += fmt.Sprintf(` AND ((receipt_cids.log_contracts && $%d::VARCHAR(66)[]`, id)
-		args = append(args, pq.Array(rctFilter.LogAddresses))
-		id++
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND ("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-		}
-		pgStr += ")"
-		// Filter on txIDs if there are any and we are matching txs
-		if rctFilter.MatchTxs && len(trxIds) > 0 {
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-		pgStr += ")"
-	} else { // If there are no contract addresses to filter on
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND (("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-			// Filter on txIDs if there are any and we are matching txs
-			if rctFilter.MatchTxs && len(trxIds) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-				args = append(args, pq.Array(trxIds))
-			}
-			pgStr += ")"
-		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
-			// If there are no contract addresses or topics to filter on,
-			// Filter on txIDs if there are any and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-	}
 
 	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth.ReceiptModel, 0)
+	receiptCids := make([]models.ReceiptModel, 0)
 	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
 }
 
@@ -397,7 +413,7 @@ func hasTopics(topics [][]string) bool {
 }
 
 // RetrieveStateCIDs retrieves and returns all of the state node cids at the provided header ID that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]eth.StateNodeModel, error) {
+func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]models.StateNodeModel, error) {
 	log.Debug("retrieving state cids for header id ", headerID)
 	args := make([]interface{}, 0, 2)
 	pgStr := `SELECT state_cids.id, state_cids.header_id,
@@ -417,12 +433,12 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
 	if !stateFilter.IntermediateNodes {
 		pgStr += ` AND state_cids.node_type = 2`
 	}
-	stateNodeCIDs := make([]eth.StateNodeModel, 0)
+	stateNodeCIDs := make([]models.StateNodeModel, 0)
 	return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
 }
 
 // RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided header id that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]eth.StorageNodeWithStateKeyModel, error) {
+func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]models.StorageNodeWithStateKeyModel, error) {
 	log.Debug("retrieving storage cids for header id ", headerID)
 	args := make([]interface{}, 0, 3)
 	pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_leaf_key, storage_cids.node_type,
@@ -450,18 +466,18 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
 	if !storageFilter.IntermediateNodes {
 		pgStr += ` AND storage_cids.node_type = 2`
 	}
-	storageNodeCIDs := make([]eth.StorageNodeWithStateKeyModel, 0)
+	storageNodeCIDs := make([]models.StorageNodeWithStateKeyModel, 0)
 	return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)
 }
 
 // RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
-func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (models.HeaderModel, []models.UncleModel, []models.TxModel, []models.ReceiptModel, error) {
 	log.Debug("retrieving block cids for block hash ", blockHash.String())
 
 	// Begin new db tx
 	tx, err := ecr.db.Beginx()
 	if err != nil {
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
 	defer func() {
 		if p := recover(); p != nil {
@@ -474,29 +490,29 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderM
 		}
 	}()
 
-	var headerCID eth.HeaderModel
+	var headerCID models.HeaderModel
 	headerCID, err = ecr.RetrieveHeaderCIDByHash(tx, blockHash)
 	if err != nil {
 		log.Error("header cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
-	var uncleCIDs []eth.UncleModel
+	var uncleCIDs []models.UncleModel
 	uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID)
 	if err != nil {
 		log.Error("uncle cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
-	var txCIDs []eth.TxModel
+	var txCIDs []models.TxModel
 	txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
 	if err != nil {
 		log.Error("tx cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
 	txIDs := make([]int64, len(txCIDs))
 	for i, txCID := range txCIDs {
 		txIDs[i] = txCID.ID
 	}
-	var rctCIDs []eth.ReceiptModel
+	var rctCIDs []models.ReceiptModel
 	rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
 	if err != nil {
 		log.Error("rct cid retrieval error")
@@ -505,13 +521,13 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderM
 }
 
 // RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
-func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (models.HeaderModel, []models.UncleModel, []models.TxModel, []models.ReceiptModel, error) {
 	log.Debug("retrieving block cids for block number ", blockNumber)
 
 	// Begin new db tx
 	tx, err := ecr.db.Beginx()
 	if err != nil {
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
 	defer func() {
 		if p := recover(); p != nil {
@@ -524,32 +540,32 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderMod
 		}
 	}()
 
-	var headerCID []eth.HeaderModel
+	var headerCID []models.HeaderModel
 	headerCID, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
 	if err != nil {
 		log.Error("header cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
 	if len(headerCID) < 1 {
-		return eth.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
+		return models.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
 	}
-	var uncleCIDs []eth.UncleModel
+	var uncleCIDs []models.UncleModel
 	uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].ID)
 	if err != nil {
 		log.Error("uncle cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
-	var txCIDs []eth.TxModel
+	var txCIDs []models.TxModel
 	txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
 	if err != nil {
 		log.Error("tx cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
 	txIDs := make([]int64, len(txCIDs))
 	for i, txCID := range txCIDs {
 		txIDs[i] = txCID.ID
 	}
-	var rctCIDs []eth.ReceiptModel
+	var rctCIDs []models.ReceiptModel
 	rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
 	if err != nil {
 		log.Error("rct cid retrieval error")
@@ -558,34 +574,33 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderMod
 }
 
 // RetrieveHeaderCIDByHash returns the header for the given block hash
-func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (eth.HeaderModel, error) {
+func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (models.HeaderModel, error) {
 	log.Debug("retrieving header cids for block hash ", blockHash.String())
 	pgStr := `SELECT * FROM eth.header_cids
 			WHERE block_hash = $1`
-	var headerCID eth.HeaderModel
+	var headerCID models.HeaderModel
 	return headerCID, tx.Get(&headerCID, pgStr, blockHash.String())
 }
 
 // RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id
-func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.TxModel, error) {
+func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]models.TxModel, error) {
 	log.Debug("retrieving tx cids for block id ", headerID)
 	pgStr := `SELECT * FROM eth.transaction_cids
 			WHERE header_id = $1
 			ORDER BY index`
-	var txCIDs []eth.TxModel
+	var txCIDs []models.TxModel
 	return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
 }
 
 // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
-func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]eth.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]models.ReceiptModel, error) {
 	log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
+	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
+ 			receipt_cids.contract, receipt_cids.contract_hash
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
 			FROM eth.receipt_cids, eth.transaction_cids
 			WHERE tx_id = ANY($1::INTEGER[])
 			AND receipt_cids.tx_id = transaction_cids.id
 			ORDER BY transaction_cids.index`
-	var rctCIDs []eth.ReceiptModel
+	var rctCIDs []models.ReceiptModel
 	return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs))
 }

pkg/eth/cid_retriever_test.go

@@ -19,17 +19,17 @@ package eth_test
 import (
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
 )
@@ -211,15 +211,17 @@ var (
 
 var _ = Describe("Retriever", func() {
 	var (
-		db        *postgres.DB
+		db          *postgres.DB
-		repo      *eth2.IPLDPublisher
+		diffIndexer *indexer.StateDiffIndexer
-		retriever *eth.CIDRetriever
+		retriever   *eth.CIDRetriever
 	)
 	BeforeEach(func() {
 		var err error
-		db, err = shared.SetupDB()
+		db, err = SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = eth2.NewIPLDPublisher(db)
+		diffIndexer, err = indexer.NewStateDiffIndexer(params.TestChainConfig, db)
+		Expect(err).ToNot(HaveOccurred())
+
 		retriever = eth.NewCIDRetriever(db)
 	})
 	AfterEach(func() {
@@ -228,10 +230,28 @@ var _ = Describe("Retriever", func() {
 
 	Describe("Retrieve", func() {
 		BeforeEach(func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Close(err)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
+			type rctCIDAndMHKeyResult struct {
+				LeafCID   string `db:"leaf_cid"`
+				LeafMhKey string `db:"leaf_mh_key"`
+			}
+			expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
+			pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+				WHERE receipt_cids.tx_id = transaction_cids.id
+				AND transaction_cids.header_id = header_cids.id
+				AND header_cids.block_number = $1
+				ORDER BY transaction_cids.index`
+			err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
 			cids, empty, err := retriever.Retrieve(openFilter, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
@@ -242,14 +262,14 @@ var _ = Describe("Retriever", func() {
 			expectedHeaderCID.ID = cids[0].Header.ID
 			expectedHeaderCID.NodeID = cids[0].Header.NodeID
 			Expect(cids[0].Header).To(Equal(expectedHeaderCID))
-			Expect(len(cids[0].Transactions)).To(Equal(3))
+			Expect(len(cids[0].Transactions)).To(Equal(4))
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
-			Expect(len(cids[0].Receipts)).To(Equal(3))
+			Expect(len(cids[0].Receipts)).To(Equal(4))
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
 			Expect(len(cids[0].StateNodes)).To(Equal(2))
 
 			for _, stateNode := range cids[0].StateNodes {
@@ -272,12 +292,23 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Applies filters from the provided config.Subscription", func() {
+			type rctCIDAndMHKeyResult struct {
+				LeafCID   string `db:"leaf_cid"`
+				LeafMhKey string `db:"leaf_mh_key"`
+			}
+			expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
+			pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+				WHERE receipt_cids.tx_id = transaction_cids.id
+				AND transaction_cids.header_id = header_cids.id
+				AND header_cids.block_number = $1
+				ORDER BY transaction_cids.index`
+			err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
 			cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids1)).To(Equal(1))
 			Expect(cids1[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids1[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids1[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids1[0].Transactions)).To(Equal(0))
 			Expect(len(cids1[0].StateNodes)).To(Equal(0))
 			Expect(len(cids1[0].StorageNodes)).To(Equal(0))
@@ -285,6 +316,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids1[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
@@ -292,7 +325,7 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids2)).To(Equal(1))
 			Expect(cids2[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids2[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids2[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids2[0].Transactions)).To(Equal(0))
 			Expect(len(cids2[0].StateNodes)).To(Equal(0))
 			Expect(len(cids2[0].StorageNodes)).To(Equal(0))
@@ -300,6 +333,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids2[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1)
@@ -307,7 +342,7 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids3)).To(Equal(1))
 			Expect(cids3[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids3[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids3[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids3[0].Transactions)).To(Equal(0))
 			Expect(len(cids3[0].StateNodes)).To(Equal(0))
 			Expect(len(cids3[0].StorageNodes)).To(Equal(0))
@@ -315,6 +350,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids3[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1)
@@ -322,7 +359,7 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids4)).To(Equal(1))
 			Expect(cids4[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids4[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids4[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids4[0].Transactions)).To(Equal(0))
 			Expect(len(cids4[0].StateNodes)).To(Equal(0))
 			Expect(len(cids4[0].StorageNodes)).To(Equal(0))
@@ -330,6 +367,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
 			expectedReceiptCID.ID = cids4[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
 			Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
@@ -337,24 +376,24 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids5)).To(Equal(1))
 			Expect(cids5[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids5[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids5[0].Header).To(Equal(models.HeaderModel{}))
-			Expect(len(cids5[0].Transactions)).To(Equal(3))
+			Expect(len(cids5[0].Transactions)).To(Equal(4))
 			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx1CID.String())).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx2CID.String())).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx3CID.String())).To(BeTrue())
 			Expect(len(cids5[0].StateNodes)).To(Equal(0))
 			Expect(len(cids5[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids5[0].Receipts)).To(Equal(3))
+			Expect(len(cids5[0].Receipts)).To(Equal(4))
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct1CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct2CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct3CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
 
 			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids6)).To(Equal(1))
 			Expect(cids6[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids6[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids6[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids6[0].Transactions)).To(Equal(1))
 			expectedTxCID := test_helpers.MockCIDWrapper.Transactions[1]
 			expectedTxCID.ID = cids6[0].Transactions[0].ID
@@ -366,6 +405,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
 			expectedReceiptCID.ID = cids6[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
 			Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids7, empty, err := retriever.Retrieve(stateFilter, 1)
@@ -373,12 +414,12 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids7)).To(Equal(1))
 			Expect(cids7[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids7[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids7[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids7[0].Transactions)).To(Equal(0))
 			Expect(len(cids7[0].Receipts)).To(Equal(0))
 			Expect(len(cids7[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids7[0].StateNodes)).To(Equal(1))
-			Expect(cids7[0].StateNodes[0]).To(Equal(eth2.StateNodeModel{
+			Expect(cids7[0].StateNodes[0]).To(Equal(models.StateNodeModel{
 				ID:       cids7[0].StateNodes[0].ID,
 				HeaderID: cids7[0].StateNodes[0].HeaderID,
 				NodeType: 2,
@@ -400,8 +441,12 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the first block that has data in the database", func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Close(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1)))
@@ -410,8 +455,12 @@ var _ = Describe("Retriever", func() {
 		It("Gets the number of the first block that has data in the database", func() {
 			payload := test_helpers.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
-			err := repo.Publish(payload)
+			tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Close(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1010101)))
@@ -422,10 +471,16 @@ var _ = Describe("Retriever", func() {
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
-			err := repo.Publish(payload1)
+			tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
-			err = repo.Publish(payload2)
+			err = tx.Close(err)
 			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Close(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(5)))
@@ -438,8 +493,11 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the latest block that has data in the database", func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
+			err = tx.Close(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1)))
@@ -448,8 +506,12 @@ var _ = Describe("Retriever", func() {
 		It("Gets the number of the latest block that has data in the database", func() {
 			payload := test_helpers.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
-			err := repo.Publish(payload)
+			tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Close(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1010101)))
@@ -460,10 +522,16 @@ var _ = Describe("Retriever", func() {
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
-			err := repo.Publish(payload1)
+			tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
-			err = repo.Publish(payload2)
+			err = tx.Close(err)
 			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Close(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1010101)))

pkg/eth/eth_state_test.go

@@ -29,17 +29,17 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 var (
@@ -74,16 +74,27 @@ var _ = Describe("eth state reading tests", func() {
 	It("test init", func() {
 		// db and type initializations
 		var err error
-		db, err = shared.SetupDB()
+		db, err = SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		transformer := eth2.NewStateDiffTransformer(chainConfig, db)
+
+		transformer, err := indexer.NewStateDiffIndexer(chainConfig, db)
+		Expect(err).ToNot(HaveOccurred())
+
 		backend, err = eth.NewEthBackend(db, &eth.Config{
 			ChainConfig: chainConfig,
-			VmConfig:    vm.Config{},
+			VMConfig:    vm.Config{},
 			RPCGasCap:   big.NewInt(10000000000), // Max gas capacity for a rpc call.
+			GroupCacheConfig: &ethServerShared.GroupCacheConfig{
+				StateDB: ethServerShared.GroupConfig{
+					Name:                   "eth_state_test",
+					CacheSizeInMB:          8,
+					CacheExpiryInMins:      60,
+					LogStatsIntervalInSecs: 0,
+				},
+			},
 		})
 		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend, nil, false)
+		api, _ = eth.NewPublicEthAPI(backend, nil, false, false, false)
 
 		// make the test blockchain (and state)
 		blocks, receipts, chain = test_helpers.MakeChain(chainLength, test_helpers.Genesis, test_helpers.TestChainGen)
@@ -135,31 +146,40 @@ var _ = Describe("eth state reading tests", func() {
 			}
 			diff, err := builder.BuildStateDiffObject(args, params)
 			Expect(err).ToNot(HaveOccurred())
-			diffRlp, err := rlp.EncodeToBytes(diff)
+			tx, err := transformer.PushBlock(block, rcts, mockTD)
 			Expect(err).ToNot(HaveOccurred())
-			blockRlp, err := rlp.EncodeToBytes(block)
+
-			Expect(err).ToNot(HaveOccurred())
+			for _, node := range diff.Nodes {
-			receiptsRlp, err := rlp.EncodeToBytes(rcts)
+				err = transformer.PushStateNode(tx, node)
-			Expect(err).ToNot(HaveOccurred())
+				Expect(err).ToNot(HaveOccurred())
-			payload := statediff.Payload{
-				StateObjectRlp:  diffRlp,
-				BlockRlp:        blockRlp,
-				ReceiptsRlp:     receiptsRlp,
-				TotalDifficulty: mockTD,
 			}
-			_, err = transformer.Transform(0, payload)
+			err = tx.Close(err)
 			Expect(err).ToNot(HaveOccurred())
 		}
 
 		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
-		indexAndPublisher := eth2.NewIPLDPublisher(db)
+		indexAndPublisher, err := indexer.NewStateDiffIndexer(chainConfig, db)
-		api = eth.NewPublicEthAPI(backend, nil, false)
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
 		Expect(err).ToNot(HaveOccurred())
+
+		tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Close(err)
+		Expect(err).ToNot(HaveOccurred())
+
 		// The non-canonical header has a child
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayloadForChild)
+		tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
 		Expect(err).ToNot(HaveOccurred())
-		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
+
+		hash := sdtypes.CodeAndCodeHash{
+			Hash: test_helpers.CodeHash,
+			Code: test_helpers.ContractCode,
+		}
+
+		err = indexAndPublisher.PushCodeAndCodeHash(tx, hash)
+		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Close(err)
 		Expect(err).ToNot(HaveOccurred())
 	})
 	defer It("test teardown", func() {
@@ -460,16 +480,16 @@ var _ = Describe("eth state reading tests", func() {
 		It("Returns empty slice if it tries to access a contract which does not exist", func() {
 			storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0))
 			Expect(err).NotTo(HaveOccurred())
-			Expect(storage).To(Equal(hexutil.Bytes(make([]byte, 32))))
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
 
 			storage, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1))
 			Expect(err).NotTo(HaveOccurred())
-			Expect(storage).To(Equal(hexutil.Bytes(make([]byte, 32))))
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
 		})
 		It("Returns empty slice if it tries to access a contract slot which does not exist", func() {
 			storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2))
 			Expect(err).NotTo(HaveOccurred())
-			Expect(storage).To(Equal(hexutil.Bytes(make([]byte, 32))))
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
 		})
 		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() {
 			// After deployment
@@ -490,7 +510,7 @@ var _ = Describe("eth state reading tests", func() {
 
 			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(5))
 			Expect(err).ToNot(HaveOccurred())
-			Expect(val).To(Equal(hexutil.Bytes{}))
+			Expect(val).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
 		})
 		It("Throws an error for a non-existing block hash", func() {
 			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithHash(randomHash, true))

pkg/eth/filterer.go

@@ -19,22 +19,21 @@ package eth
 import (
 	"bytes"
 
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
 )
 
 // Filterer interface for substituing mocks in tests
 type Filterer interface {
-	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error)
+	Filter(filter SubscriptionSettings, payload ConvertedPayload) (*IPLDs, error)
 }
 
 // ResponseFilterer satisfies the ResponseFilterer interface for ethereum
@@ -46,7 +45,7 @@ func NewResponseFilterer() *ResponseFilterer {
 }
 
 // Filter is used to filter through eth data to extract and package requested data into a Payload
-func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error) {
+func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload ConvertedPayload) (*IPLDs, error) {
 	if checkRange(filter.Start.Int64(), filter.End.Int64(), payload.Block.Number().Int64()) {
 		response := new(IPLDs)
 		response.TotalDifficulty = payload.TotalDifficulty
@@ -73,7 +72,7 @@ func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.Conve
 	return nil, nil
 }
 
-func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload eth.ConvertedPayload) error {
+func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload ConvertedPayload) error {
 	if !headerFilter.Off {
 		headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
 		if err != nil {
@@ -115,7 +114,7 @@ func checkRange(start, end, actual int64) bool {
 	return false
 }
 
-func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
+func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload ConvertedPayload) ([]common.Hash, error) {
 	var trxHashes []common.Hash
 	if !trxFilter.Off {
 		trxLen := len(payload.Block.Body().Transactions)
@@ -163,25 +162,30 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
 	return false
 }
 
-func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
+func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error {
 	if !receiptFilter.Off {
 		response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
-		for i, receipt := range payload.Receipts {
+		rctLeafCID, rctIPLDData, err := GetRctLeafNodeData(payload.Receipts)
+		if err != nil {
+			return err
+		}
+
+		for idx, receipt := range payload.Receipts {
 			// topics is always length 4
-			topics := [][]string{payload.ReceiptMetaData[i].Topic0s, payload.ReceiptMetaData[i].Topic1s, payload.ReceiptMetaData[i].Topic2s, payload.ReceiptMetaData[i].Topic3s}
+			topics := make([][]string, 4)
-			if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, payload.ReceiptMetaData[i].LogContracts, trxHashes) {
+			contracts := make([]string, len(receipt.Logs))
-				receiptBuffer := new(bytes.Buffer)
+			for _, l := range receipt.Logs {
-				if err := receipt.EncodeRLP(receiptBuffer); err != nil {
+				contracts = append(contracts, l.Address.String())
-					return err
+				for idx, t := range l.Topics {
-				}
+					topics[idx] = append(topics[idx], t.String())
-				data := receiptBuffer.Bytes()
-				cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
-				if err != nil {
-					return err
 				}
+			}
+
+			// TODO: Verify this filter logic.
+			if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) {
 				response.Receipts = append(response.Receipts, ipfs.BlockModel{
-					Data: data,
+					Data: rctIPLDData[idx],
-					CID:  cid.String(),
+					CID:  rctLeafCID[idx].String(),
 				})
 			}
 		}
@@ -253,7 +257,7 @@ func slicesShareString(slice1, slice2 []string) int {
 }
 
 // filterStateAndStorage filters state and storage nodes into the response according to the provided filters
-func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload eth.ConvertedPayload) error {
+func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
 	response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
 	response.StorageNodes = make([]StorageNode, 0)
 	stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
@@ -270,37 +274,37 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 	}
 	for _, stateNode := range payload.StateNodes {
 		if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
-			if stateNode.Type == sdtypes.Leaf || stateFilter.IntermediateNodes {
+			if stateNode.NodeType == sdtypes.Leaf || stateFilter.IntermediateNodes {
-				cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
+				cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.NodeValue, multihash.KECCAK_256)
 				if err != nil {
 					return err
 				}
 				response.StateNodes = append(response.StateNodes, StateNode{
-					StateLeafKey: stateNode.LeafKey,
+					StateLeafKey: common.BytesToHash(stateNode.LeafKey),
 					Path:         stateNode.Path,
 					IPLD: ipfs.BlockModel{
-						Data: stateNode.Value,
+						Data: stateNode.NodeValue,
 						CID:  cid.String(),
 					},
-					Type: stateNode.Type,
+					Type: stateNode.NodeType,
 				})
 			}
 		}
 		if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
 			for _, storageNode := range payload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
 				if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
-					cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
+					cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.NodeValue, multihash.KECCAK_256)
 					if err != nil {
 						return err
 					}
 					response.StorageNodes = append(response.StorageNodes, StorageNode{
-						StateLeafKey:   stateNode.LeafKey,
+						StateLeafKey:   common.BytesToHash(stateNode.LeafKey),
-						StorageLeafKey: storageNode.LeafKey,
+						StorageLeafKey: common.BytesToHash(storageNode.LeafKey),
 						IPLD: ipfs.BlockModel{
-							Data: storageNode.Value,
+							Data: storageNode.NodeValue,
 							CID:  cid.String(),
 						},
-						Type: storageNode.Type,
+						Type: storageNode.NodeType,
 						Path: storageNode.Path,
 					})
 				}
@@ -310,15 +314,52 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 	return nil
 }
 
-func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
+func checkNodeKeys(wantedKeys []common.Hash, actualKey []byte) bool {
 	// If we aren't filtering for any specific keys, all nodes are a go
 	if len(wantedKeys) == 0 {
 		return true
 	}
 	for _, key := range wantedKeys {
-		if bytes.Equal(key.Bytes(), actualKey.Bytes()) {
+		if bytes.Equal(key.Bytes(), actualKey) {
 			return true
 		}
 	}
 	return false
 }
+
+// GetRctLeafNodeData converts the receipts to receipt trie and returns the receipt leaf node IPLD data and
+// corresponding CIDs
+func GetRctLeafNodeData(rcts types.Receipts) ([]cid.Cid, [][]byte, error) {
+	receiptTrie := ipld.NewRctTrie()
+	for idx, rct := range rcts {
+		ethRct, err := ipld.NewReceipt(rct)
+		if err != nil {
+			return nil, nil, err
+		}
+		if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
+	ethRctleafNodeData := make([][]byte, len(rctLeafNodes))
+	for i, rln := range rctLeafNodes {
+		var idx uint
+
+		r := bytes.NewReader(keys[i].TrieKey)
+		err = rlp.Decode(r, &idx)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		ethRctleafNodeCids[idx] = rln.Cid()
+		ethRctleafNodeData[idx] = rln.RawData()
+	}
+
+	return ethRctleafNodeCids, ethRctleafNodeData, nil
+}

pkg/eth/filterer_test.go

@@ -19,13 +19,12 @@ package eth_test
 import (
 	"bytes"
 
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
 	"github.com/vulcanize/ipld-eth-server/pkg/shared"
@@ -49,14 +48,14 @@ var _ = Describe("Filterer", func() {
 			Expect(iplds.Header).To(Equal(test_helpers.MockIPLDs.Header))
 			var expectedEmptyUncles []ipfs.BlockModel
 			Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
-			Expect(len(iplds.Transactions)).To(Equal(3))
+			Expect(len(iplds.Transactions)).To(Equal(4))
 			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx1)).To(BeTrue())
 			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx2)).To(BeTrue())
 			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx3)).To(BeTrue())
-			Expect(len(iplds.Receipts)).To(Equal(3))
+			Expect(len(iplds.Receipts)).To(Equal(4))
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
 			Expect(len(iplds.StateNodes)).To(Equal(2))
 			for _, stateNode := range iplds.StateNodes {
 				Expect(stateNode.Type).To(Equal(sdtypes.Leaf))
@@ -88,8 +87,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds1.StateNodes)).To(Equal(0))
 			Expect(len(iplds1.Receipts)).To(Equal(1))
 			Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				CID:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload)
@@ -103,8 +102,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds2.StateNodes)).To(Equal(0))
 			Expect(len(iplds2.Receipts)).To(Equal(1))
 			Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				CID:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload)
@@ -118,8 +117,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds3.StateNodes)).To(Equal(0))
 			Expect(len(iplds3.Receipts)).To(Equal(1))
 			Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				CID:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload)
@@ -133,8 +132,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds4.StateNodes)).To(Equal(0))
 			Expect(len(iplds4.Receipts)).To(Equal(1))
 			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct2IPLD.RawData(),
+				Data: test_helpers.Rct2IPLD,
-				CID:  test_helpers.Rct2IPLD.Cid().String(),
+				CID:  test_helpers.Rct2CID.String(),
 			}))
 
 			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload)
@@ -143,16 +142,16 @@ var _ = Describe("Filterer", func() {
 			Expect(iplds5.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds5.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds5.Uncles)).To(Equal(0))
-			Expect(len(iplds5.Transactions)).To(Equal(3))
+			Expect(len(iplds5.Transactions)).To(Equal(4))
 			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx1)).To(BeTrue())
 			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx2)).To(BeTrue())
 			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx3)).To(BeTrue())
 			Expect(len(iplds5.StorageNodes)).To(Equal(0))
 			Expect(len(iplds5.StateNodes)).To(Equal(0))
-			Expect(len(iplds5.Receipts)).To(Equal(3))
+			Expect(len(iplds5.Receipts)).To(Equal(4))
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
 
 			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
@@ -166,8 +165,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds6.StateNodes)).To(Equal(0))
 			Expect(len(iplds6.Receipts)).To(Equal(1))
 			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct2IPLD.RawData(),
+				Data: test_helpers.Rct2IPLD,
-				CID:  test_helpers.Rct2IPLD.Cid().String(),
+				CID:  test_helpers.Rct2CID.String(),
 			}))
 
 			iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload)

pkg/eth/helpers.go

@@ -19,9 +19,10 @@ package eth
 import (
 	"encoding/json"
 	"fmt"
+	"os"
+
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	log "github.com/sirupsen/logrus"
-	"os"
 
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 

pkg/eth/ipld_fetcher.go

@@ -22,13 +22,11 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
@@ -104,7 +102,7 @@ func (f *IPLDFetcher) Fetch(cids CIDWrapper) (*IPLDs, error) {
 }
 
 // FetchHeaders fetches headers
-func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c eth.HeaderModel) (ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c models.HeaderModel) (ipfs.BlockModel, error) {
 	log.Debug("fetching header ipld")
 	headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
 	if err != nil {
@@ -117,7 +115,7 @@ func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c eth.HeaderModel) (ipfs.BlockMod
 }
 
 // FetchUncles fetches uncles
-func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []eth.UncleModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []models.UncleModel) ([]ipfs.BlockModel, error) {
 	log.Debug("fetching uncle iplds")
 	uncleIPLDs := make([]ipfs.BlockModel, len(cids))
 	for i, c := range cids {
@@ -134,7 +132,7 @@ func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []eth.UncleModel) ([]ipfs.Bl
 }
 
 // FetchTrxs fetches transactions
-func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []eth.TxModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []models.TxModel) ([]ipfs.BlockModel, error) {
 	log.Debug("fetching transaction iplds")
 	trxIPLDs := make([]ipfs.BlockModel, len(cids))
 	for i, c := range cids {
@@ -151,24 +149,25 @@ func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []eth.TxModel) ([]ipfs.BlockMo
 }
 
 // FetchRcts fetches receipts
-func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []eth.ReceiptModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]ipfs.BlockModel, error) {
 	log.Debug("fetching receipt iplds")
 	rctIPLDs := make([]ipfs.BlockModel, len(cids))
 	for i, c := range cids {
-		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
+		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.LeafMhKey)
 		if err != nil {
 			return nil, err
 		}
+		//nodeVal, err := DecodeLeafNode(rctBytes)
 		rctIPLDs[i] = ipfs.BlockModel{
 			Data: rctBytes,
-			CID:  c.CID,
+			CID:  c.LeafCID,
 		}
 	}
 	return rctIPLDs, nil
 }
 
 // FetchState fetches state nodes
-func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]StateNode, error) {
+func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []models.StateNodeModel) ([]StateNode, error) {
 	log.Debug("fetching state iplds")
 	stateNodes := make([]StateNode, 0, len(cids))
 	for _, stateNode := range cids {
@@ -193,7 +192,7 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]Stat
 }
 
 // FetchStorage fetches storage nodes
-func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
+func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []models.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
 	log.Debug("fetching storage iplds")
 	storageNodes := make([]StorageNode, 0, len(cids))
 	for _, storageNode := range cids {

pkg/eth/ipld_fetcher_test.go

@@ -17,12 +17,11 @@
 package eth_test
 
 import (
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
@@ -31,18 +30,30 @@ import (
 var _ = Describe("IPLDFetcher", func() {
 	var (
 		db            *postgres.DB
-		pubAndIndexer *eth2.IPLDPublisher
+		pubAndIndexer *indexer.StateDiffIndexer
 		fetcher       *eth.IPLDFetcher
 	)
 	Describe("Fetch", func() {
 		BeforeEach(func() {
-			var err error
+			var (
-			db, err = shared.SetupDB()
+				err error
+				tx  *indexer.BlockTx
+			)
+			db, err = SetupDB()
 			Expect(err).ToNot(HaveOccurred())
-			pubAndIndexer = eth2.NewIPLDPublisher(db)
+			pubAndIndexer, err = indexer.NewStateDiffIndexer(params.TestChainConfig, db)
-			err = pubAndIndexer.Publish(test_helpers.MockConvertedPayload)
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = pubAndIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			for _, node := range test_helpers.MockStateNodes {
+				err = pubAndIndexer.PushStateNode(tx, node)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Close(err)
 			Expect(err).ToNot(HaveOccurred())
 			fetcher = eth.NewIPLDFetcher(db)
+
 		})
 		AfterEach(func() {
 			eth.TearDownDB(db)

pkg/eth/ipld_retriever.go

@@ -19,15 +19,21 @@ package eth
 import (
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/statediff/trie"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	"github.com/lib/pq"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 )
 
 const (
+	// node type removed value.
+	// https://github.com/vulcanize/go-ethereum/blob/271f4d01e7e2767ffd8e0cd469bf545be96f2a84/statediff/indexer/helpers.go#L34
+	removedNode = 3
+
 	RetrieveHeadersByHashesPgStr = `SELECT cid, data
 								FROM eth.header_cids
 									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
@@ -78,33 +84,31 @@ const (
 									FROM eth.transaction_cids
 										INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
 									WHERE tx_hash = $1`
-	RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data
+	RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.leaf_cid, data
 									FROM eth.receipt_cids
 										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
-										INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+										INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
 									WHERE tx_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash
+	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
 										FROM eth.receipt_cids
 											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
 											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-											INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+											INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
 										WHERE block_hash = $1
 										ORDER BY eth.transaction_cids.index ASC`
-	RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data
+	RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.leaf_cid, data
 										FROM eth.receipt_cids
 											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
 											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-											INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+											INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
 										WHERE block_number = $1
 										ORDER BY eth.transaction_cids.index ASC`
-	RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data
+	RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.leaf_cid, data
 									FROM eth.receipt_cids
 										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
-										INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+										INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
 									WHERE tx_hash = $1`
-	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid,
+	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type
-														data,
-														was_state_removed(state_path, block_number, $2) AS removed
 												FROM eth.state_cids
 													INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
 													INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
@@ -115,12 +119,7 @@ const (
 												AND header_cids.id = (SELECT canonical_header_id(block_number))
 												ORDER BY block_number DESC
 												LIMIT 1`
-	RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid,
+	RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.node_type
-														  data,
-														  was_state_removed(state_path, block_number, (SELECT block_hash
-																									  FROM eth.header_cids
-																									  WHERE block_number = $2
-																									  LIMIT 1)) AS removed
 													FROM eth.state_cids
 														INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
 														INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
@@ -128,12 +127,7 @@ const (
 													AND block_number <= $2
 													ORDER BY block_number DESC
 													LIMIT 1`
-	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid,
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed($1, $3) AS state_leaf_removed
-																			data,
-																			was_storage_removed(storage_path, block_number, (SELECT block_hash
-																															FROM eth.header_cids
-																															WHERE block_number = $3
-																															LIMIT 1)) AS removed
 																	FROM eth.storage_cids
 																		INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
 																		INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
@@ -143,9 +137,7 @@ const (
 																	AND block_number <= $3
 																	ORDER BY block_number DESC
 																	LIMIT 1`
-	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid,
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed($1, $3) AS state_leaf_removed
-																		  data,
-																		  was_storage_removed(storage_path, block_number, $3) AS removed
 																	FROM eth.storage_cids
 																		INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
 																		INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
@@ -160,6 +152,14 @@ const (
 																	LIMIT 1`
 )
 
+var EmptyNodeValue = make([]byte, common.HashLength)
+
+type rctIpldResult struct {
+	LeafCID string `db:"leaf_cid"`
+	Data    []byte `db:"data"`
+	TxHash  string `db:"tx_hash"`
+}
+
 type ipldResult struct {
 	CID    string `db:"cid"`
 	Data   []byte `db:"data"`
@@ -327,9 +327,26 @@ func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, [
 	return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
 }
 
+// DecodeLeafNode decodes the leaf node data
+func DecodeLeafNode(node []byte) ([]byte, error) {
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return nil, err
+	}
+	ty, err := trie.CheckKeyType(nodeElements)
+	if err != nil {
+		return nil, err
+	}
+
+	if ty != sdtypes.Leaf {
+		return nil, fmt.Errorf("expected leaf node but found %s", ty)
+	}
+	return nodeElements[1].([]byte), nil
+}
+
 // RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes
 func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
+	rctResults := make([]rctIpldResult, 0)
 	hashStrs := make([]string, len(hashes))
 	for i, hash := range hashes {
 		hashStrs[i] = hash.Hex()
@@ -340,15 +357,20 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.CID
+		cids[i] = res.LeafCID
-		rcts[i] = res.Data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, err
+		}
+		rcts[i] = nodeVal
 	}
 	return cids, rcts, nil
 }
 
-// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash
+// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
+// cid returned corresponds to the leaf node data which contains the receipt.
 func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
-	rctResults := make([]ipldResult, 0)
+	rctResults := make([]rctIpldResult, 0)
 	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
 		return nil, nil, nil, err
 	}
@@ -357,39 +379,58 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string,
 	txs := make([]common.Hash, len(rctResults))
 
 	for i, res := range rctResults {
-		cids[i] = res.CID
+		cids[i] = res.LeafCID
-		rcts[i] = res.Data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		rcts[i] = nodeVal
 		txs[i] = common.HexToHash(res.TxHash)
 	}
 
 	return cids, rcts, txs, nil
 }
 
-// RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash
+// RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
+// cid returned corresponds to the leaf node data which contains the receipt.
 func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
+	rctResults := make([]rctIpldResult, 0)
 	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil {
 		return nil, nil, err
 	}
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.CID
+		cids[i] = res.LeafCID
-		rcts[i] = res.Data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, err
+		}
+		rcts[i] = nodeVal
 	}
 	return cids, rcts, nil
 }
 
-// RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash
+// RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash.
+// cid returned corresponds to the leaf node data which contains the receipt.
 func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) {
-	rctResult := new(ipldResult)
+	rctResult := new(rctIpldResult)
-	return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
+	if err := r.db.Select(&rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()); err != nil {
+		return "", nil, err
+	}
+
+	nodeVal, err := DecodeLeafNode(rctResult.Data)
+	if err != nil {
+		return "", nil, err
+	}
+	return rctResult.LeafCID, nodeVal, nil
 }
 
 type nodeInfo struct {
-	CID     string `db:"cid"`
+	CID              string `db:"cid"`
-	Data    []byte `db:"data"`
+	Data             []byte `db:"data"`
-	Removed bool   `db:"removed"`
+	NodeType         int    `db:"node_type"`
+	StateLeafRemoved bool   `db:"state_leaf_removed"`
 }
 
 // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
@@ -400,9 +441,11 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr
 	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
 		return "", nil, err
 	}
-	if accountResult.Removed {
+
-		return "", []byte{}, nil
+	if accountResult.NodeType == removedNode {
+		return "", EmptyNodeValue, nil
 	}
+
 	var i []interface{}
 	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
 		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
@@ -421,9 +464,11 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad
 	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil {
 		return "", nil, err
 	}
-	if accountResult.Removed {
+
-		return "", []byte{}, nil
+	if accountResult.NodeType == removedNode {
+		return "", EmptyNodeValue, nil
 	}
+
 	var i []interface{}
 	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
 		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
@@ -442,8 +487,8 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add
 	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
 		return "", nil, nil, err
 	}
-	if storageResult.Removed {
+	if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode {
-		return "", []byte{}, []byte{}, nil
+		return "", EmptyNodeValue, EmptyNodeValue, nil
 	}
 	var i []interface{}
 	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
@@ -464,8 +509,9 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad
 	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil {
 		return "", nil, err
 	}
-	if storageResult.Removed {
+
-		return "", []byte{}, nil
+	if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode {
+		return "", EmptyNodeValue, nil
 	}
 	var i []interface{}
 	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {

pkg/eth/test_helpers.go

@@ -17,10 +17,9 @@
 package eth
 
 import (
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 )
 
 // TearDownDB is used to tear down the watcher dbs after tests
@@ -40,13 +39,15 @@ func TearDownDB(db *postgres.DB) {
 	Expect(err).NotTo(HaveOccurred())
 	_, err = tx.Exec(`DELETE FROM blocks`)
 	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM eth.log_cids`)
+	Expect(err).NotTo(HaveOccurred())
 
 	err = tx.Commit()
 	Expect(err).NotTo(HaveOccurred())
 }
 
 // TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
-func TxModelsContainsCID(txs []eth.TxModel, cid string) bool {
+func TxModelsContainsCID(txs []models.TxModel, cid string) bool {
 	for _, tx := range txs {
 		if tx.CID == cid {
 			return true
@@ -55,10 +56,10 @@ func TxModelsContainsCID(txs []eth.TxModel, cid string) bool {
 	return false
 }
 
-// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
+// ReceiptModelsContainsCID used to check if a list of ReceiptModel contains a specific cid string
-func ReceiptModelsContainsCID(rcts []eth.ReceiptModel, cid string) bool {
+func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool {
 	for _, rct := range rcts {
-		if rct.CID == cid {
+		if rct.LeafCID == cid {
 			return true
 		}
 	}

pkg/eth/test_helpers/test_data.go

@@ -23,32 +23,31 @@ import (
 	"crypto/rand"
 	"math/big"
 
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
-	"github.com/ethereum/go-ethereum/trie"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/statediff/testhelpers"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ethereum/go-ethereum/trie"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/multiformats/go-multihash"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
-
-	eth2 "github.com/vulcanize/ipld-eth-server/pkg/eth"
 )
 
 // Test variables
 var (
 	// block data
+	db          *postgres.DB
 	BlockNumber = big.NewInt(1)
 	MockHeader  = types.Header{
 		Time:        0,
@@ -97,6 +96,8 @@ var (
 	MockChildRlp, _      = rlp.EncodeToBytes(MockChild.Header())
 	Address              = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
 	AnotherAddress       = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	AnotherAddress1      = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594")
+	AnotherAddress2      = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476596")
 	ContractAddress      = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
 	ContractHash         = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
 	MockContractByteCode = []byte{0, 1, 2, 3, 4, 5}
@@ -104,6 +105,12 @@ var (
 	mockTopic12          = common.HexToHash("0x06")
 	mockTopic21          = common.HexToHash("0x05")
 	mockTopic22          = common.HexToHash("0x07")
+	mockTopic31          = common.HexToHash("0x08")
+	mockTopic41          = common.HexToHash("0x09")
+	mockTopic42          = common.HexToHash("0x0a")
+	mockTopic43          = common.HexToHash("0x0b")
+	mockTopic51          = common.HexToHash("0x0c")
+	mockTopic61          = common.HexToHash("0x0d")
 	MockLog1             = &types.Log{
 		Address:     Address,
 		Topics:      []common.Hash{mockTopic11, mockTopic12},
@@ -120,36 +127,75 @@ var (
 		TxIndex:     1,
 		Index:       1,
 	}
+	MockLog3 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic31},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       2,
+	}
+
+	MockLog4 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic41, mockTopic42, mockTopic43},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       3,
+	}
+	MockLog5 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic51},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       4,
+	}
+	MockLog6 = &types.Log{
+		Address:     AnotherAddress2,
+		Topics:      []common.Hash{mockTopic61},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     3,
+		Index:       5,
+	}
 
 	Tx1 = GetTxnRlp(0, MockTransactions)
 	Tx2 = GetTxnRlp(1, MockTransactions)
 	Tx3 = GetTxnRlp(2, MockTransactions)
+	Tx4 = GetTxnRlp(3, MockTransactions)
 
-	Rct1 = GetRctRlp(0, MockReceipts)
+	rctCIDs, rctIPLDData, _ = eth.GetRctLeafNodeData(MockReceipts)
-	Rct2 = GetRctRlp(1, MockReceipts)
+	HeaderCID, _            = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
-	Rct3 = GetRctRlp(2, MockReceipts)
+	HeaderMhKey             = shared.MultihashKeyFromCID(HeaderCID)
-
+	Trx1CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
-	HeaderCID, _  = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
+	Trx1MhKey               = shared.MultihashKeyFromCID(Trx1CID)
-	HeaderMhKey   = shared.MultihashKeyFromCID(HeaderCID)
+	Trx2CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
-	Trx1CID, _    = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
+	Trx2MhKey               = shared.MultihashKeyFromCID(Trx2CID)
-	Trx1MhKey     = shared.MultihashKeyFromCID(Trx1CID)
+	Trx3CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
-	Trx2CID, _    = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
+	Trx3MhKey               = shared.MultihashKeyFromCID(Trx3CID)
-	Trx2MhKey     = shared.MultihashKeyFromCID(Trx2CID)
+	Trx4CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
-	Trx3CID, _    = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
+	Trx4MhKey               = shared.MultihashKeyFromCID(Trx4CID)
-	Trx3MhKey     = shared.MultihashKeyFromCID(Trx3CID)
+	Rct1CID                 = rctCIDs[0]
-	Rct1CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct1, multihash.KECCAK_256)
+	Rct1MhKey               = shared.MultihashKeyFromCID(Rct1CID)
-	Rct1MhKey     = shared.MultihashKeyFromCID(Rct1CID)
+	Rct2CID                 = rctCIDs[1]
-	Rct2CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct2, multihash.KECCAK_256)
+	Rct2MhKey               = shared.MultihashKeyFromCID(Rct2CID)
-	Rct2MhKey     = shared.MultihashKeyFromCID(Rct2CID)
+	Rct3CID                 = rctCIDs[2]
-	Rct3CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct3, multihash.KECCAK_256)
+	Rct3MhKey               = shared.MultihashKeyFromCID(Rct3CID)
-	Rct3MhKey     = shared.MultihashKeyFromCID(Rct3CID)
+	Rct4CID                 = rctCIDs[3]
-	State1CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
+	Rct4MhKey               = shared.MultihashKeyFromCID(Rct4CID)
-	State1MhKey   = shared.MultihashKeyFromCID(State1CID)
+	State1CID, _            = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
-	State2CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
+	State1MhKey             = shared.MultihashKeyFromCID(State1CID)
-	State2MhKey   = shared.MultihashKeyFromCID(State2CID)
+	State2CID, _            = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
-	StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
+	State2MhKey             = shared.MultihashKeyFromCID(State2CID)
-	StorageMhKey  = shared.MultihashKeyFromCID(StorageCID)
+	StorageCID, _           = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
-	MockTrxMeta   = []eth.TxModel{
+	StorageMhKey            = shared.MultihashKeyFromCID(StorageCID)
+	Rct1IPLD                = rctIPLDData[0]
+	Rct2IPLD                = rctIPLDData[1]
+	Rct3IPLD                = rctIPLDData[2]
+	Rct4IPLD                = rctIPLDData[3]
+	MockTrxMeta             = []models.TxModel{
 		{
 			CID:    "", // This is empty until we go to publish to ipfs
 			MhKey:  "",
@@ -177,8 +223,17 @@ var (
 			TxHash: MockTransactions[2].Hash().String(),
 			Data:   MockContractByteCode,
 		},
+		{
+			CID:    "",
+			MhKey:  "",
+			Src:    SenderAddr.Hex(),
+			Dst:    "",
+			Index:  3,
+			TxHash: MockTransactions[3].Hash().String(),
+			Data:   []byte{},
+		},
 	}
-	MockTrxMetaPostPublsh = []eth.TxModel{
+	MockTrxMetaPostPublsh = []models.TxModel{
 		{
 			CID:    Trx1CID.String(), // This is empty until we go to publish to ipfs
 			MhKey:  Trx1MhKey,
@@ -206,83 +261,67 @@ var (
 			TxHash: MockTransactions[2].Hash().String(),
 			Data:   MockContractByteCode,
 		},
-	}
-	MockRctMeta = []eth.ReceiptModel{
 		{
-			CID:   "",
+			CID:    Trx4CID.String(),
-			MhKey: "",
+			MhKey:  Trx4MhKey,
-			Topic0s: []string{
+			Src:    SenderAddr.Hex(),
-				mockTopic11.String(),
+			Dst:    AnotherAddress1.String(),
-			},
+			Index:  3,
-			Topic1s: []string{
+			TxHash: MockTransactions[3].Hash().String(),
-				mockTopic12.String(),
+			Data:   []byte{},
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				Address.String(),
-			},
-		},
-		{
-			CID:   "",
-			MhKey: "",
-			Topic0s: []string{
-				mockTopic21.String(),
-			},
-			Topic1s: []string{
-				mockTopic22.String(),
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				AnotherAddress.String(),
-			},
-		},
-		{
-			CID:          "",
-			MhKey:        "",
-			Contract:     ContractAddress.String(),
-			ContractHash: ContractHash,
-			LogContracts: []string{},
 		},
 	}
-	MockRctMetaPostPublish = []eth.ReceiptModel{
+	MockRctMeta = []models.ReceiptModel{
 		{
-			CID:   Rct1CID.String(),
+			LeafCID:      "",
-			MhKey: Rct1MhKey,
+			LeafMhKey:    "",
-			Topic0s: []string{
-				mockTopic11.String(),
-			},
-			Topic1s: []string{
-				mockTopic12.String(),
-			},
 			Contract:     "",
 			ContractHash: "",
-			LogContracts: []string{
-				Address.String(),
-			},
 		},
 		{
-			CID:   Rct2CID.String(),
+			LeafCID:      "",
-			MhKey: Rct2MhKey,
+			LeafMhKey:    "",
-			Topic0s: []string{
-				mockTopic21.String(),
-			},
-			Topic1s: []string{
-				mockTopic22.String(),
-			},
 			Contract:     "",
 			ContractHash: "",
-			LogContracts: []string{
-				AnotherAddress.String(),
-			},
 		},
 		{
-			CID:          Rct3CID.String(),
+			LeafCID:      "",
-			MhKey:        Rct3MhKey,
+			LeafMhKey:    "",
 			Contract:     ContractAddress.String(),
 			ContractHash: ContractHash,
-			LogContracts: []string{},
+		},
+		{
+			LeafCID:      "",
+			LeafMhKey:    "",
+			Contract:     "",
+			ContractHash: "",
+		},
+	}
+
+	MockRctMetaPostPublish = []models.ReceiptModel{
+		{
+			LeafCID:      Rct1CID.String(),
+			LeafMhKey:    Rct1MhKey,
+			Contract:     "",
+			ContractHash: "",
+		},
+		{
+			LeafCID:      Rct2CID.String(),
+			LeafMhKey:    Rct2MhKey,
+			Contract:     "",
+			ContractHash: "",
+		},
+		{
+			LeafCID:      Rct3CID.String(),
+			LeafMhKey:    Rct3MhKey,
+			Contract:     ContractAddress.String(),
+			ContractHash: ContractHash,
+		},
+		{
+			LeafCID:      Rct4CID.String(),
+			LeafMhKey:    Rct4MhKey,
+			Contract:     "",
+			ContractHash: "",
 		},
 	}
 
@@ -301,7 +340,7 @@ var (
 	ContractCodeHash   = crypto.Keccak256Hash(MockContractByteCode)
 	contractPath       = common.Bytes2Hex([]byte{'\x06'})
 	ContractLeafKey    = testhelpers.AddressToLeafKey(ContractAddress)
-	ContractAccount, _ = rlp.EncodeToBytes(state.Account{
+	ContractAccount, _ = rlp.EncodeToBytes(types.StateAccount{
 		Nonce:    nonce1,
 		Balance:  big.NewInt(0),
 		CodeHash: ContractCodeHash.Bytes(),
@@ -319,7 +358,7 @@ var (
 	AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
 	AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
 	AccountLeafKey  = testhelpers.Account2LeafKey
-	Account, _      = rlp.EncodeToBytes(state.Account{
+	Account, _      = rlp.EncodeToBytes(types.StateAccount{
 		Nonce:    nonce0,
 		Balance:  AccountBalance,
 		CodeHash: AccountCodeHash.Bytes(),
@@ -331,21 +370,30 @@ var (
 		Account,
 	})
 
-	MockStateNodes = []eth.TrieNode{
+	MockStateNodes = []sdtypes.StateNode{
 		{
-			LeafKey: common.BytesToHash(ContractLeafKey),
+			LeafKey:   ContractLeafKey,
-			Path:    []byte{'\x06'},
+			Path:      []byte{'\x06'},
-			Value:   ContractLeafNode,
+			NodeValue: ContractLeafNode,
-			Type:    sdtypes.Leaf,
+			NodeType:  sdtypes.Leaf,
+			StorageNodes: []sdtypes.StorageNode{
+				{
+					Path:      []byte{},
+					NodeType:  sdtypes.Leaf,
+					LeafKey:   StorageLeafKey,
+					NodeValue: StorageLeafNode,
+				},
+			},
 		},
 		{
-			LeafKey: common.BytesToHash(AccountLeafKey),
+			LeafKey:      AccountLeafKey,
-			Path:    []byte{'\x0c'},
+			Path:         []byte{'\x0c'},
-			Value:   AccountLeafNode,
+			NodeValue:    AccountLeafNode,
-			Type:    sdtypes.Leaf,
+			NodeType:     sdtypes.Leaf,
+			StorageNodes: []sdtypes.StorageNode{},
 		},
 	}
-	MockStateMetaPostPublish = []eth.StateNodeModel{
+	MockStateMetaPostPublish = []models.StateNodeModel{
 		{
 			CID:      State1CID.String(),
 			MhKey:    State1MhKey,
@@ -361,13 +409,13 @@ var (
 			StateKey: common.BytesToHash(AccountLeafKey).Hex(),
 		},
 	}
-	MockStorageNodes = map[string][]eth.TrieNode{
+	MockStorageNodes = map[string][]sdtypes.StorageNode{
 		contractPath: {
 			{
-				LeafKey: common.BytesToHash(StorageLeafKey),
+				LeafKey:   StorageLeafKey,
-				Value:   StorageLeafNode,
+				NodeValue: StorageLeafNode,
-				Type:    sdtypes.Leaf,
+				NodeType:  sdtypes.Leaf,
-				Path:    []byte{},
+				Path:      []byte{},
 			},
 		},
 	}
@@ -391,11 +439,11 @@ var (
 		StateNodes:      MockStateNodes,
 	}
 
-	Reward = eth.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
+	Reward = indexer.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
 
-	MockCIDWrapper = &eth2.CIDWrapper{
+	MockCIDWrapper = &eth.CIDWrapper{
 		BlockNumber: new(big.Int).Set(BlockNumber),
-		Header: eth.HeaderModel{
+		Header: models.HeaderModel{
 			BlockNumber:     "1",
 			BlockHash:       MockBlock.Hash().String(),
 			ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
@@ -413,9 +461,9 @@ var (
 		},
 		Transactions: MockTrxMetaPostPublsh,
 		Receipts:     MockRctMetaPostPublish,
-		Uncles:       []eth.UncleModel{},
+		Uncles:       []models.UncleModel{},
 		StateNodes:   MockStateMetaPostPublish,
-		StorageNodes: []eth.StorageNodeWithStateKeyModel{
+		StorageNodes: []models.StorageNodeWithStateKeyModel{
 			{
 				Path:       []byte{},
 				CID:        StorageCID.String(),
@@ -431,14 +479,12 @@ var (
 	Trx1IPLD, _    = blocks.NewBlockWithCid(Tx1, Trx1CID)
 	Trx2IPLD, _    = blocks.NewBlockWithCid(Tx2, Trx2CID)
 	Trx3IPLD, _    = blocks.NewBlockWithCid(Tx3, Trx3CID)
-	Rct1IPLD, _    = blocks.NewBlockWithCid(Rct1, Rct1CID)
+	Trx4IPLD, _    = blocks.NewBlockWithCid(Tx4, Trx4CID)
-	Rct2IPLD, _    = blocks.NewBlockWithCid(Rct2, Rct2CID)
-	Rct3IPLD, _    = blocks.NewBlockWithCid(Rct3, Rct3CID)
 	State1IPLD, _  = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
 	State2IPLD, _  = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
 	StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
 
-	MockIPLDs = eth2.IPLDs{
+	MockIPLDs = eth.IPLDs{
 		BlockNumber: new(big.Int).Set(BlockNumber),
 		Header: ipfs.BlockModel{
 			Data: HeaderIPLD.RawData(),
@@ -457,22 +503,30 @@ var (
 				Data: Trx3IPLD.RawData(),
 				CID:  Trx3IPLD.Cid().String(),
 			},
+			{
+				Data: Trx4IPLD.RawData(),
+				CID:  Trx4IPLD.Cid().String(),
+			},
 		},
 		Receipts: []ipfs.BlockModel{
 			{
-				Data: Rct1IPLD.RawData(),
+				Data: Rct1IPLD,
-				CID:  Rct1IPLD.Cid().String(),
+				CID:  Rct1CID.String(),
 			},
 			{
-				Data: Rct2IPLD.RawData(),
+				Data: Rct2IPLD,
-				CID:  Rct2IPLD.Cid().String(),
+				CID:  Rct2CID.String(),
 			},
 			{
-				Data: Rct3IPLD.RawData(),
+				Data: Rct3IPLD,
-				CID:  Rct3IPLD.Cid().String(),
+				CID:  Rct3CID.String(),
+			},
+			{
+				Data: Rct4IPLD,
+				CID:  Rct4CID.String(),
 			},
 		},
-		StateNodes: []eth2.StateNode{
+		StateNodes: []eth.StateNode{
 			{
 				StateLeafKey: common.BytesToHash(ContractLeafKey),
 				Type:         sdtypes.Leaf,
@@ -492,7 +546,7 @@ var (
 				Path: []byte{'\x0c'},
 			},
 		},
-		StorageNodes: []eth2.StorageNode{
+		StorageNodes: []eth.StorageNode{
 			{
 				StateLeafKey:   common.BytesToHash(ContractLeafKey),
 				StorageLeafKey: common.BytesToHash(StorageLeafKey),
@@ -516,46 +570,8 @@ var (
 		BaseFee:    big.NewInt(params.InitialBaseFee),
 	}
 
-	MockLondonTransactions, MockLondonReceipts, SenderAdd = createDynamicTransactionsAndReceipts(LondonBlockNum)
+	MockLondonTransactions, MockLondonReceipts, _ = createDynamicTransactionsAndReceipts(LondonBlockNum)
-	MockLondonBlock                                       = createNewBlock(&MockLondonHeader, MockLondonTransactions, nil, MockLondonReceipts, new(trie.Trie))
+	MockLondonBlock                               = createNewBlock(&MockLondonHeader, MockLondonTransactions, nil, MockLondonReceipts, new(trie.Trie))
-	MockLondonTrxMeta                                     = []eth.TxModel{
-		{
-			CID:    "", // This is empty until we go to publish to ipfs
-			MhKey:  "",
-			Src:    SenderAdd.Hex(),
-			Dst:    Address.String(),
-			Index:  0,
-			TxHash: MockLondonTransactions[0].Hash().String(),
-			Data:   []byte{},
-		},
-	}
-	MockLondonRctMeta = []eth.ReceiptModel{
-		{
-			CID:   "",
-			MhKey: "",
-			Topic0s: []string{
-				mockTopic11.String(),
-			},
-			Topic1s: []string{
-				mockTopic12.String(),
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				Address.String(),
-			},
-		},
-	}
-
-	MockConvertedLondonPayload = eth.ConvertedPayload{
-		TotalDifficulty: MockLondonBlock.Difficulty(),
-		Block:           MockLondonBlock,
-		Receipts:        MockLondonReceipts,
-		TxMetaData:      MockLondonTrxMeta,
-		ReceiptMetaData: MockLondonRctMeta,
-		StorageNodes:    MockStorageNodes,
-		StateNodes:      MockStateNodes,
-	}
 )
 
 func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, hasher types.TrieHasher) *types.Block {
@@ -605,8 +621,8 @@ func createDynamicTransactionsAndReceipts(blockNumber *big.Int) (types.Transacti
 	// TODO: Change the receipt type to DynamicFeeTxType once this PR is merged.
 	// https://github.com/ethereum/go-ethereum/pull/22806
 	mockReceipt1 := &types.Receipt{
-		Type:              types.AccessListTxType,
+		Type:              types.DynamicFeeTxType,
-		PostState:         common.HexToHash("0x1").Bytes(),
+		PostState:         common.HexToHash("0x0").Bytes(),
 		Status:            types.ReceiptStatusSuccessful,
 		CumulativeGasUsed: 50,
 		Logs:              []*types.Log{},
@@ -622,6 +638,7 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts,
 	trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
 	trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
 	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
+	trx4 := types.NewTransaction(3, AnotherAddress1, big.NewInt(2000), 100, big.NewInt(200), []byte{})
 	transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
 	mockCurve := elliptic.P256()
 	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
@@ -640,12 +657,16 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts,
 	if err != nil {
 		log.Fatal(err)
 	}
+	signedTrx4, err := types.SignTx(trx4, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
 	SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
 	if err != nil {
 		log.Fatal(err)
 	}
 	// make receipts
-	mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt1 := types.NewReceipt(nil, false, 50)
 
 	hash1 := signedTrx1.Hash()
 	MockLog1.TxHash = hash1
@@ -663,11 +684,17 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts,
 	mockReceipt2.GasUsed = mockReceipt2.CumulativeGasUsed - mockReceipt1.CumulativeGasUsed
 
 	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 175)
-	mockReceipt3.Logs = []*types.Log{}
+	mockReceipt3.Logs = []*types.Log{MockLog3, MockLog4, MockLog5}
 	mockReceipt3.TxHash = signedTrx3.Hash()
 	mockReceipt3.GasUsed = mockReceipt3.CumulativeGasUsed - mockReceipt2.CumulativeGasUsed
 
-	return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, SenderAddr
+	// Receipt with failed status.
+	mockReceipt4 := types.NewReceipt(nil, true, 250)
+	mockReceipt4.Logs = []*types.Log{MockLog6}
+	mockReceipt4.TxHash = signedTrx4.Hash()
+	mockReceipt4.GasUsed = mockReceipt4.CumulativeGasUsed - mockReceipt3.CumulativeGasUsed
+
+	return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr
 }
 
 func GetTxnRlp(num int, txs types.Transactions) []byte {

pkg/eth/types.go

@@ -17,14 +17,17 @@
 package eth
 
 import (
+	"errors"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
 )
 
 // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
@@ -99,18 +102,94 @@ type CallArgs struct {
 	Input                *hexutil.Bytes    `json:"input"`
 }
 
-// account indicates the overriding fields of account during the execution of
+// from retrieves the transaction sender address.
-// a message call.
+func (arg *CallArgs) from() common.Address {
-// Note, state and stateDiff can't be specified at the same time. If state is
+	if arg.From == nil {
-// set, message execution will only use the data in the given state. Otherwise
+		return common.Address{}
-// if statDiff is set, all diff will be applied first and then execute the call
+	}
-// message.
+	return *arg.From
-type account struct {
+}
-	Nonce     *hexutil.Uint64              `json:"nonce"`
+
-	Code      *hexutil.Bytes               `json:"code"`
+// data retrieves the transaction calldata. Input field is preferred.
-	Balance   **hexutil.Big                `json:"balance"`
+func (arg *CallArgs) data() []byte {
-	State     *map[common.Hash]common.Hash `json:"state"`
+	if arg.Input != nil {
-	StateDiff *map[common.Hash]common.Hash `json:"stateDiff"`
+		return *arg.Input
+	}
+	if arg.Data != nil {
+		return *arg.Data
+	}
+	return nil
+}
+
+// ToMessage converts the transaction arguments to the Message type used by the
+// core evm. This method is used in calls and traces that do not require a real
+// live transaction.
+func (arg *CallArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (types.Message, error) {
+	// Reject invalid combinations of pre- and post-1559 fee styles
+	if arg.GasPrice != nil && (arg.MaxFeePerGas != nil || arg.MaxPriorityFeePerGas != nil) {
+		return types.Message{}, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified")
+	}
+	// Set sender address or use zero address if none specified.
+	addr := arg.from()
+
+	// Set default gas & gas price if none were set
+	gas := globalGasCap
+	if gas == 0 {
+		gas = uint64(math.MaxUint64 / 2)
+	}
+	if arg.Gas != nil {
+		gas = uint64(*arg.Gas)
+	}
+	if globalGasCap != 0 && globalGasCap < gas {
+		logrus.Warn("Caller gas above allowance, capping", "requested", gas, "cap", globalGasCap)
+		gas = globalGasCap
+	}
+	var (
+		gasPrice  *big.Int
+		gasFeeCap *big.Int
+		gasTipCap *big.Int
+	)
+	if baseFee == nil {
+		// If there's no basefee, then it must be a non-1559 execution
+		gasPrice = new(big.Int)
+		if arg.GasPrice != nil {
+			gasPrice = arg.GasPrice.ToInt()
+		}
+		gasFeeCap, gasTipCap = gasPrice, gasPrice
+	} else {
+		// A basefee is provided, necessitating 1559-type execution
+		if arg.GasPrice != nil {
+			// User specified the legacy gas field, convert to 1559 gas typing
+			gasPrice = arg.GasPrice.ToInt()
+			gasFeeCap, gasTipCap = gasPrice, gasPrice
+		} else {
+			// User specified 1559 gas feilds (or none), use those
+			gasFeeCap = new(big.Int)
+			if arg.MaxFeePerGas != nil {
+				gasFeeCap = arg.MaxFeePerGas.ToInt()
+			}
+			gasTipCap = new(big.Int)
+			if arg.MaxPriorityFeePerGas != nil {
+				gasTipCap = arg.MaxPriorityFeePerGas.ToInt()
+			}
+			// Backfill the legacy gasPrice for EVM execution, unless we're all zeroes
+			gasPrice = new(big.Int)
+			if gasFeeCap.BitLen() > 0 || gasTipCap.BitLen() > 0 {
+				gasPrice = math.BigMin(new(big.Int).Add(gasTipCap, baseFee), gasFeeCap)
+			}
+		}
+	}
+	value := new(big.Int)
+	if arg.Value != nil {
+		value = arg.Value.ToInt()
+	}
+	data := arg.data()
+	var accessList types.AccessList
+	if arg.AccessList != nil {
+		accessList = *arg.AccessList
+	}
+	msg := types.NewMessage(addr, arg.To, 0, value, gas, gasPrice, gasFeeCap, gasTipCap, data, accessList, true)
+	return msg, nil
 }
 
 // IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server
@@ -146,10 +225,43 @@ type StorageNode struct {
 // Passed to IPLDFetcher
 type CIDWrapper struct {
 	BlockNumber  *big.Int
-	Header       eth.HeaderModel
+	Header       models.HeaderModel
-	Uncles       []eth.UncleModel
+	Uncles       []models.UncleModel
-	Transactions []eth.TxModel
+	Transactions []models.TxModel
-	Receipts     []eth.ReceiptModel
+	Receipts     []models.ReceiptModel
-	StateNodes   []eth.StateNodeModel
+	StateNodes   []models.StateNodeModel
-	StorageNodes []eth.StorageNodeWithStateKeyModel
+	StorageNodes []models.StorageNodeWithStateKeyModel
+}
+
+// ConvertedPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers
+// Returned by PayloadConverter
+// Passed to IPLDPublisher and ResponseFilterer
+type ConvertedPayload struct {
+	TotalDifficulty *big.Int
+	Block           *types.Block
+	TxMetaData      []models.TxModel
+	Receipts        types.Receipts
+	ReceiptMetaData []models.ReceiptModel
+	StateNodes      []sdtypes.StateNode
+	StorageNodes    map[string][]sdtypes.StorageNode
+}
+
+// LogResult represent a log.
+type LogResult struct {
+	LeafCID     string `db:"leaf_cid"`
+	ReceiptID   int64  `db:"receipt_id"`
+	Address     string `db:"address"`
+	Index       int64  `db:"index"`
+	Data        []byte `db:"log_data"`
+	Topic0      string `db:"topic0"`
+	Topic1      string `db:"topic1"`
+	Topic2      string `db:"topic2"`
+	Topic3      string `db:"topic3"`
+	LogLeafData []byte `db:"data"`
+	RctCID      string `db:"cid"`
+	RctStatus   uint64 `db:"post_status"`
+	BlockNumber string `db:"block_number"`
+	BlockHash   string `db:"block_hash"`
+	TxnIndex    int64  `db:"txn_index"`
+	TxHash      string `db:"tx_hash"`
 }

pkg/graphql/client.go

@@ -24,6 +24,8 @@ type LogResponse struct {
 	Topics      []common.Hash   `json:"topics"`
 	Data        hexutil.Bytes   `json:"data"`
 	Transaction TransactionResp `json:"transaction"`
+	ReceiptCID  string          `json:"receiptCID"`
+	Status      int32           `json:"status"`
 }
 
 type TransactionResp struct {
@@ -43,18 +45,23 @@ func NewClient(endpoint string) *Client {
 	return &Client{client: client}
 }
 
-func (c *Client) GetLogs(ctx context.Context, hash common.Hash, address common.Address) ([]LogResponse, error) {
+func (c *Client) GetLogs(ctx context.Context, hash common.Hash, address *common.Address) ([]LogResponse, error) {
-	getLogsQuery := fmt.Sprintf(`
+	params := fmt.Sprintf(`blockHash: "%s"`, hash.String())
-		query{
+	if address != nil {
-			getLogs(blockHash: "%s", contract: "%s") {
+		params += fmt.Sprintf(`, contract: "%s"`, address.String())
+	}
+
+	getLogsQuery := fmt.Sprintf(`query{
+			getLogs(%s) {
 				data
 				topics
 				transaction {
 					hash
 				}
+				status
+				receiptCID
 			}
-		}
+		}`, params)
-	`, hash.String(), address.String())
 
 	req := gqlclient.NewRequest(getLogsQuery)
 	req.Header.Set("Cache-Control", "no-cache")

pkg/graphql/graphql.go

@@ -18,6 +18,7 @@
 package graphql
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
 	"errors"
@@ -94,14 +95,18 @@ type Log struct {
 	transaction *Transaction
 	log         *types.Log
 	cid         string
-	ipldBlock   []byte
+	receiptCID  string
+	ipldBlock   []byte // log leaf node IPLD block data
+	status      uint64
 }
 
-func (l *Log) Transaction(ctx context.Context) *Transaction {
+// Transaction returns transaction that generated this log entry.
+func (l *Log) Transaction(_ context.Context) *Transaction {
 	return l.transaction
 }
 
-func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account {
+// Account returns the contract account which generated this log.
+func (l *Log) Account(_ context.Context, args BlockNumberArgs) *Account {
 	return &Account{
 		backend:       l.backend,
 		address:       l.log.Address,
@@ -109,24 +114,39 @@ func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account {
 	}
 }
 
-func (l *Log) Index(ctx context.Context) int32 {
+// Index returns the index of this log in the block
+func (l *Log) Index(_ context.Context) int32 {
 	return int32(l.log.Index)
 }
 
-func (l *Log) Topics(ctx context.Context) []common.Hash {
+// Topics returns the list of 0-4 indexed topics for the log.
+func (l *Log) Topics(_ context.Context) []common.Hash {
 	return l.log.Topics
 }
 
-func (l *Log) Data(ctx context.Context) hexutil.Bytes {
+// Data returns data of this log.
-	return hexutil.Bytes(l.log.Data)
+func (l *Log) Data(_ context.Context) hexutil.Bytes {
+	return l.log.Data
 }
 
-func (l *Log) Cid(ctx context.Context) string {
+// Cid returns cid of the leaf node of this log.
+func (l *Log) Cid(_ context.Context) string {
 	return l.cid
 }
 
-func (l *Log) IpldBlock(ctx context.Context) hexutil.Bytes {
+// IpldBlock returns IPLD block of the leaf node of this log.
-	return hexutil.Bytes(l.ipldBlock)
+func (l *Log) IpldBlock(_ context.Context) hexutil.Bytes {
+	return l.ipldBlock
+}
+
+// Status returns the status of the receipt IPLD block this Log exists in.
+func (l *Log) Status(_ context.Context) int32 {
+	return int32(l.status)
+}
+
+// ReceiptCID returns the receipt CID of the receipt IPLD block this Log exists in.
+func (l *Log) ReceiptCID(_ context.Context) string {
+	return l.receiptCID
 }
 
 // Transaction represents an Ethereum transaction.
@@ -815,16 +835,20 @@ func (b *Block) Call(ctx context.Context, args struct {
 			return nil, err
 		}
 	}
-	result, gas, failed, err := eth.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, 5*time.Second, b.backend.RPCGasCap())
+	result, err := eth.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, 5*time.Second, b.backend.RPCGasCap().Uint64())
+	if err != nil {
+		return nil, err
+	}
 	status := hexutil.Uint64(1)
-	if failed {
+	if result.Failed() {
 		status = 0
 	}
+
 	return &CallResult{
-		data:    hexutil.Bytes(result),
+		data:    result.ReturnData,
-		gasUsed: hexutil.Uint64(gas),
+		gasUsed: hexutil.Uint64(result.UsedGas),
 		status:  status,
-	}, err
+	}, nil
 }
 
 // Resolver is the top-level object in the GraphQL hierarchy.
@@ -993,6 +1017,10 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct {
 		return nil, err
 	}
 
+	if bytes.Compare(rlpValue, eth.EmptyNodeValue) == 0 {
+		return &StorageResult{value: eth.EmptyNodeValue, cid: cid, ipldBlock: ipldBlock}, nil
+	}
+
 	var value interface{}
 	err = rlp.DecodeBytes(rlpValue, &value)
 	if err != nil {
@@ -1007,40 +1035,90 @@ func (r *Resolver) GetLogs(ctx context.Context, args struct {
 	BlockHash common.Hash
 	Contract  *common.Address
 }) (*[]*Log, error) {
-	ret := make([]*Log, 0, 10)
 
-	receiptCIDs, receiptsBytes, txs, err := r.backend.IPLDRetriever.RetrieveReceiptsByBlockHash(args.BlockHash)
+	var filter eth.ReceiptFilter
+	if args.Contract != nil {
+		filter.LogAddresses = []string{args.Contract.String()}
+	}
+
+	// Begin tx
+	tx, err := r.backend.DB.Beginx()
 	if err != nil {
 		return nil, err
 	}
 
-	var logIndexInBlock uint = 0
+	filteredLogs, err := r.backend.Retriever.RetrieveFilteredGQLLogs(tx, filter, &args.BlockHash)
-	receipts := make(types.Receipts, len(receiptsBytes))
+	if err != nil {
-	for index, receiptBytes := range receiptsBytes {
+		return nil, err
-		receiptCID := receiptCIDs[index]
+	}
-		receipt := new(types.Receipt)
-		if err := rlp.DecodeBytes(receiptBytes, receipt); err != nil {
-			return nil, err
-		}
 
-		receipts[index] = receipt
+	if err = tx.Commit(); err != nil {
-		for _, log := range receipt.Logs {
+		return nil, err
-			log.Index = logIndexInBlock
+	}
-			logIndexInBlock++
 
-			if args.Contract == nil || *args.Contract == log.Address {
+	rctLog := decomposeGQLLogs(filteredLogs)
-				ret = append(ret, &Log{
+	if err != nil {
-					backend:   r.backend,
+		return nil, err
-					log:       log,
+	}
-					cid:       receiptCID,
+
-					ipldBlock: receiptBytes,
+	ret := make([]*Log, 0, 10)
-					transaction: &Transaction{
+	for _, l := range rctLog {
-						hash: txs[index],
+		ret = append(ret, &Log{
-					},
+			backend:    r.backend,
-				})
+			log:        l.Log,
-			}
+			cid:        l.CID,
-		}
+			receiptCID: l.RctCID,
+			ipldBlock:  l.LogLeafData,
+			transaction: &Transaction{
+				hash: l.Log.TxHash,
+			},
+			status: l.RctStatus,
+		})
 	}
 
 	return &ret, nil
 }
+
+type logsCID struct {
+	Log         *types.Log
+	CID         string
+	RctCID      string
+	LogLeafData []byte
+	RctStatus   uint64
+}
+
+// decomposeGQLLogs return logs for graphql.
+func decomposeGQLLogs(logCIDs []eth.LogResult) []logsCID {
+	logs := make([]logsCID, len(logCIDs))
+	for i, l := range logCIDs {
+		topics := make([]common.Hash, 0)
+		if l.Topic0 != "" {
+			topics = append(topics, common.HexToHash(l.Topic0))
+		}
+		if l.Topic1 != "" {
+			topics = append(topics, common.HexToHash(l.Topic1))
+		}
+		if l.Topic2 != "" {
+			topics = append(topics, common.HexToHash(l.Topic2))
+		}
+		if l.Topic3 != "" {
+			topics = append(topics, common.HexToHash(l.Topic3))
+		}
+
+		logs[i] = logsCID{
+			Log: &types.Log{
+				Address: common.HexToAddress(l.Address),
+				Topics:  topics,
+				Data:    l.Data,
+				Index:   uint(l.Index),
+				TxHash:  common.HexToHash(l.TxHash),
+			},
+			CID:         l.LeafCID,
+			RctCID:      l.RctCID,
+			LogLeafData: l.LogLeafData,
+			RctStatus:   l.RctStatus,
+		}
+	}
+
+	return logs
+}

pkg/graphql/graphql_test.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"os"
+	"strconv"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -28,20 +30,34 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
 	"github.com/vulcanize/ipld-eth-server/pkg/graphql"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
+// SetupDB is use to setup a db for watcher tests
+func SetupDB() (*postgres.DB, error) {
+	port, _ := strconv.Atoi(os.Getenv("DATABASE_PORT"))
+	uri := postgres.DbConnectionString(postgres.ConnectionParams{
+		User:     os.Getenv("DATABASE_USER"),
+		Password: os.Getenv("DATABASE_PASSWORD"),
+		Hostname: os.Getenv("DATABASE_HOSTNAME"),
+		Name:     os.Getenv("DATABASE_NAME"),
+		Port:     port,
+	})
+	return postgres.NewDB(uri, postgres.ConnectionConfig{}, node.Info{})
+}
+
 var _ = Describe("GraphQL", func() {
 	const (
 		gqlEndPoint = "127.0.0.1:8083"
@@ -66,13 +82,23 @@ var _ = Describe("GraphQL", func() {
 
 	It("test init", func() {
 		var err error
-		db, err = shared.SetupDB()
+		db, err = SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+
+		transformer, err := indexer.NewStateDiffIndexer(chainConfig, db)
 		Expect(err).ToNot(HaveOccurred())
-		transformer := eth2.NewStateDiffTransformer(chainConfig, db)
 		backend, err = eth.NewEthBackend(db, &eth.Config{
 			ChainConfig: chainConfig,
-			VmConfig:    vm.Config{},
+			VMConfig:    vm.Config{},
 			RPCGasCap:   big.NewInt(10000000000),
+			GroupCacheConfig: &ethServerShared.GroupCacheConfig{
+				StateDB: ethServerShared.GroupConfig{
+					Name:                   "graphql_test",
+					CacheSizeInMB:          8,
+					CacheExpiryInMins:      60,
+					LogStatsIntervalInSecs: 0,
+				},
+			},
 		})
 		Expect(err).ToNot(HaveOccurred())
 
@@ -109,35 +135,45 @@ var _ = Describe("GraphQL", func() {
 			var diff statediff.StateObject
 			diff, err = builder.BuildStateDiffObject(args, params)
 			Expect(err).ToNot(HaveOccurred())
-			diffRlp, err := rlp.EncodeToBytes(diff)
+
+			tx, err := transformer.PushBlock(block, rcts, mockTD)
 			Expect(err).ToNot(HaveOccurred())
-			blockRlp, err := rlp.EncodeToBytes(block)
+
-			Expect(err).ToNot(HaveOccurred())
+			for _, node := range diff.Nodes {
-			receiptsRlp, err := rlp.EncodeToBytes(rcts)
+				err = transformer.PushStateNode(tx, node)
-			Expect(err).ToNot(HaveOccurred())
+				Expect(err).ToNot(HaveOccurred())
-			payload := statediff.Payload{
-				StateObjectRlp:  diffRlp,
-				BlockRlp:        blockRlp,
-				ReceiptsRlp:     receiptsRlp,
-				TotalDifficulty: mockTD,
 			}
 
-			_, err = transformer.Transform(0, payload)
+			err = tx.Close(err)
 			Expect(err).ToNot(HaveOccurred())
 		}
 
 		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
-		indexAndPublisher := eth2.NewIPLDPublisher(db)
+		indexAndPublisher, err := indexer.NewStateDiffIndexer(chainConfig, db)
+		Expect(err).ToNot(HaveOccurred())
+
 		blockHash = test_helpers.MockBlock.Hash()
 		contractAddress = test_helpers.ContractAddr
 
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
+		tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Close(err)
 		Expect(err).ToNot(HaveOccurred())
 
 		// The non-canonical header has a child
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayloadForChild)
+		tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
 		Expect(err).ToNot(HaveOccurred())
-		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
+
+		ccHash := sdtypes.CodeAndCodeHash{
+			Hash: test_helpers.CodeHash,
+			Code: test_helpers.ContractCode,
+		}
+
+		err = indexAndPublisher.PushCodeAndCodeHash(tx, ccHash)
+		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Close(err)
 		Expect(err).ToNot(HaveOccurred())
 
 		graphQLServer, err = graphql.New(backend, gqlEndPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
@@ -156,7 +192,7 @@ var _ = Describe("GraphQL", func() {
 
 	Describe("eth_getLogs", func() {
 		It("Retrieves logs that matches the provided blockHash and contract address", func() {
-			logs, err := client.GetLogs(ctx, blockHash, contractAddress)
+			logs, err := client.GetLogs(ctx, blockHash, &contractAddress)
 			Expect(err).ToNot(HaveOccurred())
 
 			expectedLogs := []graphql.LogResponse{
@@ -164,13 +200,39 @@ var _ = Describe("GraphQL", func() {
 					Topics:      test_helpers.MockLog1.Topics,
 					Data:        hexutil.Bytes(test_helpers.MockLog1.Data),
 					Transaction: graphql.TransactionResp{Hash: test_helpers.MockTransactions[0].Hash()},
+					ReceiptCID:  test_helpers.Rct1CID.String(),
+					Status:      int32(test_helpers.MockReceipts[0].Status),
 				},
 			}
+
 			Expect(logs).To(Equal(expectedLogs))
 		})
 
+		It("Retrieves logs for the failed receipt status that matches the provided blockHash and another contract address", func() {
+			logs, err := client.GetLogs(ctx, blockHash, &test_helpers.AnotherAddress2)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog6.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog6.Data),
+					Transaction: graphql.TransactionResp{Hash: test_helpers.MockTransactions[3].Hash()},
+					ReceiptCID:  test_helpers.Rct4CID.String(),
+					Status:      int32(test_helpers.MockReceipts[3].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves all the logs for the receipt that matches the provided blockHash and nil contract address", func() {
+			logs, err := client.GetLogs(ctx, blockHash, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(6))
+		})
+
 		It("Retrieves logs with random hash", func() {
-			logs, err := client.GetLogs(ctx, randomHash, contractAddress)
+			logs, err := client.GetLogs(ctx, randomHash, &contractAddress)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(logs)).To(Equal(0))
 		})
@@ -208,23 +270,3 @@ var _ = Describe("GraphQL", func() {
 		})
 	})
 })
-
-func publishCode(db *postgres.DB, codeHash common.Hash, code []byte) error {
-	tx, err := db.Beginx()
-	if err != nil {
-		return err
-	}
-
-	mhKey, err := shared.MultihashKeyFromKeccak256(codeHash)
-	if err != nil {
-		_ = tx.Rollback()
-		return err
-	}
-
-	if err := shared.PublishDirect(tx, mhKey, code); err != nil {
-		_ = tx.Rollback()
-		return err
-	}
-
-	return tx.Commit()
-}

pkg/graphql/schema.go

@@ -67,11 +67,17 @@ const schema string = `
         # Transaction is the transaction that generated this log entry.
         transaction: Transaction
 
-        # CID for the Receipt IPLD block this Log exists in.
+        # CID for the leaf node IPLD block of the log.
         cid: String!
 
-        # IPLD block data for the Receipt this Log exists in.
+        # ReceiptCID for the Receipt IPLD block this Log exists in.
+        receiptCID: String!
+
+        # IPLD block data for the Log Leaf node.
         ipldBlock: Bytes!
+
+        # Status of the Receipt IPLD block this Log exists in.
+        status: Int!
     }
 
     # Transaction is an Ethereum transaction.

pkg/rpc/http.go

@@ -29,7 +29,7 @@ import (
 func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
 
 	srv := rpc.NewServer()
-	err := node.RegisterApisFromWhitelist(apis, modules, srv, false)
+	err := node.RegisterApis(apis, modules, srv, false)
 	if err != nil {
 		utils.Fatalf("Could not register HTTP API: %w", err)
 	}

pkg/rpc/ws.go

@@ -37,7 +37,7 @@ func StartWSEndpoint(endpoint string, apis []rpc.API, modules []string, wsOrigin
 
 	// Register all the APIs exposed by the services
 	handler := rpc.NewServer()
-	err = node.RegisterApisFromWhitelist(apis, modules, handler, exposeAll)
+	err = node.RegisterApis(apis, modules, handler, exposeAll)
 	if err != nil {
 		utils.Fatalf("Could not register WS API: %w", err)
 	}

pkg/serve/api.go

@@ -20,10 +20,9 @@ import (
 	"context"
 
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 )
 

pkg/serve/config.go

@@ -23,18 +23,15 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-	"github.com/vulcanize/ipld-eth-indexer/utils"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 // Env variables
@@ -51,12 +48,18 @@ const (
 	ETH_RPC_GAS_CAP         = "ETH_RPC_GAS_CAP"
 	ETH_CHAIN_CONFIG        = "ETH_CHAIN_CONFIG"
 	ETH_SUPPORTS_STATEDIFF  = "ETH_SUPPORTS_STATEDIFF"
+	ETH_FORWARD_ETH_CALLS   = "ETH_FORWARD_ETH_CALLS"
+	ETH_PROXY_ON_ERROR      = "ETH_PROXY_ON_ERROR"
+
+	VALIDATOR_ENABLED         = "VALIDATOR_ENABLED"
+	VALIDATOR_EVERY_NTH_BLOCK = "VALIDATOR_EVERY_NTH_BLOCK"
 )
 
 // Config struct
 type Config struct {
 	DB       *postgres.DB
-	DBConfig postgres.Config
+	DBConfig postgres.ConnectionConfig
+	DBParams postgres.ConnectionParams
 
 	WSEnabled  bool
 	WSEndpoint string
@@ -82,6 +85,14 @@ type Config struct {
 	EthHttpEndpoint  string
 	Client           *rpc.Client
 	SupportStateDiff bool
+	ForwardEthCalls  bool
+	ProxyOnError     bool
+
+	// Cache configuration.
+	GroupCache *ethServerShared.GroupCacheConfig
+
+	StateValidationEnabled       bool
+	StateValidationEveryNthBlock uint64
 }
 
 // NewConfig is used to initialize a watcher config from a .toml file
@@ -89,22 +100,25 @@ type Config struct {
 func NewConfig() (*Config, error) {
 	c := new(Config)
 
-	viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
+	viper.BindEnv("ethereum.httpPath", ETH_HTTP_PATH)
 	viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
 	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
 	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
 	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
+	viper.BindEnv("ethereum.forwardEthCalls", ETH_FORWARD_ETH_CALLS)
+	viper.BindEnv("ethereum.proxyOnError", ETH_PROXY_ON_ERROR)
 
-	c.DBConfig.Init()
+	c.dbInit()
-
 	ethHTTP := viper.GetString("ethereum.httpPath")
 	ethHTTPEndpoint := fmt.Sprintf("http://%s", ethHTTP)
-	nodeInfo, cli, err := shared.GetEthNodeAndClient(ethHTTPEndpoint)
+	nodeInfo, cli, err := getEthNodeAndClient(ethHTTPEndpoint)
 	if err != nil {
 		return nil, err
 	}
 	c.Client = cli
 	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
+	c.ForwardEthCalls = viper.GetBool("ethereum.forwardEthCalls")
+	c.ProxyOnError = viper.GetBool("ethereum.proxyOnError")
 	c.EthHttpEndpoint = ethHTTPEndpoint
 
 	// websocket server
@@ -184,9 +198,13 @@ func NewConfig() (*Config, error) {
 	c.IpldGraphqlEnabled = ipldGraphqlEnabled
 
 	overrideDBConnConfig(&c.DBConfig)
-	serveDB := utils.LoadPostgres(c.DBConfig, nodeInfo, false)
+	serveDB, err := postgres.NewDB(postgres.DbConnectionString(c.DBParams), c.DBConfig, nodeInfo)
-	prom.RegisterDBCollector(c.DBConfig.Name, serveDB.DB)
+	if err != nil {
-	c.DB = &serveDB
+		return nil, err
+	}
+
+	prom.RegisterDBCollector(c.DBParams.Name, serveDB.DB)
+	c.DB = serveDB
 
 	defaultSenderStr := viper.GetString("ethereum.defaultSender")
 	if defaultSenderStr != "" {
@@ -205,10 +223,15 @@ func NewConfig() (*Config, error) {
 	} else {
 		c.ChainConfig, err = eth.ChainConfig(nodeInfo.ChainID)
 	}
+
+	c.loadGroupCacheConfig()
+
+	c.loadValidatorConfig()
+
 	return c, err
 }
 
-func overrideDBConnConfig(con *postgres.Config) {
+func overrideDBConnConfig(con *postgres.ConnectionConfig) {
 	viper.BindEnv("database.server.maxIdle", SERVER_MAX_IDLE_CONNECTIONS)
 	viper.BindEnv("database.server.maxOpen", SERVER_MAX_OPEN_CONNECTIONS)
 	viper.BindEnv("database.server.maxLifetime", SERVER_MAX_CONN_LIFETIME)
@@ -216,3 +239,54 @@ func overrideDBConnConfig(con *postgres.Config) {
 	con.MaxOpen = viper.GetInt("database.server.maxOpen")
 	con.MaxLifetime = viper.GetInt("database.server.maxLifetime")
 }
+
+func (c *Config) dbInit() {
+	viper.BindEnv("database.name", DATABASE_NAME)
+	viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
+	viper.BindEnv("database.port", DATABASE_PORT)
+	viper.BindEnv("database.user", DATABASE_USER)
+	viper.BindEnv("database.password", DATABASE_PASSWORD)
+	viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
+	viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
+	viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME)
+
+	c.DBParams.Name = viper.GetString("database.name")
+	c.DBParams.Hostname = viper.GetString("database.hostname")
+	c.DBParams.Port = viper.GetInt("database.port")
+	c.DBParams.User = viper.GetString("database.user")
+	c.DBParams.Password = viper.GetString("database.password")
+	c.DBConfig.MaxIdle = viper.GetInt("database.maxIdle")
+	c.DBConfig.MaxOpen = viper.GetInt("database.maxOpen")
+	c.DBConfig.MaxLifetime = viper.GetInt("database.maxLifetime")
+}
+
+func (c *Config) loadGroupCacheConfig() {
+	viper.BindEnv("groupcache.pool.enabled", ethServerShared.GcachePoolEnabled)
+	viper.BindEnv("groupcache.pool.httpEndpoint", ethServerShared.GcachePoolHttpPath)
+	viper.BindEnv("groupcache.pool.peerHttpEndpoints", ethServerShared.GcachePoolHttpPeers)
+	viper.BindEnv("groupcache.statedb.cacheSizeInMB", ethServerShared.GcacheStatedbCacheSize)
+	viper.BindEnv("groupcache.statedb.cacheExpiryInMins", ethServerShared.GcacheStatedbCacheExpiry)
+	viper.BindEnv("groupcache.statedb.logStatsIntervalInSecs", ethServerShared.GcacheStatedbLogStatsInterval)
+
+	gcc := ethServerShared.GroupCacheConfig{}
+	gcc.Pool.Enabled = viper.GetBool("groupcache.pool.enabled")
+	if gcc.Pool.Enabled {
+		gcc.Pool.HttpEndpoint = viper.GetString("groupcache.pool.httpEndpoint")
+		gcc.Pool.PeerHttpEndpoints = viper.GetStringSlice("groupcache.pool.peerHttpEndpoints")
+	}
+
+	// Irrespective of whether the pool is enabled, we always use the hot/local cache.
+	gcc.StateDB.CacheSizeInMB = viper.GetInt("groupcache.statedb.cacheSizeInMB")
+	gcc.StateDB.CacheExpiryInMins = viper.GetInt("groupcache.statedb.cacheExpiryInMins")
+	gcc.StateDB.LogStatsIntervalInSecs = viper.GetInt("groupcache.statedb.logStatsIntervalInSecs")
+
+	c.GroupCache = &gcc
+}
+
+func (c *Config) loadValidatorConfig() {
+	viper.BindEnv("validator.enabled", VALIDATOR_ENABLED)
+	viper.BindEnv("validator.everyNthBlock", VALIDATOR_EVERY_NTH_BLOCK)
+
+	c.StateValidationEnabled = viper.GetBool("validator.enabled")
+	c.StateValidationEveryNthBlock = viper.GetUint64("validator.everyNthBlock")
+}

pkg/serve/env.go

@@ -0,0 +1,50 @@
+package serve
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/spf13/viper"
+)
+
+// Env variables
+const (
+	HTTP_TIMEOUT = "HTTP_TIMEOUT"
+
+	ETH_WS_PATH       = "ETH_WS_PATH"
+	ETH_HTTP_PATH     = "ETH_HTTP_PATH"
+	ETH_NODE_ID       = "ETH_NODE_ID"
+	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
+	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
+	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
+	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+
+	DATABASE_NAME                 = "DATABASE_NAME"
+	DATABASE_HOSTNAME             = "DATABASE_HOSTNAME"
+	DATABASE_PORT                 = "DATABASE_PORT"
+	DATABASE_USER                 = "DATABASE_USER"
+	DATABASE_PASSWORD             = "DATABASE_PASSWORD"
+	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
+)
+
+// GetEthNodeAndClient returns eth node info and client from path url
+func getEthNodeAndClient(path string) (node.Info, *rpc.Client, error) {
+	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
+	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
+	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
+	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
+	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
+
+	rpcClient, err := rpc.Dial(path)
+	if err != nil {
+		return node.Info{}, nil, err
+	}
+	return node.Info{
+		ID:           viper.GetString("ethereum.nodeID"),
+		ClientName:   viper.GetString("ethereum.clientName"),
+		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
+		NetworkID:    viper.GetString("ethereum.networkID"),
+		ChainID:      viper.GetUint64("ethereum.chainID"),
+	}, rpcClient, nil
+}

pkg/serve/service.go

@@ -21,22 +21,18 @@ import (
 	"strconv"
 	"sync"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/net"
-
-	"github.com/ethereum/go-ethereum/core/vm"
-
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	ethnode "github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	log "github.com/sirupsen/logrus"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/pkg/net"
 )
 
 const (
@@ -52,7 +48,7 @@ type Server interface {
 	APIs() []rpc.API
 	Protocols() []p2p.Protocol
 	// Pub-Sub handling event loop
-	Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.ConvertedPayload)
+	Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth.ConvertedPayload)
 	// Method to subscribe to the service
 	Subscribe(id rpc.ID, sub chan<- SubscriptionPayload, quitChan chan<- bool, params eth.SubscriptionSettings)
 	// Method to unsubscribe from the service
@@ -87,6 +83,10 @@ type Service struct {
 	supportsStateDiffing bool
 	// backend for the server
 	backend *eth.Backend
+	// whether to forward eth_calls directly to proxy node
+	forwardEthCalls bool
+	// whether to forward all calls to proxy node if they throw an error locally
+	proxyOnError bool
 }
 
 // NewServer creates a new Server using an underlying Service struct
@@ -101,12 +101,15 @@ func NewServer(settings *Config) (Server, error) {
 	sap.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
 	sap.client = settings.Client
 	sap.supportsStateDiffing = settings.SupportStateDiff
+	sap.forwardEthCalls = settings.ForwardEthCalls
+	sap.proxyOnError = settings.ProxyOnError
 	var err error
 	sap.backend, err = eth.NewEthBackend(sap.db, &eth.Config{
-		ChainConfig:   settings.ChainConfig,
+		ChainConfig:      settings.ChainConfig,
-		VmConfig:      vm.Config{},
+		VMConfig:         vm.Config{NoBaseFee: true},
-		DefaultSender: settings.DefaultSender,
+		DefaultSender:    settings.DefaultSender,
-		RPCGasCap:     settings.RPCGasCap,
+		RPCGasCap:        settings.RPCGasCap,
+		GroupCacheConfig: settings.GroupCache,
 	})
 	return sap, err
 }
@@ -133,10 +136,14 @@ func (sap *Service) APIs() []rpc.API {
 			Public:    true,
 		},
 	}
+	ethAPI, err := eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing, sap.forwardEthCalls, sap.proxyOnError)
+	if err != nil {
+		log.Fatalf("unable to create public eth api: %v", err)
+	}
 	return append(apis, rpc.API{
 		Namespace: eth.APIName,
 		Version:   eth.APIVersion,
-		Service:   eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing),
+		Service:   ethAPI,
 		Public:    true,
 	})
 }
@@ -145,7 +152,7 @@ func (sap *Service) APIs() []rpc.API {
 // It filters and sends this data to any subscribers to the service
 // This process can also be stood up alone, without an screenAndServePayload attached to a Sync process
 // and it will hang on the WaitGroup indefinitely, allowing the Service to serve historical data requests only
-func (sap *Service) Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.ConvertedPayload) {
+func (sap *Service) Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth.ConvertedPayload) {
 	sap.serveWg = wg
 	go func() {
 		wg.Add(1)
@@ -164,7 +171,7 @@ func (sap *Service) Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.
 }
 
 // filterAndServe filters the payload according to each subscription type and sends to the subscriptions
-func (sap *Service) filterAndServe(payload eth2.ConvertedPayload) {
+func (sap *Service) filterAndServe(payload eth.ConvertedPayload) {
 	log.Debug("sending eth ipld payload to subscriptions")
 	sap.Lock()
 	sap.serveWg.Add(1)
@@ -337,7 +344,7 @@ func (sap *Service) Unsubscribe(id rpc.ID) {
 func (sap *Service) Start() error {
 	log.Info("starting eth ipld server")
 	wg := new(sync.WaitGroup)
-	payloadChan := make(chan eth2.ConvertedPayload, PayloadChanBufferSize)
+	payloadChan := make(chan eth.ConvertedPayload, PayloadChanBufferSize)
 	sap.Serve(wg, payloadChan)
 	return nil
 }

pkg/shared/constants.go

@@ -19,4 +19,11 @@ package shared
 const (
 	DefaultMaxBatchSize   uint64 = 100
 	DefaultMaxBatchNumber int64  = 50
+
+	GcachePoolEnabled             = "GCACHE_POOL_ENABLED"
+	GcachePoolHttpPath            = "GCACHE_POOL_HTTP_PATH"
+	GcachePoolHttpPeers           = "GCACHE_POOL_HTTP_PEERS"
+	GcacheStatedbCacheSize        = "GCACHE_STATEDB_CACHE_SIZE"
+	GcacheStatedbCacheExpiry      = "GCACHE_STATEDB_CACHE_EXPIRY"
+	GcacheStatedbLogStatsInterval = "GCACHE_STATEDB_LOG_STATS_INTERVAL"
 )

pkg/shared/functions.go

@@ -18,13 +18,13 @@ package shared
 
 import (
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-ipfs-blockstore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	"github.com/ipfs/go-ipfs-ds-help"
+	dshelp "github.com/ipfs/go-ipfs-ds-help"
 	node "github.com/ipfs/go-ipld-format"
 	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
 )
 
 // HandleZeroAddrPointer will return an emtpy string for a nil address pointer

pkg/shared/test_helpers.go

@@ -19,7 +19,7 @@ package shared
 import (
 	"bytes"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
 )
 
 // IPLDsContainBytes used to check if a list of strings contains a particular string

pkg/shared/types.go

@@ -0,0 +1,38 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+type PoolConfig struct {
+	Enabled           bool
+	HttpEndpoint      string
+	PeerHttpEndpoints []string
+}
+
+type GroupConfig struct {
+	CacheSizeInMB          int
+	CacheExpiryInMins      int
+	LogStatsIntervalInSecs int
+
+	// Used in tests to override the cache name, to work around
+	// the "duplicate registration of group" error from groupcache
+	Name string
+}
+
+type GroupCacheConfig struct {
+	Pool    PoolConfig
+	StateDB GroupConfig
+}

scripts/run_integration_test.sh

@@ -1,13 +1,18 @@
 set -e
 set -o xtrace
 
+export ETH_FORWARD_ETH_CALLS=false
+export DB_WRITE=true
+export ETH_HTTP_PATH=""
+export ETH_PROXY_ON_ERROR=false
+
 # Clear up existing docker images and volume.
 docker-compose down --remove-orphans --volumes
 
 # Build and start the containers.
-# Note: Build only if `ipld-eth-server` code is modified. Otherwise comment this line.
+# Note: Build only if `ipld-eth-server` or other container code is modified. Otherwise comment this line.
-docker build -t ipld-eth-server_eth-server:latest .
+docker-compose -f docker-compose.test.yml -f docker-compose.yml build eth-server
-docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d db dapptools contract eth-server
+docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d ipld-eth-db dapptools contract eth-server
 
 export PGPASSWORD=password
 export DATABASE_USER=vdbm
@@ -18,4 +23,4 @@ export DATABASE_HOSTNAME=127.0.0.1
 # Wait for containers to be up and execute the integration test.
 while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
           while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
-          make integrationtest
+          make integrationtest

scripts/run_integration_test_forward_eth_calls.sh

@@ -0,0 +1,26 @@
+set -e
+set -o xtrace
+
+export ETH_FORWARD_ETH_CALLS=true
+export DB_WRITE=false
+export ETH_HTTP_PATH="dapptools:8545"
+export ETH_PROXY_ON_ERROR=false
+
+# Clear up existing docker images and volume.
+docker-compose down --remove-orphans --volumes
+
+# Build and start the containers.
+# Note: Build only if `ipld-eth-server` or other container code is modified. Otherwise comment this line.
+docker-compose -f docker-compose.test.yml -f docker-compose.yml build eth-server
+docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d ipld-eth-db dapptools contract eth-server
+
+export PGPASSWORD=password
+export DATABASE_USER=vdbm
+export DATABASE_PORT=8077
+export DATABASE_PASSWORD=password
+export DATABASE_HOSTNAME=127.0.0.1
+
+# Wait for containers to be up and execute the integration test.
+while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
+          make integrationtest

scripts/run_unit_test.sh

@@ -1,2 +1,5 @@
-docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d db
+# Clear up existing docker images and volume.
-PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 make test
+docker-compose down --remove-orphans --volumes
+
+docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d ipld-eth-db
+PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test

test/contract/contracts/GLDToken.sol

@@ -4,4 +4,7 @@ contract GLDToken is ERC20 {
     constructor() ERC20("Gold", "GLD") {
         _mint(msg.sender, 1000000000000000000000);
     }
+    function destroy() public {
+        selfdestruct(payable(msg.sender));
+    }
 }

test/contract/package-lock.json

@@ -5,6 +5,7 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "contract",
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
@@ -254,10 +255,10 @@
         "@resolver-engine/imports-fs": "^0.3.3",
         "@typechain/ethers-v5": "^2.0.0",
         "@types/mkdirp": "^0.5.2",
-        "@types/node-fetch": "^2.5.5",
+        "@types/node-fetch": "^2.6.7",
         "ethers": "^5.0.1",
         "mkdirp": "^0.5.1",
-        "node-fetch": "^2.6.0",
+        "node-fetch": "^2.6.7",
         "solc": "^0.6.3",
         "ts-generator": "^0.1.1",
         "typechain": "^3.0.0"
@@ -1506,7 +1507,7 @@
       "integrity": "sha512-dueRKfaJL4RTtSa7bWeTK1M+VH+Gns73oCgzvYfHZywRCoPSd8EkXBL0mZ9unPTveBn+D9phZBaxuzpwjWkW0g=="
     },
     "node_modules/@types/node-fetch": {
-      "version": "2.5.10",
+      "version": "2.6.7",
       "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.10.tgz",
       "integrity": "sha512-IpkX0AasN44hgEad0gEF/V6EgR5n69VEqPEgnmoM8GsIGro3PowbWs4tR6IhxUTyPLpOn+fiGG6nrQhcmoCuIQ==",
       "dev": true,
@@ -3093,7 +3094,7 @@
       "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw=="
     },
     "node_modules/follow-redirects": {
-      "version": "1.13.3",
+      "version": "1.14.7",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.3.tgz",
       "integrity": "sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==",
       "funding": [
@@ -5426,7 +5427,7 @@
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "node-fetch": "2.1.2",
+        "node-fetch": "^2.6.7",
         "whatwg-fetch": "2.0.4"
       }
     },
@@ -7532,7 +7533,7 @@
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "node-fetch": "~1.7.1"
+        "node-fetch": "^2.6.7"
       }
     },
     "node_modules/ganache-core/node_modules/fetch-ponyfill/node_modules/is-stream": {
@@ -7544,7 +7545,7 @@
       }
     },
     "node_modules/ganache-core/node_modules/fetch-ponyfill/node_modules/node-fetch": {
-      "version": "1.7.3",
+      "version": "2.6.7",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -9216,7 +9217,7 @@
       "license": "MIT"
     },
     "node_modules/ganache-core/node_modules/node-fetch": {
-      "version": "2.1.2",
+      "version": "2.6.7",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -12519,7 +12520,7 @@
         "merkle-patricia-tree": "^4.1.0",
         "mnemonist": "^0.38.0",
         "mocha": "^7.1.2",
-        "node-fetch": "^2.6.0",
+        "node-fetch": "^2.6.7",
         "qs": "^6.7.0",
         "raw-body": "^2.4.1",
         "resolve": "1.17.0",
@@ -13722,7 +13723,7 @@
       }
     },
     "node_modules/node-fetch": {
-      "version": "2.6.1",
+      "version": "2.6.7",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
       "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
       "engines": {
@@ -14696,7 +14697,7 @@
       "dependencies": {
         "command-exists": "^1.2.8",
         "commander": "3.0.2",
-        "follow-redirects": "^1.12.1",
+        "follow-redirects": "^1.14.7",
         "fs-extra": "^0.30.0",
         "js-sha3": "0.8.0",
         "memorystream": "^0.3.1",
@@ -15856,10 +15857,10 @@
         "@resolver-engine/imports-fs": "^0.3.3",
         "@typechain/ethers-v5": "^2.0.0",
         "@types/mkdirp": "^0.5.2",
-        "@types/node-fetch": "^2.5.5",
+        "@types/node-fetch": "^2.6.7",
         "ethers": "^5.0.1",
         "mkdirp": "^0.5.1",
-        "node-fetch": "^2.6.0",
+        "node-fetch": "^2.6.7",
         "solc": "^0.6.3",
         "ts-generator": "^0.1.1",
         "typechain": "^3.0.0"
@@ -16704,7 +16705,9 @@
       "resolved": "https://registry.npmjs.org/@typechain/ethers-v5/-/ethers-v5-2.0.0.tgz",
       "integrity": "sha512-0xdCkyGOzdqh4h5JSf+zoWx85IusEjDcPIwNEHP8mrWSnCae4rvrqB+/gtpdNfX7zjlFlZiMeePn2r63EI3Lrw==",
       "dev": true,
-      "requires": {}
+      "requires": {
+        "ethers": "^5.0.2"
+      }
     },
     "@types/abstract-leveldown": {
       "version": "5.0.1",
@@ -16754,7 +16757,7 @@
       "integrity": "sha512-dueRKfaJL4RTtSa7bWeTK1M+VH+Gns73oCgzvYfHZywRCoPSd8EkXBL0mZ9unPTveBn+D9phZBaxuzpwjWkW0g=="
     },
     "@types/node-fetch": {
-      "version": "2.5.10",
+      "version": "2.6.7",
       "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.10.tgz",
       "integrity": "sha512-IpkX0AasN44hgEad0gEF/V6EgR5n69VEqPEgnmoM8GsIGro3PowbWs4tR6IhxUTyPLpOn+fiGG6nrQhcmoCuIQ==",
       "dev": true,
@@ -18067,7 +18070,7 @@
       "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw=="
     },
     "follow-redirects": {
-      "version": "1.13.3",
+      "version": "1.14.7",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.3.tgz",
       "integrity": "sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA=="
     },
@@ -19807,7 +19810,7 @@
           "version": "2.2.3",
           "dev": true,
           "requires": {
-            "node-fetch": "2.1.2",
+            "node-fetch": "^2.6.7",
             "whatwg-fetch": "2.0.4"
           }
         },
@@ -21618,7 +21621,7 @@
           "version": "4.1.0",
           "dev": true,
           "requires": {
-            "node-fetch": "~1.7.1"
+            "node-fetch": "^2.6.7"
           },
           "dependencies": {
             "is-stream": {
@@ -21626,7 +21629,7 @@
               "dev": true
             },
             "node-fetch": {
-              "version": "1.7.3",
+              "version": "2.6.7",
               "dev": true,
               "requires": {
                 "encoding": "^0.1.11",
@@ -22824,7 +22827,7 @@
           "dev": true
         },
         "node-fetch": {
-          "version": "2.1.2",
+          "version": "2.6.7",
           "dev": true
         },
         "node-gyp-build": {
@@ -25307,7 +25310,7 @@
         "merkle-patricia-tree": "^4.1.0",
         "mnemonist": "^0.38.0",
         "mocha": "^7.1.2",
-        "node-fetch": "^2.6.0",
+        "node-fetch": "^2.6.7",
         "qs": "^6.7.0",
         "raw-body": "^2.4.1",
         "resolve": "1.17.0",
@@ -26211,7 +26214,7 @@
       }
     },
     "node-fetch": {
-      "version": "2.6.1",
+      "version": "2.6.7",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
       "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
     },
@@ -26930,7 +26933,7 @@
       "requires": {
         "command-exists": "^1.2.8",
         "commander": "3.0.2",
-        "follow-redirects": "^1.12.1",
+        "follow-redirects": "^1.14.7",
         "fs-extra": "^0.30.0",
         "js-sha3": "0.8.0",
         "memorystream": "^0.3.1",

test/contract/src/index.js

@@ -1,6 +1,7 @@
 const fastify = require('fastify')({ logger: true });
 const hre = require("hardhat");
 
+
 // readiness check
 fastify.get('/v1/healthz', async (req, reply) => {
     reply
@@ -22,6 +23,20 @@ fastify.get('/v1/deployContract', async (req, reply) => {
     }
 });
 
+fastify.get('/v1/destroyContract', async (req, reply) => {
+    const addr = req.query.addr;
+
+    const Token = await hre.ethers.getContractFactory("GLDToken");
+    const token = await Token.attach(addr);
+
+    await token.destroy();
+    const blockNum = await hre.ethers.provider.getBlockNumber()
+
+    return {
+        blockNumber: blockNum,
+    }
+})
+
 fastify.get('/v1/sendEth', async (req, reply) => {
     const to = req.query.to;
     const value = req.query.value;

test/direct_proxy_integration_test.go

@@ -0,0 +1,458 @@
+package integration_test
+
+import (
+	"context"
+	"math/big"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethclient"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	integration "github.com/vulcanize/ipld-eth-server/test"
+)
+
+var _ = Describe("Integration test", func() {
+	directProxyEthCalls, err := strconv.ParseBool(os.Getenv("ETH_FORWARD_ETH_CALLS"))
+	Expect(err).To(BeNil())
+	gethHttpPath := "http://127.0.0.1:8545"
+	gethClient, err := ethclient.Dial(gethHttpPath)
+	Expect(err).ToNot(HaveOccurred())
+
+	ipldEthHttpPath := "http://127.0.0.1:8081"
+	ipldClient, err := ethclient.Dial(ipldEthHttpPath)
+	Expect(err).ToNot(HaveOccurred())
+
+	ctx := context.Background()
+
+	var contract *integration.ContractDeployed
+	var erc20TotalSupply *big.Int
+	var tx *integration.Tx
+	var bigIntResult bool
+	var contractErr error
+	var txErr error
+	sleepInterval := 2 * time.Second
+
+	Describe("get Block", func() {
+		BeforeEach(func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			contract, contractErr = integration.DeployContract()
+			time.Sleep(sleepInterval)
+		})
+
+		It("get not existing block by number", func() {
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			blockNum := contract.BlockNumber + 100
+
+			gethBlock, err := gethClient.BlockByNumber(ctx, big.NewInt(int64(blockNum)))
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByNumber(ctx, big.NewInt(int64(blockNum)))
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("get not existing block by hash", func() {
+			gethBlock, err := gethClient.BlockByHash(ctx, common.HexToHash(nonExistingBlockHash))
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByHash(ctx, common.HexToHash(nonExistingBlockHash))
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("get block by number", func() {
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			blockNum := contract.BlockNumber
+
+			_, err := gethClient.BlockByNumber(ctx, big.NewInt(int64(blockNum)))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BlockByNumber(ctx, big.NewInt(int64(blockNum)))
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("get block by hash", func() {
+			_, err := gethClient.BlockByHash(ctx, common.HexToHash(contract.BlockHash))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BlockByHash(ctx, common.HexToHash(contract.BlockHash))
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Transaction", func() {
+		BeforeEach(func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			contract, contractErr = integration.DeployContract()
+			time.Sleep(sleepInterval)
+		})
+
+		It("Get tx by hash", func() {
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			_, _, err := gethClient.TransactionByHash(ctx, common.HexToHash(contract.TransactionHash))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, _, err = ipldClient.TransactionByHash(ctx, common.HexToHash(contract.TransactionHash))
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("Get tx by block hash and index", func() {
+			_, err := gethClient.TransactionInBlock(ctx, common.HexToHash(contract.BlockHash), 0)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.TransactionInBlock(ctx, common.HexToHash(contract.BlockHash), 0)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Receipt", func() {
+		BeforeEach(func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			contract, contractErr = integration.DeployContract()
+			time.Sleep(sleepInterval)
+		})
+
+		It("Get tx receipt", func() {
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			_, err := gethClient.TransactionReceipt(ctx, common.HexToHash(contract.TransactionHash))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.TransactionReceipt(ctx, common.HexToHash(contract.TransactionHash))
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("FilterLogs", func() {
+		BeforeEach(func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			contract, contractErr = integration.DeployContract()
+			time.Sleep(sleepInterval)
+		})
+
+		It("with blockhash", func() {
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			blockHash := common.HexToHash(contract.BlockHash)
+			filterQuery := ethereum.FilterQuery{
+				//Addresses: addresses,
+				BlockHash: &blockHash,
+				Topics:    [][]common.Hash{},
+			}
+
+			gethLogs, err := gethClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldLogs, err := ipldClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			// not empty list
+			Expect(gethLogs).ToNot(BeEmpty())
+			// empty list
+			Expect(ipldLogs).To(BeEmpty())
+		})
+	})
+
+	Describe("CodeAt", func() {
+		BeforeEach(func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			contract, contractErr = integration.DeployContract()
+			time.Sleep(sleepInterval)
+		})
+
+		It("gets code at non-existing address without block number", func() {
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			gethCode, err := gethClient.CodeAt(ctx, common.HexToAddress(nonExistingAddress), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, common.HexToAddress(nonExistingAddress), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of deployed contract without block number", func() {
+			_, err := gethClient.CodeAt(ctx, common.HexToAddress(contract.Address), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, common.HexToAddress(contract.Address), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldCode).To(BeEmpty())
+		})
+		It("gets code of deployed contract with block number", func() {
+			_, err := gethClient.CodeAt(ctx, common.HexToAddress(contract.Address), big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, common.HexToAddress(contract.Address), big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldCode).To(BeEmpty())
+		})
+		It("gets code of contract that doesn't exist at this height", func() {
+			gethCode, err := gethClient.CodeAt(ctx, common.HexToAddress(contract.Address), big.NewInt(int64(contract.BlockNumber-1)))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, common.HexToAddress(contract.Address), big.NewInt(int64(contract.BlockNumber-1)))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+	})
+
+	Describe("Get balance", func() {
+		address := "0x1111111111111111111111111111111111111112"
+		BeforeEach(func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			tx, txErr = integration.SendEth(address, "0.01")
+			time.Sleep(sleepInterval)
+		})
+
+		It("gets balance for an account with eth without block number", func() {
+			Expect(txErr).ToNot(HaveOccurred())
+
+			gethBalance, err := gethClient.BalanceAt(ctx, common.HexToAddress(address), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethBalance.String()).To(Equal(big.NewInt(10000000000000000).String()))
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, common.HexToAddress(address), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldBalance.String()).To(Equal(big.NewInt(0).String()))
+		})
+		It("gets balance for an account with eth with block number", func() {
+			Expect(txErr).ToNot(HaveOccurred())
+
+			_, err := gethClient.BalanceAt(ctx, common.HexToAddress(address), big.NewInt(int64(tx.BlockNumber)))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BalanceAt(ctx, common.HexToAddress(address), big.NewInt(int64(tx.BlockNumber)))
+			Expect(err).To(HaveOccurred())
+		})
+		It("gets historical balance for an account with eth with block number", func() {
+			Expect(txErr).ToNot(HaveOccurred())
+
+			_, err := gethClient.BalanceAt(ctx, common.HexToAddress(address), big.NewInt(int64(tx.BlockNumber-1)))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BalanceAt(ctx, common.HexToAddress(address), big.NewInt(int64(tx.BlockNumber-1)))
+			Expect(err).To(HaveOccurred())
+		})
+		It("gets balance for a non-existing account without block number", func() {
+			Expect(txErr).ToNot(HaveOccurred())
+
+			gethBalance, err := gethClient.BalanceAt(ctx, common.HexToAddress(nonExistingAddress), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, common.HexToAddress(nonExistingAddress), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for an non-existing block number", func() {
+			Expect(txErr).ToNot(HaveOccurred())
+
+			gethBalance, err := gethClient.BalanceAt(ctx, common.HexToAddress(address), big.NewInt(int64(tx.BlockNumber+3)))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, common.HexToAddress(nonExistingAddress), big.NewInt(int64(tx.BlockNumber+3)))
+			Expect(err).To(MatchError("header not found"))
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+	})
+
+	Describe("Get Storage", func() {
+		BeforeEach(func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			contract, contractErr = integration.DeployContract()
+			erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10)
+
+			time.Sleep(sleepInterval)
+		})
+
+		It("gets ERC20 total supply (without block number)", func() {
+			Expect(contractErr).ToNot(HaveOccurred())
+			Expect(bigIntResult).To(Equal(true))
+
+			totalSupplyIndex := "0x2"
+
+			gethStorage, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldStorage).To(Equal(make([]byte, 32)))
+		})
+
+		It("gets ERC20 total supply (with block number)", func() {
+			totalSupplyIndex := "0x2"
+
+			gethStorage, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			_, err = ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).To(HaveOccurred())
+		})
+		It("gets storage for non-existing account", func() {
+			totalSupplyIndex := "0x2"
+
+			_, err := gethClient.StorageAt(ctx, common.HexToAddress(nonExistingAddress), common.HexToHash(totalSupplyIndex), big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.StorageAt(ctx, common.HexToAddress(nonExistingAddress), common.HexToHash(totalSupplyIndex), big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).To(MatchError("header not found"))
+		})
+		It("gets storage for non-existing contract slot", func() {
+			_, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), randomHash, big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), randomHash, big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).To(MatchError("header not found"))
+		})
+		It("gets storage for non-existing contract", func() {
+			totalSupplyIndex := "0x2"
+			gethStorage, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+		It("gets storage for non-existing block number", func() {
+			blockNum := contract.BlockNumber + 100
+			totalSupplyIndex := "0x2"
+
+			gethStorage, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(int64(blockNum)))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(int64(blockNum)))
+			Expect(err).To(MatchError("header not found"))
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("get storage after self destruct", func() {
+			totalSupplyIndex := "0x2"
+
+			tx, err := integration.DestroyContract(contract.Address)
+			Expect(err).ToNot(HaveOccurred())
+
+			time.Sleep(sleepInterval)
+
+			gethStorage1, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			gethStorage2, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage1).NotTo(Equal(gethStorage2))
+			Expect(gethStorage2).To(Equal(eth.EmptyNodeValue))
+
+			_, err = ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1))
+			Expect(err).To(HaveOccurred())
+			_, err = ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+
+			// Query the current block
+			ipldStorage3, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(eth.EmptyNodeValue).To(Equal(ipldStorage3))
+		})
+	})
+
+	Describe("eth call", func() {
+		BeforeEach(func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			contract, contractErr = integration.DeployContract()
+			erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10)
+
+			time.Sleep(sleepInterval)
+		})
+
+		It("calls totalSupply() without block number", func() {
+			Expect(contractErr).ToNot(HaveOccurred())
+			Expect(bigIntResult).To(Equal(true))
+
+			contractAddress := common.HexToAddress(contract.Address)
+
+			msg := ethereum.CallMsg{
+				To:   &contractAddress,
+				Data: common.Hex2Bytes("18160ddd"), // totalSupply()
+			}
+			gethResult, err := gethClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+
+		It("calls totalSupply() with block number", func() {
+			contractAddress := common.HexToAddress(contract.Address)
+
+			msg := ethereum.CallMsg{
+				To:   &contractAddress,
+				Data: common.Hex2Bytes("18160ddd"), // totalSupply()
+			}
+			gethResult, err := gethClient.CallContract(ctx, msg, big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, big.NewInt(int64(contract.BlockNumber)))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+	})
+
+	Describe("Chain ID", func() {
+		It("Check chain id", func() {
+			if !directProxyEthCalls {
+				Skip("skipping direct-proxy-forwarding integration tests")
+			}
+			_, err := gethClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.ChainID(ctx)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

test/helper.go

@@ -14,6 +14,10 @@ type ContractDeployed struct {
 	BlockHash       string `json:"blockHash"`
 }
 
+type ContractDestroyed struct {
+	BlockNumber int64 `json:"blockNumber"`
+}
+
 type Tx struct {
 	From            string   `json:"from"`
 	To              string   `json:"to"`
@@ -43,6 +47,19 @@ func DeployContract() (*ContractDeployed, error) {
 	return &contract, nil
 }
 
+func DestroyContract(addr string) (*ContractDestroyed, error) {
+	res, err := http.Get(fmt.Sprintf("%s/v1/destroyContract?addr=%s", srvUrl, addr))
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	var data ContractDestroyed
+	decoder := json.NewDecoder(res.Body)
+
+	return &data, decoder.Decode(&data)
+}
+
 func SendEth(to string, value string) (*Tx, error) {
 	res, err := http.Get(fmt.Sprintf("%s/v1/sendEth?to=%s&value=%s", srvUrl, to, value))
 	if err != nil {

test/integration_suite_test.go

@@ -1,12 +1,12 @@
 package integration_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestIntegration(t *testing.T) {

test/integration_test.go

@@ -3,18 +3,21 @@ package integration_test
 import (
 	"context"
 	"math/big"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rlp"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	integration "github.com/vulcanize/ipld-eth-server/test"
 
-	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	integration "github.com/vulcanize/ipld-eth-server/test"
 )
 
 const nonExistingBlockHash = "0x111111111111111111111111111111111111111111111111111111111111111"
@@ -26,6 +29,8 @@ var (
 )
 
 var _ = Describe("Integration test", func() {
+	directProxyEthCalls, err := strconv.ParseBool(os.Getenv("ETH_FORWARD_ETH_CALLS"))
+	Expect(err).To(BeNil())
 	gethHttpPath := "http://127.0.0.1:8545"
 	gethClient, err := ethclient.Dial(gethHttpPath)
 	Expect(err).ToNot(HaveOccurred())
@@ -46,6 +51,9 @@ var _ = Describe("Integration test", func() {
 
 	Describe("get Block", func() {
 		BeforeEach(func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -115,6 +123,9 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Transaction", func() {
 		BeforeEach(func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -146,6 +157,9 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Receipt", func() {
 		BeforeEach(func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -173,6 +187,9 @@ var _ = Describe("Integration test", func() {
 
 	Describe("FilterLogs", func() {
 		BeforeEach(func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -203,6 +220,9 @@ var _ = Describe("Integration test", func() {
 
 	Describe("CodeAt", func() {
 		BeforeEach(func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			contract, contractErr = integration.DeployContract()
 			time.Sleep(sleepInterval)
 		})
@@ -250,6 +270,9 @@ var _ = Describe("Integration test", func() {
 	Describe("Get balance", func() {
 		address := "0x1111111111111111111111111111111111111112"
 		BeforeEach(func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			tx, txErr = integration.SendEth(address, "0.01")
 			time.Sleep(sleepInterval)
 		})
@@ -313,6 +336,9 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Get Storage", func() {
 		BeforeEach(func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			contract, contractErr = integration.DeployContract()
 			erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10)
 
@@ -391,10 +417,44 @@ var _ = Describe("Integration test", func() {
 			Expect(err).To(MatchError("header not found"))
 			Expect(gethStorage).To(Equal(ipldStorage))
 		})
+
+		It("get storage after self destruct", func() {
+			totalSupplyIndex := "0x2"
+
+			tx, err := integration.DestroyContract(contract.Address)
+			Expect(err).ToNot(HaveOccurred())
+
+			time.Sleep(sleepInterval)
+
+			gethStorage1, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			gethStorage2, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage1).NotTo(Equal(gethStorage2))
+			Expect(gethStorage2).To(Equal(eth.EmptyNodeValue))
+
+			ipldStorage1, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			ipldStorage2, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(ipldStorage1).To(Equal(gethStorage1))
+			Expect(ipldStorage2).To(Equal(gethStorage2))
+
+			// Query the current block
+			ipldStorage3, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(ipldStorage2).To(Equal(ipldStorage3))
+		})
 	})
 
 	Describe("eth call", func() {
 		BeforeEach(func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			contract, contractErr = integration.DeployContract()
 			erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10)
 
@@ -445,6 +505,9 @@ var _ = Describe("Integration test", func() {
 
 	Describe("Chain ID", func() {
 		It("Check chain id", func() {
+			if directProxyEthCalls {
+				Skip("skipping no-direct-proxy-forwarding integration tests")
+			}
 			gethChainId, err := gethClient.ChainID(ctx)
 			Expect(err).ToNot(HaveOccurred())
 

test_config/test_config.go

@@ -19,13 +19,12 @@ package test_config
 import (
 	"errors"
 
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 )
 
-var DBConfig postgres.Config
+var DBConfig postgres.ConnectionParams
 
 func init() {
 	setTestConfig()
@@ -53,7 +52,7 @@ func setTestConfig() {
 	port := vip.GetInt("database.port")
 	name := vip.GetString("database.name")
 
-	DBConfig = postgres.Config{
+	DBConfig = postgres.ConnectionParams{
 		Hostname: hn,
 		Name:     name,
 		Port:     port,

version/version.go

@@ -19,10 +19,10 @@ package version
 import "fmt"
 
 const (
-	Major = 0       // Major version component of the current release
+	Major = 2  // Major version component of the current release
-	Minor = 3       // Minor version component of the current release
+	Minor = 0  // Minor version component of the current release
-	Patch = 6       // Patch version component of the current release
+	Patch = 0  // Patch version component of the current release
-	Meta  = "alpha" // Version metadata to append to the version string
+	Meta  = "" // Version metadata to append to the version string
 )
 
 // Version holds the textual version string.