Merge pull request #30 from vulcanize/fill_gaps

Fill gaps in the Postgres IPLD database when we hit a cache miss

Makefile

@@ -51,11 +51,13 @@ CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?ssl
 #Test
 TEST_DB = vulcanize_testing
 TEST_CONNECT_STRING = postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(TEST_DB)?sslmode=disable
+TEST_CONNECT_STRING_LOCAL = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
 
 .PHONY: test
 test: | $(GINKGO) $(GOOSE)
 	go vet ./...
 	go fmt ./...
+	export PGPASSWORD=$(DATABASE_PASSWORD)
 	dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
 	createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
 	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
@@ -65,10 +67,31 @@ test: | $(GINKGO) $(GOOSE)
 integrationtest: | $(GINKGO) $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	#dropdb -h $(DATABASE_HOSTNAME) -p $(PORT) -U $(USER) -W --if-exists $(TEST_DB)
+	export PGPASSWORD=$(DATABASE_PASSWORD)
-	#createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) -W $(TEST_DB)
+	dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" up
+	createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" reset
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
+	$(GINKGO) -r integration_test/
+
+.PHONY: test_local
+test_local: | $(GINKGO) $(GOOSE)
+	go vet ./...
+	go fmt ./...
+	dropdb -h $(HOST_NAME) -p $(PORT) -U $(USER) --if-exists $(TEST_DB)
+	createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) $(TEST_DB)
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" up
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" reset
+	make migrate NAME=$(TEST_DB)
+	$(GINKGO) -r --skipPackage=integration_tests,integration
+
+.PHONY: integrationtest_local
+integrationtest_local: | $(GINKGO) $(GOOSE)
+	go vet ./...
+	go fmt ./...
+	dropdb -h $(HOST_NAME) -p $(PORT) -U $(USER) --if-exists $(TEST_DB)
+	createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) $(TEST_DB)
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" up
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" reset
 	make migrate NAME=$(TEST_DB)
 	$(GINKGO) -r integration_test/
 

db/schema.sql

@@ -347,7 +347,9 @@ CREATE TABLE eth.receipt_cids (
     topic1s character varying(66)[],
     topic2s character varying(66)[],
     topic3s character varying(66)[],
-    log_contracts character varying(66)[]
+    log_contracts character varying(66)[],
+    post_state character varying(66),
+    post_status integer
 );
 
 

environments/example.toml

@@ -20,6 +20,7 @@
     defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
     rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
     httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
     nodeID = "arch1" # $ETH_NODE_ID
     clientName = "Geth" # $ETH_CLIENT_NAME
     genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK

go.mod

@@ -25,4 +25,5 @@ require (
 )
 
 replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.14
+
 replace github.com/vulcanize/ipfs-ethdb v0.0.2-alpha => github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha

pkg/eth/api.go

@@ -18,6 +18,7 @@ package eth
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
@@ -34,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
@@ -50,21 +52,23 @@ type PublicEthAPI struct {
 	// Local db backend
 	B *Backend
 
-	// Remote node for forwarding cache misses
+	// Proxy node for forwarding cache misses
-	rpc       *rpc.Client
+	supportsStateDiff bool // Whether or not the remote node supports the statediff_writeStateDiffAt endpoint, if it does we can fill the local cache when we hit a miss
-	ethClient *ethclient.Client
+	rpc               *rpc.Client
+	ethClient         *ethclient.Client
 }
 
 // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend
-func NewPublicEthAPI(b *Backend, client *rpc.Client) *PublicEthAPI {
+func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff bool) *PublicEthAPI {
 	var ethClient *ethclient.Client
 	if client != nil {
 		ethClient = ethclient.NewClient(client)
 	}
 	return &PublicEthAPI{
-		B:         b,
+		B:                 b,
-		rpc:       client,
+		supportsStateDiff: supportsStateDiff,
-		ethClient: ethClient,
+		rpc:               client,
+		ethClient:         ethClient,
 	}
 }
 
@@ -84,6 +88,7 @@ func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.Block
 	}
 	if pea.ethClient != nil {
 		if header, err := pea.ethClient.HeaderByNumber(ctx, big.NewInt(number.Int64())); header != nil && err == nil {
+			go pea.writeStateDiffAt(number.Int64())
 			return pea.rpcMarshalHeader(header)
 		}
 	}
@@ -100,6 +105,7 @@ func (pea *PublicEthAPI) GetHeaderByHash(ctx context.Context, hash common.Hash)
 	}
 	if pea.ethClient != nil {
 		if header, err := pea.ethClient.HeaderByHash(ctx, hash); header != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
 			if res, err := pea.rpcMarshalHeader(header); err != nil {
 				return res
 			}
@@ -137,6 +143,7 @@ func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockN
 	}
 	if pea.ethClient != nil {
 		if block, err := pea.ethClient.BlockByNumber(ctx, big.NewInt(number.Int64())); block != nil && err == nil {
+			go pea.writeStateDiffAt(number.Int64())
 			return pea.rpcMarshalBlock(block, true, fullTx)
 		}
 	}
@@ -152,6 +159,7 @@ func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, f
 	}
 	if pea.ethClient != nil {
 		if block, err := pea.ethClient.BlockByHash(ctx, hash); block != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
 			return pea.rpcMarshalBlock(block, true, fullTx)
 		}
 	}
@@ -179,6 +187,7 @@ func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, bloc
 	}
 	if pea.rpc != nil {
 		if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockNumberAndIndex", blockNr, index); uncle != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
 		}
 	}
@@ -200,6 +209,7 @@ func (pea *PublicEthAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockH
 	}
 	if pea.rpc != nil {
 		if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockHashAndIndex", blockHash, index); uncle != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
 		}
 	}
@@ -215,6 +225,7 @@ func (pea *PublicEthAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr
 	if pea.rpc != nil {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockNumber", blockNr); num != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return num
 		}
 	}
@@ -230,6 +241,7 @@ func (pea *PublicEthAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash
 	if pea.rpc != nil {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockHash", blockHash); num != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return num
 		}
 	}
@@ -251,6 +263,7 @@ func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common
 	if pea.rpc != nil {
 		var num *hexutil.Uint64
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getTransactionCount", address, blockNrOrHash); num != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return num, nil
 		}
 	}
@@ -275,6 +288,7 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, b
 	if pea.rpc != nil {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByNumber", blockNr); num != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return num
 		}
 	}
@@ -290,6 +304,7 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByHash(ctx context.Context, blo
 	if pea.rpc != nil {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByHash", blockHash); num != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return num
 		}
 	}
@@ -304,6 +319,7 @@ func (pea *PublicEthAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context
 	if pea.rpc != nil {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return tx
 		}
 	}
@@ -318,6 +334,7 @@ func (pea *PublicEthAPI) GetTransactionByBlockHashAndIndex(ctx context.Context,
 	if pea.rpc != nil {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return tx
 		}
 	}
@@ -332,6 +349,7 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Cont
 	if pea.rpc != nil {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return tx
 		}
 	}
@@ -346,6 +364,7 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Contex
 	if pea.rpc != nil {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return tx
 		}
 	}
@@ -362,6 +381,7 @@ func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.H
 	if pea.rpc != nil {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); tx != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
 			return tx, nil
 		}
 	}
@@ -378,6 +398,7 @@ func (pea *PublicEthAPI) GetRawTransactionByHash(ctx context.Context, hash commo
 	if pea.rpc != nil {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); tx != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
 			return tx, nil
 		}
 	}
@@ -398,6 +419,7 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.
 	}
 	if pea.rpc != nil {
 		if receipt := pea.remoteGetTransactionReceipt(ctx, hash); receipt != nil {
+			go pea.writeStateDiffFor(hash)
 			return receipt, nil
 		}
 	}
@@ -489,6 +511,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
 		if arg, err := toFilterArg(crit); err == nil {
 			var res []*types.Log
 			if err := pea.rpc.CallContext(ctx, &res, "eth_getLogs", arg); err == nil {
+				go pea.writeStateDiffWithCriteria(crit)
 				return res, nil
 			}
 		}
@@ -601,6 +624,7 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address,
 	if pea.rpc != nil {
 		var res *hexutil.Big
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getBalance", address, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return res, nil
 		}
 	}
@@ -626,6 +650,7 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres
 	if pea.rpc != nil {
 		var res hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return res, nil
 		}
 	}
@@ -641,6 +666,7 @@ func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, bl
 	if pea.rpc != nil {
 		var res hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return res, nil
 		}
 	}
@@ -656,6 +682,7 @@ func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, s
 	if pea.rpc != nil {
 		var res *AccountResult
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getProof", address, storageKeys, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return res, nil
 		}
 	}
@@ -726,6 +753,7 @@ func (pea *PublicEthAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash
 	if (failed || err != nil) && pea.rpc != nil {
 		var hex hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return hex, nil
 		}
 	}
@@ -845,6 +873,89 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 	return result.Return(), result.UsedGas, result.Failed(), err
 }
 
+// writeStateDiffAtOrFor calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffAtOrFor(blockNrOrHash rpc.BlockNumberOrHash) {
+	// short circuit right away if the proxy doesn't support diffing
+	if !pea.supportsStateDiff {
+		return
+	}
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		pea.writeStateDiffAt(blockNr.Int64())
+		return
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		pea.writeStateDiffFor(hash)
+	}
+}
+
+// writeStateDiffWithCriteria calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffWithCriteria(crit ethereum.FilterQuery) {
+	// short circuit right away if the proxy doesn't support diffing
+	if !pea.supportsStateDiff {
+		return
+	}
+	if crit.BlockHash != nil {
+		pea.writeStateDiffFor(*crit.BlockHash)
+		return
+	}
+	var start, end int64
+	if crit.FromBlock != nil {
+		start = crit.FromBlock.Int64()
+	}
+	if crit.ToBlock != nil {
+		end = crit.ToBlock.Int64()
+	} else {
+		end = start
+	}
+	for i := start; i <= end; i++ {
+		pea.writeStateDiffAt(i)
+	}
+}
+
+// writeStateDiffAt calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffAt(height int64) {
+	if !pea.supportsStateDiff {
+		return
+	}
+	// we use a separate context than the one provided by the client
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	var data json.RawMessage
+	params := statediff.Params{
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		IncludeBlock:             true,
+		IncludeReceipts:          true,
+		IncludeTD:                true,
+		IncludeCode:              true,
+	}
+	if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffAt", uint64(height), params); err != nil {
+		logrus.Errorf("writeStateDiffAt %d faild with err %s", height, err.Error())
+	}
+}
+
+// writeStateDiffFor calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffFor(blockHash common.Hash) {
+	if !pea.supportsStateDiff {
+		return
+	}
+	// we use a separate context than the one provided by the client
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	var data json.RawMessage
+	params := statediff.Params{
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		IncludeBlock:             true,
+		IncludeReceipts:          true,
+		IncludeTD:                true,
+		IncludeCode:              true,
+	}
+	if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffFor", blockHash, params); err != nil {
+		logrus.Errorf("writeStateDiffFor %s faild with err %s", blockHash.Hex(), err.Error())
+	}
+}
+
 // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field
 func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
 	fields, err := RPCMarshalBlock(b, inclTx, fullTx)

pkg/eth/api_test.go

@@ -189,7 +189,7 @@ var _ = Describe("API", func() {
 		indexAndPublisher := eth2.NewIPLDPublisher(db)
 		backend, err := eth.NewEthBackend(db, &eth.Config{})
 		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend, nil)
+		api = eth.NewPublicEthAPI(backend, nil, false)
 		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
 		Expect(err).ToNot(HaveOccurred())
 		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)

pkg/eth/backend.go

@@ -20,12 +20,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/ethereum/go-ethereum/trie"
 	"math/big"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
-
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
@@ -40,8 +38,9 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/vulcanize/ipfs-ethdb"
+	"github.com/ethereum/go-ethereum/trie"
 
+	"github.com/vulcanize/ipfs-ethdb"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 	shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 

pkg/eth/cid_retriever_test.go

@@ -17,9 +17,10 @@
 package eth_test
 
 import (
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"

pkg/eth/eth_state_test.go

@@ -19,7 +19,6 @@ package eth_test
 import (
 	"bytes"
 	"context"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 	"io/ioutil"
 	"math/big"
 
@@ -38,6 +37,7 @@ import (
 
 	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
@@ -83,7 +83,7 @@ var _ = Describe("eth state reading tests", func() {
 			RPCGasCap:   big.NewInt(10000000000),
 		})
 		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend, nil)
+		api = eth.NewPublicEthAPI(backend, nil, false)
 
 		// make the test blockchain (and state)
 		blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen)
@@ -153,7 +153,7 @@ var _ = Describe("eth state reading tests", func() {
 
 		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
 		indexAndPublisher := eth2.NewIPLDPublisher(db)
-		api = eth.NewPublicEthAPI(backend, nil)
+		api = eth.NewPublicEthAPI(backend, nil, false)
 		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
 		Expect(err).ToNot(HaveOccurred())
 		// The non-canonical header has a child

pkg/eth/filterer.go

@@ -18,6 +18,7 @@ package eth
 
 import (
 	"bytes"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	"github.com/ethereum/go-ethereum/common"

pkg/eth/filterer_test.go

@@ -18,6 +18,7 @@ package eth_test
 
 import (
 	"bytes"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	. "github.com/onsi/ginkgo"

pkg/eth/helpers.go

@@ -18,6 +18,7 @@ package eth
 
 import (
 	"fmt"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	"github.com/ethereum/go-ethereum/params"

pkg/eth/test_helpers/test_data.go

@@ -20,9 +20,10 @@ import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
+	"math/big"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/ethereum/go-ethereum/trie"
-	"math/big"
 
 	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 

pkg/graphql/service.go

@@ -18,11 +18,11 @@ package graphql
 
 import (
 	"fmt"
-	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/node"
 	"net"
 	"net/http"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/graph-gophers/graphql-go"

pkg/rpc/http.go

@@ -18,6 +18,7 @@ package rpc
 
 import (
 	"fmt"
+
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"

pkg/rpc/ws.go

@@ -17,12 +17,13 @@
 package rpc
 
 import (
-	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/node"
 	"net"
 	"net/http"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
+
 	"github.com/vulcanize/ipld-eth-server/pkg/prom"
 )
 

pkg/serve/config.go

@@ -48,19 +48,21 @@ const (
 
 	ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR"
 	ETH_RPC_GAS_CAP         = "ETH_RPC_GAS_CAP"
+	ETH_SUPPORTS_STATEDIFF  = "ETH_SUPPORTS_STATEDIFF"
 )
 
 // Config struct
 type Config struct {
-	DB            *postgres.DB
+	DB               *postgres.DB
-	DBConfig      postgres.Config
+	DBConfig         postgres.Config
-	WSEndpoint    string
+	WSEndpoint       string
-	HTTPEndpoint  string
+	HTTPEndpoint     string
-	IPCEndpoint   string
+	IPCEndpoint      string
-	ChainConfig   *params.ChainConfig
+	ChainConfig      *params.ChainConfig
-	DefaultSender *common.Address
+	DefaultSender    *common.Address
-	RPCGasCap     *big.Int
+	RPCGasCap        *big.Int
-	Client        *rpc.Client
+	Client           *rpc.Client
+	SupportStateDiff bool
 }
 
 // NewConfig is used to initialize a watcher config from a .toml file
@@ -74,6 +76,7 @@ func NewConfig() (*Config, error) {
 	viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
 	viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
 	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
+	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
 
 	c.DBConfig.Init()
 
@@ -83,6 +86,7 @@ func NewConfig() (*Config, error) {
 		return nil, err
 	}
 	c.Client = cli
+	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
 
 	wsPath := viper.GetString("server.wsPath")
 	if wsPath == "" {

pkg/serve/service.go

@@ -80,6 +80,8 @@ type Service struct {
 	serveWg *sync.WaitGroup
 	// rpc client for forwarding cache misses
 	client *rpc.Client
+	// whether the proxied client supports state diffing
+	supportsStateDiffing bool
 	// backend for the server
 	backend *eth.Backend
 }
@@ -94,6 +96,8 @@ func NewServer(settings *Config) (Server, error) {
 	sap.QuitChan = make(chan bool)
 	sap.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
 	sap.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
+	sap.client = settings.Client
+	sap.supportsStateDiffing = settings.SupportStateDiff
 	var err error
 	sap.backend, err = eth.NewEthBackend(sap.db, &eth.Config{
 		ChainConfig:   settings.ChainConfig,
@@ -141,7 +145,7 @@ func (sap *Service) APIs() []rpc.API {
 	return append(apis, rpc.API{
 		Namespace: eth.APIName,
 		Version:   eth.APIVersion,
-		Service:   eth.NewPublicEthAPI(sap.backend, sap.client),
+		Service:   eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing),
 		Public:    true,
 	})
 }

version/version.go

@@ -21,7 +21,7 @@ import "fmt"
 const (
 	Major = 0       // Major version component of the current release
 	Minor = 3       // Minor version component of the current release
-	Patch = 0       // Patch version component of the current release
+	Patch = 1       // Patch version component of the current release
 	Meta  = "alpha" // Version metadata to append to the version string
 )