Merge pull request #30 from vulcanize/fill_gaps

Fill gaps in the Postgres IPLD database when we hit a cache miss
This commit is contained in:
Ian Norden 2021-02-24 16:27:59 -06:00 committed by GitHub
commit a181a5c25b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 189 additions and 37 deletions

View File

@ -51,11 +51,13 @@ CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?ssl
#Test #Test
TEST_DB = vulcanize_testing TEST_DB = vulcanize_testing
TEST_CONNECT_STRING = postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(TEST_DB)?sslmode=disable TEST_CONNECT_STRING = postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(TEST_DB)?sslmode=disable
TEST_CONNECT_STRING_LOCAL = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
.PHONY: test .PHONY: test
test: | $(GINKGO) $(GOOSE) test: | $(GINKGO) $(GOOSE)
go vet ./... go vet ./...
go fmt ./... go fmt ./...
export PGPASSWORD=$(DATABASE_PASSWORD)
dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB) dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB) createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up $(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
@ -65,10 +67,31 @@ test: | $(GINKGO) $(GOOSE)
integrationtest: | $(GINKGO) $(GOOSE) integrationtest: | $(GINKGO) $(GOOSE)
go vet ./... go vet ./...
go fmt ./... go fmt ./...
#dropdb -h $(DATABASE_HOSTNAME) -p $(PORT) -U $(USER) -W --if-exists $(TEST_DB) export PGPASSWORD=$(DATABASE_PASSWORD)
#createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) -W $(TEST_DB) dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" up createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" reset $(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
$(GINKGO) -r integration_test/
.PHONY: test_local
test_local: | $(GINKGO) $(GOOSE)
go vet ./...
go fmt ./...
dropdb -h $(HOST_NAME) -p $(PORT) -U $(USER) --if-exists $(TEST_DB)
createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) $(TEST_DB)
$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" up
$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" reset
make migrate NAME=$(TEST_DB)
$(GINKGO) -r --skipPackage=integration_tests,integration
.PHONY: integrationtest_local
integrationtest_local: | $(GINKGO) $(GOOSE)
go vet ./...
go fmt ./...
dropdb -h $(HOST_NAME) -p $(PORT) -U $(USER) --if-exists $(TEST_DB)
createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) $(TEST_DB)
$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" up
$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" reset
make migrate NAME=$(TEST_DB) make migrate NAME=$(TEST_DB)
$(GINKGO) -r integration_test/ $(GINKGO) -r integration_test/

View File

@ -347,7 +347,9 @@ CREATE TABLE eth.receipt_cids (
topic1s character varying(66)[], topic1s character varying(66)[],
topic2s character varying(66)[], topic2s character varying(66)[],
topic3s character varying(66)[], topic3s character varying(66)[],
log_contracts character varying(66)[] log_contracts character varying(66)[],
post_state character varying(66),
post_status integer
); );

View File

@ -20,6 +20,7 @@
defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
nodeID = "arch1" # $ETH_NODE_ID nodeID = "arch1" # $ETH_NODE_ID
clientName = "Geth" # $ETH_CLIENT_NAME clientName = "Geth" # $ETH_CLIENT_NAME
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK

1
go.mod
View File

@ -25,4 +25,5 @@ require (
) )
replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.14 replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.14
replace github.com/vulcanize/ipfs-ethdb v0.0.2-alpha => github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha replace github.com/vulcanize/ipfs-ethdb v0.0.2-alpha => github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha

View File

@ -18,6 +18,7 @@ package eth
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"math" "math"
@ -34,6 +35,7 @@ import (
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-eth-indexer/pkg/eth" "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
@ -50,21 +52,23 @@ type PublicEthAPI struct {
// Local db backend // Local db backend
B *Backend B *Backend
// Remote node for forwarding cache misses // Proxy node for forwarding cache misses
rpc *rpc.Client supportsStateDiff bool // Whether or not the remote node supports the statediff_writeStateDiffAt endpoint, if it does we can fill the local cache when we hit a miss
ethClient *ethclient.Client rpc *rpc.Client
ethClient *ethclient.Client
} }
// NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend
func NewPublicEthAPI(b *Backend, client *rpc.Client) *PublicEthAPI { func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff bool) *PublicEthAPI {
var ethClient *ethclient.Client var ethClient *ethclient.Client
if client != nil { if client != nil {
ethClient = ethclient.NewClient(client) ethClient = ethclient.NewClient(client)
} }
return &PublicEthAPI{ return &PublicEthAPI{
B: b, B: b,
rpc: client, supportsStateDiff: supportsStateDiff,
ethClient: ethClient, rpc: client,
ethClient: ethClient,
} }
} }
@ -84,6 +88,7 @@ func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.Block
} }
if pea.ethClient != nil { if pea.ethClient != nil {
if header, err := pea.ethClient.HeaderByNumber(ctx, big.NewInt(number.Int64())); header != nil && err == nil { if header, err := pea.ethClient.HeaderByNumber(ctx, big.NewInt(number.Int64())); header != nil && err == nil {
go pea.writeStateDiffAt(number.Int64())
return pea.rpcMarshalHeader(header) return pea.rpcMarshalHeader(header)
} }
} }
@ -100,6 +105,7 @@ func (pea *PublicEthAPI) GetHeaderByHash(ctx context.Context, hash common.Hash)
} }
if pea.ethClient != nil { if pea.ethClient != nil {
if header, err := pea.ethClient.HeaderByHash(ctx, hash); header != nil && err == nil { if header, err := pea.ethClient.HeaderByHash(ctx, hash); header != nil && err == nil {
go pea.writeStateDiffFor(hash)
if res, err := pea.rpcMarshalHeader(header); err != nil { if res, err := pea.rpcMarshalHeader(header); err != nil {
return res return res
} }
@ -137,6 +143,7 @@ func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockN
} }
if pea.ethClient != nil { if pea.ethClient != nil {
if block, err := pea.ethClient.BlockByNumber(ctx, big.NewInt(number.Int64())); block != nil && err == nil { if block, err := pea.ethClient.BlockByNumber(ctx, big.NewInt(number.Int64())); block != nil && err == nil {
go pea.writeStateDiffAt(number.Int64())
return pea.rpcMarshalBlock(block, true, fullTx) return pea.rpcMarshalBlock(block, true, fullTx)
} }
} }
@ -152,6 +159,7 @@ func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, f
} }
if pea.ethClient != nil { if pea.ethClient != nil {
if block, err := pea.ethClient.BlockByHash(ctx, hash); block != nil && err == nil { if block, err := pea.ethClient.BlockByHash(ctx, hash); block != nil && err == nil {
go pea.writeStateDiffFor(hash)
return pea.rpcMarshalBlock(block, true, fullTx) return pea.rpcMarshalBlock(block, true, fullTx)
} }
} }
@ -179,6 +187,7 @@ func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, bloc
} }
if pea.rpc != nil { if pea.rpc != nil {
if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockNumberAndIndex", blockNr, index); uncle != nil && err == nil { if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockNumberAndIndex", blockNr, index); uncle != nil && err == nil {
go pea.writeStateDiffAt(blockNr.Int64())
return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false) return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
} }
} }
@ -200,6 +209,7 @@ func (pea *PublicEthAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockH
} }
if pea.rpc != nil { if pea.rpc != nil {
if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockHashAndIndex", blockHash, index); uncle != nil && err == nil { if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockHashAndIndex", blockHash, index); uncle != nil && err == nil {
go pea.writeStateDiffFor(blockHash)
return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false) return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
} }
} }
@ -215,6 +225,7 @@ func (pea *PublicEthAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr
if pea.rpc != nil { if pea.rpc != nil {
var num *hexutil.Uint var num *hexutil.Uint
if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockNumber", blockNr); num != nil && err == nil { if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockNumber", blockNr); num != nil && err == nil {
go pea.writeStateDiffAt(blockNr.Int64())
return num return num
} }
} }
@ -230,6 +241,7 @@ func (pea *PublicEthAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash
if pea.rpc != nil { if pea.rpc != nil {
var num *hexutil.Uint var num *hexutil.Uint
if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockHash", blockHash); num != nil && err == nil { if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockHash", blockHash); num != nil && err == nil {
go pea.writeStateDiffFor(blockHash)
return num return num
} }
} }
@ -251,6 +263,7 @@ func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common
if pea.rpc != nil { if pea.rpc != nil {
var num *hexutil.Uint64 var num *hexutil.Uint64
if err := pea.rpc.CallContext(ctx, &num, "eth_getTransactionCount", address, blockNrOrHash); num != nil && err == nil { if err := pea.rpc.CallContext(ctx, &num, "eth_getTransactionCount", address, blockNrOrHash); num != nil && err == nil {
go pea.writeStateDiffAtOrFor(blockNrOrHash)
return num, nil return num, nil
} }
} }
@ -275,6 +288,7 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, b
if pea.rpc != nil { if pea.rpc != nil {
var num *hexutil.Uint var num *hexutil.Uint
if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByNumber", blockNr); num != nil && err == nil { if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByNumber", blockNr); num != nil && err == nil {
go pea.writeStateDiffAt(blockNr.Int64())
return num return num
} }
} }
@ -290,6 +304,7 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByHash(ctx context.Context, blo
if pea.rpc != nil { if pea.rpc != nil {
var num *hexutil.Uint var num *hexutil.Uint
if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByHash", blockHash); num != nil && err == nil { if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByHash", blockHash); num != nil && err == nil {
go pea.writeStateDiffFor(blockHash)
return num return num
} }
} }
@ -304,6 +319,7 @@ func (pea *PublicEthAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context
if pea.rpc != nil { if pea.rpc != nil {
var tx *RPCTransaction var tx *RPCTransaction
if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil { if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
go pea.writeStateDiffAt(blockNr.Int64())
return tx return tx
} }
} }
@ -318,6 +334,7 @@ func (pea *PublicEthAPI) GetTransactionByBlockHashAndIndex(ctx context.Context,
if pea.rpc != nil { if pea.rpc != nil {
var tx *RPCTransaction var tx *RPCTransaction
if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil { if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
go pea.writeStateDiffFor(blockHash)
return tx return tx
} }
} }
@ -332,6 +349,7 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Cont
if pea.rpc != nil { if pea.rpc != nil {
var tx hexutil.Bytes var tx hexutil.Bytes
if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil { if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
go pea.writeStateDiffAt(blockNr.Int64())
return tx return tx
} }
} }
@ -346,6 +364,7 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Contex
if pea.rpc != nil { if pea.rpc != nil {
var tx hexutil.Bytes var tx hexutil.Bytes
if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil { if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
go pea.writeStateDiffFor(blockHash)
return tx return tx
} }
} }
@ -362,6 +381,7 @@ func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.H
if pea.rpc != nil { if pea.rpc != nil {
var tx *RPCTransaction var tx *RPCTransaction
if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); tx != nil && err == nil { if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); tx != nil && err == nil {
go pea.writeStateDiffFor(hash)
return tx, nil return tx, nil
} }
} }
@ -378,6 +398,7 @@ func (pea *PublicEthAPI) GetRawTransactionByHash(ctx context.Context, hash commo
if pea.rpc != nil { if pea.rpc != nil {
var tx hexutil.Bytes var tx hexutil.Bytes
if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); tx != nil && err == nil { if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); tx != nil && err == nil {
go pea.writeStateDiffFor(hash)
return tx, nil return tx, nil
} }
} }
@ -398,6 +419,7 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.
} }
if pea.rpc != nil { if pea.rpc != nil {
if receipt := pea.remoteGetTransactionReceipt(ctx, hash); receipt != nil { if receipt := pea.remoteGetTransactionReceipt(ctx, hash); receipt != nil {
go pea.writeStateDiffFor(hash)
return receipt, nil return receipt, nil
} }
} }
@ -489,6 +511,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
if arg, err := toFilterArg(crit); err == nil { if arg, err := toFilterArg(crit); err == nil {
var res []*types.Log var res []*types.Log
if err := pea.rpc.CallContext(ctx, &res, "eth_getLogs", arg); err == nil { if err := pea.rpc.CallContext(ctx, &res, "eth_getLogs", arg); err == nil {
go pea.writeStateDiffWithCriteria(crit)
return res, nil return res, nil
} }
} }
@ -601,6 +624,7 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address,
if pea.rpc != nil { if pea.rpc != nil {
var res *hexutil.Big var res *hexutil.Big
if err := pea.rpc.CallContext(ctx, &res, "eth_getBalance", address, blockNrOrHash); res != nil && err == nil { if err := pea.rpc.CallContext(ctx, &res, "eth_getBalance", address, blockNrOrHash); res != nil && err == nil {
go pea.writeStateDiffAtOrFor(blockNrOrHash)
return res, nil return res, nil
} }
} }
@ -626,6 +650,7 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres
if pea.rpc != nil { if pea.rpc != nil {
var res hexutil.Bytes var res hexutil.Bytes
if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil { if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil {
go pea.writeStateDiffAtOrFor(blockNrOrHash)
return res, nil return res, nil
} }
} }
@ -641,6 +666,7 @@ func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, bl
if pea.rpc != nil { if pea.rpc != nil {
var res hexutil.Bytes var res hexutil.Bytes
if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil { if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil {
go pea.writeStateDiffAtOrFor(blockNrOrHash)
return res, nil return res, nil
} }
} }
@ -656,6 +682,7 @@ func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, s
if pea.rpc != nil { if pea.rpc != nil {
var res *AccountResult var res *AccountResult
if err := pea.rpc.CallContext(ctx, &res, "eth_getProof", address, storageKeys, blockNrOrHash); res != nil && err == nil { if err := pea.rpc.CallContext(ctx, &res, "eth_getProof", address, storageKeys, blockNrOrHash); res != nil && err == nil {
go pea.writeStateDiffAtOrFor(blockNrOrHash)
return res, nil return res, nil
} }
} }
@ -726,6 +753,7 @@ func (pea *PublicEthAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash
if (failed || err != nil) && pea.rpc != nil { if (failed || err != nil) && pea.rpc != nil {
var hex hexutil.Bytes var hex hexutil.Bytes
if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil { if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil {
go pea.writeStateDiffAtOrFor(blockNrOrHash)
return hex, nil return hex, nil
} }
} }
@ -845,6 +873,89 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
return result.Return(), result.UsedGas, result.Failed(), err return result.Return(), result.UsedGas, result.Failed(), err
} }
// writeStateDiffAtOrFor calls out to the proxy statediffing geth client to fill in a gap in the index
func (pea *PublicEthAPI) writeStateDiffAtOrFor(blockNrOrHash rpc.BlockNumberOrHash) {
// short circuit right away if the proxy doesn't support diffing
if !pea.supportsStateDiff {
return
}
if blockNr, ok := blockNrOrHash.Number(); ok {
pea.writeStateDiffAt(blockNr.Int64())
return
}
if hash, ok := blockNrOrHash.Hash(); ok {
pea.writeStateDiffFor(hash)
}
}
// writeStateDiffWithCriteria calls out to the proxy statediffing geth client to fill in a gap in the index
func (pea *PublicEthAPI) writeStateDiffWithCriteria(crit ethereum.FilterQuery) {
// short circuit right away if the proxy doesn't support diffing
if !pea.supportsStateDiff {
return
}
if crit.BlockHash != nil {
pea.writeStateDiffFor(*crit.BlockHash)
return
}
var start, end int64
if crit.FromBlock != nil {
start = crit.FromBlock.Int64()
}
if crit.ToBlock != nil {
end = crit.ToBlock.Int64()
} else {
end = start
}
for i := start; i <= end; i++ {
pea.writeStateDiffAt(i)
}
}
// writeStateDiffAt calls out to the proxy statediffing geth client to fill in a gap in the index
func (pea *PublicEthAPI) writeStateDiffAt(height int64) {
if !pea.supportsStateDiff {
return
}
// we use a separate context than the one provided by the client
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
var data json.RawMessage
params := statediff.Params{
IntermediateStateNodes: true,
IntermediateStorageNodes: true,
IncludeBlock: true,
IncludeReceipts: true,
IncludeTD: true,
IncludeCode: true,
}
if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffAt", uint64(height), params); err != nil {
logrus.Errorf("writeStateDiffAt %d faild with err %s", height, err.Error())
}
}
// writeStateDiffFor calls out to the proxy statediffing geth client to fill in a gap in the index
func (pea *PublicEthAPI) writeStateDiffFor(blockHash common.Hash) {
if !pea.supportsStateDiff {
return
}
// we use a separate context than the one provided by the client
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
var data json.RawMessage
params := statediff.Params{
IntermediateStateNodes: true,
IntermediateStorageNodes: true,
IncludeBlock: true,
IncludeReceipts: true,
IncludeTD: true,
IncludeCode: true,
}
if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffFor", blockHash, params); err != nil {
logrus.Errorf("writeStateDiffFor %s faild with err %s", blockHash.Hex(), err.Error())
}
}
// rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field
func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
fields, err := RPCMarshalBlock(b, inclTx, fullTx) fields, err := RPCMarshalBlock(b, inclTx, fullTx)

View File

@ -189,7 +189,7 @@ var _ = Describe("API", func() {
indexAndPublisher := eth2.NewIPLDPublisher(db) indexAndPublisher := eth2.NewIPLDPublisher(db)
backend, err := eth.NewEthBackend(db, &eth.Config{}) backend, err := eth.NewEthBackend(db, &eth.Config{})
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
api = eth.NewPublicEthAPI(backend, nil) api = eth.NewPublicEthAPI(backend, nil, false)
err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload) err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode) err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)

View File

@ -20,12 +20,10 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"github.com/ethereum/go-ethereum/trie"
"math/big" "math/big"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/ethash"
@ -40,8 +38,9 @@ import (
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/vulcanize/ipfs-ethdb" "github.com/ethereum/go-ethereum/trie"
"github.com/vulcanize/ipfs-ethdb"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres" "github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared" shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared"

View File

@ -17,9 +17,10 @@
package eth_test package eth_test
import ( import (
"math/big"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared" "github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"math/big"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"

View File

@ -19,7 +19,6 @@ package eth_test
import ( import (
"bytes" "bytes"
"context" "context"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"io/ioutil" "io/ioutil"
"math/big" "math/big"
@ -38,6 +37,7 @@ import (
eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth" eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres" "github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"github.com/vulcanize/ipld-eth-server/pkg/eth" "github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers" "github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
@ -83,7 +83,7 @@ var _ = Describe("eth state reading tests", func() {
RPCGasCap: big.NewInt(10000000000), RPCGasCap: big.NewInt(10000000000),
}) })
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
api = eth.NewPublicEthAPI(backend, nil) api = eth.NewPublicEthAPI(backend, nil, false)
// make the test blockchain (and state) // make the test blockchain (and state)
blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen) blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen)
@ -153,7 +153,7 @@ var _ = Describe("eth state reading tests", func() {
// Insert some non-canonical data into the database so that we test our ability to discern canonicity // Insert some non-canonical data into the database so that we test our ability to discern canonicity
indexAndPublisher := eth2.NewIPLDPublisher(db) indexAndPublisher := eth2.NewIPLDPublisher(db)
api = eth.NewPublicEthAPI(backend, nil) api = eth.NewPublicEthAPI(backend, nil, false)
err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload) err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
// The non-canonical header has a child // The non-canonical header has a child

View File

@ -18,6 +18,7 @@ package eth
import ( import (
"bytes" "bytes"
sdtypes "github.com/ethereum/go-ethereum/statediff/types" sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"

View File

@ -18,6 +18,7 @@ package eth_test
import ( import (
"bytes" "bytes"
sdtypes "github.com/ethereum/go-ethereum/statediff/types" sdtypes "github.com/ethereum/go-ethereum/statediff/types"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"

View File

@ -18,6 +18,7 @@ package eth
import ( import (
"fmt" "fmt"
sdtypes "github.com/ethereum/go-ethereum/statediff/types" sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"

View File

@ -20,9 +20,10 @@ import (
"crypto/ecdsa" "crypto/ecdsa"
"crypto/elliptic" "crypto/elliptic"
"crypto/rand" "crypto/rand"
"math/big"
sdtypes "github.com/ethereum/go-ethereum/statediff/types" sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"math/big"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared" "github.com/vulcanize/ipld-eth-indexer/pkg/shared"

View File

@ -18,11 +18,11 @@ package graphql
import ( import (
"fmt" "fmt"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/node"
"net" "net"
"net/http" "net/http"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/graph-gophers/graphql-go" "github.com/graph-gophers/graphql-go"

View File

@ -18,6 +18,7 @@ package rpc
import ( import (
"fmt" "fmt"
"github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"

View File

@ -17,12 +17,13 @@
package rpc package rpc
import ( import (
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/node"
"net" "net"
"net/http" "net/http"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/vulcanize/ipld-eth-server/pkg/prom" "github.com/vulcanize/ipld-eth-server/pkg/prom"
) )

View File

@ -48,19 +48,21 @@ const (
ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR" ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR"
ETH_RPC_GAS_CAP = "ETH_RPC_GAS_CAP" ETH_RPC_GAS_CAP = "ETH_RPC_GAS_CAP"
ETH_SUPPORTS_STATEDIFF = "ETH_SUPPORTS_STATEDIFF"
) )
// Config struct // Config struct
type Config struct { type Config struct {
DB *postgres.DB DB *postgres.DB
DBConfig postgres.Config DBConfig postgres.Config
WSEndpoint string WSEndpoint string
HTTPEndpoint string HTTPEndpoint string
IPCEndpoint string IPCEndpoint string
ChainConfig *params.ChainConfig ChainConfig *params.ChainConfig
DefaultSender *common.Address DefaultSender *common.Address
RPCGasCap *big.Int RPCGasCap *big.Int
Client *rpc.Client Client *rpc.Client
SupportStateDiff bool
} }
// NewConfig is used to initialize a watcher config from a .toml file // NewConfig is used to initialize a watcher config from a .toml file
@ -74,6 +76,7 @@ func NewConfig() (*Config, error) {
viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH) viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR) viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP) viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
c.DBConfig.Init() c.DBConfig.Init()
@ -83,6 +86,7 @@ func NewConfig() (*Config, error) {
return nil, err return nil, err
} }
c.Client = cli c.Client = cli
c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
wsPath := viper.GetString("server.wsPath") wsPath := viper.GetString("server.wsPath")
if wsPath == "" { if wsPath == "" {

View File

@ -80,6 +80,8 @@ type Service struct {
serveWg *sync.WaitGroup serveWg *sync.WaitGroup
// rpc client for forwarding cache misses // rpc client for forwarding cache misses
client *rpc.Client client *rpc.Client
// whether the proxied client supports state diffing
supportsStateDiffing bool
// backend for the server // backend for the server
backend *eth.Backend backend *eth.Backend
} }
@ -94,6 +96,8 @@ func NewServer(settings *Config) (Server, error) {
sap.QuitChan = make(chan bool) sap.QuitChan = make(chan bool)
sap.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription) sap.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
sap.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings) sap.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
sap.client = settings.Client
sap.supportsStateDiffing = settings.SupportStateDiff
var err error var err error
sap.backend, err = eth.NewEthBackend(sap.db, &eth.Config{ sap.backend, err = eth.NewEthBackend(sap.db, &eth.Config{
ChainConfig: settings.ChainConfig, ChainConfig: settings.ChainConfig,
@ -141,7 +145,7 @@ func (sap *Service) APIs() []rpc.API {
return append(apis, rpc.API{ return append(apis, rpc.API{
Namespace: eth.APIName, Namespace: eth.APIName,
Version: eth.APIVersion, Version: eth.APIVersion,
Service: eth.NewPublicEthAPI(sap.backend, sap.client), Service: eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing),
Public: true, Public: true,
}) })
} }

View File

@ -21,7 +21,7 @@ import "fmt"
const ( const (
Major = 0 // Major version component of the current release Major = 0 // Major version component of the current release
Minor = 3 // Minor version component of the current release Minor = 3 // Minor version component of the current release
Patch = 0 // Patch version component of the current release Patch = 1 // Patch version component of the current release
Meta = "alpha" // Version metadata to append to the version string Meta = "alpha" // Version metadata to append to the version string
) )