From b208281ad6f9c00d350b4d3b73ada1894a4b73f3 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 30 Oct 2020 11:54:22 -0500 Subject: [PATCH] optimize GetStorageAt; GetStorageAt unit tests --- pkg/eth/api.go | 10 +- pkg/eth/api_test.go | 135 ++++++++----- pkg/eth/backend.go | 28 +++ .../{eth_call_test.go => eth_state_test.go} | 66 ++++-- pkg/eth/ipld_retriever.go | 191 +++++++++++++++--- pkg/eth/test_helpers/chain_maker.go | 20 +- 6 files changed, 334 insertions(+), 116 deletions(-) rename pkg/eth/{eth_call_test.go => eth_state_test.go} (67%) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 3ad6401a..8be45588 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -619,13 +619,9 @@ func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Add // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { - state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state != nil && err == nil { - res := state.GetState(address, common.HexToHash(key)) - err = state.Error() - if err == nil { - return res[:], nil - } + storageVal, err := pea.B.GetStorageByNumberOrHash(ctx, address, common.HexToHash(key), blockNrOrHash) + if storageVal != nil && err == nil { + return storageVal, nil } if pea.rpc != nil { var res hexutil.Bytes diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index f7d052dd..f910c7e5 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -43,6 +43,7 @@ var ( randomAddr = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f") randomHash = crypto.Keccak256Hash(randomAddr.Bytes()) number = rpc.BlockNumber(test_helpers.BlockNumber.Int64()) + wrongNumber = rpc.BlockNumber(number + 1) blockHash = test_helpers.MockBlock.Header().Hash() ctx = context.Background() expectedBlock = map[string]interface{}{ @@ -177,18 +178,17 @@ var ( var _ = Describe("API", func() { var ( - db *postgres.DB - indexAndPublisher *eth2.IPLDPublisher - backend *eth.Backend - api *eth.PublicEthAPI + db *postgres.DB + api *eth.PublicEthAPI ) // Test db setup, rather than using BeforeEach we only need to setup once since the tests do not mutate the database - It("", func() { + // Note: if you focus one of the tests be sure to focus this and the defered It() + It("test init", func() { var err error db, err = shared.SetupDB() Expect(err).ToNot(HaveOccurred()) - indexAndPublisher = eth2.NewIPLDPublisher(db) - backend, err = eth.NewEthBackend(db, ð.Config{}) + indexAndPublisher := eth2.NewIPLDPublisher(db) + backend, err := eth.NewEthBackend(db, ð.Config{}) Expect(err).ToNot(HaveOccurred()) api = eth.NewPublicEthAPI(backend, nil) err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload) @@ -203,13 +203,13 @@ var _ = Describe("API", func() { expectedBlock["uncles"] = uncleHashes }) // Single test db tear down at end of all tests - defer It("", func() { eth.TearDownDB(db) }) + defer It("test teardown", func() { eth.TearDownDB(db) }) /* Headers and blocks */ - Describe("GetHeaderByNumber", func() { + Describe("eth_getHeaderByNumber", func() { It("Retrieves a header by number", func() { header, err := api.GetHeaderByNumber(ctx, number) Expect(err).ToNot(HaveOccurred()) @@ -217,7 +217,7 @@ var _ = Describe("API", func() { }) It("Throws an error if a header cannot be found", func() { - header, err := api.GetHeaderByNumber(ctx, rpc.BlockNumber(number+1)) + header, err := api.GetHeaderByNumber(ctx, wrongNumber) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) Expect(header).To(BeNil()) @@ -226,7 +226,19 @@ var _ = Describe("API", func() { }) }) - Describe("BlockNumber", func() { + Describe("eth_getHeaderByHash", func() { + It("Retrieves a header by hash", func() { + header := api.GetHeaderByHash(ctx, blockHash) + Expect(header).To(Equal(expectedHeader)) + }) + + It("Throws an error if a header cannot be found", func() { + header := api.GetHeaderByHash(ctx, randomHash) + Expect(header).To(BeNil()) + }) + }) + + Describe("eth_blockNumber", func() { It("Retrieves the head block number", func() { bn := api.BlockNumber() ubn := (uint64)(bn) @@ -235,9 +247,8 @@ var _ = Describe("API", func() { }) }) - Describe("GetBlockByNumber", func() { - It("Retrieves a block by number", func() { - // without full txs + Describe("eth_getBlockByNumber", func() { + It("Retrieves a block by number, without full txs", func() { block, err := api.GetBlockByNumber(ctx, number, false) Expect(err).ToNot(HaveOccurred()) transactionHashes := make([]interface{}, len(test_helpers.MockBlock.Transactions())) @@ -248,8 +259,9 @@ var _ = Describe("API", func() { for key, val := range expectedBlock { Expect(val).To(Equal(block[key])) } - // with full txs - block, err = api.GetBlockByNumber(ctx, number, true) + }) + It("Retrieves a block by number, with full txs", func() { + block, err := api.GetBlockByNumber(ctx, number, true) Expect(err).ToNot(HaveOccurred()) transactions := make([]interface{}, len(test_helpers.MockBlock.Transactions())) for i, trx := range test_helpers.MockBlock.Transactions() { @@ -260,11 +272,15 @@ var _ = Describe("API", func() { Expect(val).To(Equal(block[key])) } }) + It("Throws an error if a block cannot be found", func() { + _, err := api.GetBlockByNumber(ctx, wrongNumber, false) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) }) - Describe("GetBlockByHash", func() { - It("Retrieves a block by hash", func() { - // without full txs + Describe("eth_getBlockByHash", func() { + It("Retrieves a block by hash, without full txs", func() { block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), false) Expect(err).ToNot(HaveOccurred()) transactionHashes := make([]interface{}, len(test_helpers.MockBlock.Transactions())) @@ -275,8 +291,9 @@ var _ = Describe("API", func() { for key, val := range expectedBlock { Expect(val).To(Equal(block[key])) } - // with full txs - block, err = api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true) + }) + It("Retrieves a block by hash, with full txs", func() { + block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true) Expect(err).ToNot(HaveOccurred()) transactions := make([]interface{}, len(test_helpers.MockBlock.Transactions())) for i, trx := range test_helpers.MockBlock.Transactions() { @@ -287,6 +304,11 @@ var _ = Describe("API", func() { Expect(val).To(Equal(block[key])) } }) + It("Throws an error if a block cannot be found", func() { + _, err := api.GetBlockByHash(ctx, randomHash, false) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) }) /* @@ -295,7 +317,7 @@ var _ = Describe("API", func() { */ - Describe("GetUncleByBlockNumberAndIndex", func() { + Describe("eth_getUncleByBlockNumberAndIndex", func() { It("Retrieves the uncle at the provided index in the canoncial block with the provided hash", func() { uncle1, err := api.GetUncleByBlockNumberAndIndex(ctx, number, 0) Expect(err).ToNot(HaveOccurred()) @@ -304,9 +326,19 @@ var _ = Describe("API", func() { Expect(err).ToNot(HaveOccurred()) Expect(uncle2).To(Equal(expectedUncle2)) }) + It("Throws an error if an block for blocknumber cannot be found", func() { + _, err := api.GetUncleByBlockNumberAndIndex(ctx, wrongNumber, 0) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Returns `nil` if an uncle at the provided index does not exist for the block found for the provided block number", func() { + uncle, err := api.GetUncleByBlockNumberAndIndex(ctx, number, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(uncle).To(BeNil()) + }) }) - Describe("GetUncleByBlockHashAndIndex", func() { + Describe("eth_getUncleByBlockHashAndIndex", func() { It("Retrieves the uncle at the provided index in the block with the provided hash", func() { uncle1, err := api.GetUncleByBlockHashAndIndex(ctx, blockHash, 0) Expect(err).ToNot(HaveOccurred()) @@ -315,16 +347,26 @@ var _ = Describe("API", func() { Expect(err).ToNot(HaveOccurred()) Expect(uncle2).To(Equal(expectedUncle2)) }) + It("Throws an error if an block for blockhash cannot be found", func() { + _, err := api.GetUncleByBlockHashAndIndex(ctx, randomHash, 0) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Returns `nil` if an uncle at the provided index does not exist for the block with the provided hash", func() { + uncle, err := api.GetUncleByBlockHashAndIndex(ctx, blockHash, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(uncle).To(BeNil()) + }) }) - Describe("GetUncleCountByBlockNumber", func() { + Describe("eth_getUncleCountByBlockNumber", func() { It("Retrieves the number of uncles for the canonical block with the provided number", func() { count := api.GetUncleCountByBlockNumber(ctx, number) Expect(uint64(*count)).To(Equal(uint64(2))) }) }) - Describe("GetUncleCountByBlockHash", func() { + Describe("eth_getUncleCountByBlockHash", func() { It("Retrieves the number of uncles for the block with the provided hash", func() { count := api.GetUncleCountByBlockHash(ctx, blockHash) Expect(uint64(*count)).To(Equal(uint64(2))) @@ -337,7 +379,7 @@ var _ = Describe("API", func() { */ - Describe("GetTransactionCount", func() { + Describe("eth_getTransactionCount", func() { It("Retrieves the number of transactions the given address has sent for the given block number", func() { count, err := api.GetTransactionCount(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithNumber(number)) Expect(err).ToNot(HaveOccurred()) @@ -358,21 +400,21 @@ var _ = Describe("API", func() { }) }) - Describe("GetBlockTransactionCountByNumber", func() { + Describe("eth_getBlockTransactionCountByNumber", func() { It("Retrieves the number of transactions in the canonical block with the provided number", func() { count := api.GetBlockTransactionCountByNumber(ctx, number) Expect(uint64(*count)).To(Equal(uint64(3))) }) }) - Describe("GetBlockTransactionCountByHash", func() { + Describe("eth_getBlockTransactionCountByHash", func() { It("Retrieves the number of transactions in the block with the provided hash ", func() { count := api.GetBlockTransactionCountByHash(ctx, blockHash) Expect(uint64(*count)).To(Equal(uint64(3))) }) }) - Describe("GetTransactionByBlockNumberAndIndex", func() { + Describe("eth_getTransactionByBlockNumberAndIndex", func() { It("Retrieves the tx with the provided index in the canonical block with the provided block number", func() { tx := api.GetTransactionByBlockNumberAndIndex(ctx, number, 0) Expect(tx).ToNot(BeNil()) @@ -388,7 +430,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetTransactionByBlockHashAndIndex", func() { + Describe("eth_getTransactionByBlockHashAndIndex", func() { It("Retrieves the tx with the provided index in the block with the provided hash", func() { tx := api.GetTransactionByBlockHashAndIndex(ctx, blockHash, 0) Expect(tx).ToNot(BeNil()) @@ -404,7 +446,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetRawTransactionByBlockNumberAndIndex", func() { + Describe("eth_getRawTransactionByBlockNumberAndIndex", func() { It("Retrieves the raw tx with the provided index in the canonical block with the provided block number", func() { tx := api.GetRawTransactionByBlockNumberAndIndex(ctx, number, 0) Expect(tx).ToNot(BeNil()) @@ -420,7 +462,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetRawTransactionByBlockHashAndIndex", func() { + Describe("eth_getRawTransactionByBlockHashAndIndex", func() { It("Retrieves the raw tx with the provided index in the block with the provided hash", func() { tx := api.GetRawTransactionByBlockHashAndIndex(ctx, blockHash, 0) Expect(tx).ToNot(BeNil()) @@ -436,7 +478,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetTransactionByHash", func() { + Describe("eth_getTransactionByHash", func() { It("Retrieves a transaction by hash", func() { hash := test_helpers.MockTransactions[0].Hash() tx, err := api.GetTransactionByHash(ctx, hash) @@ -459,7 +501,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetRawTransactionByHash", func() { + Describe("eth_getRawTransactionByHash", func() { It("Retrieves a raw transaction by hash", func() { hash := test_helpers.MockTransactions[0].Hash() tx, err := api.GetRawTransactionByHash(ctx, hash) @@ -488,7 +530,7 @@ var _ = Describe("API", func() { */ - Describe("GetTransactionReceipt", func() { + Describe("eth_getTransactionReceipt", func() { It("Retrieves a receipt by tx hash", func() { hash := test_helpers.MockTransactions[0].Hash() rct, err := api.GetTransactionReceipt(ctx, hash) @@ -511,7 +553,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetLogs", func() { + Describe("eth_getLogs", func() { It("Retrieves receipt logs that match the provided topics within the provided range", func() { crit := ethereum.FilterQuery{ Topics: [][]common.Hash{ @@ -888,7 +930,7 @@ var _ = Describe("API", func() { */ - Describe("GetBalance", func() { + Describe("eth_getBalance", func() { It("Retrieves the eth balance for the provided account address at the block with the provided number", func() { bal, err := api.GetBalance(ctx, test_helpers.AccountAddresss, rpc.BlockNumberOrHashWithNumber(number)) Expect(err).ToNot(HaveOccurred()) @@ -913,17 +955,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetStorageAt", func() { - It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() { - /* - val, err := api.GetStorageAt(ctx, test_helpers.ContractAddress, common.Bytes2Hex(test_helpers.StorageLeafKey), rpc.BlockNumberOrHashWithNumber(number)) - Expect(err).ToNot(HaveOccurred()) - Expect(val).To(Equal((hexutil.Bytes)(test_helpers.StorageValue))) - */ - }) - }) - - Describe("GetCode", func() { + Describe("eth_getCode", func() { It("Retrieves the code for the provided contract address at the block with the provided number", func() { code, err := api.GetCode(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithNumber(number)) Expect(err).ToNot(HaveOccurred()) @@ -938,13 +970,6 @@ var _ = Describe("API", func() { _, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blockHash, true)) Expect(err).To(HaveOccurred()) }) - - }) - - Describe("GetProof", func() { - It("Retrieves the Merkle-proof for a given account and optionally some storage keys at the block with the provided hash or number", func() { - - }) }) }) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index dc68cd32..0cb4c52c 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -22,6 +22,8 @@ import ( "fmt" "math/big" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus" @@ -608,6 +610,32 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has return code, err } +// GetStorageByNumberOrHash returns the storage value for the provided contract address an storage key at the block corresponding to the provided number or hash +func (b *Backend) GetStorageByNumberOrHash(ctx context.Context, address common.Address, storageLeafKey common.Hash, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.GetStorageByNumber(ctx, address, storageLeafKey, uint64(blockNr.Int64())) + } + if hash, ok := blockNrOrHash.Hash(); ok { + return b.GetStorageByHash(ctx, address, storageLeafKey, hash) + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +// GetStorageByNumber returns the storage value for the provided contract address an storage key at the block corresponding to the provided number +func (b *Backend) GetStorageByNumber(ctx context.Context, address common.Address, storageLeafKey common.Hash, number uint64) (hexutil.Bytes, error) { + hash := b.GetCanonicalHash(number) + if hash == (common.Hash{}) { + return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number) + } + return b.GetStorageByHash(ctx, address, storageLeafKey, hash) +} + +// GetStorageByHash returns the storage value for the provided contract address an storage key at the block corresponding to the provided hash +func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, storageLeafKey, hash common.Hash) (hexutil.Bytes, error) { + _, storageRlp, err := b.IPLDRetriever.RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address, storageLeafKey, hash) + return storageRlp, err +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/eth_call_test.go b/pkg/eth/eth_state_test.go similarity index 67% rename from pkg/eth/eth_call_test.go rename to pkg/eth/eth_state_test.go index c5360d16..6709764d 100644 --- a/pkg/eth/eth_call_test.go +++ b/pkg/eth/eth_state_test.go @@ -59,28 +59,23 @@ func init() { } } -var _ = Describe("eth_call", func() { +var _ = Describe("eth state reading tests", func() { var ( blocks []*types.Block receipts []types.Receipts chain *core.BlockChain db *postgres.DB - transformer *eth2.StateDiffTransformer - backend *eth.Backend api *eth.PublicEthAPI - builder statediff.Builder - pams statediff.Params chainConfig = params.TestChainConfig mockTD = big.NewInt(1337) ) - - BeforeEach(func() { + It("test init", func() { // db and type initializations var err error db, err = shared.SetupDB() Expect(err).ToNot(HaveOccurred()) - transformer = eth2.NewStateDiffTransformer(chainConfig, db) - backend, err = eth.NewEthBackend(db, ð.Config{ + transformer := eth2.NewStateDiffTransformer(chainConfig, db) + backend, err := eth.NewEthBackend(db, ð.Config{ ChainConfig: chainConfig, VmConfig: vm.Config{}, RPCGasCap: big.NewInt(10000000000), @@ -90,12 +85,12 @@ var _ = Describe("eth_call", func() { // make the test blockchain (and state) blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen) - pams = statediff.Params{ + params := statediff.Params{ IntermediateStateNodes: true, IntermediateStorageNodes: true, } // iterate over the blocks, generating statediff payloads, and transforming the data into Postgres - builder = statediff.NewBuilder(chain.StateCache()) + builder := statediff.NewBuilder(chain.StateCache()) for i, block := range blocks { var args statediff.Args var rcts types.Receipts @@ -115,7 +110,7 @@ var _ = Describe("eth_call", func() { } rcts = receipts[i-1] } - diff, err := builder.BuildStateDiffObject(args, pams) + diff, err := builder.BuildStateDiffObject(args, params) Expect(err).ToNot(HaveOccurred()) diffRlp, err := rlp.EncodeToBytes(diff) Expect(err).ToNot(HaveOccurred()) @@ -133,10 +128,11 @@ var _ = Describe("eth_call", func() { Expect(err).ToNot(HaveOccurred()) } }) - AfterEach(func() { + defer It("test teardown", func() { eth.TearDownDB(db) chain.Stop() }) + Describe("eth_call", func() { It("Applies call args (tx data) on top of state, returning the result (e.g. a Getter method call)", func() { data, err := parsedABI.Pack("data") @@ -146,7 +142,7 @@ var _ = Describe("eth_call", func() { To: &test_helpers.ContractAddr, Data: &bdata, } - // Before contract deployment + // Before contract deployment, returns nil res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(0), nil) Expect(err).ToNot(HaveOccurred()) Expect(res).To(BeNil()) @@ -177,4 +173,46 @@ var _ = Describe("eth_call", func() { Expect(res).To(Equal(expectedRes)) }) }) + + Describe("eth_getStorageAt", func() { + It("Throws an error if it tries to access a contract which does not exist", func() { + _, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + _, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Throws an error if it tries to access a contract slot which does not exist", func() { + _, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() { + // After deployment + val, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(2)) + Expect(err).ToNot(HaveOccurred()) + expectedRes := hexutil.Bytes(common.Hex2Bytes("01")) + Expect(val).To(Equal(expectedRes)) + + val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(3)) + Expect(err).ToNot(HaveOccurred()) + expectedRes = hexutil.Bytes(common.Hex2Bytes("03")) + Expect(val).To(Equal(expectedRes)) + + val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(4)) + Expect(err).ToNot(HaveOccurred()) + expectedRes = hexutil.Bytes(common.Hex2Bytes("09")) + Expect(val).To(Equal(expectedRes)) + + val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(5)) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(hexutil.Bytes{})) + }) + }) + + Describe("eth_getProof", func() { + + }) }) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index a5c2a1aa..057fd6f9 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -19,6 +19,8 @@ package eth import ( "fmt" + "github.com/vulcanize/ipld-eth-server/pkg/shared" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" @@ -28,62 +30,78 @@ import ( ) const ( - RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids + RetrieveIPLDpgStr = `SELECT data + FROM public.blocks + WHERE key = $1` + RetrieveHeadersByHashesPgStr = `SELECT cid, data + FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_number = $1` - RetrieveHeaderByHashPgStr = `SELECT cid, data FROM eth.header_cids + RetrieveHeaderByHashPgStr = `SELECT cid, data + FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_hash = $1` - RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids + RetrieveUnclesByHashesPgStr = `SELECT cid, data + FROM eth.uncle_cids INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks WHERE uncle_cids.header_id = header_cids.id AND uncle_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks + RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data + FROM eth.uncle_cids, eth.header_cids, public.blocks WHERE uncle_cids.header_id = header_cids.id AND uncle_cids.mh_key = blocks.key AND block_number = $1` - RetrieveUncleByHashPgStr = `SELECT cid, data FROM eth.uncle_cids + RetrieveUncleByHashPgStr = `SELECT cid, data + FROM eth.uncle_cids INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = $1` - RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids + RetrieveTransactionsByHashesPgStr = `SELECT cid, data + FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks + RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data + FROM eth.transaction_cids, eth.header_cids, public.blocks WHERE transaction_cids.header_id = header_cids.id AND transaction_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks + RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data + FROM eth.transaction_cids, eth.header_cids, public.blocks WHERE transaction_cids.header_id = header_cids.id AND transaction_cids.mh_key = blocks.key AND block_number = $1` RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = $1` - RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks + RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data + FROM eth.receipt_cids, eth.transaction_cids, public.blocks WHERE receipt_cids.mh_key = blocks.key AND receipt_cids.tx_id = transaction_cids.id AND tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks + RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data + FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id AND receipt_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks + RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data + FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id AND receipt_cids.mh_key = blocks.key AND block_number = $1` - RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids + RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data + FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids WHERE receipt_cids.mh_key = blocks.key AND receipt_cids.tx_id = transaction_cids.id AND tx_hash = $1` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data + FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 @@ -93,13 +111,59 @@ const ( AND header_cids.id = (SELECT canonical_header(block_number)) ORDER BY block_number DESC LIMIT 1` - RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks + RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data + FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 AND block_number <= $2 ORDER BY block_number DESC LIMIT 1` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data + FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND storage_cids.mh_key = blocks.key + AND state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= $3 + ORDER BY block_number DESC + LIMIT 1` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data + FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND storage_cids.mh_key = blocks.key + AND state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` + retrieveStorageInfoPgStr = `SELECT storage_cids.cid, data, storage_path, block_number + FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND storage_cids.mh_key = blocks.key + AND state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` + wasNodeDeletedpgStr = `SELECT exists(SELECT * + FROM eth.storage_cids, eth.state_cids, eth.header_cids + WHERE storage_cids.state_id = state_cids.id + AND storage_path = $1 + AND block_number > $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND storage_cids.node_type = 3)` ) type ipldResult struct { @@ -323,6 +387,7 @@ func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, } // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash +// TODO: ensure this handles deleted accounts appropriately func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) { accountResult := new(ipldResult) leafKey := crypto.Keccak256Hash(address.Bytes()) @@ -340,25 +405,87 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr } // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number -// This can return multiple results if we have two versions of state in the database at the provided height -func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) ([]string, [][]byte, error) { - accountResults := make([]ipldResult, 0) +// This can return a non-canonical account +func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) (string, []byte, error) { + accountResult := new(ipldResult) leafKey := crypto.Keccak256Hash(address.Bytes()) - if err := r.db.Get(&accountResults, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil { - return nil, nil, err + if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil { + return "", nil, err } - cids := make([]string, len(accountResults)) - accounts := make([][]byte, len(accountResults)) - for i, res := range accountResults { - cids[i] = res.CID - var iface []interface{} - if err := rlp.DecodeBytes(res.Data, &iface); err != nil { - return nil, nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) - } - if len(iface) != 2 { - return nil, nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements") - } - accounts[i] = iface[1].([]byte) + var i []interface{} + if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { + return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) } - return cids, accounts, nil + if len(i) != 2 { + return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements") + } + return accountResult.CID, i[1].([]byte), nil +} + +type storageInfo struct { + CID string `db:"cid"` + Data []byte `db:"data"` + Path []byte `db:"storage_path"` + BlockNumber uint64 `db:"block_number"` +} + +// RetrieveStorageAtByAddressAndStorageKeyAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block hash +func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address common.Address, storageLeafKey, hash common.Hash) (string, []byte, error) { + // Begin tx + tx, err := r.db.Beginx() + if err != nil { + return "", nil, err + } + defer func() { + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else if err != nil { + shared.Rollback(tx) + } else { + err = tx.Commit() + } + }() + + storageResult := new(storageInfo) + stateLeafKey := crypto.Keccak256Hash(address.Bytes()) + if err := tx.Get(storageResult, retrieveStorageInfoPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil { + return "", nil, err + } + + deleted := false + if err := tx.Get(&deleted, wasNodeDeletedpgStr, storageResult.Path, storageResult.BlockNumber, hash.Hex()); err != nil { + return "", nil, err + } + if deleted { + return "", []byte{}, nil + } + var i []interface{} + if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { + err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error()) + return "", nil, err + } + if len(i) != 2 { + err = fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") + return "", nil, err + } + return storageResult.CID, i[1].([]byte), err +} + +// RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block number +// This can retrun a non-canonical value +func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(address common.Address, storageLeafKey common.Hash, number uint64) (string, []byte, error) { + storageResult := new(ipldResult) + stateLeafKey := crypto.Keccak256Hash(address.Bytes()) + if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil { + return "", nil, err + } + var i []interface{} + if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { + return "", nil, fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") + } + return storageResult.CID, i[1].([]byte), nil } diff --git a/pkg/eth/test_helpers/chain_maker.go b/pkg/eth/test_helpers/chain_maker.go index 2e587a9d..e694ce67 100644 --- a/pkg/eth/test_helpers/chain_maker.go +++ b/pkg/eth/test_helpers/chain_maker.go @@ -37,12 +37,16 @@ var ( TestBankFunds = big.NewInt(100000000) Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds) - Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 - Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e - ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032") - ContractAddr common.Address + Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032") + ContractAddr common.Address + IndexZero = "0000000000000000000000000000000000000000000000000000000000000000" + IndexOne = "0000000000000000000000000000000000000000000000000000000000000001" + ContractSlotPosition = common.FromHex(IndexOne) + ContractSlotKeyHash = crypto.Keccak256Hash(ContractSlotPosition) ) /* test function signatures @@ -88,8 +92,8 @@ func TestChainGen(i int, block *core.BlockGen) { case 3: block.SetCoinbase(Account2Addr) data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000009") - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) - block.AddTx(tx1) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + block.AddTx(tx) case 4: block.SetCoinbase(Account1Addr) data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000000")