From 70c539c8d7f21ecbbbfff0cf3b280e62fa76cfdc Mon Sep 17 00:00:00 2001 From: n0cte Date: Thu, 19 Aug 2021 11:47:42 +0300 Subject: [PATCH 01/14] fix encoding when storage is empty --- pkg/eth/api.go | 3 +++ pkg/graphql/graphql.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index e9e9c922..2c0ca3ab 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -709,6 +709,9 @@ func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Add func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { storageVal, err := pea.B.GetStorageByNumberOrHash(ctx, address, common.HexToHash(key), blockNrOrHash) if storageVal != nil && err == nil { + if len(storageVal) == 0 { + return make([]byte, 32), nil + } var value common.Hash _, content, _, err := rlp.Split(storageVal) if err == io.ErrUnexpectedEOF { diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index d12838e2..561abc29 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -1010,6 +1010,9 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct { return nil, err } + if len(rlpValue) == 0 { + return &StorageResult{value: make([]byte, 32), cid: cid, ipldBlock: ipldBlock}, nil + } var value interface{} err = rlp.DecodeBytes(rlpValue, &value) From bca33381dd835d97823c634092130664888a44bd Mon Sep 17 00:00:00 2001 From: n0cte Date: Thu, 19 Aug 2021 13:12:23 +0300 Subject: [PATCH 02/14] fix value checking for function GetStorageAt --- pkg/eth/eth_state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 8ee4a12f..89099c5a 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -506,7 +506,7 @@ var _ = Describe("eth state reading tests", func() { val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(5)) Expect(err).ToNot(HaveOccurred()) - Expect(val).To(Equal(hexutil.Bytes{})) + Expect(val).To(Equal(hexutil.Bytes(make([]byte, 32)))) }) It("Throws an error for a non-existing block hash", func() { _, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithHash(randomHash, true)) From 3a216b2ca33b36f8c8a19e09ac014b6cf1e30237 Mon Sep 17 00:00:00 2001 From: n0cte Date: Wed, 1 Sep 2021 02:10:42 +0300 Subject: [PATCH 03/14] add data length check --- pkg/eth/ipld_retriever.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 38f1b0ae..03be27c1 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -438,9 +438,11 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil { return "", nil, err } + if accountResult.NodeType == removedNode { return "", []byte{}, nil } + var i []interface{} if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) @@ -459,9 +461,11 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil { return "", nil, err } + if accountResult.NodeType == removedNode { return "", []byte{}, nil } + var i []interface{} if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) @@ -502,6 +506,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil { return "", nil, err } + if storageResult.NodeType == removedNode { return "", []byte{}, nil } From 1057b001f1b06e656ad2502a8d1086311bd8c7b8 Mon Sep 17 00:00:00 2001 From: n0cte Date: Mon, 6 Sep 2021 17:44:06 +0300 Subject: [PATCH 04/14] add test. comparison storages after contract destruction --- test/contract/contracts/GLDToken.sol | 3 +++ test/contract/src/index.js | 9 +++++++++ test/helper.go | 19 +++++++++++++++++++ test/integration_test.go | 24 ++++++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/test/contract/contracts/GLDToken.sol b/test/contract/contracts/GLDToken.sol index 4cd3205f..596baad3 100644 --- a/test/contract/contracts/GLDToken.sol +++ b/test/contract/contracts/GLDToken.sol @@ -4,4 +4,7 @@ contract GLDToken is ERC20 { constructor() ERC20("Gold", "GLD") { _mint(msg.sender, 1000000000000000000000); } + function destroy() public { + selfdestruct(payable(msg.sender)); + } } diff --git a/test/contract/src/index.js b/test/contract/src/index.js index 37639542..b5f865de 100644 --- a/test/contract/src/index.js +++ b/test/contract/src/index.js @@ -22,6 +22,15 @@ fastify.get('/v1/deployContract', async (req, reply) => { } }); +fastify.get('/v1/destoyContract', async (req, reply) => { + const addr = req.query.addr; + + const Token = await hre.ethers.getContractFactory("GLDToken"); + const token = await Token.attach(addr); + + return token.destroy(); +}) + fastify.get('/v1/sendEth', async (req, reply) => { const to = req.query.to; const value = req.query.value; diff --git a/test/helper.go b/test/helper.go index e458bfde..a79b2679 100644 --- a/test/helper.go +++ b/test/helper.go @@ -14,6 +14,12 @@ type ContractDeployed struct { BlockHash string `json:"blockHash"` } +type ContractDestoyed struct { + TransactionHash string `json:"txHash"` + BlockNumber int64 `json:"blockNumber"` + BlockHash string `json:"blockHash"` +} + type Tx struct { From string `json:"from"` To string `json:"to"` @@ -43,6 +49,19 @@ func DeployContract() (*ContractDeployed, error) { return &contract, nil } +func DestoyContract(addr string) (*ContractDestoyed, error) { + res, err := http.Get(fmt.Sprintf("%s/v1/destoyContract?addr=%s", srvUrl, addr)) + if err != nil { + return nil, err + } + defer res.Body.Close() + + var data ContractDestoyed + decoder := json.NewDecoder(res.Body) + + return &data, decoder.Decode(&data) +} + func SendEth(to string, value string) (*Tx, error) { res, err := http.Get(fmt.Sprintf("%s/v1/sendEth?to=%s&value=%s", srvUrl, to, value)) if err != nil { diff --git a/test/integration_test.go b/test/integration_test.go index 43cb8626..38f1d73f 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -391,6 +391,30 @@ var _ = Describe("Integration test", func() { Expect(err).To(MatchError("header not found")) Expect(gethStorage).To(Equal(ipldStorage)) }) + + It("get storage after selfdestruct", func() { + totalSupplyIndex := "0x2" + zeroHash := make([]byte, 32) + + tx, err := integration.DestoyContract(contract.Address) + Expect(err).ToNot(HaveOccurred()) + + gethStorage1, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1)) + Expect(err).ToNot(HaveOccurred()) + gethStorage2, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber)) + Expect(err).ToNot(HaveOccurred()) + + Expect(gethStorage1).NotTo(Equal(gethStorage2)) + Expect(gethStorage2).To(Equal(zeroHash)) + + ipldStorage1, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1)) + Expect(err).ToNot(HaveOccurred()) + ipldStorage2, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber)) + Expect(err).ToNot(HaveOccurred()) + + Expect(ipldStorage1).To(Equal(gethStorage1)) + Expect(ipldStorage2).To(Equal(gethStorage2)) + }) }) Describe("eth call", func() { From 54db8f23e07488805d416406c93934d4eeba9b72 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Wed, 22 Sep 2021 16:14:35 +0530 Subject: [PATCH 05/14] Refactor the code --- pkg/eth/api.go | 3 --- pkg/eth/eth_state_test.go | 8 ++++---- pkg/eth/ipld_retriever.go | 10 ++++++---- pkg/graphql/graphql.go | 7 +++++-- test/integration_test.go | 8 ++++---- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 2c0ca3ab..e9e9c922 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -709,9 +709,6 @@ func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Add func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { storageVal, err := pea.B.GetStorageByNumberOrHash(ctx, address, common.HexToHash(key), blockNrOrHash) if storageVal != nil && err == nil { - if len(storageVal) == 0 { - return make([]byte, 32), nil - } var value common.Hash _, content, _, err := rlp.Split(storageVal) if err == io.ErrUnexpectedEOF { diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 89099c5a..6ff70c25 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -476,16 +476,16 @@ var _ = Describe("eth state reading tests", func() { It("Returns empty slice if it tries to access a contract which does not exist", func() { storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0)) Expect(err).NotTo(HaveOccurred()) - Expect(storage).To(Equal(hexutil.Bytes(make([]byte, 32)))) + Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue))) storage, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1)) Expect(err).NotTo(HaveOccurred()) - Expect(storage).To(Equal(hexutil.Bytes(make([]byte, 32)))) + Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue))) }) It("Returns empty slice if it tries to access a contract slot which does not exist", func() { storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2)) Expect(err).NotTo(HaveOccurred()) - Expect(storage).To(Equal(hexutil.Bytes(make([]byte, 32)))) + Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue))) }) It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() { // After deployment @@ -506,7 +506,7 @@ var _ = Describe("eth state reading tests", func() { val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(5)) Expect(err).ToNot(HaveOccurred()) - Expect(val).To(Equal(hexutil.Bytes(make([]byte, 32)))) + Expect(val).To(Equal(hexutil.Bytes(eth.EmptyNodeValue))) }) It("Throws an error for a non-existing block hash", func() { _, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithHash(randomHash, true)) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 03be27c1..b28fb9dc 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -152,6 +152,8 @@ const ( LIMIT 1` ) +var EmptyNodeValue = make([]byte, common.HashLength) + type rctIpldResult struct { LeafCID string `db:"leaf_cid"` Data []byte `db:"data"` @@ -440,7 +442,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr } if accountResult.NodeType == removedNode { - return "", []byte{}, nil + return "", EmptyNodeValue, nil } var i []interface{} @@ -463,7 +465,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad } if accountResult.NodeType == removedNode { - return "", []byte{}, nil + return "", EmptyNodeValue, nil } var i []interface{} @@ -485,7 +487,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add return "", nil, nil, err } if storageResult.NodeType == removedNode { - return "", []byte{}, []byte{}, nil + return "", EmptyNodeValue, EmptyNodeValue, nil } var i []interface{} if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { @@ -508,7 +510,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad } if storageResult.NodeType == removedNode { - return "", []byte{}, nil + return "", EmptyNodeValue, nil } var i []interface{} if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index 561abc29..1f36ef33 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -18,6 +18,7 @@ package graphql import ( + "bytes" "context" "database/sql" "errors" @@ -31,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" + "github.com/vulcanize/ipld-eth-server/pkg/eth" ) @@ -1010,8 +1012,9 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct { return nil, err } - if len(rlpValue) == 0 { - return &StorageResult{value: make([]byte, 32), cid: cid, ipldBlock: ipldBlock}, nil + + if bytes.Compare(rlpValue, eth.EmptyNodeValue) == 0 { + return &StorageResult{value: eth.EmptyNodeValue, cid: cid, ipldBlock: ipldBlock}, nil } var value interface{} diff --git a/test/integration_test.go b/test/integration_test.go index 38f1d73f..f2663568 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -9,12 +9,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rlp" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - integration "github.com/vulcanize/ipld-eth-server/test" - "github.com/ethereum/go-ethereum/ethclient" + "github.com/vulcanize/ipld-eth-server/pkg/eth" + integration "github.com/vulcanize/ipld-eth-server/test" ) const nonExistingBlockHash = "0x111111111111111111111111111111111111111111111111111111111111111" @@ -394,7 +395,6 @@ var _ = Describe("Integration test", func() { It("get storage after selfdestruct", func() { totalSupplyIndex := "0x2" - zeroHash := make([]byte, 32) tx, err := integration.DestoyContract(contract.Address) Expect(err).ToNot(HaveOccurred()) @@ -405,7 +405,7 @@ var _ = Describe("Integration test", func() { Expect(err).ToNot(HaveOccurred()) Expect(gethStorage1).NotTo(Equal(gethStorage2)) - Expect(gethStorage2).To(Equal(zeroHash)) + Expect(gethStorage2).To(Equal(eth.EmptyNodeValue)) ipldStorage1, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1)) Expect(err).ToNot(HaveOccurred()) From 81546618d3b7dc9a1944d46f35d6fbda7911e0cb Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Wed, 22 Sep 2021 16:28:57 +0530 Subject: [PATCH 06/14] Add sleep interval. --- test/integration_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration_test.go b/test/integration_test.go index f2663568..354972fc 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -399,6 +399,8 @@ var _ = Describe("Integration test", func() { tx, err := integration.DestoyContract(contract.Address) Expect(err).ToNot(HaveOccurred()) + time.Sleep(sleepInterval) + gethStorage1, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1)) Expect(err).ToNot(HaveOccurred()) gethStorage2, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber)) From f6a6a294bbaa63d4f96438b0de6bba511e3dd2df Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Wed, 22 Sep 2021 16:30:10 +0530 Subject: [PATCH 07/14] Fix typo. --- test/contract/src/index.js | 2 +- test/helper.go | 8 ++++---- test/integration_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/contract/src/index.js b/test/contract/src/index.js index b5f865de..6a15b704 100644 --- a/test/contract/src/index.js +++ b/test/contract/src/index.js @@ -22,7 +22,7 @@ fastify.get('/v1/deployContract', async (req, reply) => { } }); -fastify.get('/v1/destoyContract', async (req, reply) => { +fastify.get('/v1/destroyContract', async (req, reply) => { const addr = req.query.addr; const Token = await hre.ethers.getContractFactory("GLDToken"); diff --git a/test/helper.go b/test/helper.go index a79b2679..c17f4283 100644 --- a/test/helper.go +++ b/test/helper.go @@ -14,7 +14,7 @@ type ContractDeployed struct { BlockHash string `json:"blockHash"` } -type ContractDestoyed struct { +type ContractDestroyed struct { TransactionHash string `json:"txHash"` BlockNumber int64 `json:"blockNumber"` BlockHash string `json:"blockHash"` @@ -49,14 +49,14 @@ func DeployContract() (*ContractDeployed, error) { return &contract, nil } -func DestoyContract(addr string) (*ContractDestoyed, error) { - res, err := http.Get(fmt.Sprintf("%s/v1/destoyContract?addr=%s", srvUrl, addr)) +func DestroyContract(addr string) (*ContractDestroyed, error) { + res, err := http.Get(fmt.Sprintf("%s/v1/destroyContract?addr=%s", srvUrl, addr)) if err != nil { return nil, err } defer res.Body.Close() - var data ContractDestoyed + var data ContractDestroyed decoder := json.NewDecoder(res.Body) return &data, decoder.Decode(&data) diff --git a/test/integration_test.go b/test/integration_test.go index 354972fc..609f04d4 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -396,7 +396,7 @@ var _ = Describe("Integration test", func() { It("get storage after selfdestruct", func() { totalSupplyIndex := "0x2" - tx, err := integration.DestoyContract(contract.Address) + tx, err := integration.DestroyContract(contract.Address) Expect(err).ToNot(HaveOccurred()) time.Sleep(sleepInterval) From a1781b1eeb224fdde46202458294d7c2d937ce82 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Wed, 22 Sep 2021 18:23:11 +0530 Subject: [PATCH 08/14] Get the block number after the contract is destroyed. --- test/contract/src/index.js | 7 ++++++- test/integration_test.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/contract/src/index.js b/test/contract/src/index.js index 6a15b704..6339a1d9 100644 --- a/test/contract/src/index.js +++ b/test/contract/src/index.js @@ -1,6 +1,7 @@ const fastify = require('fastify')({ logger: true }); const hre = require("hardhat"); + // readiness check fastify.get('/v1/healthz', async (req, reply) => { reply @@ -28,7 +29,11 @@ fastify.get('/v1/destroyContract', async (req, reply) => { const Token = await hre.ethers.getContractFactory("GLDToken"); const token = await Token.attach(addr); - return token.destroy(); + await token.destroy(); + const blockNum = await hre.ethers.provider.getBlockNumber() + return { + blockNumber: blockNum, + } }) fastify.get('/v1/sendEth', async (req, reply) => { diff --git a/test/integration_test.go b/test/integration_test.go index 609f04d4..8e7a8c26 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -393,7 +393,7 @@ var _ = Describe("Integration test", func() { Expect(gethStorage).To(Equal(ipldStorage)) }) - It("get storage after selfdestruct", func() { + It("get storage after self destruct", func() { totalSupplyIndex := "0x2" tx, err := integration.DestroyContract(contract.Address) From f09f665b11245b5ea4002aea6223a76ee2205740 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 23 Sep 2021 16:06:09 +0530 Subject: [PATCH 09/14] Fix empty value when contract is deleted. --- db/migrations/00014_create_stored_functions.sql | 17 +++++++++++++++++ docker-compose.yml | 1 + pkg/eth/ipld_retriever.go | 15 ++++++++------- scripts/run_intregration_test.sh | 6 +++--- test/contract/src/index.js | 4 ++++ test/helper.go | 5 ++--- test/integration_test.go | 15 ++++++++++++++- 7 files changed, 49 insertions(+), 14 deletions(-) diff --git a/db/migrations/00014_create_stored_functions.sql b/db/migrations/00014_create_stored_functions.sql index 1e9ad468..f5457b8b 100644 --- a/db/migrations/00014_create_stored_functions.sql +++ b/db/migrations/00014_create_stored_functions.sql @@ -1,4 +1,20 @@ -- +goose Up + + +-- +goose StatementBegin +-- returns if a storage node at the provided path was removed in the range >= the provided height and <= the provided block hash +CREATE OR REPLACE FUNCTION was_state_leaf_removed(state_leaf_key BYTEA, block_num BIGINT) RETURNS BOOLEAN +AS $$ +SELECT exists(SELECT 1 + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_leaf_key = state_leaf_key + AND block_number <= block_num + AND state_cids.node_type = 3 + LIMIT 1); +$$ LANGUAGE SQL; +-- +goose StatementEnd + -- +goose StatementBegin CREATE TYPE child_result AS ( has_child BOOLEAN, @@ -115,6 +131,7 @@ LANGUAGE 'plpgsql'; -- +goose StatementEnd -- +goose Down +DROP FUNCTION was_state_leaf_removed; DROP FUNCTION canonical_header_id; DROP FUNCTION canonical_header_from_array; DROP FUNCTION has_child; diff --git a/docker-compose.yml b/docker-compose.yml index c24619bc..2ef02cfd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,7 @@ services: - vdb_db_eth_server:/var/lib/postgresql/data ports: - "127.0.0.1:8077:5432" + command: ["postgres", "-c", "log_statement=all"] eth-server: restart: unless-stopped diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index b28fb9dc..7c82e3ba 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -127,7 +127,7 @@ const ( AND block_number <= $2 ORDER BY block_number DESC LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed(state_leaf_key, block_number) AS state_leaf_removed FROM eth.storage_cids INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) @@ -137,7 +137,7 @@ const ( AND block_number <= $3 ORDER BY block_number DESC LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed(state_leaf_key, block_number) AS state_leaf_removed FROM eth.storage_cids INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) @@ -427,9 +427,10 @@ func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, } type nodeInfo struct { - CID string `db:"cid"` - Data []byte `db:"data"` - NodeType int `db:"node_type"` + CID string `db:"cid"` + Data []byte `db:"data"` + NodeType int `db:"node_type"` + StateLeafRemoved bool `db:"state_leaf_removed"` } // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash @@ -486,7 +487,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { return "", nil, nil, err } - if storageResult.NodeType == removedNode { + if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode { return "", EmptyNodeValue, EmptyNodeValue, nil } var i []interface{} @@ -509,7 +510,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad return "", nil, err } - if storageResult.NodeType == removedNode { + if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode { return "", EmptyNodeValue, nil } var i []interface{} diff --git a/scripts/run_intregration_test.sh b/scripts/run_intregration_test.sh index f9056dc4..c81ca89b 100755 --- a/scripts/run_intregration_test.sh +++ b/scripts/run_intregration_test.sh @@ -5,9 +5,9 @@ set -o xtrace docker-compose down --remove-orphans --volumes # Build and start the containers. -# Note: Build only if `ipld-eth-server` code is modified. Otherwise comment this line. -docker build -t ipld-eth-server_eth-server:latest . -docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d db dapptools contract eth-server +# Note: Build only if `ipld-eth-server` or other container code is modified. Otherwise comment this line. +docker-compose -f docker-compose.test.yml -f docker-compose.yml build eth-server +docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d db dapptools contract eth-server 2>&1 | tee docker.logs export PGPASSWORD=password export DATABASE_USER=vdbm diff --git a/test/contract/src/index.js b/test/contract/src/index.js index 6339a1d9..e484d769 100644 --- a/test/contract/src/index.js +++ b/test/contract/src/index.js @@ -15,6 +15,8 @@ fastify.get('/v1/deployContract', async (req, reply) => { const token = await GLDToken.deploy(); await token.deployed(); + console.log(`Deployed block ${token.deployTransaction.blockNumber}`) + return { address: token.address, txHash: token.deployTransaction.hash, @@ -31,6 +33,8 @@ fastify.get('/v1/destroyContract', async (req, reply) => { await token.destroy(); const blockNum = await hre.ethers.provider.getBlockNumber() + console.log(`Destroyed block ${blockNum}`) + return { blockNumber: blockNum, } diff --git a/test/helper.go b/test/helper.go index c17f4283..1cc741d5 100644 --- a/test/helper.go +++ b/test/helper.go @@ -15,9 +15,7 @@ type ContractDeployed struct { } type ContractDestroyed struct { - TransactionHash string `json:"txHash"` - BlockNumber int64 `json:"blockNumber"` - BlockHash string `json:"blockHash"` + BlockNumber int64 `json:"blockNumber"` } type Tx struct { @@ -56,6 +54,7 @@ func DestroyContract(addr string) (*ContractDestroyed, error) { } defer res.Body.Close() + fmt.Println(res.Body) var data ContractDestroyed decoder := json.NewDecoder(res.Body) diff --git a/test/integration_test.go b/test/integration_test.go index 8e7a8c26..ca43dad9 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -2,6 +2,7 @@ package integration_test import ( "context" + "fmt" "math/big" "time" @@ -317,6 +318,8 @@ var _ = Describe("Integration test", func() { contract, contractErr = integration.DeployContract() erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10) + fmt.Printf("Deployed address: %d\n", contract.BlockNumber) + time.Sleep(sleepInterval) }) @@ -393,7 +396,7 @@ var _ = Describe("Integration test", func() { Expect(gethStorage).To(Equal(ipldStorage)) }) - It("get storage after self destruct", func() { + FIt("get storage after self destruct", func() { totalSupplyIndex := "0x2" tx, err := integration.DestroyContract(contract.Address) @@ -401,6 +404,10 @@ var _ = Describe("Integration test", func() { time.Sleep(sleepInterval) + fmt.Printf("Destroyed address: %d\n", tx.BlockNumber) + + fmt.Printf("Contract Address: %s \n", contract.Address) + gethStorage1, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1)) Expect(err).ToNot(HaveOccurred()) gethStorage2, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber)) @@ -416,6 +423,12 @@ var _ = Describe("Integration test", func() { Expect(ipldStorage1).To(Equal(gethStorage1)) Expect(ipldStorage2).To(Equal(gethStorage2)) + + // Query the current block + ipldStorage3, err := ipldClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), nil) + Expect(err).ToNot(HaveOccurred()) + + Expect(ipldStorage2).To(Equal(ipldStorage3)) }) }) From 500bba43b41690f7e5a149fffbaeb586bcdbf2be Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Fri, 24 Sep 2021 15:34:09 +0530 Subject: [PATCH 10/14] Fix SQL query to check for empty storage value. --- .../00014_create_stored_functions.sql | 32 ++++++++++++------- docker-compose.yml | 3 +- pkg/eth/ipld_retriever.go | 4 +-- scripts/run_intregration_test.sh | 2 +- test/contract/src/index.js | 3 -- test/helper.go | 1 - test/integration_test.go | 9 +----- 7 files changed, 25 insertions(+), 29 deletions(-) diff --git a/db/migrations/00014_create_stored_functions.sql b/db/migrations/00014_create_stored_functions.sql index f5457b8b..778aeb3b 100644 --- a/db/migrations/00014_create_stored_functions.sql +++ b/db/migrations/00014_create_stored_functions.sql @@ -1,18 +1,26 @@ -- +goose Up - - -- +goose StatementBegin --- returns if a storage node at the provided path was removed in the range >= the provided height and <= the provided block hash -CREATE OR REPLACE FUNCTION was_state_leaf_removed(state_leaf_key BYTEA, block_num BIGINT) RETURNS BOOLEAN +-- returns if a state leaf node was removed within the provided block number +CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying) RETURNS boolean + LANGUAGE plpgsql AS $$ -SELECT exists(SELECT 1 - FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE state_leaf_key = state_leaf_key - AND block_number <= block_num - AND state_cids.node_type = 3 - LIMIT 1); -$$ LANGUAGE SQL; +DECLARE + rec RECORD; +BEGIN + FOR rec IN SELECT state_cids.node_type + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_leaf_key = key + AND block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = hash) + ORDER BY state_cids.id DESC LIMIT 1 + LOOP + IF rec.node_type = 3 THEN + RETURN TRUE; + END IF; + END LOOP; + RETURN FALSE; +END; +$$; -- +goose StatementEnd -- +goose StatementBegin diff --git a/docker-compose.yml b/docker-compose.yml index 2ef02cfd..badd03ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: restart: on-failure depends_on: - db - image: vulcanize/statediff-migrations:v0.7.0 + image: vulcanize/statediff-migrations:0.7.0 environment: DATABASE_USER: vdbm DATABASE_NAME: vulcanize_public @@ -39,7 +39,6 @@ services: - vdb_db_eth_server:/var/lib/postgresql/data ports: - "127.0.0.1:8077:5432" - command: ["postgres", "-c", "log_statement=all"] eth-server: restart: unless-stopped diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 7c82e3ba..08e11884 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -127,7 +127,7 @@ const ( AND block_number <= $2 ORDER BY block_number DESC LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed(state_leaf_key, block_number) AS state_leaf_removed + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed($1, $3) AS state_leaf_removed FROM eth.storage_cids INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) @@ -137,7 +137,7 @@ const ( AND block_number <= $3 ORDER BY block_number DESC LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed(state_leaf_key, block_number) AS state_leaf_removed + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed($1, $3) AS state_leaf_removed FROM eth.storage_cids INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) diff --git a/scripts/run_intregration_test.sh b/scripts/run_intregration_test.sh index c81ca89b..929e6297 100755 --- a/scripts/run_intregration_test.sh +++ b/scripts/run_intregration_test.sh @@ -7,7 +7,7 @@ docker-compose down --remove-orphans --volumes # Build and start the containers. # Note: Build only if `ipld-eth-server` or other container code is modified. Otherwise comment this line. docker-compose -f docker-compose.test.yml -f docker-compose.yml build eth-server -docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d db dapptools contract eth-server 2>&1 | tee docker.logs +docker-compose -f docker-compose.test.yml -f docker-compose.yml up -d db dapptools contract eth-server export PGPASSWORD=password export DATABASE_USER=vdbm diff --git a/test/contract/src/index.js b/test/contract/src/index.js index e484d769..b20d54a2 100644 --- a/test/contract/src/index.js +++ b/test/contract/src/index.js @@ -15,8 +15,6 @@ fastify.get('/v1/deployContract', async (req, reply) => { const token = await GLDToken.deploy(); await token.deployed(); - console.log(`Deployed block ${token.deployTransaction.blockNumber}`) - return { address: token.address, txHash: token.deployTransaction.hash, @@ -33,7 +31,6 @@ fastify.get('/v1/destroyContract', async (req, reply) => { await token.destroy(); const blockNum = await hre.ethers.provider.getBlockNumber() - console.log(`Destroyed block ${blockNum}`) return { blockNumber: blockNum, diff --git a/test/helper.go b/test/helper.go index 1cc741d5..5cd8ee2a 100644 --- a/test/helper.go +++ b/test/helper.go @@ -54,7 +54,6 @@ func DestroyContract(addr string) (*ContractDestroyed, error) { } defer res.Body.Close() - fmt.Println(res.Body) var data ContractDestroyed decoder := json.NewDecoder(res.Body) diff --git a/test/integration_test.go b/test/integration_test.go index ca43dad9..d250fcce 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -2,7 +2,6 @@ package integration_test import ( "context" - "fmt" "math/big" "time" @@ -318,8 +317,6 @@ var _ = Describe("Integration test", func() { contract, contractErr = integration.DeployContract() erc20TotalSupply, bigIntResult = new(big.Int).SetString("1000000000000000000000", 10) - fmt.Printf("Deployed address: %d\n", contract.BlockNumber) - time.Sleep(sleepInterval) }) @@ -396,7 +393,7 @@ var _ = Describe("Integration test", func() { Expect(gethStorage).To(Equal(ipldStorage)) }) - FIt("get storage after self destruct", func() { + It("get storage after self destruct", func() { totalSupplyIndex := "0x2" tx, err := integration.DestroyContract(contract.Address) @@ -404,10 +401,6 @@ var _ = Describe("Integration test", func() { time.Sleep(sleepInterval) - fmt.Printf("Destroyed address: %d\n", tx.BlockNumber) - - fmt.Printf("Contract Address: %s \n", contract.Address) - gethStorage1, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber-1)) Expect(err).ToNot(HaveOccurred()) gethStorage2, err := gethClient.StorageAt(ctx, common.HexToAddress(contract.Address), common.HexToHash(totalSupplyIndex), big.NewInt(tx.BlockNumber)) From 7eb7849b5eab20bef8c3ea153e51175ee71c1f35 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Fri, 24 Sep 2021 17:56:50 +0530 Subject: [PATCH 11/14] Self review --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index badd03ca..c24619bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: restart: on-failure depends_on: - db - image: vulcanize/statediff-migrations:0.7.0 + image: vulcanize/statediff-migrations:v0.7.0 environment: DATABASE_USER: vdbm DATABASE_NAME: vulcanize_public From e81da697bc4c55f3b65bf3b6aad1024ade938968 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 27 Sep 2021 17:14:27 +0530 Subject: [PATCH 12/14] Fix query and add debug logs. --- .../00014_create_stored_functions.sql | 32 +++++++------------ docker-compose.yml | 1 + 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/db/migrations/00014_create_stored_functions.sql b/db/migrations/00014_create_stored_functions.sql index 778aeb3b..e0374636 100644 --- a/db/migrations/00014_create_stored_functions.sql +++ b/db/migrations/00014_create_stored_functions.sql @@ -1,26 +1,18 @@ -- +goose Up -- +goose StatementBegin -- returns if a state leaf node was removed within the provided block number -CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying) RETURNS boolean - LANGUAGE plpgsql -AS $$ -DECLARE - rec RECORD; -BEGIN - FOR rec IN SELECT state_cids.node_type - FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE state_leaf_key = key - AND block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = hash) - ORDER BY state_cids.id DESC LIMIT 1 - LOOP - IF rec.node_type = 3 THEN - RETURN TRUE; - END IF; - END LOOP; - RETURN FALSE; -END; -$$; +CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying) + RETURNS boolean AS $$ + SELECT state_cids.node_type = 3 + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_leaf_key = key + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + ORDER BY block_number DESC LIMIT 1; +$$ +language sql; -- +goose StatementEnd -- +goose StatementBegin diff --git a/docker-compose.yml b/docker-compose.yml index c24619bc..2ef02cfd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,7 @@ services: - vdb_db_eth_server:/var/lib/postgresql/data ports: - "127.0.0.1:8077:5432" + command: ["postgres", "-c", "log_statement=all"] eth-server: restart: unless-stopped From b0674df3ce9cef2dff2f0294c5998108b15262ac Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 27 Sep 2021 17:53:42 +0530 Subject: [PATCH 13/14] Remove parallel migration in integration test. --- docker-compose.yml | 1 + entrypoint.sh | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2ef02cfd..0137fba2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,7 @@ services: DATABASE_USER: "vdbm" DATABASE_PASSWORD: "password" ETH_CHAIN_ID: 4 + RUN_DB_MIGRATION: "no" volumes: - type: bind source: ./chain.json diff --git a/entrypoint.sh b/entrypoint.sh index 2c9e9821..0efef13f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,18 +4,21 @@ # Construct the connection string for postgres VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable -# Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -echo "Running database migrations" -./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up -rv=$? -if [ $rv != 0 ]; then - echo "Could not run migrations. Are the database details correct?" - exit 1 +if [ "$RUN_DB_MIGRATION" != "no" ] +then + # Run the DB migrations + echo "Connecting with: $VDB_PG_CONNECT" + echo "Running database migrations" + ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up + rv=$? + + if [ $rv != 0 ]; then + echo "Could not run migrations. Are the database details correct?" + exit 1 + fi fi - echo "Beginning the ipld-eth-server process" echo running: ./ipld-eth-server ${VDB_COMMAND} --config=config.toml From 13e0a5cd19a594b428c32301ba363dcd5fb8de6b Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Mon, 27 Sep 2021 21:55:43 +0530 Subject: [PATCH 14/14] Update statediff migration docker version. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0137fba2..15c73bce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: restart: on-failure depends_on: - db - image: vulcanize/statediff-migrations:v0.7.0 + image: vulcanize/statediff-migrations:v0.8.0 environment: DATABASE_USER: vdbm DATABASE_NAME: vulcanize_public