From 9072ebc91e536352b253aea1bb1236e20a29b60a Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 3 Oct 2023 12:22:54 +0800 Subject: [PATCH 1/6] fix test chain.json also: clear eth.header_cids script patches --- scripts/compare-statediffs.sh | 4 +++- test/ci-chain.json | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/compare-statediffs.sh b/scripts/compare-statediffs.sh index 9442862..7555279 100755 --- a/scripts/compare-statediffs.sh +++ b/scripts/compare-statediffs.sh @@ -48,7 +48,7 @@ export DATABASE_PASSWORD="password" export ETH_NODE_ID=test-node export ETH_CLIENT_NAME=test-client export ETH_NETWORK_ID=test-network -export ETH_CHAIN_ID=4242 +export ETH_CHAIN_ID=1212 export SERVICE_HTTP_PATH='127.0.0.1:8545' export LOG_LEVEL=debug @@ -65,6 +65,7 @@ clear_table() { } tables=( + eth.header_cids eth.log_cids eth.receipt_cids eth.state_cids @@ -86,6 +87,7 @@ run_service() { service_binary=$1 service_output_dir=$2 + mkdir -p $service_output_dir $service_binary serve & until grep "HTTP endpoint opened" $LOG_FILE diff --git a/test/ci-chain.json b/test/ci-chain.json index d2ee2f5..44d2cd2 100644 --- a/test/ci-chain.json +++ b/test/ci-chain.json @@ -1,5 +1,5 @@ { - "chainId": 41337, + "chainId": 1212, "homesteadBlock": 0, "eip150Block": 0, "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000", @@ -9,6 +9,11 @@ "constantinopleBlock": 0, "petersburgBlock": 0, "istanbulBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "mergeForkBlock": 0, + "terminalTotalDifficulty": 0, + "mergeNetsplitBlock": 0, "clique": { "period": 5, "epoch": 30000 -- 2.45.2 From b305f541a4c2aada90d033931f479dce141ac49b Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 3 Oct 2023 13:02:54 +0800 Subject: [PATCH 2/6] deferred rollback --- pkg/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/service.go b/pkg/service.go index 6785079..6add701 100644 --- a/pkg/service.go +++ b/pkg/service.go @@ -404,6 +404,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p if err != nil { return err } + defer tx.RollbackOnFailure(err) // defer handling of commit/rollback for any return case output := func(node sdtypes.StateLeafNode) error { return sds.indexer.PushStateNode(tx, node, block.Hash().String()) -- 2.45.2 From ad9e41012c9ccba2bed45000000f5b93dd71816a Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 3 Oct 2023 13:29:31 +0800 Subject: [PATCH 3/6] set respective eth chain ids --- .gitea/workflows/tests.yml | 2 ++ scripts/compare-statediffs.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 8f1f1e9..3672e86 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -52,6 +52,7 @@ jobs: LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient LOG_FILE: ./server-log ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05" + ETH_CHAIN_ID: 41337 timeout-minutes: 30 run: | ./eth-statediff-service --config ./test/ci-config.toml serve & @@ -121,6 +122,7 @@ jobs: LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614" ETH_CHAIN_CONFIG: ./eth-statediff-service/test/ci-chain.json + ETH_CHAIN_ID: 1212 run: | until ready_query='select max(version_id) from goose_db_version;' diff --git a/scripts/compare-statediffs.sh b/scripts/compare-statediffs.sh index 7555279..d0b6687 100755 --- a/scripts/compare-statediffs.sh +++ b/scripts/compare-statediffs.sh @@ -48,7 +48,6 @@ export DATABASE_PASSWORD="password" export ETH_NODE_ID=test-node export ETH_CLIENT_NAME=test-client export ETH_NETWORK_ID=test-network -export ETH_CHAIN_ID=1212 export SERVICE_HTTP_PATH='127.0.0.1:8545' export LOG_LEVEL=debug -- 2.45.2 From 07fdae78fdfc4b39bf9accbf0c65141de83a7e21 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 3 Oct 2023 19:21:30 +0800 Subject: [PATCH 4/6] chain id for int. test --- .gitea/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 3672e86..8066a3b 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -52,7 +52,7 @@ jobs: LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient LOG_FILE: ./server-log ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05" - ETH_CHAIN_ID: 41337 + ETH_CHAIN_ID: 4242 timeout-minutes: 30 run: | ./eth-statediff-service --config ./test/ci-config.toml serve & @@ -72,6 +72,8 @@ jobs: [[ "$(count_results eth.header_cids)" = 33 ]] [[ "$(count_results eth.state_cids)" = 21 ]] [[ "$(count_results eth.storage_cids)" = 18 ]] + [[ "$(count_results eth.transaction_cids)" = 10 ]] + [[ "$(count_results eth.receipt_cids)" = 10 ]] compliance-test: name: Run compliance tests -- 2.45.2 From fd6fb995400eb9825a1f25e4cce08c69186701e8 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 3 Oct 2023 19:33:14 +0800 Subject: [PATCH 5/6] guard output sinks --- pkg/service.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/service.go b/pkg/service.go index 6add701..1ba687b 100644 --- a/pkg/service.go +++ b/pkg/service.go @@ -404,12 +404,18 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p if err != nil { return err } - defer tx.RollbackOnFailure(err) // defer handling of commit/rollback for any return case + defer tx.RollbackOnFailure(err) + + var nodeMtx, ipldMtx sync.Mutex output := func(node sdtypes.StateLeafNode) error { + nodeMtx.Lock() + defer nodeMtx.Unlock() return sds.indexer.PushStateNode(tx, node, block.Hash().String()) } - codeOutput := func(c sdtypes.IPLD) error { + ipldOutput := func(c sdtypes.IPLD) error { + ipldMtx.Lock() + defer ipldMtx.Unlock() return sds.indexer.PushIPLD(tx, c) } prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t)) @@ -419,7 +425,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p OldStateRoot: parentRoot, BlockNumber: block.Number(), BlockHash: block.Hash(), - }, params, output, codeOutput) + }, params, output, ipldOutput) prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t)) t = time.Now() err = tx.Submit() -- 2.45.2 From f4fbbee9ed528308b96754daf364bcb8b96785d3 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 3 Oct 2023 19:55:41 +0800 Subject: [PATCH 6/6] use different fixture --- .gitea/workflows/tests.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 8066a3b..db3f398 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -48,19 +48,18 @@ jobs: - name: Run basic integration test env: DATABASE_TYPE: postgres - LEVELDB_PATH: ./fixtures/chaindata/_data/small - LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient + LEVELDB_PATH: ./fixtures/chaindata/_data/small2 + LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient LOG_FILE: ./server-log - ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05" - ETH_CHAIN_ID: 4242 - timeout-minutes: 30 + ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614" + timeout-minutes: 20 run: | ./eth-statediff-service --config ./test/ci-config.toml serve & sleep 10 - ./scripts/request-range.sh 0 32 || (E=$?; cat $LOG_FILE; exit $E) + ./scripts/request-range.sh 0 10 || (E=$?; cat $LOG_FILE; exit $E) - until grep "Finished processing block 32" $LOG_FILE + until grep "Finished processing block 10" $LOG_FILE do sleep 1; done count_results() { @@ -69,11 +68,11 @@ jobs: psql -tA cerc_testing -U vdbm -c "$query" } set -x - [[ "$(count_results eth.header_cids)" = 33 ]] - [[ "$(count_results eth.state_cids)" = 21 ]] - [[ "$(count_results eth.storage_cids)" = 18 ]] - [[ "$(count_results eth.transaction_cids)" = 10 ]] - [[ "$(count_results eth.receipt_cids)" = 10 ]] + [[ "$(count_results eth.header_cids)" = 11 ]] + [[ "$(count_results eth.state_cids)" = 287 ]] + [[ "$(count_results eth.storage_cids)" = 31 ]] + [[ "$(count_results eth.transaction_cids)" = 144 ]] + [[ "$(count_results eth.receipt_cids)" = 144 ]] compliance-test: name: Run compliance tests @@ -124,7 +123,6 @@ jobs: LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614" ETH_CHAIN_CONFIG: ./eth-statediff-service/test/ci-chain.json - ETH_CHAIN_ID: 1212 run: | until ready_query='select max(version_id) from goose_db_version;' -- 2.45.2