cerc-io/eth-statediff-service
16
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix concurrency and tests (#4)

Browse Source 
1. Synchronizes pushes to indexer - The Tx object is not threadsafe.
2. Fixes the chain config for the CI test fixtures, which were not accurate for the data, and causing processing errors

Reviewed-on: #4
This commit is contained in:
Roy Crihfield 2023-10-04 07:44:43 +00:00
parent 167cd2839c
commit 161febc47a
4 changed files with 28 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.gitea/workflows/tests.yml
View File

@ -48,18 +48,18 @@ jobs:
- name: Run basic integration test - name: Run basic integration test
env: env:
DATABASE_TYPE: postgres DATABASE_TYPE: postgres
LEVELDB_PATH: ./fixtures/chaindata/_data/small LEVELDB_PATH: ./fixtures/chaindata/_data/small2
LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
LOG_FILE: ./server-log LOG_FILE: ./server-log
ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05" ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
timeout-minutes: 30 timeout-minutes: 20
run: | run: |
./eth-statediff-service --config ./test/ci-config.toml serve & ./eth-statediff-service --config ./test/ci-config.toml serve &
sleep 10 sleep 10
./scripts/request-range.sh 0 32 || (E=$?; cat $LOG_FILE; exit $E) ./scripts/request-range.sh 0 10 || (E=$?; cat $LOG_FILE; exit $E)
until grep "Finished processing block 32" $LOG_FILE until grep "Finished processing block 10" $LOG_FILE
do sleep 1; done do sleep 1; done
count_results() { count_results() {
@ -68,9 +68,11 @@ jobs:
psql -tA cerc_testing -U vdbm -c "$query" psql -tA cerc_testing -U vdbm -c "$query"
} }
set -x set -x
[[ "$(count_results eth.header_cids)" = 33 ]] [[ "$(count_results eth.header_cids)" = 11 ]]
[[ "$(count_results eth.state_cids)" = 21 ]] [[ "$(count_results eth.state_cids)" = 287 ]]
[[ "$(count_results eth.storage_cids)" = 18 ]] [[ "$(count_results eth.storage_cids)" = 31 ]]
[[ "$(count_results eth.transaction_cids)" = 144 ]]
[[ "$(count_results eth.receipt_cids)" = 144 ]]
compliance-test: compliance-test:
name: Run compliance tests name: Run compliance tests

11
pkg/service.go
View File

@ -405,10 +405,17 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
return err return err
} }
// defer handling of commit/rollback for any return case // defer handling of commit/rollback for any return case
defer tx.RollbackOnFailure(err)
var nodeMtx, ipldMtx sync.Mutex
output := func(node sdtypes.StateLeafNode) error { output := func(node sdtypes.StateLeafNode) error {
nodeMtx.Lock()
defer nodeMtx.Unlock()
return sds.indexer.PushStateNode(tx, node, block.Hash().String()) return sds.indexer.PushStateNode(tx, node, block.Hash().String())
} }
codeOutput := func(c sdtypes.IPLD) error { ipldOutput := func(c sdtypes.IPLD) error {
ipldMtx.Lock()
defer ipldMtx.Unlock()
return sds.indexer.PushIPLD(tx, c) return sds.indexer.PushIPLD(tx, c)
} }
prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t)) prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
@ -418,7 +425,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
OldStateRoot: parentRoot, OldStateRoot: parentRoot,
BlockNumber: block.Number(), BlockNumber: block.Number(),
BlockHash: block.Hash(), BlockHash: block.Hash(),
}, params, output, codeOutput) }, params, output, ipldOutput)
prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t)) prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t))
t = time.Now() t = time.Now()
err = tx.Submit() err = tx.Submit()

3
scripts/compare-statediffs.sh
View File

@ -48,7 +48,6 @@ export DATABASE_PASSWORD="password"
export ETH_NODE_ID=test-node export ETH_NODE_ID=test-node
export ETH_CLIENT_NAME=test-client export ETH_CLIENT_NAME=test-client
export ETH_NETWORK_ID=test-network export ETH_NETWORK_ID=test-network
export ETH_CHAIN_ID=4242
export SERVICE_HTTP_PATH='127.0.0.1:8545' export SERVICE_HTTP_PATH='127.0.0.1:8545'
export LOG_LEVEL=debug export LOG_LEVEL=debug
@ -65,6 +64,7 @@ clear_table() {
} }
tables=( tables=(
eth.header_cids
eth.log_cids eth.log_cids
eth.receipt_cids eth.receipt_cids
eth.state_cids eth.state_cids
@ -86,6 +86,7 @@ run_service() {
service_binary=$1 service_binary=$1
service_output_dir=$2 service_output_dir=$2
mkdir -p $service_output_dir
$service_binary serve & $service_binary serve &
until grep "HTTP endpoint opened" $LOG_FILE until grep "HTTP endpoint opened" $LOG_FILE

7
test/ci-chain.json
View File

@ -1,5 +1,5 @@
{ {
"chainId": 41337, "chainId": 1212,
"homesteadBlock": 0, "homesteadBlock": 0,
"eip150Block": 0, "eip150Block": 0,
"eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000", "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
@ -9,6 +9,11 @@
"constantinopleBlock": 0, "constantinopleBlock": 0,
"petersburgBlock": 0, "petersburgBlock": 0,
"istanbulBlock": 0, "istanbulBlock": 0,
"berlinBlock": 0,
"londonBlock": 0,
"mergeForkBlock": 0,
"terminalTotalDifficulty": 0,
"mergeNetsplitBlock": 0,
"clique": { "clique": {
"period": 5, "period": 5,
"epoch": 30000 "epoch": 30000