Merge pull request 'Pinning the older version of Delve' (#267) from fix/pin-dlv-version into v5

Reviewed-on: #267
Reviewed-by: Roy Crihfield <roy@manteia.ltd>

.dockerignore

@@ -12,6 +12,7 @@ integration
 test
 scripts
 Dockerfile
+environments
 
 **/node_modules
 build

.github/workflows/tests.yaml

@@ -12,7 +12,9 @@ on:
       - ci-test
 
 env:
-  SO_VERSION: v1.1.0-87fffca-202404110321
+  SO_VERSION: v1.1.0-36d4969-202407091537
+  FIXTURENET_ETH_STACKS_REF: main
+  SYSTEM_TESTS_REF: main
 
 jobs:
   test:
@@ -49,7 +51,15 @@ jobs:
         env:
           DEBIAN_FRONTEND: noninteractive
         run: apt-get update && apt-get install -y jq
-      - name: Install Python
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: 3.11
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
         uses: actions/setup-python@v4
         with:
           python-version: 3.11
@@ -60,11 +70,18 @@ jobs:
           ref: ${{ env.SO_VERSION }}
           path: ./stack-orchestrator
       - run: pip install ./stack-orchestrator
+      - name: Clone fixturenet stack repo
+        uses: actions/checkout@v4
+        with:
+          repository: cerc-io/fixturenet-eth-stacks
+          ref: ${{ env.FIXTURENET_ETH_STACKS_REF }}
+          path: ./fixturenet-eth-stacks
+          progress: false
 
       - name: Run testnet stack
         env:
           CERC_GO_AUTH_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
-        run: ./scripts/integration-setup.sh
+        run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth
       - name: Run server
         env:
           ETH_FORWARD_ETH_CALLS: false
@@ -75,11 +92,31 @@ jobs:
           go install github.com/onsi/ginkgo/v2/ginkgo
           ginkgo -v --label-filter '!proxy' -r ./integration
 
+      - name: Clone system-tests
+        uses: actions/checkout@v4
+        with:
+          repository: cerc-io/system-tests
+          ref: ${{ env.SYSTEM_TESTS_REF }}
+          path: ./system-tests
+          token: ${{ secrets.CICD_REPO_TOKEN }}
+          progress: false
+      - name: Run system tests
+        working-directory: ./system-tests
+        # Work around dependency conflict in system-tests:
+        # web3 uses an older eth-account until (unreleased) v7
+        run: |
+          pip3 install pytest
+          pip3 install -r requirements.txt
+          pip3 install --no-deps 'eth-account>=0.12.3,<0.13'
+          pip3 install 'pydantic>=2.0.0'
+          # Skips tests that require Blob indexing
+          python3 -m pytest -vv -m "not blob_db"
+
       - name: Run testnet stack without statediff
         env:
           CERC_RUN_STATEDIFF: false
           SKIP_BUILD: 1
-        run: ./scripts/integration-setup.sh
+        run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth
       - name: Run server with call forwarding
         env:
           ETH_FORWARD_ETH_CALLS: true

Dockerfile

@@ -1,9 +1,9 @@
-FROM golang:1.21-alpine as debugger
+FROM golang:1.21-alpine AS debugger
 
 # Include dlv
-RUN go install github.com/go-delve/delve/cmd/dlv@latest
+RUN go install github.com/go-delve/delve/cmd/dlv@v1.22.1
 
-FROM golang:1.21-alpine as builder
+FROM golang:1.21-alpine AS builder
 
 RUN apk --update --no-cache add gcc musl-dev binutils-gold git
 
@@ -29,7 +29,6 @@ RUN GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -
 FROM alpine
 
 ARG USER="vdm"
-ARG CONFIG_FILE="./environments/example.toml"
 
 RUN adduser -Du 5000 $USER
 WORKDIR /app
@@ -38,13 +37,10 @@ USER $USER
 
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/$CONFIG_FILE config.toml
 COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/entrypoint.sh .
 
-
 # keep binaries immutable
 COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-server/ipld-eth-server ipld-eth-server
-COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-server/environments environments
 
 # Allow for debugging
 COPY --from=debugger  /go/bin/dlv /usr/local/bin/

cmd/root.go

@@ -95,7 +95,7 @@ func init() {
 
 func initConfig() {
 	if cfgFile == "" && envFile == "" {
-		log.Fatal("No configuration file specified, use --config , --env flag to provide configuration")
+		log.Warn("No configuration file specified, use --config , --env flag to provide configuration")
 	}
 
 	if cfgFile != "" {

cmd/serve.go

@@ -60,6 +60,7 @@ func serve() {
 		logWithCommand.Fatal(err)
 	}
 	logWithCommand.Debugf("server config: %+v", serverConfig)
+	logWithCommand.Debugf("chain config: %+v", serverConfig.ChainConfig)
 	server, err := s.NewServer(serverConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
@@ -276,10 +277,10 @@ func init() {
 	serveCmd.PersistentFlags().String("eth-client-name", "Geth", "eth client name")
 	serveCmd.PersistentFlags().String("eth-genesis-block", "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
 	serveCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
-	serveCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
+	serveCmd.PersistentFlags().String("eth-chain-id", "", "eth chain id")
+	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
 	serveCmd.PersistentFlags().String("eth-default-sender", "", "default sender address")
 	serveCmd.PersistentFlags().String("eth-rpc-gas-cap", "", "rpc gas cap (for eth_Call execution)")
-	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
 	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether the proxy ethereum client supports statediffing endpoints")
 	serveCmd.PersistentFlags().Bool("eth-forward-eth-calls", false, "whether to immediately forward eth_calls to proxy client")
 	serveCmd.PersistentFlags().Bool("eth-proxy-on-error", true, "whether to forward all failed calls to proxy client")

entrypoint.sh

@@ -1,17 +1,8 @@
 #!/bin/sh
 
-echo "Beginning the ipld-eth-server process"
-
 START_CMD="./ipld-eth-server"
 if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
     START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec `pwd`/ipld-eth-server --continue --"
 fi
 
-echo running: $START_CMD ${VDB_COMMAND} --config=`pwd`/config.toml
+exec $START_CMD ${VDB_COMMAND:-serve}
-$START_CMD ${VDB_COMMAND} --config=`pwd`/config.toml
-rv=$?
-
-if [ $rv != 0 ]; then
-  echo "ipld-eth-server startup failed"
-  exit 1
-fi

environments/testing.toml

@@ -1,5 +1,5 @@
 [database]
-  name     = "vulcanize_testing"
+  name     = "cerc_testing"
   hostname = "localhost"
   port     = 5432
 

go.mod

@@ -4,10 +4,10 @@ go 1.21
 
 require (
 	github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha
-	github.com/cerc-io/eth-iterator-utils v0.2.0
+	github.com/cerc-io/eth-iterator-utils v0.3.1
 	github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha
-	github.com/cerc-io/ipld-eth-statedb v0.1.0
+	github.com/cerc-io/ipld-eth-statedb v0.1.1
-	github.com/cerc-io/plugeth-statediff v0.2.1
+	github.com/cerc-io/plugeth-statediff v0.3.2
 	github.com/ethereum/go-ethereum v1.13.14
 	github.com/google/uuid v1.6.0
 	github.com/graph-gophers/graphql-go v1.3.0

go.sum

@@ -81,16 +81,16 @@ github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2
 github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha h1:gpEk3BQQhnkilXxP12q77LSiIAvXfTdQ+4xxIv/JWiI=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha/go.mod h1:xmazdaZ/CUaXt52Mvd0KU7dcbJM3glvJzA2mEpVMgbM=
-github.com/cerc-io/eth-iterator-utils v0.2.0 h1:wikAfWZ0fAqLqUy/Ud/a1n9p/arVeKG8P8tRjZE7oBg=
+github.com/cerc-io/eth-iterator-utils v0.3.1 h1:h4Bp0+fUiwkyug1uCEO2LZr2qxoW1yKszV2EO/2CDB0=
-github.com/cerc-io/eth-iterator-utils v0.2.0/go.mod h1:wDUJvwKDSOdqTIyeG+yXJ2ckzc9f2Fem614fV61DBcg=
+github.com/cerc-io/eth-iterator-utils v0.3.1/go.mod h1:UNrjsP5bApZkqqqfU7nmnPN/dIIo9GOUUD79tmoX/s4=
-github.com/cerc-io/eth-testing v0.4.0 h1:ivGbXnEqlXMt/3m3jbsPJaVT7ZDTenFQWCryt1Rd/Jk=
+github.com/cerc-io/eth-testing v0.5.1 h1:xxcQf9ymJS0911yWIrUiGvCvqfvEjYmHvhBJkCD/whs=
-github.com/cerc-io/eth-testing v0.4.0/go.mod h1:CVsmHjFldX9gwaQSQwGmKbmh0g6Dq+bsqB2CxBf9zbk=
+github.com/cerc-io/eth-testing v0.5.1/go.mod h1:p86je2PjSM7u8Qd7rMIG/Zw+tQlBoS5Emkh1ECnC5t0=
 github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha h1:+XhYHvzC3zFIvcWEb466SNDfeLrvcW3xe/d0cbVVVRA=
 github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha/go.mod h1:w5g07b6Uz+c6r/ySml58TEOJdUYAibYYF05H5ULVv2I=
-github.com/cerc-io/ipld-eth-statedb v0.1.0 h1:K2Xy8hgmkSRmHaqmw0CZf9hWX0r8/r0563O7osmyVBE=
+github.com/cerc-io/ipld-eth-statedb v0.1.1 h1:QvhdO9jZqQu+NmiilW4Ef4N3qbdN4+FRHwVVbHg7Q/w=
-github.com/cerc-io/ipld-eth-statedb v0.1.0/go.mod h1:Lo4TKVs7bTGr2b7L9ccmo/EMV96/nGE7T/GYliuOHlM=
+github.com/cerc-io/ipld-eth-statedb v0.1.1/go.mod h1:+BF5xGXodtUFlNZ9W3JYsUHRmPyso+td8IgB+x/bYjM=
-github.com/cerc-io/plugeth-statediff v0.2.1 h1:zUbkazJW0omFGg7z/9MJnttox4VRurLW1pFytDlRQjM=
+github.com/cerc-io/plugeth-statediff v0.3.2 h1:GuOUqDT6nJRCikyaNxDI2pA7TRnOTWOcyJGlJtwSzHY=
-github.com/cerc-io/plugeth-statediff v0.2.1/go.mod h1:n85L2n8Q3bQVlAVFnyk2soCLJW+YFKGe3DVRauEaS2s=
+github.com/cerc-io/plugeth-statediff v0.3.2/go.mod h1:r6Mzc6k4V9KD+iN9AXa/LmmRISDsQnnIwKzZMFjJ+eE=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

pkg/eth/api.go

@@ -660,6 +660,12 @@ func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash co
 	if receipt.Logs == nil {
 		fields["logs"] = []*types.Log{}
 	}
+
+	if tx.Type() == types.BlobTxType {
+		fields["blobGasUsed"] = hexutil.Uint64(receipt.BlobGasUsed)
+		fields["blobGasPrice"] = (*hexutil.Big)(receipt.BlobGasPrice)
+	}
+
 	// If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation
 	if receipt.ContractAddress != (common.Address{}) {
 		fields["contractAddress"] = receipt.ContractAddress
@@ -1245,7 +1251,8 @@ func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx boo
 	if inclTx {
 		td, err := pea.B.GetTd(b.Hash())
 		if err != nil {
-			log.Errorf("error getting td for block with hash and number %s, %s: %s", b.Hash().String(), b.Number().String(), err)
+			err = fmt.Errorf("error getting TD for block at (%s, %s): %s", b.Number(), b.Hash(), err)
+			log.Error(err)
 			return nil, err
 		}
 		fields["totalDifficulty"] = (*hexutil.Big)(td)

pkg/eth/api_test/api_test.go

@@ -50,7 +50,7 @@ var (
 	blockHash      = test_helpers.MockBlock.Header().Hash()
 	baseFee        = test_helpers.MockLondonBlock.BaseFee()
 	ctx            = context.Background()
-	chainConfig    = &*params.TestChainConfig
+	chainConfig    = &*params.MergedTestChainConfig
 
 	expectedBlock = map[string]interface{}{
 		"number":           (*hexutil.Big)(test_helpers.MockBlock.Number()),
@@ -390,7 +390,7 @@ var _ = Describe("API", func() {
 			Expect(block).To(BeZero())
 		})
 		It("Fetch BaseFee from london block by block hash, returns `nil` for legacy block", func() {
-			block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true)
+			block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), false)
 			Expect(err).ToNot(HaveOccurred())
 			_, ok := block["baseFeePerGas"]
 			Expect(ok).To(Equal(false))

pkg/eth/backend.go

@@ -271,7 +271,7 @@ func (b *Backend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.Blo
 func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Block, error) {
 	number, err := b.NormalizeBlockNumber(blockNumber)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to normalize block number: %w", err)
 	}
 	canonicalHash, err := b.GetCanonicalHash(uint64(number))
 	if err != nil {
@@ -349,11 +349,16 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 		return nil, err
 	}
 
-	// Placeholder for withdrawal processing (TODO: https://git.vdb.to/cerc-io/ipld-eth-server/pulls/265)
+	// Fetch withdrawals
 	var withdrawals types.Withdrawals
 	if b.Config.ChainConfig.IsShanghai(header.Number, header.Time) {
-		// All blocks after Shanghai must include a withdrawals root.
+		withdrawals, err = b.GetWithdrawals(tx, hash, blockNumber)
-		withdrawals = make(types.Withdrawals, 0)
+		if err != nil && err != sql.ErrNoRows {
+			log.Error("error fetching withdrawals: ", err)
+			return nil, err
+		}
+	} else if len(withdrawals) > 0 {
+		return nil, errors.New("withdrawals set before Shanghai activation")
 	}
 
 	// Compose everything together into a complete block
@@ -501,6 +506,23 @@ func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash,
 	return rcts, nil
 }
 
+// GetWithdrawals retrieves transactions for a provided block hash and number
+func (b *Backend) GetWithdrawals(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Withdrawals, error) {
+	_, rlpBytes, err := b.Retriever.RetrieveWithdrawals(tx, hash, number)
+	if err != nil {
+		return nil, err
+	}
+
+	withdrawals := make(types.Withdrawals, len(rlpBytes))
+	for i, bytes := range rlpBytes {
+		withdrawals[i] = new(types.Withdrawal)
+		if err := rlp.DecodeBytes(bytes, withdrawals[i]); err != nil {
+			return nil, err
+		}
+	}
+	return withdrawals, nil
+}
+
 // GetTransaction retrieves a tx by hash
 // It also returns the blockhash, blocknumber, and tx index associated with the transaction
 func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (bool, *types.Transaction, common.Hash, uint64, uint64, error) {

pkg/eth/retriever.go

@@ -506,6 +506,24 @@ func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (
 	return cids, rcts, txs, nil
 }
 
+// RetrieveWithdrawals returns the CIDs and RLP bytes for the withdrawals corresponding to the
+// provided block hash, number.  Returned CIDs correspond to the leaf node data which contains the
+// withdrawal object.
+func (r *Retriever) RetrieveWithdrawals(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
+	results := make([]ipldResult, 0)
+	if err := tx.Select(&results, RetrieveWithdrawalsPgStr, hash.Hex(), number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(results))
+	withdrawals := make([][]byte, len(results))
+
+	for i, res := range results {
+		cids[i] = res.CID
+		withdrawals[i] = res.Data
+	}
+	return cids, withdrawals, nil
+}
+
 // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
 // TODO: ensure this handles deleted accounts appropriately
 func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (StateAccountRecord, error) {

pkg/eth/sql.go

@@ -107,6 +107,21 @@ WHERE header_cids.block_hash = $1
 	AND blocks.key = receipt_cids.cid
 ORDER BY eth.transaction_cids.index ASC
 `
+	RetrieveWithdrawalsPgStr = `
+SELECT withdrawal_cids.cid,
+	   blocks.data
+  FROM eth.withdrawal_cids
+       JOIN eth.header_cids
+           ON header_cids.block_hash = $1
+           AND header_cids.block_number = $2
+           AND header_cids.canonical
+           AND withdrawal_cids.block_number = header_cids.block_number
+           AND withdrawal_cids.header_id = header_cids.block_hash
+       JOIN ipld.blocks
+           ON blocks.block_number = header_cids.block_number
+           AND blocks.key = withdrawal_cids.cid
+ ORDER BY eth.withdrawal_cids.index ASC`
+
 	RetrieveAccountByLeafKeyAndBlockHashPgStr = `
 SELECT state_cids.nonce,
 	state_cids.balance,

pkg/eth/test_helpers/test_data.go

@@ -53,6 +53,10 @@ var (
 		Extra:       []byte{},
 	}
 	MockTransactions, MockReceipts, SenderAddr = createLegacyTransactionsAndReceipts()
+	MockWithdrawals                            = types.Withdrawals{
+		{Index: 0, Validator: 1, Address: Address, Amount: 1000000000},
+		{Index: 1, Validator: 5, Address: AnotherAddress, Amount: 2000000000},
+	}
 	MockUncles = []*types.Header{
 		{
 			Time:        1,
@@ -75,7 +79,7 @@ var (
 			ParentHash:  Genesis.Hash(),
 		},
 	}
-	MockBlock       = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, trie.NewEmpty(nil))
+	MockBlock       = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, nil, trie.NewEmpty(nil))
 	MockChildHeader = types.Header{
 		Time:        0,
 		Number:      big.NewInt(BlockNumber1 + 1),
@@ -326,11 +330,11 @@ var (
 			Extra:       []byte{},
 		},
 	}
-	MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, MockLondonUncles, MockLondonReceipts, trie.NewEmpty(nil))
+	MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, MockLondonUncles, MockLondonReceipts, MockWithdrawals, trie.NewEmpty(nil))
 )
 
-func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, hasher types.TrieHasher) *types.Block {
+func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, withdrawals types.Withdrawals, hasher types.TrieHasher) *types.Block {
-	block := types.NewBlock(header, txs, uncles, receipts, hasher)
+	block := types.NewBlockWithWithdrawals(header, txs, uncles, receipts, withdrawals, hasher)
 	bHash := block.Hash()
 	for _, r := range receipts {
 		for _, l := range r.Logs {
@@ -455,7 +459,7 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts,
 func getReceiptCIDs(rcts []*types.Receipt) ([]cid.Cid, error) {
 	cids := make([]cid.Cid, len(rcts))
 	for i, rct := range rcts {
-		ethRct, err := ipld.NewReceipt(rct)
+		ethRct, err := ipld.EncodeReceipt(rct)
 		if err != nil {
 			return nil, err
 		}

pkg/serve/config.go

@@ -128,9 +128,10 @@ func NewConfig() (*Config, error) {
 	viper.BindEnv("server.ipcPath", SERVER_IPC_PATH)
 	viper.BindEnv("server.graphqlPath", SERVER_GRAPHQL_PATH)
 
+	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
 	viper.BindEnv("ethereum.httpPath", ETH_HTTP_PATH)
 	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
-	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
 	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
 	viper.BindEnv("ethereum.stateDiffTimeout", ETH_STATEDIFF_TIMEOUT)
 	viper.BindEnv("ethereum.forwardEthCalls", ETH_FORWARD_ETH_CALLS)
@@ -143,11 +144,32 @@ func NewConfig() (*Config, error) {
 	c.dbInit()
 	ethHTTP := viper.GetString("ethereum.httpPath")
 	ethHTTPEndpoint := fmt.Sprintf("http://%s", ethHTTP)
-	nodeInfo, cli, err := getEthNodeAndClient(ethHTTPEndpoint)
+
-	c.NodeNetworkID = nodeInfo.NetworkID
+	// At least one of chain ID and chain config must be passed.
+	// If both are passed, the chain ID must match the config.
+	chainID := viper.GetUint64("ethereum.chainID")
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+	var err error
+	if chainConfigPath != "" {
+		if c.ChainConfig, err = utils.LoadConfig(chainConfigPath); err != nil {
+			return nil, err
+		}
+		// Only validate the chain ID if it was actually passed
+		if viper.GetString("ethereum.chainID") != "" && c.ChainConfig.ChainID.Uint64() != chainID {
+			return nil, fmt.Errorf("passed chain ID %d does not match chain config chain ID %d",
+				chainID, c.ChainConfig.ChainID.Uint64())
+		}
+	} else {
+		if c.ChainConfig, err = utils.ChainConfig(chainID); err != nil {
+			return nil, err
+		}
+	}
+
+	nodeInfo, cli, err := getEthNodeAndClient(ethHTTPEndpoint, chainID)
 	if err != nil {
 		return nil, err
 	}
+	c.NodeNetworkID = nodeInfo.NetworkID
 	c.Client = cli
 	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
 	c.ForwardEthCalls = viper.GetBool("ethereum.forwardEthCalls")
@@ -237,12 +259,6 @@ func NewConfig() (*Config, error) {
 	if c.StateDiffTimeout < 0 {
 		return nil, errors.New("ethereum.stateDiffTimeout < 0")
 	}
-	chainConfigPath := viper.GetString("ethereum.chainConfig")
-	if chainConfigPath != "" {
-		c.ChainConfig, err = utils.LoadConfig(chainConfigPath)
-	} else {
-		c.ChainConfig, err = utils.ChainConfig(nodeInfo.ChainID)
-	}
 
 	c.loadGroupCacheConfig()
 
@@ -312,12 +328,11 @@ func (c *Config) loadValidatorConfig() {
 }
 
 // GetEthNodeAndClient returns eth node info and client from path url
-func getEthNodeAndClient(path string) (node.Info, *rpc.Client, error) {
+func getEthNodeAndClient(path string, chainid uint64) (node.Info, *rpc.Client, error) {
 	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
 	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
 	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
 	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
-	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
 
 	rpcClient, err := rpc.Dial(path)
 	if err != nil {
@@ -328,6 +343,6 @@ func getEthNodeAndClient(path string) (node.Info, *rpc.Client, error) {
 		ClientName:   viper.GetString("ethereum.clientName"),
 		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
 		NetworkID:    viper.GetString("ethereum.networkID"),
-		ChainID:      viper.GetUint64("ethereum.chainID"),
+		ChainID:      chainid,
 	}, rpcClient, nil
 }

pkg/shared/test_helpers.go

@@ -27,6 +27,7 @@ import (
 	"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
 	"github.com/cerc-io/plugeth-statediff/indexer/models"
 	"github.com/cerc-io/plugeth-statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/jmoiron/sqlx"
@@ -55,30 +56,7 @@ func SetupDB() *sqlx.DB {
 
 // TearDownDB is used to tear down the watcher dbs after tests
 func TearDownDB(db *sqlx.DB) {
-	tx, err := db.Beginx()
+	err := test_helpers.ClearSqlxDB(db)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM nodes`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM ipld.blocks`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.header_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.uncle_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.transaction_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.receipt_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.state_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.storage_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.log_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth_meta.watched_addresses`)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = tx.Commit()
 	Expect(err).NotTo(HaveOccurred())
 }
 

scripts/run-test-stack.sh

@@ -1,43 +1,46 @@
 #!/bin/bash
-# Builds and deploys a stack with only what we need.
-# This script assumes we are running in the project root.
 
 set -e
 
-laconic_so="${LACONIC_SO:-laconic-so} --stack $(readlink -f test) --verbose"
+stack_dir=$(readlink -f "$1")
+[[ -d "$stack_dir" ]]
+
+laconic_so="laconic-so --verbose --stack $stack_dir"
 
 CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
+# By default assume we are running in the project root.
+export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}"
 
-# Prevent conflicting tty output
+# Don't run geth/plugeth in the debugger, it will swallow error backtraces
-export BUILDKIT_PROGRESS=plain
-
-# By default assume we are running in the project root
-export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-..}"
-# v5 migrations only go up to version 18
-echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 >> $CONFIG_DIR/stack.env
-# Pass this in so we can run eth_call forwarding tests, which expect no IPLD DB
-echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env
-# don't run plugeth in the debugger
 echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
+# Passing this lets us run eth_call forwarding tests without running ipld-eth-db
+echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env
 
 set -x
 
 if [[ -z $SKIP_BUILD ]]; then
+  # Prevent conflicting tty output
+  export BUILDKIT_PROGRESS=plain
+
+  # The server itself will be run separately
   $laconic_so setup-repositories \
     --exclude git.vdb.to/cerc-io/ipld-eth-server
-    # Assume the tested image has been built separately
   $laconic_so build-containers \
     --exclude cerc/ipld-eth-server
 fi
 
-$laconic_so deploy \
+if ! $laconic_so deploy \
   --exclude ipld-eth-server \
   --env-file $CONFIG_DIR/stack.env \
   --cluster test up
+then
+  $laconic_so deploy --cluster test logs
+  exit 1
+fi
 
 set +x
 
-# Get IPv4 endpoint of geth file server
+# Get IPv4 endpoint of geth bootnode file server
 bootnode_endpoint=$(docker port test-fixturenet-eth-bootnode-geth-1 9898 | head -1)
 
 # Extract the chain config and ID from genesis file

test/compose-db.yml

@@ -5,7 +5,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.4.0-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"

test/compose-server.yml

@@ -4,6 +4,8 @@ services:
   ipld-eth-server:
     restart: unless-stopped
     image: cerc/ipld-eth-server:local
+    build:
+      context: ..
     networks:
       - test_default
     environment:

test/stack.yml

@@ -1,23 +0,0 @@
-version: "1.2"
-name: fixturenet-plugeth-tx
-description: "Plugeth Ethereum Fixturenet for testing ipld-eth-server"
-repos:
-  - git.vdb.to/cerc-io/plugeth@v1.13.14-cerc-2
-  - git.vdb.to/cerc-io/plugeth-statediff
-  - git.vdb.to/cerc-io/lighthouse
-  - git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha
-  - git.vdb.to/cerc-io/ipld-eth-server
-containers:
-  - cerc/plugeth-statediff
-  - cerc/plugeth
-  - cerc/fixturenet-eth-genesis
-  - cerc/fixturenet-plugeth-plugeth
-  - cerc/lighthouse
-  - cerc/lighthouse-cli
-  - cerc/fixturenet-eth-lighthouse
-  - cerc/ipld-eth-db
-  - cerc/ipld-eth-server
-pods:
-  - fixturenet-plugeth
-  - ipld-eth-db
-  - ipld-eth-server