Index blob hashes (#27)

Reviewed-on: #27

.gitea/workflows/test.yml

@@ -9,7 +9,9 @@ on:
       - ci-test
 
 env:
-  SO_VERSION: v1.1.0-87fffca-202404110321
+  SO_VERSION: v1.1.0-36d4969-202407091537
+  FIXTURENET_ETH_STACKS_REF: main
+  SYSTEM_TESTS_REF: main
 
 jobs:
   unit-tests:
@@ -31,34 +33,49 @@ jobs:
     name: Run integration tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: ./plugeth-statediff
+          progress: false
       - name: Build docker image
         run: |
           docker build ./plugeth-statediff -t cerc/plugeth-statediff:local
 
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: 3.11
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - name: "Print Python version"
+        run: python3 --version
       - name: Install stack-orchestrator
         run: |
           curl -L -O https://github.com/cerc-io/stack-orchestrator/releases/download/$SO_VERSION/laconic-so
           chmod +x laconic-so
+          echo PATH="$PATH:$(pwd)" >> $GITHUB_ENV
       - name: Clone system-tests
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: cerc-io/system-tests
-          ref: v0.1.0-20240411
+          ref: ${{ env.SYSTEM_TESTS_REF }}
           path: ./system-tests
           token: ${{ secrets.CICD_REPO_TOKEN }}
+          progress: false
+      - name: Clone fixturenet stack repo
+        uses: actions/checkout@v4
+        with:
+          repository: cerc-io/fixturenet-eth-stacks
+          ref: ${{ env.FIXTURENET_ETH_STACKS_REF }}
+          path: ./fixturenet-eth-stacks
+          progress: false
       - name: Run testnet stack
         working-directory: ./plugeth-statediff
-        env:
-          LACONIC_SO: ../laconic-so
-        run: ./scripts/integration-setup.sh
-      - name: Install Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.11
-
+        run: ./scripts/run-test-stack.sh
       - name: Run tests
         working-directory: ./system-tests
         run: |
@@ -70,13 +87,14 @@ jobs:
     name: Run compliance tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: ./plugeth-statediff
-      - uses: actions/checkout@v3
+      - name: Check out compliance tests
+        uses: actions/checkout@v4
         with:
           repository: cerc-io/eth-statediff-compliance
-          ref: v0.2.0
+          ref: v0.3.0
           path: ./eth-statediff-compliance
           token: ${{ secrets.CICD_REPO_TOKEN }}
       - uses: actions/setup-go@v4

Dockerfile

@@ -1,6 +1,6 @@
 # Using image with same alpine as plugeth,
 # but go 1.21 to evade https://github.com/Consensys/gnark-crypto/issues/468
-FROM golang:1.21-alpine3.18 as builder
+FROM golang:1.21-alpine as builder
 
 RUN apk add --no-cache gcc musl-dev binutils-gold linux-headers git
 

adapt/util.go

@@ -7,14 +7,16 @@ import (
 )
 
 func ChainConfig(cc *plugeth_params.ChainConfig) *params.ChainConfig {
-	return &params.ChainConfig{
-		ChainID:             cc.ChainID,
-		HomesteadBlock:      cc.HomesteadBlock,
-		DAOForkBlock:        cc.DAOForkBlock,
-		DAOForkSupport:      cc.DAOForkSupport,
-		EIP150Block:         cc.EIP150Block,
-		EIP155Block:         cc.EIP155Block,
-		EIP158Block:         cc.EIP158Block,
+	ret := &params.ChainConfig{
+		ChainID: cc.ChainID,
+
+		HomesteadBlock: cc.HomesteadBlock,
+		DAOForkBlock:   cc.DAOForkBlock,
+		DAOForkSupport: cc.DAOForkSupport,
+		EIP150Block:    cc.EIP150Block,
+		EIP155Block:    cc.EIP155Block,
+		EIP158Block:    cc.EIP158Block,
+
 		ByzantiumBlock:      cc.ByzantiumBlock,
 		ConstantinopleBlock: cc.ConstantinopleBlock,
 		PetersburgBlock:     cc.PetersburgBlock,
@@ -22,5 +24,23 @@ func ChainConfig(cc *plugeth_params.ChainConfig) *params.ChainConfig {
 		MuirGlacierBlock:    cc.MuirGlacierBlock,
 		BerlinBlock:         cc.BerlinBlock,
 		LondonBlock:         cc.LondonBlock,
+
+		ArrowGlacierBlock:  cc.ArrowGlacierBlock,
+		GrayGlacierBlock:   cc.GrayGlacierBlock,
+		MergeNetsplitBlock: cc.MergeNetsplitBlock,
+
+		ShanghaiTime: cc.ShanghaiTime,
+		CancunTime:   cc.CancunTime,
+		PragueTime:   cc.PragueTime,
+
+		TerminalTotalDifficulty:       cc.TerminalTotalDifficulty,
+		TerminalTotalDifficultyPassed: cc.TerminalTotalDifficultyPassed,
 	}
+	if cc.Ethash != nil {
+		ret.Ethash = &params.EthashConfig{}
+	}
+	if cc.Clique != nil {
+		ret.Clique = &params.CliqueConfig{cc.Clique.Period, cc.Clique.Epoch}
+	}
+	return ret
 }

go.mod

@@ -3,8 +3,8 @@ module github.com/cerc-io/plugeth-statediff
 go 1.21
 
 require (
-	github.com/cerc-io/eth-iterator-utils v0.2.0
-	github.com/cerc-io/eth-testing v0.4.0
+	github.com/cerc-io/eth-iterator-utils v0.3.1
+	github.com/cerc-io/eth-testing v0.5.1
 	github.com/ethereum/go-ethereum v1.13.14
 	github.com/georgysavva/scany v0.2.9
 	github.com/golang/mock v1.6.0

go.sum

@@ -35,10 +35,10 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cerc-io/eth-iterator-utils v0.2.0 h1:wikAfWZ0fAqLqUy/Ud/a1n9p/arVeKG8P8tRjZE7oBg=
-github.com/cerc-io/eth-iterator-utils v0.2.0/go.mod h1:wDUJvwKDSOdqTIyeG+yXJ2ckzc9f2Fem614fV61DBcg=
-github.com/cerc-io/eth-testing v0.4.0 h1:ivGbXnEqlXMt/3m3jbsPJaVT7ZDTenFQWCryt1Rd/Jk=
-github.com/cerc-io/eth-testing v0.4.0/go.mod h1:CVsmHjFldX9gwaQSQwGmKbmh0g6Dq+bsqB2CxBf9zbk=
+github.com/cerc-io/eth-iterator-utils v0.3.1 h1:h4Bp0+fUiwkyug1uCEO2LZr2qxoW1yKszV2EO/2CDB0=
+github.com/cerc-io/eth-iterator-utils v0.3.1/go.mod h1:UNrjsP5bApZkqqqfU7nmnPN/dIIo9GOUUD79tmoX/s4=
+github.com/cerc-io/eth-testing v0.5.1 h1:xxcQf9ymJS0911yWIrUiGvCvqfvEjYmHvhBJkCD/whs=
+github.com/cerc-io/eth-testing v0.5.1/go.mod h1:p86je2PjSM7u8Qd7rMIG/Zw+tQlBoS5Emkh1ECnC5t0=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=

indexer/constructor.go

@@ -70,12 +70,12 @@ func NewStateDiffIndexer(
 		var driver sql.Driver
 		switch pgc.Driver {
 		case postgres.PGX:
-			driver, err = postgres.NewPGXDriver(ctx, pgc, nodeInfo)
+			driver, err = postgres.ConnectPGXDriver(ctx, pgc, nodeInfo)
 			if err != nil {
 				return nil, nil, err
 			}
 		case postgres.SQLX:
-			driver, err = postgres.NewSQLXDriver(ctx, pgc, nodeInfo)
+			driver, err = postgres.ConnectSQLXDriver(ctx, pgc, nodeInfo)
 			if err != nil {
 				return nil, nil, err
 			}

indexer/database/dump/indexer.go

@@ -82,7 +82,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	}
 
 	// Generate the block iplds
-	txNodes, rctNodes, logNodes, err := ipld.FromBlockAndReceipts(block, receipts)
+	txNodes, rctNodes, logNodes, wdNodes, err := ipld.FromBlockAndReceipts(block, receipts)
 	if err != nil {
 		return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
 	}
@@ -123,15 +123,17 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
 	t = time.Now()
 	// Publish and index receipts and txs
-	err = sdi.processReceiptsAndTxs(blockTx, processArgs{
+	err = sdi.processObjects(blockTx, processArgs{
 		headerID:    headerID,
 		blockNumber: block.Number(),
 		blockTime:   block.Time(),
 		receipts:    receipts,
 		txs:         transactions,
+		withdrawals: block.Withdrawals(),
 		rctNodes:    rctNodes,
 		txNodes:     txNodes,
 		logNodes:    logNodes,
+		wdNodes:     wdNodes,
 	})
 	if err != nil {
 		return nil, err
@@ -151,7 +153,7 @@ func (sdi *StateDiffIndexer) PushHeader(batch interfaces.Batch, header *types.He
 	if !ok {
 		return "", fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
 	}
-	headerNode, err := ipld.NewEthHeader(header)
+	headerNode, err := ipld.EncodeHeader(header)
 	if err != nil {
 		return "", err
 	}
@@ -173,6 +175,7 @@ func (sdi *StateDiffIndexer) PushHeader(batch interfaces.Batch, header *types.He
 		Timestamp:       header.Time,
 		Coinbase:        header.Coinbase.String(),
 		Canonical:       true,
+		WithdrawalsRoot: shared.MaybeStringHash(header.WithdrawalsHash),
 	}
 	_, err = fmt.Fprintf(sdi.dump, "%+v\r\n", mod)
 	return headerID, err
@@ -225,13 +228,15 @@ type processArgs struct {
 	blockTime   uint64
 	receipts    types.Receipts
 	txs         types.Transactions
-	rctNodes    []*ipld.EthReceipt
-	txNodes     []*ipld.EthTx
-	logNodes    [][]*ipld.EthLog
+	withdrawals types.Withdrawals
+	rctNodes    []ipld.IPLD
+	txNodes     []ipld.IPLD
+	logNodes    [][]ipld.IPLD
+	wdNodes     []ipld.IPLD
 }
 
-// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
-func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error {
+// processObjects publishes and indexes receipt and transaction IPLDs in Postgres
+func (sdi *StateDiffIndexer) processObjects(tx *BatchTx, args processArgs) error {
 	// Process receipts and txs
 	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber, args.blockTime)
 	for i, receipt := range args.receipts {
@@ -268,6 +273,20 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
 			return err
 		}
 
+		if trx.Type() == types.BlobTxType {
+			blobHashes := trx.BlobHashes()
+			for i, hash := range blobHashes {
+				bhModel := models.BlobHashModel{
+					TxHash:   trxID,
+					Index:    uint64(i),
+					BlobHash: hash,
+				}
+				if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", bhModel); err != nil {
+					return err
+				}
+			}
+		}
+
 		// this is the contract address if this receipt is for a contract creation tx
 		contract := shared.HandleZeroAddr(receipt.ContractAddress)
 
@@ -314,6 +333,23 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
 			return err
 		}
 	}
+	// Process withdrawals
+	for i, withdrawal := range args.withdrawals {
+		wdNode := args.wdNodes[i]
+		tx.cacheIPLD(wdNode)
+		wdModel := models.WithdrawalModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			CID:         wdNode.Cid().String(),
+			Index:       withdrawal.Index,
+			Validator:   withdrawal.Validator,
+			Address:     withdrawal.Address.String(),
+			Amount:      withdrawal.Amount,
+		}
+		if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", wdModel); err != nil {
+			return err
+		}
+	}
 
 	return nil
 }

indexer/database/file/csv_indexer_legacy_test.go

@@ -24,6 +24,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -55,6 +56,8 @@ func setupLegacyCSVIndexer(t *testing.T) {
 func setupLegacyCSV(t *testing.T) {
 	setupLegacyCSVIndexer(t)
 	test.SetupLegacyTestData(t, ind)
+	t.Cleanup(func() { tearDownCSV(t) })
+	time.Sleep(delayForDockerSync)
 }
 
 func dumpCSVFileData(t *testing.T) {
@@ -64,7 +67,7 @@ func dumpCSVFileData(t *testing.T) {
 
 	localOutputDir := filepath.Join(workingDir, file.CSVTestConfig.OutputDir)
 
-	for _, tbl := range file.Tables {
+	for _, tbl := range schema.EthTables {
 		err := test_helpers.DedupFile(file.TableFilePath(localOutputDir, tbl.Name))
 		require.NoError(t, err)
 
@@ -89,6 +92,7 @@ func dumpCSVFileData(t *testing.T) {
 func resetAndDumpWatchedAddressesCSVFileData(t *testing.T) {
 	test_helpers.TearDownDB(t, db)
 
+	time.Sleep(delayForDockerSync)
 	outputFilePath := filepath.Join(dbDirectory, file.CSVTestConfig.WatchedAddressesFilePath)
 	stmt := fmt.Sprintf(pgCopyStatement, schema.TableWatchedAddresses.Name, outputFilePath)
 
@@ -111,7 +115,6 @@ func TestLegacyCSVFileIndexer(t *testing.T) {
 	t.Run("Publish and index header IPLDs", func(t *testing.T) {
 		setupLegacyCSV(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.TestLegacyIndexer(t, db)
 	})

indexer/database/file/csv_indexer_test.go

@@ -21,6 +21,7 @@ import (
 	"math/big"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -30,6 +31,9 @@ import (
 	"github.com/cerc-io/plugeth-statediff/indexer/test"
 )
 
+// docker bind mount is slow to sync files
+var delayForDockerSync = 1 * time.Second
+
 func setupCSVIndexer(t *testing.T) {
 	if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) {
 		err := os.RemoveAll(file.CSVTestConfig.OutputDir)
@@ -53,18 +57,21 @@ func setupCSVIndexer(t *testing.T) {
 func setupCSV(t *testing.T) {
 	setupCSVIndexer(t)
 	test.SetupTestData(t, ind)
+	t.Cleanup(func() { tearDownCSV(t) })
+	time.Sleep(delayForDockerSync)
 }
 
 func setupCSVNonCanonical(t *testing.T) {
 	setupCSVIndexer(t)
 	test.SetupTestDataNonCanonical(t, ind)
+	t.Cleanup(func() { tearDownCSV(t) })
+	time.Sleep(delayForDockerSync)
 }
 
 func TestCSVFileIndexer(t *testing.T) {
 	t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
 		setupCSV(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexHeaderIPLDs(t, db)
 	})
@@ -72,7 +79,6 @@ func TestCSVFileIndexer(t *testing.T) {
 	t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
 		setupCSV(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexTransactionIPLDs(t, db)
 	})
@@ -80,7 +86,6 @@ func TestCSVFileIndexer(t *testing.T) {
 	t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
 		setupCSV(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexLogIPLDs(t, db)
 	})
@@ -88,15 +93,20 @@ func TestCSVFileIndexer(t *testing.T) {
 	t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
 		setupCSV(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexReceiptIPLDs(t, db)
 	})
 
+	t.Run("Publish and index withdrawal IPLDs in a single tx", func(t *testing.T) {
+		setupCSV(t)
+		dumpCSVFileData(t)
+
+		test.DoTestPublishAndIndexWithdrawalIPLDs(t, db)
+	})
+
 	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
 		setupCSV(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexStateIPLDs(t, db)
 	})
@@ -104,7 +114,6 @@ func TestCSVFileIndexer(t *testing.T) {
 	t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
 		setupCSV(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexStorageIPLDs(t, db)
 	})
@@ -114,7 +123,6 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
 	t.Run("Publish and index header", func(t *testing.T) {
 		setupCSVNonCanonical(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.TestPublishAndIndexHeaderNonCanonical(t, db)
 	})
@@ -122,7 +130,6 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
 	t.Run("Publish and index transactions", func(t *testing.T) {
 		setupCSVNonCanonical(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexTransactionsNonCanonical(t, db)
 	})
@@ -130,7 +137,6 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
 	t.Run("Publish and index receipts", func(t *testing.T) {
 		setupCSVNonCanonical(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexReceiptsNonCanonical(t, db)
 	})
@@ -138,7 +144,6 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
 	t.Run("Publish and index logs", func(t *testing.T) {
 		setupCSVNonCanonical(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexLogsNonCanonical(t, db)
 	})
@@ -146,7 +151,6 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
 	t.Run("Publish and index state nodes", func(t *testing.T) {
 		setupCSVNonCanonical(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexStateNonCanonical(t, db)
 	})
@@ -154,7 +158,6 @@ func TestCSVFileIndexerNonCanonical(t *testing.T) {
 	t.Run("Publish and index storage nodes", func(t *testing.T) {
 		setupCSVNonCanonical(t)
 		dumpCSVFileData(t)
-		defer tearDownCSV(t)
 
 		test.DoTestPublishAndIndexStorageNonCanonical(t, db)
 	})

indexer/database/file/csv_writer.go

@@ -36,22 +36,8 @@ import (
 	sdtypes "github.com/cerc-io/plugeth-statediff/types"
 )
 
-var (
-	Tables = []*schema.Table{
-		&schema.TableIPLDBlock,
-		&schema.TableNodeInfo,
-		&schema.TableHeader,
-		&schema.TableStateNode,
-		&schema.TableStorageNode,
-		&schema.TableUncle,
-		&schema.TableTransaction,
-		&schema.TableReceipt,
-		&schema.TableLog,
-	}
-)
-
 type tableRow struct {
-	table  schema.Table
+	table  *schema.Table
 	values []interface{}
 }
 
@@ -134,7 +120,7 @@ func NewCSVWriter(path string, watchedAddressesFilePath string, diff bool) (*CSV
 		return nil, fmt.Errorf("unable to create directory '%s': %w", path, err)
 	}
 
-	writers, err := makeFileWriters(path, Tables)
+	writers, err := makeFileWriters(path, schema.EthTables)
 	if err != nil {
 		return nil, err
 	}
@@ -164,7 +150,7 @@ func (csw *CSVWriter) Loop() {
 		for {
 			select {
 			case row := <-csw.rows:
-				err := csw.writers.write(&row.table, row.values...)
+				err := csw.writers.write(row.table, row.values...)
 				if err != nil {
 					panic(fmt.Sprintf("error writing csv buffer: %v", err))
 				}
@@ -204,13 +190,13 @@ func (csw *CSVWriter) Close() error {
 func (csw *CSVWriter) upsertNode(node nodeinfo.Info) {
 	var values []interface{}
 	values = append(values, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID)
-	csw.rows <- tableRow{schema.TableNodeInfo, values}
+	csw.rows <- tableRow{&schema.TableNodeInfo, values}
 }
 
 func (csw *CSVWriter) upsertIPLD(ipld models.IPLDModel) {
 	var values []interface{}
 	values = append(values, ipld.BlockNumber, ipld.Key, ipld.Data)
-	csw.rows <- tableRow{schema.TableIPLDBlock, values}
+	csw.rows <- tableRow{&schema.TableIPLDBlock, values}
 }
 
 func (csw *CSVWriter) upsertIPLDDirect(blockNumber, key string, value []byte) {
@@ -231,11 +217,25 @@ func (csw *CSVWriter) upsertIPLDNode(blockNumber string, i ipld.IPLD) {
 
 func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) {
 	var values []interface{}
-	values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
-		header.TotalDifficulty, header.NodeIDs, header.Reward, header.StateRoot, header.TxRoot,
-		header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.Coinbase,
-		header.Canonical)
-	csw.rows <- tableRow{schema.TableHeader, values}
+	values = append(values,
+		header.BlockNumber,
+		header.BlockHash,
+		header.ParentHash,
+		header.CID,
+		header.TotalDifficulty,
+		header.NodeIDs,
+		header.Reward,
+		header.StateRoot,
+		header.TxRoot,
+		header.RctRoot,
+		header.UnclesHash,
+		header.Bloom,
+		strconv.FormatUint(header.Timestamp, 10),
+		header.Coinbase,
+		header.Canonical,
+		header.WithdrawalsRoot,
+	)
+	csw.rows <- tableRow{&schema.TableHeader, values}
 	metrics.IndexerMetrics.BlocksCounter.Inc(1)
 }
 
@@ -243,22 +243,29 @@ func (csw *CSVWriter) upsertUncleCID(uncle models.UncleModel) {
 	var values []interface{}
 	values = append(values, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
 		uncle.Reward, uncle.Index)
-	csw.rows <- tableRow{schema.TableUncle, values}
+	csw.rows <- tableRow{&schema.TableUncle, values}
 }
 
 func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) {
 	var values []interface{}
 	values = append(values, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
 		transaction.Src, transaction.Index, transaction.Type, transaction.Value)
-	csw.rows <- tableRow{schema.TableTransaction, values}
+	csw.rows <- tableRow{&schema.TableTransaction, values}
 	metrics.IndexerMetrics.TransactionsCounter.Inc(1)
 }
 
+func (csw *CSVWriter) upsertBlobHash(blobHash models.BlobHashModel) {
+	var values []interface{}
+	values = append(values, blobHash.TxHash, blobHash.Index, blobHash.BlobHash)
+	csw.rows <- tableRow{&schema.TableBlobHash, values}
+	metrics.IndexerMetrics.BlobHashCounter.Inc(1)
+}
+
 func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
 	var values []interface{}
 	values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
 		rct.PostState, rct.PostStatus)
-	csw.rows <- tableRow{schema.TableReceipt, values}
+	csw.rows <- tableRow{&schema.TableReceipt, values}
 	metrics.IndexerMetrics.ReceiptsCounter.Inc(1)
 }
 
@@ -267,11 +274,26 @@ func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) {
 		var values []interface{}
 		values = append(values, l.BlockNumber, l.HeaderID, l.CID, l.ReceiptID, l.Address, l.Index, l.Topic0,
 			l.Topic1, l.Topic2, l.Topic3)
-		csw.rows <- tableRow{schema.TableLog, values}
+		csw.rows <- tableRow{&schema.TableLog, values}
 		metrics.IndexerMetrics.LogsCounter.Inc(1)
 	}
 }
 
+func (csw *CSVWriter) upsertWithdrawalCID(withdrawal models.WithdrawalModel) {
+	var values []interface{}
+	values = append(values,
+		withdrawal.BlockNumber,
+		withdrawal.HeaderID,
+		withdrawal.CID,
+		withdrawal.Index,
+		withdrawal.Validator,
+		withdrawal.Address,
+		withdrawal.Amount,
+	)
+	csw.rows <- tableRow{&schema.TableWithdrawal, values}
+	metrics.IndexerMetrics.WithdrawalsCounter.Inc(1)
+}
+
 func (csw *CSVWriter) upsertStateCID(stateNode models.StateNodeModel) {
 	balance := stateNode.Balance
 	if stateNode.Removed {
@@ -281,14 +303,14 @@ func (csw *CSVWriter) upsertStateCID(stateNode models.StateNodeModel) {
 	var values []interface{}
 	values = append(values, stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID,
 		csw.isDiff, balance, strconv.FormatUint(stateNode.Nonce, 10), stateNode.CodeHash, stateNode.StorageRoot, stateNode.Removed)
-	csw.rows <- tableRow{schema.TableStateNode, values}
+	csw.rows <- tableRow{&schema.TableStateNode, values}
 }
 
 func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
 	var values []interface{}
 	values = append(values, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID,
 		csw.isDiff, storageCID.Value, storageCID.Removed)
-	csw.rows <- tableRow{schema.TableStorageNode, values}
+	csw.rows <- tableRow{&schema.TableStorageNode, values}
 }
 
 // LoadWatchedAddresses loads watched addresses from a file

indexer/database/file/indexer.go

@@ -84,7 +84,7 @@ func NewStateDiffIndexer(chainConfig *params.ChainConfig, config Config, nodeInf
 		if _, err := os.Stat(outputDir); !errors.Is(err, os.ErrNotExist) {
 			return nil, fmt.Errorf("cannot create output directory, directory (%s) already exists", outputDir)
 		}
-		log.Info("Writing statediff CSV files to directory", "file", outputDir)
+		log.Info("Writing statediff CSV files", "directory", outputDir)
 
 		if watchedAddressesFilePath == "" {
 			watchedAddressesFilePath = defaultWatchedAddressesCSVFilePath
@@ -156,7 +156,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	}
 
 	// Generate the block iplds
-	txNodes, rctNodes, logNodes, err := ipld.FromBlockAndReceipts(block, receipts)
+	txNodes, rctNodes, logNodes, wdNodes, err := ipld.FromBlockAndReceipts(block, receipts)
 	if err != nil {
 		return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
 	}
@@ -191,21 +191,26 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	t = time.Now()
 
 	// write uncles
-	sdi.processUncles(headerID, block.Number(), block.UncleHash(), block.Uncles())
+	err = sdi.processUncles(headerID, block.Number(), block.UncleHash(), block.Uncles())
+	if err != nil {
+		return nil, err
+	}
 	tDiff = time.Since(t)
 	metrics.IndexerMetrics.UncleProcessingTimer.Update(tDiff)
 	traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
 	t = time.Now()
 
-	// write receipts and txs
-	err = sdi.processReceiptsAndTxs(processArgs{
+	err = sdi.processObjects(processArgs{
 		headerID:    headerID,
 		blockNumber: block.Number(),
+		blockTime:   block.Time(),
 		receipts:    receipts,
 		txs:         transactions,
+		withdrawals: block.Withdrawals(),
 		rctNodes:    rctNodes,
 		txNodes:     txNodes,
 		logNodes:    logNodes,
+		wdNodes:     wdNodes,
 	})
 	if err != nil {
 		return nil, err
@@ -222,7 +227,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 // it returns the headerID
 func (sdi *StateDiffIndexer) PushHeader(_ interfaces.Batch, header *types.Header, reward, td *big.Int) (string, error) {
 	// Process the header
-	headerNode, err := ipld.NewEthHeader(header)
+	headerNode, err := ipld.EncodeHeader(header)
 	if err != nil {
 		return "", err
 	}
@@ -245,6 +250,7 @@ func (sdi *StateDiffIndexer) PushHeader(_ interfaces.Batch, header *types.Header
 		Timestamp:       header.Time,
 		Coinbase:        header.Coinbase.String(),
 		Canonical:       true,
+		WithdrawalsRoot: shared.MaybeStringHash(header.WithdrawalsHash),
 	})
 	return headerID, nil
 }
@@ -293,13 +299,15 @@ type processArgs struct {
 	blockTime   uint64
 	receipts    types.Receipts
 	txs         types.Transactions
-	rctNodes    []*ipld.EthReceipt
-	txNodes     []*ipld.EthTx
-	logNodes    [][]*ipld.EthLog
+	withdrawals types.Withdrawals
+	rctNodes    []ipld.IPLD
+	txNodes     []ipld.IPLD
+	logNodes    [][]ipld.IPLD
+	wdNodes     []ipld.IPLD
 }
 
-// processReceiptsAndTxs writes receipt and tx IPLD insert SQL stmts to a file
-func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
+// processObjects writes receipt and tx IPLD insert SQL stmts to a file
+func (sdi *StateDiffIndexer) processObjects(args processArgs) error {
 	// Process receipts and txs
 	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber, args.blockTime)
 	for i, receipt := range args.receipts {
@@ -334,6 +342,17 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
 		}
 		sdi.fileWriter.upsertTransactionCID(txModel)
 
+		if trx.Type() == types.BlobTxType {
+			blobHashes := trx.BlobHashes()
+			for i, hash := range blobHashes {
+				sdi.fileWriter.upsertBlobHash(models.BlobHashModel{
+					TxHash:   txID,
+					Index:    uint64(i),
+					BlobHash: hash,
+				})
+			}
+		}
+
 		// this is the contract address if this receipt is for a contract creation tx
 		contract := shared.HandleZeroAddr(receipt.ContractAddress)
 
@@ -376,6 +395,21 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
 		}
 		sdi.fileWriter.upsertLogCID(logDataSet)
 	}
+	// Process withdrawals
+	for i, wd := range args.withdrawals {
+		wdNode := args.wdNodes[i]
+		sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), wdNode)
+		wdModel := models.WithdrawalModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			CID:         wdNode.Cid().String(),
+			Index:       wd.Index,
+			Validator:   wd.Validator,
+			Address:     wd.Address.String(),
+			Amount:      wd.Amount,
+		}
+		sdi.fileWriter.upsertWithdrawalCID(wdModel)
+	}
 
 	return nil
 }

indexer/database/file/interfaces.go

@@ -39,8 +39,10 @@ type FileWriter interface {
 	upsertHeaderCID(header models.HeaderModel)
 	upsertUncleCID(uncle models.UncleModel)
 	upsertTransactionCID(transaction models.TxModel)
+	upsertBlobHash(models.BlobHashModel)
 	upsertReceiptCID(rct *models.ReceiptModel)
 	upsertLogCID(logs []*models.LogsModel)
+	upsertWithdrawalCID(models.WithdrawalModel)
 	upsertStateCID(stateNode models.StateNodeModel)
 	upsertStorageCID(storageCID models.StorageNodeModel)
 	upsertIPLD(ipld models.IPLDModel)

indexer/database/file/sql_indexer_test.go

@@ -93,6 +93,14 @@ func TestSQLFileIndexer(t *testing.T) {
 		test.DoTestPublishAndIndexReceiptIPLDs(t, db)
 	})
 
+	t.Run("Publish and index withdrawal IPLDs in a single tx", func(t *testing.T) {
+		setup(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.DoTestPublishAndIndexWithdrawalIPLDs(t, db)
+	})
+
 	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
 		setup(t)
 		dumpFileData(t)

indexer/database/file/sql_writer.go

@@ -32,6 +32,7 @@ import (
 	"github.com/cerc-io/plugeth-statediff/indexer/ipld"
 	"github.com/cerc-io/plugeth-statediff/indexer/models"
 	nodeinfo "github.com/cerc-io/plugeth-statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
 	"github.com/cerc-io/plugeth-statediff/types"
 )
 
@@ -145,8 +146,8 @@ const (
 	ipldInsert = "INSERT INTO ipld.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n"
 
 	headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, " +
-		"state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase, canonical) VALUES " +
-		"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %t);\n"
+		"state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase, canonical, withdrawals_root) VALUES " +
+		"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %t, '%s');\n"
 
 	uncleInsert = "INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, index) VALUES " +
 		"('%s', '%s', '%s', '%s', '%s', '%s', %d);\n"
@@ -167,6 +168,11 @@ const (
 		"removed, diff, val) VALUES ('%s', '%s', '%s', '%s', '%s', %t, %t, '\\x%x');\n"
 )
 
+var (
+	withdrawalsInsert = schema.TableWithdrawal.FmtStringInsert() + ";\n"
+	blobHashesInsert  = schema.TableBlobHash.FmtStringInsert() + ";\n"
+)
+
 func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) {
 	sqw.stmts <- []byte(fmt.Sprintf(nodeInsert, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID))
 }
@@ -192,9 +198,24 @@ func (sqw *SQLWriter) upsertIPLDNode(blockNumber string, i ipld.IPLD) {
 }
 
 func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
-	stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
-		header.TotalDifficulty, formatPostgresStringArray(header.NodeIDs), header.Reward, header.StateRoot, header.TxRoot,
-		header.RctRoot, header.UnclesHash, header.Bloom, header.Timestamp, header.Coinbase, header.Canonical)
+	stmt := fmt.Sprintf(headerInsert,
+		header.BlockNumber,
+		header.BlockHash,
+		header.ParentHash,
+		header.CID,
+		header.TotalDifficulty,
+		formatPostgresStringArray(header.NodeIDs),
+		header.Reward,
+		header.StateRoot,
+		header.TxRoot,
+		header.RctRoot,
+		header.UnclesHash,
+		header.Bloom,
+		header.Timestamp,
+		header.Coinbase,
+		header.Canonical,
+		header.WithdrawalsRoot,
+	)
 	sqw.stmts <- []byte(stmt)
 	metrics.IndexerMetrics.BlocksCounter.Inc(1)
 }
@@ -210,6 +231,11 @@ func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) {
 	metrics.IndexerMetrics.TransactionsCounter.Inc(1)
 }
 
+func (sqw *SQLWriter) upsertBlobHash(bh models.BlobHashModel) {
+	sqw.stmts <- []byte(fmt.Sprintf(blobHashesInsert, bh.TxHash, bh.Index, bh.BlobHash))
+	metrics.IndexerMetrics.BlobHashCounter.Inc(1)
+}
+
 func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) {
 	sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
 		rct.PostState, rct.PostStatus))
@@ -224,6 +250,19 @@ func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) {
 	}
 }
 
+func (sqw *SQLWriter) upsertWithdrawalCID(withdrawal models.WithdrawalModel) {
+	sqw.stmts <- []byte(fmt.Sprintf(withdrawalsInsert,
+		withdrawal.BlockNumber,
+		withdrawal.HeaderID,
+		withdrawal.CID,
+		withdrawal.Index,
+		withdrawal.Validator,
+		withdrawal.Address,
+		withdrawal.Amount,
+	))
+	metrics.IndexerMetrics.WithdrawalsCounter.Inc(1)
+}
+
 func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) {
 	balance := stateNode.Balance
 	if stateNode.Removed {

indexer/database/metrics/metrics.go

@@ -52,10 +52,14 @@ type IndexerMetricsHandles struct {
 	BlocksCounter metrics.Counter
 	// The total number of processed transactions
 	TransactionsCounter metrics.Counter
+	// The total number of indexed blob hashes
+	BlobHashCounter metrics.Counter
 	// The total number of processed receipts
 	ReceiptsCounter metrics.Counter
 	// The total number of processed logs
 	LogsCounter metrics.Counter
+	// The total number of processed logs
+	WithdrawalsCounter metrics.Counter
 	// The total number of access list entries processed
 	AccessListEntriesCounter metrics.Counter
 	// Time spent waiting for free postgres tx
@@ -88,8 +92,10 @@ func RegisterIndexerMetrics(reg metrics.Registry) IndexerMetricsHandles {
 	ctx := IndexerMetricsHandles{
 		BlocksCounter:                     metrics.NewCounter(),
 		TransactionsCounter:               metrics.NewCounter(),
+		BlobHashCounter:                   metrics.NewCounter(),
 		ReceiptsCounter:                   metrics.NewCounter(),
 		LogsCounter:                       metrics.NewCounter(),
+		WithdrawalsCounter:                metrics.NewCounter(),
 		AccessListEntriesCounter:          metrics.NewCounter(),
 		FreePostgresTimer:                 metrics.NewTimer(),
 		PostgresCommitTimer:               metrics.NewTimer(),
@@ -111,8 +117,10 @@ func RegisterIndexerMetrics(reg metrics.Registry) IndexerMetricsHandles {
 	subsys := "indexer"
 	reg.Register(metricName(subsys, "blocks"), ctx.BlocksCounter)
 	reg.Register(metricName(subsys, "transactions"), ctx.TransactionsCounter)
+	reg.Register(metricName(subsys, "blob_hashes"), ctx.BlobHashCounter)
 	reg.Register(metricName(subsys, "receipts"), ctx.ReceiptsCounter)
 	reg.Register(metricName(subsys, "logs"), ctx.LogsCounter)
+	reg.Register(metricName(subsys, "withdrawals"), ctx.WithdrawalsCounter)
 	reg.Register(metricName(subsys, "access_list_entries"), ctx.AccessListEntriesCounter)
 	reg.Register(metricName(subsys, "t_free_postgres"), ctx.FreePostgresTimer)
 	reg.Register(metricName(subsys, "t_postgres_commit"), ctx.PostgresCommitTimer)

indexer/database/sql/indexer.go

@@ -105,7 +105,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	}
 
 	// Generate the block iplds
-	txNodes, rctNodes, logNodes, err := ipld.FromBlockAndReceipts(block, receipts)
+	txNodes, rctNodes, logNodes, wdNodes, err := ipld.FromBlockAndReceipts(block, receipts)
 	if err != nil {
 		return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %w", err)
 	}
@@ -148,16 +148,18 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	}
 	metrics2.IndexerMetrics.UncleProcessingTimer.Update(time.Since(t))
 	t = time.Now()
-	// Publish and index receipts and txs
-	err = sdi.processReceiptsAndTxs(batch, processArgs{
+
+	err = sdi.processObjects(batch, processArgs{
 		headerID:    headerID,
 		blockNumber: block.Number(),
 		blockTime:   block.Time(),
 		receipts:    receipts,
+		withdrawals: block.Withdrawals(),
 		txs:         transactions,
 		rctNodes:    rctNodes,
 		txNodes:     txNodes,
 		logNodes:    logNodes,
+		wdNodes:     wdNodes,
 	})
 	if err != nil {
 		return nil, err
@@ -185,7 +187,7 @@ func (sdi *StateDiffIndexer) PushHeader(batch interfaces.Batch, header *types.He
 		return "", fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
 	}
 	// Process the header
-	headerNode, err := ipld.NewEthHeader(header)
+	headerNode, err := ipld.EncodeHeader(header)
 	if err != nil {
 		return "", err
 	}
@@ -208,6 +210,7 @@ func (sdi *StateDiffIndexer) PushHeader(batch interfaces.Batch, header *types.He
 		Timestamp:       header.Time,
 		Coinbase:        header.Coinbase.String(),
 		Canonical:       true,
+		WithdrawalsRoot: shared.MaybeStringHash(header.WithdrawalsHash),
 	})
 }
 
@@ -258,13 +261,15 @@ type processArgs struct {
 	blockTime   uint64
 	receipts    types.Receipts
 	txs         types.Transactions
-	rctNodes    []*ipld.EthReceipt
-	txNodes     []*ipld.EthTx
-	logNodes    [][]*ipld.EthLog
+	withdrawals types.Withdrawals
+	rctNodes    []ipld.IPLD
+	txNodes     []ipld.IPLD
+	logNodes    [][]ipld.IPLD
+	wdNodes     []ipld.IPLD
 }
 
-// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
-func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error {
+// processObjects publishes and indexes receipt and transaction IPLDs in Postgres
+func (sdi *StateDiffIndexer) processObjects(tx *BatchTx, args processArgs) error {
 	// Process receipts and txs
 	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber, args.blockTime)
 	for i, receipt := range args.receipts {
@@ -301,6 +306,19 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
 			return err
 		}
 
+		if trx.Type() == types.BlobTxType {
+			blobHashes := trx.BlobHashes()
+			for i, hash := range blobHashes {
+				if err := sdi.dbWriter.upsertBlobHash(tx.dbtx, models.BlobHashModel{
+					TxHash:   txID,
+					Index:    uint64(i),
+					BlobHash: hash,
+				}); err != nil {
+					return err
+				}
+			}
+		}
+
 		// this is the contract address if this receipt is for a contract creation tx
 		contract := shared.HandleZeroAddr(receipt.ContractAddress)
 
@@ -348,7 +366,23 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
 			return err
 		}
 	}
-
+	// Process withdrawals
+	for i, withdrawal := range args.withdrawals {
+		wdNode := args.wdNodes[i]
+		tx.cacheIPLD(wdNode)
+		wdModel := models.WithdrawalModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			CID:         wdNode.Cid().String(),
+			Index:       withdrawal.Index,
+			Validator:   withdrawal.Validator,
+			Address:     withdrawal.Address.String(),
+			Amount:      withdrawal.Amount,
+		}
+		if err := sdi.dbWriter.upsertWithdrawalCID(tx.dbtx, wdModel); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 

indexer/database/sql/interfaces.go

@@ -52,8 +52,10 @@ type Statements interface {
 	SetCanonicalHeaderStm() string
 	InsertUncleStm() string
 	InsertTxStm() string
+	InsertBlobHashStm() string
 	InsertRctStm() string
 	InsertLogStm() string
+	InsertWithdrawalStm() string
 	InsertStateStm() string
 	InsertStorageStm() string
 	InsertIPLDStm() string

indexer/database/sql/pgx_indexer_test.go

@@ -96,6 +96,14 @@ func TestPGXIndexer(t *testing.T) {
 		test.DoTestPublishAndIndexReceiptIPLDs(t, db)
 	})
 
+	t.Run("Publish and index withdrawal IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.DoTestPublishAndIndexWithdrawalIPLDs(t, db)
+	})
+
 	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
 		setupPGX(t)
 		defer tearDown(t)

indexer/database/sql/postgres/database.go

@@ -18,6 +18,7 @@ package postgres
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
 	"github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
@@ -43,7 +44,9 @@ type DB struct {
 
 // MaxHeaderStm satisfies the sql.Statements interface
 func (db *DB) MaxHeaderStm() string {
-	return fmt.Sprintf("SELECT block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase FROM %s ORDER BY block_number DESC LIMIT 1", schema.TableHeader.Name)
+	return fmt.Sprintf("SELECT %s FROM %s ORDER BY block_number DESC LIMIT 1",
+		strings.Join(schema.TableHeader.ColumnNames(), ","),
+		schema.TableHeader.Name)
 }
 
 // ExistsHeaderStm satisfies the sql.Statements interface
@@ -59,7 +62,7 @@ func (db *DB) DetectGapsStm() string {
 // InsertHeaderStm satisfies the sql.Statements interface
 // Stm == Statement
 func (db *DB) InsertHeaderStm() string {
-	return schema.TableHeader.ToInsertStatement(db.upsert)
+	return schema.TableHeader.PreparedInsert(db.upsert)
 }
 
 // SetCanonicalHeaderStm satisfies the sql.Statements interface
@@ -70,37 +73,47 @@ func (db *DB) SetCanonicalHeaderStm() string {
 
 // InsertUncleStm satisfies the sql.Statements interface
 func (db *DB) InsertUncleStm() string {
-	return schema.TableUncle.ToInsertStatement(db.upsert)
+	return schema.TableUncle.PreparedInsert(db.upsert)
 }
 
 // InsertTxStm satisfies the sql.Statements interface
 func (db *DB) InsertTxStm() string {
-	return schema.TableTransaction.ToInsertStatement(db.upsert)
+	return schema.TableTransaction.PreparedInsert(db.upsert)
+}
+
+// InsertBlobHashStm satisfies the sql.Statements interface
+func (db *DB) InsertBlobHashStm() string {
+	return schema.TableBlobHash.PreparedInsert(db.upsert)
 }
 
 // InsertRctStm satisfies the sql.Statements interface
 func (db *DB) InsertRctStm() string {
-	return schema.TableReceipt.ToInsertStatement(db.upsert)
+	return schema.TableReceipt.PreparedInsert(db.upsert)
 }
 
 // InsertLogStm satisfies the sql.Statements interface
 func (db *DB) InsertLogStm() string {
-	return schema.TableLog.ToInsertStatement(db.upsert)
+	return schema.TableLog.PreparedInsert(db.upsert)
+}
+
+// InsertLogStm satisfies the sql.Statements interface
+func (db *DB) InsertWithdrawalStm() string {
+	return schema.TableWithdrawal.PreparedInsert(db.upsert)
 }
 
 // InsertStateStm satisfies the sql.Statements interface
 func (db *DB) InsertStateStm() string {
-	return schema.TableStateNode.ToInsertStatement(db.upsert)
+	return schema.TableStateNode.PreparedInsert(db.upsert)
 }
 
 // InsertStorageStm satisfies the sql.Statements interface
 func (db *DB) InsertStorageStm() string {
-	return schema.TableStorageNode.ToInsertStatement(db.upsert)
+	return schema.TableStorageNode.PreparedInsert(db.upsert)
 }
 
 // InsertIPLDStm satisfies the sql.Statements interface
 func (db *DB) InsertIPLDStm() string {
-	return schema.TableIPLDBlock.ToInsertStatement(db.upsert)
+	return schema.TableIPLDBlock.PreparedInsert(db.upsert)
 }
 
 // InsertIPLDsStm satisfies the sql.Statements interface

indexer/database/sql/postgres/pgx.go

@@ -37,7 +37,6 @@ type PGXDriver struct {
 	ctx      context.Context
 	pool     *pgxpool.Pool
 	nodeInfo node.Info
-	nodeID   string
 	config   Config
 }
 
@@ -50,21 +49,25 @@ func ConnectPGX(ctx context.Context, config Config) (*pgxpool.Pool, error) {
 	return pgxpool.ConnectConfig(ctx, pgConf)
 }
 
-// NewPGXDriver returns a new pgx driver
+// ConnectPGXDriver returns a new pgx driver
 // it initializes the connection pool and creates the node info table
-func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
+func ConnectPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
 	dbPool, err := ConnectPGX(ctx, config)
 	if err != nil {
 		return nil, ErrDBConnectionFailed(err)
 	}
-	pg := &PGXDriver{ctx: ctx, pool: dbPool, nodeInfo: node, config: config}
-	nodeErr := pg.createNode()
+	pg := NewPGXDriver(ctx, dbPool, config)
+	nodeErr := pg.createNode(node)
 	if nodeErr != nil {
 		return &PGXDriver{}, ErrUnableToSetNode(nodeErr)
 	}
 	return pg, nil
 }
 
+func NewPGXDriver(ctx context.Context, pool *pgxpool.Pool, config Config) *PGXDriver {
+	return &PGXDriver{ctx: ctx, pool: pool, config: config}
+}
+
 // MakeConfig creates a pgxpool.Config from the provided Config
 func MakeConfig(config Config) (*pgxpool.Config, error) {
 	conf, err := pgxpool.ParseConfig("")
@@ -102,19 +105,19 @@ func MakeConfig(config Config) (*pgxpool.Config, error) {
 	return conf, nil
 }
 
-func (pgx *PGXDriver) createNode() error {
+func (pgx *PGXDriver) createNode(nodeInfo node.Info) error {
 	_, err := pgx.pool.Exec(
 		pgx.ctx,
 		createNodeStm,
-		pgx.nodeInfo.GenesisBlock,
-		pgx.nodeInfo.NetworkID,
-		pgx.nodeInfo.ID,
-		pgx.nodeInfo.ClientName,
-		pgx.nodeInfo.ChainID)
+		nodeInfo.GenesisBlock,
+		nodeInfo.NetworkID,
+		nodeInfo.ID,
+		nodeInfo.ClientName,
+		nodeInfo.ChainID)
 	if err != nil {
 		return ErrUnableToSetNode(err)
 	}
-	pgx.nodeID = pgx.nodeInfo.ID
+	pgx.nodeInfo = nodeInfo
 	return nil
 }
 
@@ -155,7 +158,7 @@ func (pgx *PGXDriver) Stats() metrics.DbStats {
 
 // NodeID satisfies sql.Database
 func (pgx *PGXDriver) NodeID() string {
-	return pgx.nodeID
+	return pgx.nodeInfo.ID
 }
 
 // Close satisfies sql.Database/io.Closer

indexer/database/sql/postgres/pgx_test.go

@@ -94,7 +94,7 @@ func TestPostgresPGX(t *testing.T) {
 
 	t.Run("throws error when can't connect to the database", func(t *testing.T) {
 		goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
-		_, err := postgres.NewPGXDriver(ctx, postgres.Config{}, goodInfo)
+		_, err := postgres.ConnectPGXDriver(ctx, postgres.Config{}, goodInfo)
 		if err == nil {
 			t.Fatal("Expected an error")
 		}
@@ -106,7 +106,7 @@ func TestPostgresPGX(t *testing.T) {
 		badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
 		badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
 
-		_, err := postgres.NewPGXDriver(ctx, pgConfig, badInfo)
+		_, err := postgres.ConnectPGXDriver(ctx, pgConfig, badInfo)
 		if err == nil {
 			t.Fatal("Expected an error")
 		}

indexer/database/sql/postgres/sqlx.go

@@ -34,7 +34,6 @@ type SQLXDriver struct {
 	ctx      context.Context
 	db       *sqlx.DB
 	nodeInfo node.Info
-	nodeID   string
 }
 
 // ConnectSQLX initializes and returns a SQLX connection pool for postgres
@@ -53,32 +52,36 @@ func ConnectSQLX(ctx context.Context, config Config) (*sqlx.DB, error) {
 	return db, nil
 }
 
-// NewSQLXDriver returns a new sqlx driver for Postgres
+// ConnectSQLXDriver returns a new sqlx driver for Postgres
 // it initializes the connection pool and creates the node info table
-func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
+func ConnectSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
 	db, err := ConnectSQLX(ctx, config)
 	if err != nil {
 		return nil, err
 	}
-	driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node}
-	if err := driver.createNode(); err != nil {
+	driver := NewSQLXDriver(ctx, db)
+	if err := driver.createNode(node); err != nil {
 		return nil, err
 	}
 	return driver, nil
 }
 
-func (driver *SQLXDriver) createNode() error {
+func NewSQLXDriver(ctx context.Context, db *sqlx.DB) *SQLXDriver {
+	return &SQLXDriver{ctx: ctx, db: db}
+}
+
+func (driver *SQLXDriver) createNode(nodeInfo node.Info) error {
 	_, err := driver.db.Exec(
 		createNodeStm,
-		driver.nodeInfo.GenesisBlock,
-		driver.nodeInfo.NetworkID,
-		driver.nodeInfo.ID,
-		driver.nodeInfo.ClientName,
-		driver.nodeInfo.ChainID)
+		nodeInfo.GenesisBlock,
+		nodeInfo.NetworkID,
+		nodeInfo.ID,
+		nodeInfo.ClientName,
+		nodeInfo.ChainID)
 	if err != nil {
 		return ErrUnableToSetNode(err)
 	}
-	driver.nodeID = driver.nodeInfo.ID
+	driver.nodeInfo = nodeInfo
 	return nil
 }
 
@@ -118,7 +121,7 @@ func (driver *SQLXDriver) Stats() metrics.DbStats {
 
 // NodeID satisfies sql.Database
 func (driver *SQLXDriver) NodeID() string {
-	return driver.nodeID
+	return driver.nodeInfo.ID
 }
 
 // Close satisfies sql.Database/io.Closer

indexer/database/sql/postgres/sqlx_test.go

@@ -97,7 +97,7 @@ func TestPostgresSQLX(t *testing.T) {
 
 	t.Run("throws error when can't connect to the database", func(t *testing.T) {
 		goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
-		_, err := postgres.NewSQLXDriver(ctx, postgres.Config{}, goodInfo)
+		_, err := postgres.ConnectSQLXDriver(ctx, postgres.Config{}, goodInfo)
 		if err == nil {
 			t.Fatal("Expected an error")
 		}
@@ -109,7 +109,7 @@ func TestPostgresSQLX(t *testing.T) {
 		badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
 		badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
 
-		_, err := postgres.NewSQLXDriver(ctx, pgConfig, badInfo)
+		_, err := postgres.ConnectSQLXDriver(ctx, pgConfig, badInfo)
 		if err == nil {
 			t.Fatal("Expected an error")
 		}

indexer/database/sql/postgres/test_helpers.go

@@ -30,7 +30,7 @@ func SetupSQLXDB() (sql.Database, error) {
 		return nil, err
 	}
 	conf.MaxIdle = 0
-	driver, err := NewSQLXDriver(context.Background(), conf, node.Info{})
+	driver, err := ConnectSQLXDriver(context.Background(), conf, node.Info{})
 	if err != nil {
 		return nil, err
 	}
@@ -39,7 +39,7 @@ func SetupSQLXDB() (sql.Database, error) {
 
 // SetupPGXDB is used to setup a pgx db for tests
 func SetupPGXDB(config Config) (sql.Database, error) {
-	driver, err := NewPGXDriver(context.Background(), config, node.Info{})
+	driver, err := ConnectPGXDriver(context.Background(), config, node.Info{})
 	if err != nil {
 		return nil, err
 	}

indexer/database/sql/sqlx_indexer_test.go

@@ -82,6 +82,14 @@ func TestSQLXIndexer(t *testing.T) {
 		test.DoTestPublishAndIndexReceiptIPLDs(t, db)
 	})
 
+	t.Run("Publish and index withdrawal IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.DoTestPublishAndIndexWithdrawalIPLDs(t, db)
+	})
+
 	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
 		setupSQLX(t)
 		defer tearDown(t)

indexer/database/sql/writer.go

@@ -95,6 +95,7 @@ func (w *Writer) maxHeader() (*models.HeaderModel, error) {
 		&model.Timestamp,
 		&model.Coinbase,
 		&model.Canonical,
+		&model.WithdrawalsRoot,
 	)
 	model.BlockNumber = strconv.FormatUint(number, 10)
 	model.TotalDifficulty = strconv.FormatUint(td, 10)
@@ -125,6 +126,7 @@ func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error {
 		header.Timestamp,
 		header.Coinbase,
 		header.Canonical,
+		header.WithdrawalsRoot,
 	)
 	if err != nil {
 		return insertError{"eth.header_cids", err, w.db.InsertHeaderStm(), header}
@@ -207,6 +209,30 @@ func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
 	return nil
 }
 
+/*
+INSERT INTO eth.blob_hashes (tx_hash, index, blob_hash) VALUES ($1, $2, $3)
+ON CONFLICT (tx_hash, index) DO NOTHING
+*/
+func (w *Writer) upsertBlobHash(tx Tx, blobHash models.BlobHashModel) error {
+	if w.useCopyForTx(tx) {
+		rows := toRows(toRow(blobHash.TxHash, blobHash.Index, blobHash.BlobHash))
+		_, err := tx.CopyFrom(w.db.Context(), schema.TableBlobHash.TableName(), schema.TableBlobHash.ColumnNames(), rows)
+		if err != nil {
+			return insertError{"eth.blob_hashes", err, "COPY", blobHash}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertBlobHashStm(),
+			blobHash.TxHash,
+			blobHash.Index,
+			blobHash.BlobHash,
+		)
+		if err != nil {
+			return insertError{"eth.blob_hashes", err, w.db.InsertBlobHashStm(), blobHash}
+		}
+	}
+	return nil
+}
+
 /*
 INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, post_status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
 ON CONFLICT (tx_id, header_id, block_number) DO NOTHING
@@ -286,6 +312,41 @@ func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
 	return nil
 }
 
+func (w *Writer) upsertWithdrawalCID(tx Tx, withdrawal models.WithdrawalModel) error {
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseUint(withdrawal.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.withdrawal_cids", err, "COPY", withdrawal}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), schema.TableWithdrawal.TableName(), schema.TableWithdrawal.ColumnNames(),
+			toRows(toRow(blockNum,
+				withdrawal.HeaderID,
+				withdrawal.CID,
+				withdrawal.Index,
+				withdrawal.Validator,
+				withdrawal.Address,
+				withdrawal.Amount)))
+		if err != nil {
+			return insertError{"eth.withdrawal_cids", err, "COPY", withdrawal}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertWithdrawalStm(),
+			withdrawal.BlockNumber,
+			withdrawal.HeaderID,
+			withdrawal.CID,
+			withdrawal.Index,
+			withdrawal.Validator,
+			withdrawal.Address,
+			withdrawal.Amount)
+		if err != nil {
+			return insertError{"eth.withdrawal_cids", err, w.db.InsertWithdrawalStm(), withdrawal}
+		}
+	}
+	metrics.IndexerMetrics.WithdrawalsCounter.Inc(1)
+	return nil
+}
+
 /*
 INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
 ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING

indexer/ipld/encode.go

@@ -0,0 +1,102 @@
+// VulcanizeDB
+// Copyright © 2024 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipld
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	mh "github.com/multiformats/go-multihash"
+)
+
+// EncodeHeader converts a *types.Header into an IPLD node
+func EncodeHeader(header *types.Header) (IPLD, error) {
+	headerRLP, err := rlp.EncodeToBytes(header)
+	if err != nil {
+		return nil, err
+	}
+	c, err := RawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &node{
+		cid:     c,
+		rawdata: headerRLP,
+	}, nil
+}
+
+// EncodeTx converts a *types.Transaction to an IPLD node
+func EncodeTx(tx *types.Transaction) (IPLD, error) {
+	txRaw, err := tx.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	c, err := RawdataToCid(MEthTx, txRaw, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &node{
+		cid:     c,
+		rawdata: txRaw,
+	}, nil
+}
+
+// EncodeReceipt converts a types.Receipt to an IPLD node
+func EncodeReceipt(receipt *types.Receipt) (IPLD, error) {
+	rctRaw, err := receipt.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	c, err := RawdataToCid(MEthTxReceipt, rctRaw, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &node{
+		cid:     c,
+		rawdata: rctRaw,
+	}, nil
+}
+
+// EncodeLog converts a Log to an IPLD node
+func EncodeLog(log *types.Log) (IPLD, error) {
+	logRaw, err := rlp.EncodeToBytes(log)
+	if err != nil {
+		return nil, err
+	}
+	c, err := RawdataToCid(MEthLog, logRaw, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &node{
+		cid:     c,
+		rawdata: logRaw,
+	}, nil
+}
+
+func EncodeWithdrawal(w *types.Withdrawal) (IPLD, error) {
+	wRaw, err := rlp.EncodeToBytes(w)
+	if err != nil {
+		return nil, err
+	}
+	c, err := RawdataToCid(MEthWithdrawal, wRaw, mh.KECCAK_256)
+	if err != nil {
+		return nil, err
+	}
+	return &node{
+		cid:     c,
+		rawdata: wRaw,
+	}, nil
+}

indexer/ipld/eth_header.go

@@ -1,60 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package ipld
-
-import (
-	"github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
-)
-
-// EthHeader (eth-block, codec 0x90), represents an ethereum block header
-type EthHeader struct {
-	cid     cid.Cid
-	rawdata []byte
-}
-
-// Static (compile time) check that EthHeader satisfies the node.Node interface.
-var _ IPLD = (*EthHeader)(nil)
-
-// NewEthHeader converts a *types.Header into an EthHeader IPLD node
-func NewEthHeader(header *types.Header) (*EthHeader, error) {
-	headerRLP, err := rlp.EncodeToBytes(header)
-	if err != nil {
-		return nil, err
-	}
-	c, err := RawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256)
-	if err != nil {
-		return nil, err
-	}
-	return &EthHeader{
-		cid:     c,
-		rawdata: headerRLP,
-	}, nil
-}
-
-// RawData returns the binary of the RLP encode of the block header.
-func (b *EthHeader) RawData() []byte {
-	return b.rawdata
-}
-
-// Cid returns the cid of the block header.
-func (b *EthHeader) Cid() cid.Cid {
-	return b.cid
-}

indexer/ipld/eth_log.go

@@ -1,43 +0,0 @@
-package ipld
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-)
-
-// EthLog (eth-log, codec 0x9a), represents an ethereum block header
-type EthLog struct {
-	rawData []byte
-	cid     cid.Cid
-}
-
-// Static (compile time) check that EthLog satisfies the node.Node interface.
-var _ IPLD = (*EthLog)(nil)
-
-// NewLog create a new EthLog IPLD node
-func NewLog(log *types.Log) (*EthLog, error) {
-	logRaw, err := rlp.EncodeToBytes(log)
-	if err != nil {
-		return nil, err
-	}
-	c, err := RawdataToCid(MEthLog, logRaw, mh.KECCAK_256)
-	if err != nil {
-		return nil, err
-	}
-	return &EthLog{
-		cid:     c,
-		rawData: logRaw,
-	}, nil
-}
-
-// RawData returns the binary of the RLP encode of the log.
-func (l *EthLog) RawData() []byte {
-	return l.rawData
-}
-
-// Cid returns the cid of the receipt log.
-func (l *EthLog) Cid() cid.Cid {
-	return l.cid
-}

indexer/ipld/eth_parser.go

@@ -22,25 +22,29 @@ import (
 
 // FromBlockAndReceipts takes a block and processes it
 // to return it a set of IPLD nodes for further processing.
-func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) ([]*EthTx, []*EthReceipt, [][]*EthLog, error) {
+func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) ([]IPLD, []IPLD, [][]IPLD, []IPLD, error) {
 	// Process the txs
 	txNodes, err := processTransactions(block.Transactions())
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, nil, nil, err
+	}
+	withdrawalNodes, err := processWithdrawals(block.Withdrawals())
+	if err != nil {
+		return nil, nil, nil, nil, err
 	}
 
 	// Process the receipts and logs
 	rctNodes, logNodes, err := processReceiptsAndLogs(receipts)
 
-	return txNodes, rctNodes, logNodes, err
+	return txNodes, rctNodes, logNodes, withdrawalNodes, err
 }
 
 // processTransactions will take the found transactions in a parsed block body
 // to return IPLD node slices for eth-tx
-func processTransactions(txs []*types.Transaction) ([]*EthTx, error) {
-	var ethTxNodes []*EthTx
+func processTransactions(txs []*types.Transaction) ([]IPLD, error) {
+	var ethTxNodes []IPLD
 	for _, tx := range txs {
-		ethTx, err := NewEthTx(tx)
+		ethTx, err := EncodeTx(tx)
 		if err != nil {
 			return nil, err
 		}
@@ -50,12 +54,25 @@ func processTransactions(txs []*types.Transaction) ([]*EthTx, error) {
 	return ethTxNodes, nil
 }
 
+func processWithdrawals(withdrawals []*types.Withdrawal) ([]IPLD, error) {
+	var withdrawalNodes []IPLD
+	for _, withdrawal := range withdrawals {
+		ethW, err := EncodeWithdrawal(withdrawal)
+		if err != nil {
+			return nil, err
+		}
+		withdrawalNodes = append(withdrawalNodes, ethW)
+	}
+
+	return withdrawalNodes, nil
+}
+
 // processReceiptsAndLogs will take in receipts
 // to return IPLD node slices for eth-rct and eth-log
-func processReceiptsAndLogs(rcts []*types.Receipt) ([]*EthReceipt, [][]*EthLog, error) {
+func processReceiptsAndLogs(rcts []*types.Receipt) ([]IPLD, [][]IPLD, error) {
 	// Pre allocating memory.
-	ethRctNodes := make([]*EthReceipt, len(rcts))
-	ethLogNodes := make([][]*EthLog, len(rcts))
+	ethRctNodes := make([]IPLD, len(rcts))
+	ethLogNodes := make([][]IPLD, len(rcts))
 
 	for idx, rct := range rcts {
 		logNodes, err := processLogs(rct.Logs)
@@ -63,7 +80,7 @@ func processReceiptsAndLogs(rcts []*types.Receipt) ([]*EthReceipt, [][]*EthLog,
 			return nil, nil, err
 		}
 
-		ethRct, err := NewReceipt(rct)
+		ethRct, err := EncodeReceipt(rct)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -75,10 +92,10 @@ func processReceiptsAndLogs(rcts []*types.Receipt) ([]*EthReceipt, [][]*EthLog,
 	return ethRctNodes, ethLogNodes, nil
 }
 
-func processLogs(logs []*types.Log) ([]*EthLog, error) {
-	logNodes := make([]*EthLog, len(logs))
+func processLogs(logs []*types.Log) ([]IPLD, error) {
+	logNodes := make([]IPLD, len(logs))
 	for idx, log := range logs {
-		logNode, err := NewLog(log)
+		logNode, err := EncodeLog(log)
 		if err != nil {
 			return nil, err
 		}

indexer/ipld/eth_parser_test.go

@@ -92,7 +92,7 @@ func loadBlockData(t *testing.T) []testCase {
 func TestFromBlockAndReceipts(t *testing.T) {
 	testCases := loadBlockData(t)
 	for _, tc := range testCases {
-		_, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts)
+		_, _, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts)
 		if err != nil {
 			t.Fatalf("error generating IPLDs from block and receipts, err %v, kind %s, block hash %s", err, tc.kind, tc.block.Hash())
 		}

indexer/ipld/eth_receipt.go

@@ -1,58 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package ipld
-
-import (
-	"github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-type EthReceipt struct {
-	rawdata []byte
-	cid     cid.Cid
-}
-
-// Static (compile time) check that EthReceipt satisfies the node.Node interface.
-var _ IPLD = (*EthReceipt)(nil)
-
-// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node
-func NewReceipt(receipt *types.Receipt) (*EthReceipt, error) {
-	rctRaw, err := receipt.MarshalBinary()
-	if err != nil {
-		return nil, err
-	}
-	c, err := RawdataToCid(MEthTxReceipt, rctRaw, mh.KECCAK_256)
-	if err != nil {
-		return nil, err
-	}
-	return &EthReceipt{
-		cid:     c,
-		rawdata: rctRaw,
-	}, nil
-}
-
-// RawData returns the binary of the RLP encode of the receipt.
-func (r *EthReceipt) RawData() []byte {
-	return r.rawdata
-}
-
-// Cid returns the cid of the receipt.
-func (r *EthReceipt) Cid() cid.Cid {
-	return r.cid
-}

indexer/ipld/eth_tx.go

@@ -1,59 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package ipld
-
-import (
-	"github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-// EthTx (eth-tx codec 0x93) represents an ethereum transaction
-type EthTx struct {
-	cid     cid.Cid
-	rawdata []byte
-}
-
-// Static (compile time) check that EthTx satisfies the node.Node interface.
-var _ IPLD = (*EthTx)(nil)
-
-// NewEthTx converts a *types.Transaction to an EthTx IPLD node
-func NewEthTx(tx *types.Transaction) (*EthTx, error) {
-	txRaw, err := tx.MarshalBinary()
-	if err != nil {
-		return nil, err
-	}
-	c, err := RawdataToCid(MEthTx, txRaw, mh.KECCAK_256)
-	if err != nil {
-		return nil, err
-	}
-	return &EthTx{
-		cid:     c,
-		rawdata: txRaw,
-	}, nil
-}
-
-// RawData returns the binary of the RLP encode of the transaction.
-func (t *EthTx) RawData() []byte {
-	return t.rawdata
-}
-
-// Cid returns the cid of the transaction.
-func (t *EthTx) Cid() cid.Cid {
-	return t.cid
-}

indexer/ipld/interface.go

@@ -2,7 +2,25 @@ package ipld
 
 import "github.com/ipfs/go-cid"
 
+// Check that node satisfies the IPLD Node interface.
+var _ IPLD = (*node)(nil)
+
+type node struct {
+	cid     cid.Cid
+	rawdata []byte
+}
+
 type IPLD interface {
 	Cid() cid.Cid
 	RawData() []byte
 }
+
+// RawData returns the RLP encoded bytes of the node.
+func (b node) RawData() []byte {
+	return b.rawdata
+}
+
+// Cid returns the CID of the node.
+func (b node) Cid() cid.Cid {
+	return b.cid
+}

indexer/ipld/shared.go

@@ -37,6 +37,7 @@ const (
 	MEthStorageTrie     = 0x98
 	MEthLogTrie         = 0x99
 	MEthLog             = 0x9a
+	MEthWithdrawal      = 0x9b // TODO add to multicodec registry
 )
 
 // RawdataToCid takes the desired codec and a slice of bytes

indexer/mocks/test_data.go

@@ -19,8 +19,8 @@ package mocks
 import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
-	"crypto/rand"
 	"math/big"
+	"math/rand"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -39,6 +39,9 @@ import (
 
 // Test variables
 var (
+	// RNG for deterministically generated keys
+	rng = rand.New(rand.NewSource(0))
+
 	// block data
 	TestChainConfig = params.MainnetChainConfig
 	BlockNumber     = TestChainConfig.LondonBlock
@@ -47,7 +50,7 @@ var (
 	// canonical block at London height
 	// includes 5 transactions: 3 Legacy + 1 EIP-2930 + 1 EIP-1559
 	MockHeader = types.Header{
-		Time:        0,
+		Time:        BlockTime,
 		Number:      new(big.Int).Set(BlockNumber),
 		Root:        common.HexToHash("0x0"),
 		TxHash:      common.HexToHash("0x0"),
@@ -58,15 +61,19 @@ var (
 		Coinbase:    common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476777"),
 	}
 	MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts(TestChainConfig, BlockNumber, BlockTime)
-	MockBlock                                  = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, trie.NewEmpty(nil))
-	MockHeaderRlp, _                           = rlp.EncodeToBytes(MockBlock.Header())
+	MockWithdrawals                            = types.Withdrawals{
+		{Index: 0, Validator: 1, Address: Address, Amount: 1000000000},
+		{Index: 1, Validator: 5, Address: AnotherAddress, Amount: 2000000000},
+	}
+	MockBlock        = types.NewBlockWithWithdrawals(&MockHeader, MockTransactions, nil, MockReceipts, MockWithdrawals, trie.NewEmpty(nil))
+	MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header())
 
 	// non-canonical block at London height
 	// includes 2nd and 5th transactions from the canonical block
 	MockNonCanonicalHeader            = MockHeader
 	MockNonCanonicalBlockTransactions = types.Transactions{MockTransactions[1], MockTransactions[4]}
 	MockNonCanonicalBlockReceipts     = createNonCanonicalBlockReceipts(TestChainConfig, BlockNumber, BlockTime, MockNonCanonicalBlockTransactions)
-	MockNonCanonicalBlock             = types.NewBlock(&MockNonCanonicalHeader, MockNonCanonicalBlockTransactions, nil, MockNonCanonicalBlockReceipts, trie.NewEmpty(nil))
+	MockNonCanonicalBlock             = types.NewBlockWithWithdrawals(&MockNonCanonicalHeader, MockNonCanonicalBlockTransactions, nil, MockNonCanonicalBlockReceipts, MockWithdrawals[:1], trie.NewEmpty(nil))
 	MockNonCanonicalHeaderRlp, _      = rlp.EncodeToBytes(MockNonCanonicalBlock.Header())
 
 	// non-canonical block at London height + 1
@@ -86,7 +93,7 @@ var (
 	}
 	MockNonCanonicalBlock2Transactions = types.Transactions{MockTransactions[2], MockTransactions[4]}
 	MockNonCanonicalBlock2Receipts     = createNonCanonicalBlockReceipts(TestChainConfig, Block2Number, BlockTime, MockNonCanonicalBlock2Transactions)
-	MockNonCanonicalBlock2             = types.NewBlock(&MockNonCanonicalHeader2, MockNonCanonicalBlock2Transactions, nil, MockNonCanonicalBlock2Receipts, trie.NewEmpty(nil))
+	MockNonCanonicalBlock2             = types.NewBlockWithWithdrawals(&MockNonCanonicalHeader2, MockNonCanonicalBlock2Transactions, nil, MockNonCanonicalBlock2Receipts, types.Withdrawals{}, trie.NewEmpty(nil))
 	MockNonCanonicalHeader2Rlp, _      = rlp.EncodeToBytes(MockNonCanonicalBlock2.Header())
 
 	Address                     = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
@@ -348,7 +355,10 @@ func NewLegacyData(config *params.ChainConfig) *LegacyData {
 
 	mockTransactions, mockReceipts, senderAddr := createLegacyTransactionsAndReceipts(config, blockNumber)
 	mockBlock := types.NewBlock(&mockHeader, mockTransactions, nil, mockReceipts, trie.NewEmpty(nil))
-	mockHeaderRlp, _ := rlp.EncodeToBytes(mockBlock.Header())
+	mockHeaderRlp, err := rlp.EncodeToBytes(mockBlock.Header())
+	if err != nil {
+		panic(err)
+	}
 	contractAddress := crypto.CreateAddress(senderAddr, mockTransactions[2].Nonce())
 
 	return &LegacyData{
@@ -388,7 +398,7 @@ func createLegacyTransactionsAndReceipts(config *params.ChainConfig, blockNumber
 	blockTime := uint64(0)
 	transactionSigner := types.MakeSigner(config, blockNumber, blockTime)
 	mockCurve := elliptic.P256()
-	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rng)
 	if err != nil {
 		log.Crit(err.Error())
 	}
@@ -424,13 +434,15 @@ func createLegacyTransactionsAndReceipts(config *params.ChainConfig, blockNumber
 	return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, senderAddr
 }
 
-// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+// createTransactionsAndReceipts generates signed mock transactions and mock receipts with mock logs, and returns the address of the sender with them.
 func createTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big.Int, blockTime uint64) (types.Transactions, types.Receipts, common.Address) {
+	const txCount = 6
 	// make transactions
-	trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
-	trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
-	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
-	trx4 := types.NewTx(&types.AccessListTx{
+	txs := make(types.Transactions, txCount)
+	txs[0] = types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
+	txs[1] = types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
+	txs[2] = types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
+	txs[3] = types.NewTx(&types.AccessListTx{
 		ChainID:  config.ChainID,
 		Nonce:    0,
 		GasPrice: big.NewInt(100),
@@ -443,7 +455,7 @@ func createTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big.
 			AccessListEntry2,
 		},
 	})
-	trx5 := types.NewTx(&types.DynamicFeeTx{
+	txs[4] = types.NewTx(&types.DynamicFeeTx{
 		ChainID:   config.ChainID,
 		Nonce:     0,
 		GasTipCap: big.NewInt(100),
@@ -457,74 +469,84 @@ func createTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big.
 			AccessListEntry2,
 		},
 	})
+	txs[5] = types.NewTx(&types.BlobTx{
+		ChainID:    uint256.MustFromBig(config.ChainID),
+		Nonce:      0,
+		GasTipCap:  uint256.NewInt(100),
+		GasFeeCap:  uint256.NewInt(100),
+		Gas:        50,
+		To:         AnotherAddress,
+		Value:      uint256.NewInt(0),
+		BlobFeeCap: uint256.NewInt(1e6),
+		BlobHashes: []common.Hash{
+			common.HexToHash("0x0100000000000000000000000000000000000000000000000000000000000001"),
+			common.HexToHash("0x0100000000000000000000000000000000000000000000000000000000000002"),
+		},
+	})
 
 	transactionSigner := types.MakeSigner(config, blockNumber, blockTime)
 	mockCurve := elliptic.P256()
-	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rng)
 	if err != nil {
 		log.Crit(err.Error())
 	}
-	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
-	if err != nil {
-		log.Crit(err.Error())
+	var signedTxs types.Transactions
+	for _, tx := range txs {
+		signed, err := types.SignTx(tx, transactionSigner, mockPrvKey)
+		if err != nil {
+			log.Crit(err.Error())
+		}
+		signedTxs = append(signedTxs, signed)
 	}
-	signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey)
-	if err != nil {
-		log.Crit(err.Error())
-	}
-	signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey)
-	if err != nil {
-		log.Crit(err.Error())
-	}
-	signedTrx4, err := types.SignTx(trx4, transactionSigner, mockPrvKey)
-	if err != nil {
-		log.Crit(err.Error())
-	}
-	signedTrx5, err := types.SignTx(trx5, transactionSigner, mockPrvKey)
-	if err != nil {
-		log.Crit(err.Error())
-	}
-
-	senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	senderAddr, err := types.Sender(transactionSigner, signedTxs[0]) // same for both trx
 	if err != nil {
 		log.Crit(err.Error())
 	}
 
 	// make receipts
-	mockReceipt1 := types.NewReceipt(nil, false, 50)
-	mockReceipt1.Logs = []*types.Log{MockLog1}
-	mockReceipt1.TxHash = signedTrx1.Hash()
-	mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
-	mockReceipt2.Logs = []*types.Log{MockLog2, ShortLog1}
-	mockReceipt2.TxHash = signedTrx2.Hash()
-	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75)
-	mockReceipt3.Logs = []*types.Log{}
-	mockReceipt3.TxHash = signedTrx3.Hash()
-	mockReceipt4 := &types.Receipt{
+	receipts := make(types.Receipts, txCount)
+	receipts[0] = types.NewReceipt(nil, false, 50)
+	receipts[0].Logs = []*types.Log{MockLog1}
+	receipts[0].TxHash = signedTxs[0].Hash()
+	receipts[1] = types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	receipts[1].Logs = []*types.Log{MockLog2, ShortLog1}
+	receipts[1].TxHash = signedTxs[1].Hash()
+	receipts[2] = types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75)
+	receipts[2].Logs = []*types.Log{}
+	receipts[2].TxHash = signedTxs[2].Hash()
+	receipts[3] = &types.Receipt{
 		Type:              types.AccessListTxType,
 		PostState:         common.HexToHash("0x3").Bytes(),
 		Status:            types.ReceiptStatusSuccessful,
 		CumulativeGasUsed: 175,
 		Logs:              []*types.Log{MockLog3, MockLog4, ShortLog2},
-		TxHash:            signedTrx4.Hash(),
+		TxHash:            signedTxs[3].Hash(),
 	}
-	mockReceipt5 := &types.Receipt{
+	receipts[4] = &types.Receipt{
 		Type:              types.DynamicFeeTxType,
 		PostState:         common.HexToHash("0x3").Bytes(),
 		Status:            types.ReceiptStatusSuccessful,
 		CumulativeGasUsed: 175,
 		Logs:              []*types.Log{},
-		TxHash:            signedTrx5.Hash(),
+		TxHash:            signedTxs[4].Hash(),
+	}
+	receipts[5] = &types.Receipt{
+		Type:              types.BlobTxType,
+		PostState:         common.HexToHash("0x3").Bytes(),
+		Status:            types.ReceiptStatusSuccessful,
+		CumulativeGasUsed: 175,
+		Logs:              []*types.Log{},
+		TxHash:            signedTxs[5].Hash(),
 	}
 
-	return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4, signedTrx5}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4, mockReceipt5}, senderAddr
+	return signedTxs, receipts, senderAddr
 }
 
 // createNonCanonicalBlockReceipts is a helper function to generate mock receipts with mock logs for non-canonical blocks
 func createNonCanonicalBlockReceipts(config *params.ChainConfig, blockNumber *big.Int, blockTime uint64, transactions types.Transactions) types.Receipts {
 	transactionSigner := types.MakeSigner(config, blockNumber, blockTime)
 	mockCurve := elliptic.P256()
-	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rng)
 	if err != nil {
 		log.Crit(err.Error())
 	}

indexer/models/models.go

@@ -16,7 +16,10 @@
 
 package models
 
-import "github.com/lib/pq"
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/lib/pq"
+)
 
 // IPLDModel is the db model for ipld.blocks
 type IPLDModel struct {
@@ -42,6 +45,7 @@ type HeaderModel struct {
 	Timestamp       uint64         `db:"timestamp"`
 	Coinbase        string         `db:"coinbase"`
 	Canonical       bool           `db:"canonical"`
+	WithdrawalsRoot string         `db:"withdrawals_root"`
 }
 
 // UncleModel is the db model for eth.uncle_cids
@@ -105,7 +109,7 @@ type StorageNodeModel struct {
 	Value       []byte `db:"val"`
 }
 
-// LogsModel is the db model for eth.logs
+// LogsModel is the db model for eth.log_cids
 type LogsModel struct {
 	BlockNumber string `db:"block_number"`
 	HeaderID    string `db:"header_id"`
@@ -118,3 +122,21 @@ type LogsModel struct {
 	Topic2      string `db:"topic2"`
 	Topic3      string `db:"topic3"`
 }
+
+// WithdrawalModel is the db model for eth.withdrawal_cids
+type WithdrawalModel struct {
+	BlockNumber string `db:"block_number"`
+	HeaderID    string `db:"header_id"`
+	CID         string `db:"cid"`
+	Index       uint64 `db:"index"`
+	Validator   uint64 `db:"validator"`
+	Address     string `db:"address"`
+	Amount      uint64 `db:"amount"`
+}
+
+// BlobHashModel is the DB model for eth.blob_hashes
+type BlobHashModel struct {
+	TxHash   string      `db:"tx_hash"`
+	Index    uint64      `db:"index"`
+	BlobHash common.Hash `db:"blob_hash"`
+}

indexer/shared/functions.go

@@ -35,3 +35,12 @@ func HandleZeroAddr(to common.Address) string {
 	}
 	return to.String()
 }
+
+// MaybeStringHash calls String on its argument and returns a pointer to the result.
+// When passed nil, it returns nil.
+func MaybeStringHash(hash *common.Hash) string {
+	if hash == nil {
+		return ""
+	}
+	return hash.String()
+}

indexer/shared/schema/schema.go

@@ -16,6 +16,25 @@
 
 package schema
 
+var EthTables = []*Table{
+	&TableIPLDBlock,
+	&TableNodeInfo,
+	&TableHeader,
+	&TableStateNode,
+	&TableStorageNode,
+	&TableUncle,
+	&TableTransaction,
+	&TableReceipt,
+	&TableLog,
+	&TableWithdrawal,
+	&TableBlobHash,
+}
+
+var AllTables = append(
+	EthTables,
+	&TableWatchedAddresses,
+)
+
 var TableIPLDBlock = Table{
 	Name: `ipld.blocks`,
 	Columns: []Column{
@@ -52,9 +71,10 @@ var TableHeader = Table{
 		{Name: "receipt_root", Type: Dvarchar},
 		{Name: "uncles_hash", Type: Dvarchar},
 		{Name: "bloom", Type: Dbytea},
-		{Name: "timestamp", Type: Dnumeric},
+		{Name: "timestamp", Type: Dbigint},
 		{Name: "coinbase", Type: Dvarchar},
 		{Name: "canonical", Type: Dboolean},
+		{Name: "withdrawals_root", Type: Dvarchar},
 	},
 	UpsertClause: OnConflict("block_number", "block_hash").Set(
 		"parent_hash",
@@ -70,6 +90,7 @@ var TableHeader = Table{
 		"timestamp",
 		"coinbase",
 		"canonical",
+		"withdrawals_root",
 	)}
 
 var TableStateNode = Table{
@@ -165,6 +186,29 @@ var TableLog = Table{
 	UpsertClause: OnConflict("block_number", "header_id", "rct_id", "index"),
 }
 
+var TableWithdrawal = Table{
+	Name: "eth.withdrawal_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "header_id", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "index", Type: Dinteger},
+		{Name: "validator", Type: Dinteger},
+		{Name: "address", Type: Dvarchar},
+		{Name: "amount", Type: Dinteger},
+	},
+	UpsertClause: OnConflict("block_number", "header_id", "index"),
+}
+
+var TableBlobHash = Table{
+	Name: "eth.blob_hashes",
+	Columns: []Column{
+		{Name: "tx_hash", Type: Dvarchar},
+		{Name: "index", Type: Dinteger},
+		{Name: "blob_hash", Type: Dbytea},
+	},
+}
+
 var TableWatchedAddresses = Table{
 	Name: "eth_meta.watched_addresses",
 	Columns: []Column{

indexer/shared/schema/table.go

@@ -53,34 +53,6 @@ type Table struct {
 	UpsertClause ConflictClause
 }
 
-type colfmt = func(interface{}) string
-
-func (tbl *Table) ToCsvRow(args ...interface{}) []string {
-	var row []string
-	for i, col := range tbl.Columns {
-		value := col.Type.formatter()(args[i])
-
-		if col.Array {
-			valueList := funk.Map(args[i], col.Type.formatter()).([]string)
-			value = fmt.Sprintf("{%s}", strings.Join(valueList, ","))
-		}
-
-		row = append(row, value)
-	}
-	return row
-}
-
-func (tbl *Table) VarcharColumns() []string {
-	columns := funk.Filter(tbl.Columns, func(col Column) bool {
-		return col.Type == Dvarchar
-	}).([]Column)
-
-	columnNames := funk.Map(columns, func(col Column) string {
-		return col.Name
-	}).([]string)
-	return columnNames
-}
-
 func OnConflict(target ...string) ConflictClause {
 	return ConflictClause{Target: target}
 }
@@ -89,35 +61,6 @@ func (c ConflictClause) Set(fields ...string) ConflictClause {
 	return c
 }
 
-// ToInsertStatement returns a Postgres-compatible SQL insert statement for the table
-// using positional placeholders
-func (tbl *Table) ToInsertStatement(upsert bool) string {
-	var colnames, placeholders []string
-	for i, col := range tbl.Columns {
-		colnames = append(colnames, col.Name)
-		placeholders = append(placeholders, fmt.Sprintf("$%d", i+1))
-	}
-	suffix := fmt.Sprintf("ON CONFLICT (%s)", strings.Join(tbl.UpsertClause.Target, ", "))
-	if upsert && len(tbl.UpsertClause.Update) != 0 {
-		var update_placeholders []string
-		for _, name := range tbl.UpsertClause.Update {
-			i := funk.IndexOf(tbl.Columns, func(col Column) bool { return col.Name == name })
-			update_placeholders = append(update_placeholders, fmt.Sprintf("$%d", i+1))
-		}
-		suffix += fmt.Sprintf(
-			" DO UPDATE SET (%s) = (%s)",
-			strings.Join(tbl.UpsertClause.Update, ", "), strings.Join(update_placeholders, ", "),
-		)
-	} else {
-		suffix += " DO NOTHING"
-	}
-
-	return fmt.Sprintf(
-		"INSERT INTO %s (%s) VALUES (%s) %s",
-		tbl.Name, strings.Join(colnames, ", "), strings.Join(placeholders, ", "), suffix,
-	)
-}
-
 // TableName returns a pgx-compatible table name.
 func (tbl *Table) TableName() []string {
 	return strings.Split(tbl.Name, ".")
@@ -132,11 +75,45 @@ func (tbl *Table) ColumnNames() []string {
 	return names
 }
 
+// PreparedInsert returns a pgx/sqlx-compatible SQL prepared insert statement for the table
+// using positional placeholders.
+// If upsert is true, include an ON CONFLICT clause handling column updates.
+func (tbl *Table) PreparedInsert(upsert bool) string {
+	var colnames, placeholders []string
+	for i, col := range tbl.Columns {
+		colnames = append(colnames, col.Name)
+		placeholders = append(placeholders, fmt.Sprintf("$%d", i+1))
+	}
+	suffix := " ON CONFLICT"
+	if len(tbl.UpsertClause.Target) > 0 {
+		suffix += fmt.Sprintf(" (%s)", strings.Join(tbl.UpsertClause.Target, ", "))
+	}
+	if upsert && len(tbl.UpsertClause.Update) != 0 {
+		var update_placeholders []string
+		for _, name := range tbl.UpsertClause.Update {
+			update_placeholders = append(update_placeholders, "EXCLUDED."+name)
+		}
+		suffix += fmt.Sprintf(
+			" DO UPDATE SET (%s) = ROW(%s)",
+			strings.Join(tbl.UpsertClause.Update, ", "), strings.Join(update_placeholders, ", "),
+		)
+	} else {
+		suffix += " DO NOTHING"
+	}
+
+	return fmt.Sprintf(
+		"INSERT INTO %s (%s) VALUES (%s)",
+		tbl.Name, strings.Join(colnames, ", "), strings.Join(placeholders, ", "),
+	) + suffix
+}
+
+type colfmt = func(interface{}) string
+
 func sprintf(f string) colfmt {
 	return func(x interface{}) string { return fmt.Sprintf(f, x) }
 }
 
-func (typ colType) formatter() colfmt {
+func (typ colType) csvFormatter() colfmt {
 	switch typ {
 	case Dinteger:
 		return sprintf("%d")
@@ -157,6 +134,61 @@ func (typ colType) formatter() colfmt {
 		return sprintf("%s")
 	case Dtext:
 		return sprintf("%s")
+	default:
+		panic("invalid column type")
 	}
-	panic("unreachable")
+}
+
+// ToCsvRow converts a list of values to a list of strings suitable for CSV output.
+func (tbl *Table) ToCsvRow(args ...interface{}) []string {
+	var row []string
+	for i, col := range tbl.Columns {
+		value := col.Type.csvFormatter()(args[i])
+
+		if col.Array {
+			valueList := funk.Map(args[i], col.Type.csvFormatter()).([]string)
+			value = fmt.Sprintf("{%s}", strings.Join(valueList, ","))
+		}
+
+		row = append(row, value)
+	}
+	return row
+}
+
+// VarcharColumns returns the names of columns with type VARCHAR.
+func (tbl *Table) VarcharColumns() []string {
+	columns := funk.Filter(tbl.Columns, func(col Column) bool {
+		return col.Type == Dvarchar
+	}).([]Column)
+
+	columnNames := funk.Map(columns, func(col Column) string {
+		return col.Name
+	}).([]string)
+	return columnNames
+}
+
+func formatSpec(typ colType) string {
+	switch typ {
+	case Dinteger:
+		return "%d"
+	case Dboolean:
+		return "%t"
+	case Dbytea:
+		return `'\x%x'`
+	default:
+		return "'%s'"
+	}
+}
+
+// FmtStringInsert returns a format string for creating a Postgres insert statement.
+func (tbl *Table) FmtStringInsert() string {
+	var colnames, placeholders []string
+	for _, col := range tbl.Columns {
+		colnames = append(colnames, col.Name)
+		placeholders = append(placeholders, formatSpec(col.Type))
+	}
+	return fmt.Sprintf(
+		"INSERT INTO %s (%s) VALUES (%s);",
+		tbl.Name, strings.Join(colnames, ", "), strings.Join(placeholders, ", "),
+	)
 }

indexer/shared/schema/table_test.go

@@ -8,47 +8,55 @@ import (
 	. "github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
 )
 
-var testHeaderTable = Table{
-	Name: "eth.header_cids",
-	Columns: []Column{
-		{Name: "block_number", Type: Dbigint},
-		{Name: "block_hash", Type: Dvarchar},
-		{Name: "parent_hash", Type: Dvarchar},
-		{Name: "cid", Type: Dtext},
-		{Name: "td", Type: Dnumeric},
-		{Name: "node_id", Type: Dvarchar},
-		{Name: "reward", Type: Dnumeric},
-		{Name: "state_root", Type: Dvarchar},
-		{Name: "tx_root", Type: Dvarchar},
-		{Name: "receipt_root", Type: Dvarchar},
-		{Name: "uncle_root", Type: Dvarchar},
-		{Name: "bloom", Type: Dbytea},
-		{Name: "timestamp", Type: Dnumeric},
-		{Name: "mh_key", Type: Dtext},
-		{Name: "times_validated", Type: Dinteger},
-		{Name: "coinbase", Type: Dvarchar},
-	},
-	UpsertClause: OnConflict("block_hash", "block_number").Set(
-		"parent_hash",
-		"cid",
-		"td",
-		"node_id",
-		"reward",
-		"state_root",
-		"tx_root",
-		"receipt_root",
-		"uncle_root",
-		"bloom",
-		"timestamp",
-		"mh_key",
-		"times_validated",
-		"coinbase",
-	),
-}
+var (
+	testTable = Table{
+		Name: "test_table",
+		Columns: []Column{
+			{Name: "id", Type: Dbigint},
+			{Name: "name", Type: Dvarchar},
+			{Name: "age", Type: Dinteger},
+		},
+	}
+	testTableWithConflictClause = Table{
+		Name: "test_table_conflict",
+		Columns: []Column{
+			{Name: "id", Type: Dbigint},
+			{Name: "name", Type: Dvarchar},
+			{Name: "age", Type: Dinteger},
+		},
+		UpsertClause: OnConflict("id").Set("name", "age"),
+	}
+)
+
+const (
+	expectedHeaderPreparedWithUpsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase, canonical, withdrawals_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase, canonical, withdrawals_root) = ROW(EXCLUDED.parent_hash, EXCLUDED.cid, EXCLUDED.td, EXCLUDED.node_ids, EXCLUDED.reward, EXCLUDED.state_root, EXCLUDED.tx_root, EXCLUDED.receipt_root, EXCLUDED.uncles_hash, EXCLUDED.bloom, EXCLUDED.timestamp, EXCLUDED.coinbase, EXCLUDED.canonical, EXCLUDED.withdrawals_root)"
+
+	expectedHeaderPreparedWithoutUpsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase, canonical, withdrawals_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_number, block_hash) DO NOTHING"
+
+	expectedHeaderFmtString = `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase, canonical, withdrawals_root) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\x%x', '%s', '%s', %t, '%s');`
+)
 
 func TestTable(t *testing.T) {
-	headerUpsert := `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)`
-	headerNoUpsert := `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_hash, block_number) DO NOTHING`
-	require.Equal(t, headerNoUpsert, testHeaderTable.ToInsertStatement(false))
-	require.Equal(t, headerUpsert, testHeaderTable.ToInsertStatement(true))
+	require.Equal(t,
+		"INSERT INTO test_table (id, name, age) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
+		testTable.PreparedInsert(true),
+	)
+	require.Equal(t,
+		"INSERT INTO test_table (id, name, age) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
+		testTable.PreparedInsert(false),
+	)
+	require.Equal(t, "INSERT INTO test_table (id, name, age) VALUES ('%s', '%s', %d);", testTable.FmtStringInsert())
+
+	require.Equal(t,
+		"INSERT INTO test_table_conflict (id, name, age) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET (name, age) = ROW(EXCLUDED.name, EXCLUDED.age)",
+		testTableWithConflictClause.PreparedInsert(true),
+	)
+	require.Equal(t,
+		"INSERT INTO test_table_conflict (id, name, age) VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING",
+		testTableWithConflictClause.PreparedInsert(false),
+	)
+
+	require.Equal(t, expectedHeaderPreparedWithUpsert, TableHeader.PreparedInsert(true))
+	require.Equal(t, expectedHeaderPreparedWithoutUpsert, TableHeader.PreparedInsert(false))
+	require.Equal(t, expectedHeaderFmtString, TableHeader.FmtStringInsert())
 }

indexer/test/test.go

@@ -33,10 +33,9 @@ import (
 	"github.com/cerc-io/plugeth-statediff/indexer/mocks"
 	"github.com/cerc-io/plugeth-statediff/indexer/models"
 	"github.com/cerc-io/plugeth-statediff/indexer/shared"
-	"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
 )
 
-// SetupTestData indexes a single mock block along with it's state nodes
+// SetupTestData indexes a single mock block along with its state nodes
 func SetupTestData(t *testing.T, ind interfaces.StateDiffIndexer) {
 	var tx interfaces.Batch
 	tx, err = ind.PushBlock(
@@ -110,16 +109,14 @@ func DoTestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	require.Equal(t, 5, len(trxs))
-	expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String()))
-	expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String()))
-	expectTrue(t, test_helpers.ListContainsString(trxs, trx3CID.String()))
-	expectTrue(t, test_helpers.ListContainsString(trxs, trx4CID.String()))
-	expectTrue(t, test_helpers.ListContainsString(trxs, trx5CID.String()))
+	require.Equal(t, len(txCIDs), len(trxs))
+	for _, c := range txCIDs {
+		require.Contains(t, trxs, c.String())
+	}
 
 	transactions := mocks.MockBlock.Transactions()
 	type txResult struct {
-		TxType uint8 `db:"tx_type"`
+		TxType int `db:"tx_type"`
 		Value  string
 	}
 	for _, c := range trxs {
@@ -133,9 +130,11 @@ func DoTestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
 			t.Fatal(err)
 		}
 		txTypeAndValueStr := `SELECT tx_type, CAST(value as TEXT) FROM eth.transaction_cids WHERE cid = $1`
+		txBlobHashQuery := `SELECT blob_hash FROM eth.blob_hashes WHERE tx_hash = $1`
+		txBlobIndexQuery := `SELECT index FROM eth.blob_hashes WHERE tx_hash = $1`
 		switch c {
-		case trx1CID.String():
-			require.Equal(t, tx1, data)
+		case txCIDs[0].String():
+			require.Equal(t, encodedTxs[0], data)
 			txRes := new(txResult)
 			err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
 			if err != nil {
@@ -147,8 +146,8 @@ func DoTestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
 			if txRes.Value != transactions[0].Value().String() {
 				t.Fatalf("expected tx value %s got %s", transactions[0].Value().String(), txRes.Value)
 			}
-		case trx2CID.String():
-			require.Equal(t, tx2, data)
+		case txCIDs[1].String():
+			require.Equal(t, encodedTxs[1], data)
 			txRes := new(txResult)
 			err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
 			if err != nil {
@@ -160,8 +159,8 @@ func DoTestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
 			if txRes.Value != transactions[1].Value().String() {
 				t.Fatalf("expected tx value %s got %s", transactions[1].Value().String(), txRes.Value)
 			}
-		case trx3CID.String():
-			require.Equal(t, tx3, data)
+		case txCIDs[2].String():
+			require.Equal(t, encodedTxs[2], data)
 			txRes := new(txResult)
 			err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
 			if err != nil {
@@ -173,8 +172,8 @@ func DoTestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
 			if txRes.Value != transactions[2].Value().String() {
 				t.Fatalf("expected tx value %s got %s", transactions[2].Value().String(), txRes.Value)
 			}
-		case trx4CID.String():
-			require.Equal(t, tx4, data)
+		case txCIDs[3].String():
+			require.Equal(t, encodedTxs[3], data)
 			txRes := new(txResult)
 			err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
 			if err != nil {
@@ -186,8 +185,8 @@ func DoTestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
 			if txRes.Value != transactions[3].Value().String() {
 				t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value)
 			}
-		case trx5CID.String():
-			require.Equal(t, tx5, data)
+		case txCIDs[4].String():
+			require.Equal(t, encodedTxs[4], data)
 			txRes := new(txResult)
 			err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
 			if err != nil {
@@ -199,6 +198,28 @@ func DoTestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
 			if txRes.Value != transactions[4].Value().String() {
 				t.Fatalf("expected tx value %s got %s", transactions[4].Value().String(), txRes.Value)
 			}
+		case txCIDs[5].String():
+			require.Equal(t, encodedTxs[5], data)
+			var txRes txResult
+			err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
+			if err != nil {
+				t.Fatal(err)
+			}
+			require.Equal(t, types.BlobTxType, txRes.TxType)
+			require.Equal(t, transactions[5].Value().String(), txRes.Value)
+
+			var txBlobHashes []common.Hash
+			var txBlobIndices []uint64
+			err = db.Select(context.Background(), &txBlobHashes, txBlobHashQuery, transactions[5].Hash().String())
+			if err != nil {
+				t.Fatal(err)
+			}
+			require.Equal(t, transactions[5].BlobHashes(), txBlobHashes)
+			err = db.Select(context.Background(), &txBlobIndices, txBlobIndexQuery, transactions[5].Hash().String())
+			if err != nil {
+				t.Fatal(err)
+			}
+			require.Equal(t, []uint64{0, 1}, txBlobIndices)
 		}
 	}
 }
@@ -256,12 +277,10 @@ func DoTestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	require.Equal(t, 5, len(rcts))
-	expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String()))
-	expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String()))
-	expectTrue(t, test_helpers.ListContainsString(rcts, rct3CID.String()))
-	expectTrue(t, test_helpers.ListContainsString(rcts, rct4CID.String()))
-	expectTrue(t, test_helpers.ListContainsString(rcts, rct5CID.String()))
+	require.Equal(t, len(rctCIDs), len(rcts))
+	for _, c := range rctCIDs {
+		require.Contains(t, rcts, c.String())
+	}
 
 	for idx, c := range rcts {
 		result := make([]models.IPLDModel, 0)
@@ -290,8 +309,8 @@ func DoTestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
 
 		postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE cid = $1`
 		switch c {
-		case rct1CID.String():
-			require.Equal(t, rct1, data)
+		case rctCIDs[0].String():
+			require.Equal(t, encodedRcts[0], data)
 			var postStatus uint64
 			pgStr = `SELECT post_status FROM eth.receipt_cids WHERE cid = $1`
 			err = db.Get(context.Background(), &postStatus, pgStr, c)
@@ -299,38 +318,81 @@ func DoTestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
 				t.Fatal(err)
 			}
 			require.Equal(t, mocks.ExpectedPostStatus, postStatus)
-		case rct2CID.String():
-			require.Equal(t, rct2, data)
+		case rctCIDs[1].String():
+			require.Equal(t, encodedRcts[1], data)
 			var postState string
 			err = db.Get(context.Background(), &postState, postStatePgStr, c)
 			if err != nil {
 				t.Fatal(err)
 			}
 			require.Equal(t, mocks.ExpectedPostState1, postState)
-		case rct3CID.String():
-			require.Equal(t, rct3, data)
+		case rctCIDs[2].String():
+			require.Equal(t, encodedRcts[2], data)
 			var postState string
 			err = db.Get(context.Background(), &postState, postStatePgStr, c)
 			if err != nil {
 				t.Fatal(err)
 			}
 			require.Equal(t, mocks.ExpectedPostState2, postState)
-		case rct4CID.String():
-			require.Equal(t, rct4, data)
+		case rctCIDs[3].String():
+			require.Equal(t, encodedRcts[3], data)
 			var postState string
 			err = db.Get(context.Background(), &postState, postStatePgStr, c)
 			if err != nil {
 				t.Fatal(err)
 			}
 			require.Equal(t, mocks.ExpectedPostState3, postState)
-		case rct5CID.String():
-			require.Equal(t, rct5, data)
+		case rctCIDs[4].String():
+			require.Equal(t, encodedRcts[4], data)
 			var postState string
 			err = db.Get(context.Background(), &postState, postStatePgStr, c)
 			if err != nil {
 				t.Fatal(err)
 			}
 			require.Equal(t, mocks.ExpectedPostState3, postState)
+		case rctCIDs[5].String():
+			require.Equal(t, encodedRcts[5], data)
+			var postState string
+			err = db.Get(context.Background(), &postState, postStatePgStr, c)
+			if err != nil {
+				t.Fatal(err)
+			}
+			require.Equal(t, mocks.ExpectedPostState3, postState)
+		}
+	}
+}
+
+func DoTestPublishAndIndexWithdrawalIPLDs(t *testing.T, db sql.Database) {
+	// check that withdrawals were properly indexed and published
+	wds := make([]string, 0)
+	pgStr := `SELECT withdrawal_cids.cid FROM eth.withdrawal_cids
+		INNER JOIN eth.header_cids ON (withdrawal_cids.header_id = header_cids.block_hash)
+		WHERE header_cids.block_number = $1
+		ORDER BY withdrawal_cids.index`
+	err = db.Select(context.Background(), &wds, pgStr, mocks.BlockNumber.Uint64())
+	if err != nil {
+		t.Fatal(err)
+	}
+	require.Equal(t, 2, len(wds))
+	require.Contains(t, wds, wd1CID.String())
+	require.Contains(t, wds, wd2CID.String())
+
+	for _, c := range wds {
+		dc, err := cid.Decode(c)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var data []byte
+		err = db.Get(context.Background(), &data, ipfsPgGet, dc.String(), mocks.BlockNumber.Uint64())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		switch c {
+		case wd1CID.String():
+			require.Equal(t, wd1, data)
+		case wd2CID.String():
+			require.Equal(t, wd2, data)
 		}
 	}
 }
@@ -594,7 +656,7 @@ func SetupTestDataNonCanonical(t *testing.T, ind interfaces.StateDiffIndexer) {
 func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
 	// check indexed headers
 	pgStr := `SELECT CAST(block_number as TEXT), block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT),
-			tx_root, receipt_root, uncles_hash, coinbase
+			tx_root, receipt_root, uncles_hash, coinbase, withdrawals_root
 			FROM eth.header_cids
 			ORDER BY block_number`
 	headerRes := make([]models.HeaderModel, 0)
@@ -616,6 +678,7 @@ func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
 			RctRoot:         mockBlock.ReceiptHash().String(),
 			UnclesHash:      mockBlock.UncleHash().String(),
 			Coinbase:        mocks.MockHeader.Coinbase.String(),
+			WithdrawalsRoot: shared.MaybeStringHash(mockBlock.Header().WithdrawalsHash),
 		},
 		{
 			BlockNumber:     mockNonCanonicalBlock.Number().String(),
@@ -626,6 +689,7 @@ func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
 			RctRoot:         mockNonCanonicalBlock.ReceiptHash().String(),
 			UnclesHash:      mockNonCanonicalBlock.UncleHash().String(),
 			Coinbase:        mocks.MockNonCanonicalHeader.Coinbase.String(),
+			WithdrawalsRoot: shared.MaybeStringHash(mockNonCanonicalBlock.Header().WithdrawalsHash),
 		},
 		{
 			BlockNumber:     mockNonCanonicalBlock2.Number().String(),
@@ -636,6 +700,7 @@ func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
 			RctRoot:         mockNonCanonicalBlock2.ReceiptHash().String(),
 			UnclesHash:      mockNonCanonicalBlock2.UncleHash().String(),
 			Coinbase:        mocks.MockNonCanonicalHeader2.Coinbase.String(),
+			WithdrawalsRoot: shared.MaybeStringHash(mockNonCanonicalBlock2.Header().WithdrawalsHash),
 		},
 	}
 	expectedRes[0].Reward = shared.CalcEthBlockReward(mockBlock.Header(), mockBlock.Uncles(), mockBlock.Transactions(), mocks.MockReceipts).String()
@@ -686,62 +751,19 @@ func DoTestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database
 
 	// expected transactions in the canonical block
 	mockBlockTxs := mocks.MockBlock.Transactions()
-	expectedBlockTxs := []models.TxModel{
-		{
+	expectedBlockTxs := make([]models.TxModel, len(mockBlockTxs))
+	for i, tx := range mockBlockTxs {
+		expectedBlockTxs[i] = models.TxModel{
 			BlockNumber: mockBlock.Number().String(),
 			HeaderID:    mockBlock.Hash().String(),
-			TxHash:      mockBlockTxs[0].Hash().String(),
-			CID:         trx1CID.String(),
-			Dst:         shared.HandleZeroAddrPointer(mockBlockTxs[0].To()),
+			TxHash:      tx.Hash().String(),
+			CID:         txCIDs[i].String(),
+			Dst:         shared.HandleZeroAddrPointer(tx.To()),
 			Src:         mocks.SenderAddr.String(),
-			Index:       0,
-			Type:        mockBlockTxs[0].Type(),
-			Value:       mockBlockTxs[0].Value().String(),
-		},
-		{
-			BlockNumber: mockBlock.Number().String(),
-			HeaderID:    mockBlock.Hash().String(),
-			TxHash:      mockBlockTxs[1].Hash().String(),
-			CID:         trx2CID.String(),
-			Dst:         shared.HandleZeroAddrPointer(mockBlockTxs[1].To()),
-			Src:         mocks.SenderAddr.String(),
-			Index:       1,
-			Type:        mockBlockTxs[1].Type(),
-			Value:       mockBlockTxs[1].Value().String(),
-		},
-		{
-			BlockNumber: mockBlock.Number().String(),
-			HeaderID:    mockBlock.Hash().String(),
-			TxHash:      mockBlockTxs[2].Hash().String(),
-			CID:         trx3CID.String(),
-			Dst:         shared.HandleZeroAddrPointer(mockBlockTxs[2].To()),
-			Src:         mocks.SenderAddr.String(),
-			Index:       2,
-			Type:        mockBlockTxs[2].Type(),
-			Value:       mockBlockTxs[2].Value().String(),
-		},
-		{
-			BlockNumber: mockBlock.Number().String(),
-			HeaderID:    mockBlock.Hash().String(),
-			TxHash:      mockBlockTxs[3].Hash().String(),
-			CID:         trx4CID.String(),
-			Dst:         shared.HandleZeroAddrPointer(mockBlockTxs[3].To()),
-			Src:         mocks.SenderAddr.String(),
-			Index:       3,
-			Type:        mockBlockTxs[3].Type(),
-			Value:       mockBlockTxs[3].Value().String(),
-		},
-		{
-			BlockNumber: mockBlock.Number().String(),
-			HeaderID:    mockBlock.Hash().String(),
-			TxHash:      mockBlockTxs[4].Hash().String(),
-			CID:         trx5CID.String(),
-			Dst:         shared.HandleZeroAddrPointer(mockBlockTxs[4].To()),
-			Src:         mocks.SenderAddr.String(),
-			Index:       4,
-			Type:        mockBlockTxs[4].Type(),
-			Value:       mockBlockTxs[4].Value().String(),
-		},
+			Index:       int64(i),
+			Type:        tx.Type(),
+			Value:       tx.Value().String(),
+		}
 	}
 
 	// expected transactions in the non-canonical block at London height
@@ -751,7 +773,7 @@ func DoTestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database
 			BlockNumber: mockNonCanonicalBlock.Number().String(),
 			HeaderID:    mockNonCanonicalBlock.Hash().String(),
 			TxHash:      mockNonCanonicalBlockTxs[0].Hash().String(),
-			CID:         trx2CID.String(),
+			CID:         txCIDs[1].String(),
 			Dst:         mockNonCanonicalBlockTxs[0].To().String(),
 			Src:         mocks.SenderAddr.String(),
 			Index:       0,
@@ -762,7 +784,7 @@ func DoTestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database
 			BlockNumber: mockNonCanonicalBlock.Number().String(),
 			HeaderID:    mockNonCanonicalBlock.Hash().String(),
 			TxHash:      mockNonCanonicalBlockTxs[1].Hash().String(),
-			CID:         trx5CID.String(),
+			CID:         txCIDs[4].String(),
 			Dst:         mockNonCanonicalBlockTxs[1].To().String(),
 			Src:         mocks.SenderAddr.String(),
 			Index:       1,
@@ -778,7 +800,7 @@ func DoTestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database
 			BlockNumber: mockNonCanonicalBlock2.Number().String(),
 			HeaderID:    mockNonCanonicalBlock2.Hash().String(),
 			TxHash:      mockNonCanonicalBlock2Txs[0].Hash().String(),
-			CID:         trx3CID.String(),
+			CID:         txCIDs[2].String(),
 			Dst:         "",
 			Src:         mocks.SenderAddr.String(),
 			Index:       0,
@@ -789,7 +811,7 @@ func DoTestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database
 			BlockNumber: mockNonCanonicalBlock2.Number().String(),
 			HeaderID:    mockNonCanonicalBlock2.Hash().String(),
 			TxHash:      mockNonCanonicalBlock2Txs[1].Hash().String(),
-			CID:         trx5CID.String(),
+			CID:         txCIDs[4].String(),
 			Dst:         mockNonCanonicalBlock2Txs[1].To().String(),
 			Src:         mocks.SenderAddr.String(),
 			Index:       1,
@@ -826,14 +848,12 @@ func DoTestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database
 	// check indexed IPLD blocks
 	var data []byte
 
-	txCIDs := []cid.Cid{trx1CID, trx2CID, trx3CID, trx4CID, trx5CID}
-	txRLPs := [][]byte{tx1, tx2, tx3, tx4, tx5}
 	for i, txCID := range txCIDs {
 		err = db.Get(context.Background(), &data, ipfsPgGet, txCID.String(), mocks.BlockNumber.Uint64())
 		if err != nil {
 			t.Fatal(err)
 		}
-		require.Equal(t, txRLPs[i], data)
+		require.Equal(t, encodedTxs[i], data)
 	}
 }
 
@@ -849,11 +869,10 @@ func DoTestPublishAndIndexReceiptsNonCanonical(t *testing.T, db sql.Database) {
 	}
 
 	// expected receipts in the canonical block
-	rctCids := []cid.Cid{rct1CID, rct2CID, rct3CID, rct4CID, rct5CID}
 	expectedBlockRctsMap := make(map[string]models.ReceiptModel, len(mocks.MockReceipts))
 	for i, mockBlockRct := range mocks.MockReceipts {
-		rctModel := createRctModel(mockBlockRct, rctCids[i], mockBlock.Number().String())
-		expectedBlockRctsMap[rctCids[i].String()] = rctModel
+		rctModel := createRctModel(mockBlockRct, rctCIDs[i], mockBlock.Number().String())
+		expectedBlockRctsMap[rctCIDs[i].String()] = rctModel
 	}
 
 	// expected receipts in the non-canonical block at London height
@@ -908,10 +927,8 @@ func DoTestPublishAndIndexReceiptsNonCanonical(t *testing.T, db sql.Database) {
 	// check indexed rct IPLD blocks
 	var data []byte
 
-	rctRLPs := [][]byte{
-		rct1, rct2, rct3, rct4, rct5, nonCanonicalBlockRct1, nonCanonicalBlockRct2,
-	}
-	for i, rctCid := range append(rctCids, nonCanonicalBlockRctCids...) {
+	rctRLPs := append(encodedRcts, nonCanonicalBlockRct1, nonCanonicalBlockRct2)
+	for i, rctCid := range append(rctCIDs, nonCanonicalBlockRctCids...) {
 		err = db.Get(context.Background(), &data, ipfsPgGet, rctCid.String(), mocks.BlockNumber.Uint64())
 		if err != nil {
 			t.Fatal(err)

indexer/test/test_init.go

@@ -37,13 +37,14 @@ var (
 					WHERE key = $1 AND block_number = $2`
 	watchedAddressesPgGet = `SELECT *
 					FROM eth_meta.watched_addresses`
-	tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5            []byte
+	encodedTxs, encodedRcts                                          [][]byte
+	wd1, wd2                                                         []byte
 	nonCanonicalBlockRct1, nonCanonicalBlockRct2                     []byte
 	nonCanonicalBlock2Rct1, nonCanonicalBlock2Rct2                   []byte
 	mockBlock, mockNonCanonicalBlock, mockNonCanonicalBlock2         *types.Block
 	headerCID, mockNonCanonicalHeaderCID, mockNonCanonicalHeader2CID cid.Cid
-	trx1CID, trx2CID, trx3CID, trx4CID, trx5CID                      cid.Cid
-	rct1CID, rct2CID, rct3CID, rct4CID, rct5CID                      cid.Cid
+	txCIDs, rctCIDs                                                  []cid.Cid
+	wd1CID, wd2CID                                                   cid.Cid
 	nonCanonicalBlockRct1CID, nonCanonicalBlockRct2CID               cid.Cid
 	nonCanonicalBlock2Rct1CID, nonCanonicalBlock2Rct2CID             cid.Cid
 	state1CID, state2CID, storageCID                                 cid.Cid
@@ -62,56 +63,36 @@ func init() {
 	mockNonCanonicalBlock2 = mocks.MockNonCanonicalBlock2
 	nonCanonicalBlock2Rcts := mocks.MockNonCanonicalBlock2Receipts
 
-	// encode mock receipts
+	// encode mock txs and receipts
 	buf := new(bytes.Buffer)
-	txs.EncodeIndex(0, buf)
-	tx1 = make([]byte, buf.Len())
-	copy(tx1, buf.Bytes())
+	encodedTxs = make([][]byte, len(txs))
+	encodedRcts = make([][]byte, len(rcts))
+
+	for i := 0; i < len(txs); i++ {
+		txs.EncodeIndex(i, buf)
+		tx := make([]byte, buf.Len())
+		copy(tx, buf.Bytes())
+		buf.Reset()
+		encodedTxs[i] = tx
+	}
+
+	for i := 0; i < len(rcts); i++ {
+		rcts.EncodeIndex(i, buf)
+		rct := make([]byte, buf.Len())
+		copy(rct, buf.Bytes())
+		buf.Reset()
+		encodedRcts[i] = rct
+	}
+
+	// encode mock withdrawals
+	mocks.MockWithdrawals.EncodeIndex(0, buf)
+	wd1 = make([]byte, buf.Len())
+	copy(wd1, buf.Bytes())
 	buf.Reset()
 
-	txs.EncodeIndex(1, buf)
-	tx2 = make([]byte, buf.Len())
-	copy(tx2, buf.Bytes())
-	buf.Reset()
-
-	txs.EncodeIndex(2, buf)
-	tx3 = make([]byte, buf.Len())
-	copy(tx3, buf.Bytes())
-	buf.Reset()
-
-	txs.EncodeIndex(3, buf)
-	tx4 = make([]byte, buf.Len())
-	copy(tx4, buf.Bytes())
-	buf.Reset()
-
-	txs.EncodeIndex(4, buf)
-	tx5 = make([]byte, buf.Len())
-	copy(tx5, buf.Bytes())
-	buf.Reset()
-
-	rcts.EncodeIndex(0, buf)
-	rct1 = make([]byte, buf.Len())
-	copy(rct1, buf.Bytes())
-	buf.Reset()
-
-	rcts.EncodeIndex(1, buf)
-	rct2 = make([]byte, buf.Len())
-	copy(rct2, buf.Bytes())
-	buf.Reset()
-
-	rcts.EncodeIndex(2, buf)
-	rct3 = make([]byte, buf.Len())
-	copy(rct3, buf.Bytes())
-	buf.Reset()
-
-	rcts.EncodeIndex(3, buf)
-	rct4 = make([]byte, buf.Len())
-	copy(rct4, buf.Bytes())
-	buf.Reset()
-
-	rcts.EncodeIndex(4, buf)
-	rct5 = make([]byte, buf.Len())
-	copy(rct5, buf.Bytes())
+	mocks.MockWithdrawals.EncodeIndex(1, buf)
+	wd2 = make([]byte, buf.Len())
+	copy(wd2, buf.Bytes())
 	buf.Reset()
 
 	// encode mock receipts for non-canonical blocks
@@ -138,19 +119,23 @@ func init() {
 	headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockHeaderRlp, multihash.KECCAK_256)
 	mockNonCanonicalHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeaderRlp, multihash.KECCAK_256)
 	mockNonCanonicalHeader2CID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeader2Rlp, multihash.KECCAK_256)
-	trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx1, multihash.KECCAK_256)
-	trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx2, multihash.KECCAK_256)
-	trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx3, multihash.KECCAK_256)
-	trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx4, multihash.KECCAK_256)
-	trx5CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx5, multihash.KECCAK_256)
+
+	for i := 0; i < len(txs); i++ {
+		tx, _ := ipld.RawdataToCid(ipld.MEthTx, encodedTxs[i], multihash.KECCAK_256)
+		txCIDs = append(txCIDs, tx)
+	}
+
 	state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.ContractLeafNode, multihash.KECCAK_256)
 	state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256)
 	storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256)
-	rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct1, multihash.KECCAK_256)
-	rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct2, multihash.KECCAK_256)
-	rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct3, multihash.KECCAK_256)
-	rct4CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct4, multihash.KECCAK_256)
-	rct5CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct5, multihash.KECCAK_256)
+
+	for i := 0; i < len(rcts); i++ {
+		rct, _ := ipld.RawdataToCid(ipld.MEthTxReceipt, encodedRcts[i], multihash.KECCAK_256)
+		rctCIDs = append(rctCIDs, rct)
+	}
+
+	wd1CID, _ = ipld.RawdataToCid(ipld.MEthWithdrawal, wd1, multihash.KECCAK_256)
+	wd2CID, _ = ipld.RawdataToCid(ipld.MEthWithdrawal, wd2, multihash.KECCAK_256)
 
 	// create raw receipts for non-canonical blocks
 	nonCanonicalBlockRct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, nonCanonicalBlockRct1, multihash.KECCAK_256)

indexer/test_helpers/test_helpers.go

@@ -19,22 +19,16 @@ package test_helpers
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"os"
 	"testing"
 
 	"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
+	"github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
+	"github.com/jmoiron/sqlx"
 )
 
-// ListContainsString used to check if a list of strings contains a particular string
-func ListContainsString(sss []string, s string) bool {
-	for _, str := range sss {
-		if s == str {
-			return true
-		}
-	}
-	return false
-}
-
 // DedupFile removes duplicates from the given file
 func DedupFile(filePath string) error {
 	f, err := os.OpenFile(filePath, os.O_CREATE|os.O_RDONLY, os.ModePerm)
@@ -49,9 +43,6 @@ func DedupFile(filePath string) error {
 		s := sc.Text()
 		stmts[s] = struct{}{}
 	}
-	if err != nil {
-		return err
-	}
 
 	f.Close()
 
@@ -70,30 +61,30 @@ func DedupFile(filePath string) error {
 
 // TearDownDB is used to tear down the watcher dbs after tests
 func TearDownDB(t *testing.T, db sql.Database) {
-	ctx := context.Background()
-	tx, err := db.Begin(ctx)
+	err := ClearDB(db)
 	if err != nil {
 		t.Fatal(err)
 	}
+}
 
-	statements := []string{
-		`TRUNCATE nodes`,
-		`TRUNCATE ipld.blocks`,
-		`TRUNCATE eth.header_cids`,
-		`TRUNCATE eth.uncle_cids`,
-		`TRUNCATE eth.transaction_cids`,
-		`TRUNCATE eth.receipt_cids`,
-		`TRUNCATE eth.state_cids`,
-		`TRUNCATE eth.storage_cids`,
-		`TRUNCATE eth.log_cids`,
-		`TRUNCATE eth_meta.watched_addresses`,
+func ClearSqlxDB(sqlxdb *sqlx.DB) error {
+	driver := postgres.NewSQLXDriver(context.Background(), sqlxdb)
+	db := postgres.NewPostgresDB(driver, false)
+	return ClearDB(db)
+}
+
+func ClearDB(db sql.Database) error {
+	ctx := context.Background()
+	tx, err := db.Begin(ctx)
+	if err != nil {
+		return err
 	}
-	for _, stm := range statements {
+
+	for _, tbl := range schema.AllTables {
+		stm := fmt.Sprintf("TRUNCATE %s", tbl.Name)
 		if _, err = tx.Exec(ctx, stm); err != nil {
-			t.Fatal(err)
+			return fmt.Errorf("error executing `%s`: %w", stm, err)
 		}
 	}
-	if err = tx.Commit(ctx); err != nil {
-		t.Fatal(err)
-	}
+	return tx.Commit(ctx)
 }

mainnet_tests/builder_test.go

@@ -20,7 +20,7 @@ import (
 	"math/big"
 	"testing"
 
-	"github.com/cerc-io/eth-testing/chaindata/mainnet"
+	"github.com/cerc-io/eth-testing/chains/mainnet"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"

scripts/run-test-stack.sh

@@ -1,23 +1,21 @@
 #!/bin/bash
-# Builds and deploys a stack with only what we need.
-# This script assumes we are running in the project root.
 
-set -e
+set -ex
 
 # Note: stack path should be absolute, otherwise SO looks for it in packaged stacks
-laconic_so="${LACONIC_SO:-laconic-so} --stack $(readlink -f test) --verbose"
+stack_dir=$(readlink -f "${1:-../fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth}")
+
+[[ -d "$stack_dir" ]]
 
 CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
-
-# Point stack-orchestrator to the multi-project root
+# By default assume we are running in the project root.
 export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}"
 
-# v5 migrations only go up to version 20
-echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=20 >> $CONFIG_DIR/stack.env
-# don't run plugeth in the debugger
+laconic_so="laconic-so --verbose --stack $stack_dir"
+
+# Don't run geth/plugeth in the debugger, it will swallow error backtraces
 echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
 
-set -x
 
 if [[ -z $SKIP_BUILD ]]; then
   $laconic_so setup-repositories \
@@ -27,6 +25,10 @@ if [[ -z $SKIP_BUILD ]]; then
     --exclude cerc/plugeth-statediff
 fi
 
-$laconic_so deploy \
-    --env-file $CONFIG_DIR/stack.env \
-    --cluster test up
+if ! $laconic_so deploy \
+  --env-file $CONFIG_DIR/stack.env \
+  --cluster test up
+then
+  $laconic_so deploy --cluster test logs
+  exit 1
+fi

test/compose.yml

@@ -3,7 +3,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.4.0-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"

test/stack.yml

@@ -2,10 +2,10 @@ version: "1.2"
 name: fixturenet-plugeth-tx
 description: "Plugeth Ethereum Fixturenet for testing plugeth-statediff"
 repos:
-  - git.vdb.to/cerc-io/plugeth@v1.5.0-cerc-2
+  - git.vdb.to/cerc-io/plugeth@v1.13.14-cerc-2
   - git.vdb.to/cerc-io/plugeth-statediff
   - git.vdb.to/cerc-io/lighthouse
-  - git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha
+  - git.vdb.to/cerc-io/ipld-eth-db@v5.4.0-alpha
 containers:
   - cerc/plugeth-statediff
   - cerc/plugeth

test_helpers/chaingen/gen.go

@@ -11,8 +11,10 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/holiman/uint256"
 )
 
 const secondsPerBlock = 12
@@ -148,3 +150,58 @@ func (gen *GenContext) createTx(from common.Address, to *common.Address, amount
 	}
 	return types.SignTx(tx, signer, priv)
 }
+
+func (gen *GenContext) createBlobTx(
+	from common.Address,
+	to common.Address,
+	amount *uint256.Int,
+	gasLimit uint64,
+	blobData []byte,
+) (*types.Transaction, error) {
+	signer := types.MakeSigner(gen.ChainConfig, gen.block.Number(), gen.time)
+	nonce := gen.block.TxNonce(from)
+	priv, ok := gen.Keys[from]
+	if !ok {
+		return nil, errors.New("No private key for sender address" + from.String())
+	}
+
+	if !gen.ChainConfig.IsCancun(gen.block.Number(), gen.time) {
+		return nil, errors.New("blob tx is only supported from Cancun fork")
+	}
+
+	sidecar := MakeSidecar([][]byte{blobData})
+	tx := types.NewTx(&types.BlobTx{
+		ChainID:    uint256.MustFromBig(gen.ChainConfig.ChainID),
+		Nonce:      nonce,
+		To:         to,
+		Gas:        gasLimit,
+		GasTipCap:  uint256.NewInt(50),
+		GasFeeCap:  uint256.NewInt(1000000000),
+		Value:      amount,
+		BlobFeeCap: uint256.NewInt(1000000),
+		BlobHashes: sidecar.BlobHashes(),
+		Sidecar:    sidecar,
+	})
+	return types.SignTx(tx, signer, priv)
+}
+
+// From go-ethereum/cmd/devp2p/internal/ethtest/chain.go
+func MakeSidecar(data [][]byte) *types.BlobTxSidecar {
+	var (
+		blobs       = make([]kzg4844.Blob, len(data))
+		commitments []kzg4844.Commitment
+		proofs      []kzg4844.Proof
+	)
+	for i := range blobs {
+		copy(blobs[i][:], data[i])
+		c, _ := kzg4844.BlobToCommitment(blobs[i])
+		p, _ := kzg4844.ComputeBlobProof(blobs[i], c)
+		commitments = append(commitments, c)
+		proofs = append(proofs, p)
+	}
+	return &types.BlobTxSidecar{
+		Blobs:       blobs,
+		Commitments: commitments,
+		Proofs:      proofs,
+	}
+}

test_helpers/db.go

@@ -1,33 +0,0 @@
-package test_helpers
-
-import (
-	"fmt"
-
-	"github.com/jmoiron/sqlx"
-)
-
-// ClearDB is used to empty the IPLD-ETH tables after tests
-func ClearDB(db *sqlx.DB) error {
-	tx, err := db.Beginx()
-	if err != nil {
-		return err
-	}
-	statements := []string{
-		`TRUNCATE nodes`,
-		`TRUNCATE ipld.blocks`,
-		`TRUNCATE eth.header_cids`,
-		`TRUNCATE eth.uncle_cids`,
-		`TRUNCATE eth.transaction_cids`,
-		`TRUNCATE eth.receipt_cids`,
-		`TRUNCATE eth.state_cids`,
-		`TRUNCATE eth.storage_cids`,
-		`TRUNCATE eth.log_cids`,
-		`TRUNCATE eth_meta.watched_addresses`,
-	}
-	for _, stm := range statements {
-		if _, err = tx.Exec(stm); err != nil {
-			return fmt.Errorf("error executing `%s`: %w", stm, err)
-		}
-	}
-	return tx.Commit()
-}

utils/iterator_test.go

@@ -3,7 +3,7 @@ package utils_test
 import (
 	"testing"
 
-	"github.com/cerc-io/eth-testing/chaindata/mainnet"
+	"github.com/cerc-io/eth-testing/chains/mainnet"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"