Copy package from geth fork

indexer/database/sql/batch_tx.go

@@ -0,0 +1,126 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+
+	"github.com/lib/pq"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+)
+
+const startingCacheCapacity = 1024 * 24
+
+// BatchTx wraps a sql tx with the state necessary for building the tx concurrently during trie difference iteration
+type BatchTx struct {
+	BlockNumber      string
+	ctx              context.Context
+	dbtx             Tx
+	stm              string
+	quit             chan struct{}
+	iplds            chan models.IPLDModel
+	ipldCache        models.IPLDBatch
+	removedCacheFlag *uint32
+	// Tracks expected cache size and ensures cache is caught up before flush
+	cacheWg sync.WaitGroup
+
+	submit func(blockTx *BatchTx, err error) error
+}
+
+// Submit satisfies indexer.AtomicTx
+func (tx *BatchTx) Submit(err error) error {
+	return tx.submit(tx, err)
+}
+
+func (tx *BatchTx) flush() error {
+	tx.cacheWg.Wait()
+	_, err := tx.dbtx.Exec(tx.ctx, tx.stm, pq.Array(tx.ipldCache.BlockNumbers), pq.Array(tx.ipldCache.Keys),
+		pq.Array(tx.ipldCache.Values))
+	if err != nil {
+		log.Debug(insertError{"ipld.blocks", err, tx.stm,
+			struct {
+				blockNumbers []string
+				keys         []string
+				values       [][]byte
+			}{
+				tx.ipldCache.BlockNumbers,
+				tx.ipldCache.Keys,
+				tx.ipldCache.Values,
+			}}.Error())
+		return insertError{"ipld.blocks", err, tx.stm, "too many arguments; use debug mode for full list"}
+	}
+	tx.ipldCache = models.IPLDBatch{}
+	return nil
+}
+
+// run in background goroutine to synchronize concurrent appends to the ipldCache
+func (tx *BatchTx) cache() {
+	for {
+		select {
+		case i := <-tx.iplds:
+			tx.ipldCache.BlockNumbers = append(tx.ipldCache.BlockNumbers, i.BlockNumber)
+			tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
+			tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
+			tx.cacheWg.Done()
+		case <-tx.quit:
+			tx.ipldCache = models.IPLDBatch{}
+			return
+		}
+	}
+}
+
+func (tx *BatchTx) cacheDirect(key string, value []byte) {
+	tx.cacheWg.Add(1)
+	tx.iplds <- models.IPLDModel{
+		BlockNumber: tx.BlockNumber,
+		Key:         key,
+		Data:        value,
+	}
+}
+
+func (tx *BatchTx) cacheIPLD(i ipld.IPLD) {
+	tx.cacheWg.Add(1)
+	tx.iplds <- models.IPLDModel{
+		BlockNumber: tx.BlockNumber,
+		Key:         i.Cid().String(),
+		Data:        i.RawData(),
+	}
+}
+
+func (tx *BatchTx) cacheRemoved(key string, value []byte) {
+	if atomic.LoadUint32(tx.removedCacheFlag) == 0 {
+		atomic.StoreUint32(tx.removedCacheFlag, 1)
+		tx.cacheWg.Add(1)
+		tx.iplds <- models.IPLDModel{
+			BlockNumber: tx.BlockNumber,
+			Key:         key,
+			Data:        value,
+		}
+	}
+}
+
+// rollback sql transaction and log any error
+func rollback(ctx context.Context, tx Tx) {
+	if err := tx.Rollback(ctx); err != nil {
+		log.Error(err.Error())
+	}
+}

indexer/database/sql/indexer.go

@@ -0,0 +1,583 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package sql provides an interface for pushing and indexing IPLD objects into a sql database
+// Metrics for reporting processing and connection stats are defined in ./metrics.go
+
+package sql
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math/big"
+	"time"
+
+	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
+
+// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of an SQL sql
+type StateDiffIndexer struct {
+	ctx         context.Context
+	chainConfig *params.ChainConfig
+	dbWriter    *Writer
+}
+
+// NewStateDiffIndexer creates a sql implementation of interfaces.StateDiffIndexer
+func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, db Database) (*StateDiffIndexer, error) {
+	return &StateDiffIndexer{
+		ctx:         ctx,
+		chainConfig: chainConfig,
+		dbWriter:    NewWriter(db),
+	}, nil
+}
+
+// ReportDBMetrics is a reporting function to run as goroutine
+func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) {
+	if !metrics.Enabled {
+		return
+	}
+	ticker := time.NewTicker(delay)
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				metrics2.DBMetrics.Update(sdi.dbWriter.db.Stats())
+			case <-quit:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+}
+
+// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
+// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
+func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
+	start, t := time.Now(), time.Now()
+	blockHash := block.Hash()
+	blockHashStr := blockHash.String()
+	height := block.NumberU64()
+	traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
+	transactions := block.Transactions()
+	// Derive any missing fields
+	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.BaseFee(), transactions); err != nil {
+		return nil, err
+	}
+
+	// Generate the block iplds
+	headerNode, txNodes, rctNodes, logNodes, err := ipld.FromBlockAndReceipts(block, receipts)
+	if err != nil {
+		return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
+	}
+
+	if len(txNodes) != len(rctNodes) {
+		return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d)", len(txNodes), len(rctNodes))
+	}
+
+	// Calculate reward
+	var reward *big.Int
+	// in PoA networks block reward is 0
+	if sdi.chainConfig.Clique != nil {
+		reward = big.NewInt(0)
+	} else {
+		reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
+	}
+	t = time.Now()
+
+	// Begin new DB tx for everything
+	tx := NewDelayedTx(sdi.dbWriter.db)
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(sdi.ctx, tx)
+			panic(p)
+		} else if err != nil {
+			rollback(sdi.ctx, tx)
+		}
+	}()
+	blockTx := &BatchTx{
+		removedCacheFlag: new(uint32),
+		ctx:              sdi.ctx,
+		BlockNumber:      block.Number().String(),
+		stm:              sdi.dbWriter.db.InsertIPLDsStm(),
+		iplds:            make(chan models.IPLDModel),
+		quit:             make(chan struct{}),
+		ipldCache: models.IPLDBatch{
+			BlockNumbers: make([]string, 0, startingCacheCapacity),
+			Keys:         make([]string, 0, startingCacheCapacity),
+			Values:       make([][]byte, 0, startingCacheCapacity),
+		},
+		dbtx: tx,
+		// handle transaction commit or rollback for any return case
+		submit: func(self *BatchTx, err error) error {
+			defer func() {
+				close(self.quit)
+				close(self.iplds)
+			}()
+			if p := recover(); p != nil {
+				log.Info("panic detected before tx submission, rolling back the tx", "panic", p)
+				rollback(sdi.ctx, tx)
+				panic(p)
+			} else if err != nil {
+				log.Info("error detected before tx submission, rolling back the tx", "error", err)
+				rollback(sdi.ctx, tx)
+			} else {
+				tDiff := time.Since(t)
+				metrics2.IndexerMetrics.StateStoreCodeProcessingTimer.Update(tDiff)
+				traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
+				t = time.Now()
+				if err := self.flush(); err != nil {
+					rollback(sdi.ctx, tx)
+					traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
+					log.Debug(traceMsg)
+					return err
+				}
+				err = tx.Commit(sdi.ctx)
+				tDiff = time.Since(t)
+				metrics2.IndexerMetrics.PostgresCommitTimer.Update(tDiff)
+				traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
+			}
+			traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
+			log.Debug(traceMsg)
+			return err
+		},
+	}
+	go blockTx.cache()
+
+	tDiff := time.Since(t)
+	metrics2.IndexerMetrics.FreePostgresTimer.Update(tDiff)
+
+	traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
+	t = time.Now()
+
+	// Publish and index header, collect headerID
+	var headerID string
+	headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty)
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics2.IndexerMetrics.HeaderProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+	// Publish and index uncles
+	err = sdi.processUncles(blockTx, headerID, block.Number(), block.UncleHash(), block.Uncles())
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics2.IndexerMetrics.UncleProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+	// Publish and index receipts and txs
+	err = sdi.processReceiptsAndTxs(blockTx, processArgs{
+		headerID:    headerID,
+		blockNumber: block.Number(),
+		receipts:    receipts,
+		txs:         transactions,
+		rctNodes:    rctNodes,
+		txNodes:     txNodes,
+		logNodes:    logNodes,
+	})
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics2.IndexerMetrics.TxAndRecProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+
+	return blockTx, err
+}
+
+// processHeader publishes and indexes a header IPLD in Postgres
+// it returns the headerID
+func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode ipld.IPLD, reward, td *big.Int) (string, error) {
+	tx.cacheIPLD(headerNode)
+
+	var baseFee *string
+	if header.BaseFee != nil {
+		baseFee = new(string)
+		*baseFee = header.BaseFee.String()
+	}
+	headerID := header.Hash().String()
+	// index header
+	return headerID, sdi.dbWriter.upsertHeaderCID(tx.dbtx, models.HeaderModel{
+		CID:             headerNode.Cid().String(),
+		ParentHash:      header.ParentHash.String(),
+		BlockNumber:     header.Number.String(),
+		BlockHash:       headerID,
+		TotalDifficulty: td.String(),
+		Reward:          reward.String(),
+		Bloom:           header.Bloom.Bytes(),
+		StateRoot:       header.Root.String(),
+		RctRoot:         header.ReceiptHash.String(),
+		TxRoot:          header.TxHash.String(),
+		UnclesHash:      header.UncleHash.String(),
+		Timestamp:       header.Time,
+		Coinbase:        header.Coinbase.String(),
+	})
+}
+
+// processUncles publishes and indexes uncle IPLDs in Postgres
+func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, unclesHash common.Hash, uncles []*types.Header) error {
+	// publish and index uncles
+	uncleEncoding, err := rlp.EncodeToBytes(uncles)
+	if err != nil {
+		return err
+	}
+	preparedHash := crypto.Keccak256Hash(uncleEncoding)
+	if !bytes.Equal(preparedHash.Bytes(), unclesHash.Bytes()) {
+		return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.Hex(), unclesHash.Hex())
+	}
+	unclesCID, err := ipld.RawdataToCid(ipld.MEthHeaderList, uncleEncoding, multihash.KECCAK_256)
+	if err != nil {
+		return err
+	}
+	tx.cacheDirect(unclesCID.String(), uncleEncoding)
+	for i, uncle := range uncles {
+		var uncleReward *big.Int
+		// in PoA networks uncle reward is 0
+		if sdi.chainConfig.Clique != nil {
+			uncleReward = big.NewInt(0)
+		} else {
+			uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncle.Number.Uint64())
+		}
+		uncle := models.UncleModel{
+			BlockNumber: blockNumber.String(),
+			HeaderID:    headerID,
+			CID:         unclesCID.String(),
+			ParentHash:  uncle.ParentHash.String(),
+			BlockHash:   uncle.Hash().String(),
+			Reward:      uncleReward.String(),
+			Index:       int64(i),
+		}
+		if err := sdi.dbWriter.upsertUncleCID(tx.dbtx, uncle); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// processArgs bundles arguments to processReceiptsAndTxs
+type processArgs struct {
+	headerID    string
+	blockNumber *big.Int
+	receipts    types.Receipts
+	txs         types.Transactions
+	rctNodes    []*ipld.EthReceipt
+	txNodes     []*ipld.EthTx
+	logNodes    [][]*ipld.EthLog
+}
+
+// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
+func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error {
+	// Process receipts and txs
+	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
+	for i, receipt := range args.receipts {
+		txNode := args.txNodes[i]
+		tx.cacheIPLD(txNode)
+		tx.cacheIPLD(args.rctNodes[i])
+
+		// index tx
+		trx := args.txs[i]
+		txID := trx.Hash().String()
+
+		var val string
+		if trx.Value() != nil {
+			val = trx.Value().String()
+		}
+
+		// derive sender for the tx that corresponds with this receipt
+		from, err := types.Sender(signer, trx)
+		if err != nil {
+			return fmt.Errorf("error deriving tx sender: %v", err)
+		}
+		txModel := models.TxModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			Dst:         shared.HandleZeroAddrPointer(trx.To()),
+			Src:         shared.HandleZeroAddr(from),
+			TxHash:      txID,
+			Index:       int64(i),
+			CID:         txNode.Cid().String(),
+			Type:        trx.Type(),
+			Value:       val,
+		}
+		if err := sdi.dbWriter.upsertTransactionCID(tx.dbtx, txModel); err != nil {
+			return err
+		}
+
+		// this is the contract address if this receipt is for a contract creation tx
+		contract := shared.HandleZeroAddr(receipt.ContractAddress)
+
+		rctModel := &models.ReceiptModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			TxID:        txID,
+			Contract:    contract,
+			CID:         args.rctNodes[i].Cid().String(),
+		}
+		if len(receipt.PostState) == 0 {
+			rctModel.PostStatus = receipt.Status
+		} else {
+			rctModel.PostState = common.BytesToHash(receipt.PostState).String()
+		}
+
+		if err := sdi.dbWriter.upsertReceiptCID(tx.dbtx, rctModel); err != nil {
+			return err
+		}
+
+		// index logs
+		logDataSet := make([]*models.LogsModel, len(receipt.Logs))
+		for idx, l := range receipt.Logs {
+			tx.cacheIPLD(args.logNodes[i][idx])
+			topicSet := make([]string, 4)
+			for ti, topic := range l.Topics {
+				topicSet[ti] = topic.Hex()
+			}
+
+			logDataSet[idx] = &models.LogsModel{
+				BlockNumber: args.blockNumber.String(),
+				HeaderID:    args.headerID,
+				ReceiptID:   txID,
+				Address:     l.Address.String(),
+				Index:       int64(l.Index),
+				CID:         args.logNodes[i][idx].Cid().String(),
+				Topic0:      topicSet[0],
+				Topic1:      topicSet[1],
+				Topic2:      topicSet[2],
+				Topic3:      topicSet[3],
+			}
+		}
+
+		if err := sdi.dbWriter.upsertLogCID(tx.dbtx, logDataSet); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql
+func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateLeafNode, headerID string) error {
+	tx, ok := batch.(*BatchTx)
+	if !ok {
+		return fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
+	}
+	// publish the state node
+	var stateModel models.StateNodeModel
+	if stateNode.Removed {
+		tx.cacheRemoved(shared.RemovedNodeStateCID, []byte{})
+		stateModel = models.StateNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			CID:         shared.RemovedNodeStateCID,
+			Removed:     true,
+		}
+	} else {
+		stateModel = models.StateNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			CID:         stateNode.AccountWrapper.CID,
+			Removed:     false,
+			Balance:     stateNode.AccountWrapper.Account.Balance.String(),
+			Nonce:       stateNode.AccountWrapper.Account.Nonce,
+			CodeHash:    common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
+			StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
+		}
+	}
+
+	// index the state node
+	if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil {
+		return err
+	}
+
+	// if there are any storage nodes associated with this node, publish and index them
+	for _, storageNode := range stateNode.StorageDiff {
+		if storageNode.Removed {
+			tx.cacheRemoved(shared.RemovedNodeStorageCID, []byte{})
+			storageModel := models.StorageNodeModel{
+				BlockNumber: tx.BlockNumber,
+				HeaderID:    headerID,
+				StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+				StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
+				CID:         shared.RemovedNodeStorageCID,
+				Removed:     true,
+				Value:       []byte{},
+			}
+			if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
+				return err
+			}
+			continue
+		}
+		storageModel := models.StorageNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
+			CID:         storageNode.CID,
+			Removed:     false,
+			Value:       storageNode.Value,
+		}
+		if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PushIPLD publishes iplds to ipld.blocks
+func (sdi *StateDiffIndexer) PushIPLD(batch interfaces.Batch, ipld sdtypes.IPLD) error {
+	tx, ok := batch.(*BatchTx)
+	if !ok {
+		return fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
+	}
+	tx.cacheDirect(ipld.CID, ipld.Content)
+	return nil
+}
+
+// Close satisfies io.Closer
+func (sdi *StateDiffIndexer) Close() error {
+	return sdi.dbWriter.Close()
+}
+
+// Update the known gaps table with the gap information.
+
+// LoadWatchedAddresses reads watched addresses from the database
+func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
+	addressStrings := make([]string, 0)
+	pgStr := "SELECT address FROM eth_meta.watched_addresses"
+	err := sdi.dbWriter.db.Select(sdi.ctx, &addressStrings, pgStr)
+	if err != nil {
+		return nil, fmt.Errorf("error loading watched addresses: %v", err)
+	}
+
+	watchedAddresses := []common.Address{}
+	for _, addressString := range addressStrings {
+		watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString))
+	}
+
+	return watchedAddresses, nil
+}
+
+// InsertWatchedAddresses inserts the given addresses in the database
+func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) (err error) {
+	tx := NewDelayedTx(sdi.dbWriter.db)
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(sdi.ctx, tx)
+			panic(p)
+		} else if err != nil {
+			rollback(sdi.ctx, tx)
+		} else {
+			err = tx.Commit(sdi.ctx)
+		}
+	}()
+
+	for _, arg := range args {
+		_, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`,
+			arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
+		if err != nil {
+			return fmt.Errorf("error inserting watched_addresses entry: %v", err)
+		}
+	}
+
+	return err
+}
+
+// RemoveWatchedAddresses removes the given watched addresses from the database
+func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) (err error) {
+	tx := NewDelayedTx(sdi.dbWriter.db)
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(sdi.ctx, tx)
+			panic(p)
+		} else if err != nil {
+			rollback(sdi.ctx, tx)
+		} else {
+			err = tx.Commit(sdi.ctx)
+		}
+	}()
+
+	for _, arg := range args {
+		_, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses WHERE address = $1`, arg.Address)
+		if err != nil {
+			return fmt.Errorf("error removing watched_addresses entry: %v", err)
+		}
+	}
+
+	return err
+}
+
+// SetWatchedAddresses clears and inserts the given addresses in the database
+func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) (err error) {
+	tx := NewDelayedTx(sdi.dbWriter.db)
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(sdi.ctx, tx)
+			panic(p)
+		} else if err != nil {
+			rollback(sdi.ctx, tx)
+		} else {
+			err = tx.Commit(sdi.ctx)
+		}
+	}()
+
+	_, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`)
+	if err != nil {
+		return fmt.Errorf("error setting watched_addresses table: %v", err)
+	}
+
+	for _, arg := range args {
+		_, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`,
+			arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
+		if err != nil {
+			return fmt.Errorf("error setting watched_addresses table: %v", err)
+		}
+	}
+
+	return err
+}
+
+// ClearWatchedAddresses clears all the watched addresses from the database
+func (sdi *StateDiffIndexer) ClearWatchedAddresses() error {
+	_, err := sdi.dbWriter.db.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`)
+	if err != nil {
+		return fmt.Errorf("error clearing watched_addresses table: %v", err)
+	}
+
+	return nil
+}

indexer/database/sql/indexer_shared_test.go

@@ -0,0 +1,28 @@
+package sql_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+var (
+	db  sql.Database
+	err error
+	ind interfaces.StateDiffIndexer
+)
+
+func checkTxClosure(t *testing.T, idle, inUse, open int64) {
+	require.Equal(t, idle, db.Stats().Idle())
+	require.Equal(t, inUse, db.Stats().InUse())
+	require.Equal(t, open, db.Stats().Open())
+}
+
+func tearDown(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+	require.NoError(t, ind.Close())
+}

indexer/database/sql/interfaces.go

@@ -0,0 +1,88 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"context"
+	"io"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+)
+
+// Database interfaces required by the sql indexer
+type Database interface {
+	Driver
+	Statements
+}
+
+// Driver interface has all the methods required by a driver implementation to support the sql indexer
+type Driver interface {
+	UseCopyFrom() bool
+	QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
+	Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
+	Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error
+	Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error
+	Begin(ctx context.Context) (Tx, error)
+	Stats() metrics.DbStats
+	NodeID() string
+	Context() context.Context
+	io.Closer
+}
+
+// Statements interface to accommodate different SQL query syntax
+type Statements interface {
+	InsertHeaderStm() string
+	InsertUncleStm() string
+	InsertTxStm() string
+	InsertRctStm() string
+	InsertLogStm() string
+	InsertStateStm() string
+	InsertStorageStm() string
+	InsertIPLDStm() string
+	InsertIPLDsStm() string
+
+	// Table/column descriptions for use with CopyFrom and similar commands.
+	LogTableName() []string
+	LogColumnNames() []string
+	RctTableName() []string
+	RctColumnNames() []string
+	StateTableName() []string
+	StateColumnNames() []string
+	StorageTableName() []string
+	StorageColumnNames() []string
+	TxTableName() []string
+	TxColumnNames() []string
+}
+
+// Tx interface to accommodate different concrete SQL transaction types
+type Tx interface {
+	QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
+	Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
+	CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error)
+	Commit(ctx context.Context) error
+	Rollback(ctx context.Context) error
+}
+
+// ScannableRow interface to accommodate different concrete row types
+type ScannableRow interface {
+	Scan(dest ...interface{}) error
+}
+
+// Result interface to accommodate different concrete result types
+type Result interface {
+	RowsAffected() (int64, error)
+}

indexer/database/sql/lazy_tx.go

@@ -0,0 +1,106 @@
+package sql
+
+import (
+	"context"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// Changing this to 1 would make sure only sequential COPYs were combined.
+const copyFromCheckLimit = 100
+
+type DelayedTx struct {
+	cache []interface{}
+	db    Database
+}
+type cachedStmt struct {
+	sql  string
+	args []interface{}
+}
+
+type copyFrom struct {
+	tableName   []string
+	columnNames []string
+	rows        [][]interface{}
+}
+
+func (cf *copyFrom) appendRows(rows [][]interface{}) {
+	cf.rows = append(cf.rows, rows...)
+}
+
+func (cf *copyFrom) matches(tableName []string, columnNames []string) bool {
+	return reflect.DeepEqual(cf.tableName, tableName) && reflect.DeepEqual(cf.columnNames, columnNames)
+}
+
+func NewDelayedTx(db Database) *DelayedTx {
+	return &DelayedTx{db: db}
+}
+
+func (tx *DelayedTx) QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow {
+	return tx.db.QueryRow(ctx, sql, args...)
+}
+
+func (tx *DelayedTx) findPrevCopyFrom(tableName []string, columnNames []string, limit int) (*copyFrom, int) {
+	for pos, count := len(tx.cache)-1, 0; pos >= 0 && count < limit; pos, count = pos-1, count+1 {
+		prevCopy, ok := tx.cache[pos].(*copyFrom)
+		if ok && prevCopy.matches(tableName, columnNames) {
+			return prevCopy, count
+		}
+	}
+	return nil, -1
+}
+
+func (tx *DelayedTx) CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error) {
+	if prevCopy, distance := tx.findPrevCopyFrom(tableName, columnNames, copyFromCheckLimit); nil != prevCopy {
+		log.Trace("statediff lazy_tx : Appending to COPY", "table", tableName,
+			"current", len(prevCopy.rows), "new", len(rows), "distance", distance)
+		prevCopy.appendRows(rows)
+	} else {
+		tx.cache = append(tx.cache, &copyFrom{tableName, columnNames, rows})
+	}
+
+	return 0, nil
+}
+
+func (tx *DelayedTx) Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) {
+	tx.cache = append(tx.cache, cachedStmt{sql, args})
+	return nil, nil
+}
+
+func (tx *DelayedTx) Commit(ctx context.Context) error {
+	base, err := tx.db.Begin(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(ctx, base)
+			panic(p)
+		} else if err != nil {
+			rollback(ctx, base)
+		}
+	}()
+	for _, item := range tx.cache {
+		switch item := item.(type) {
+		case *copyFrom:
+			_, err := base.CopyFrom(ctx, item.tableName, item.columnNames, item.rows)
+			if err != nil {
+				log.Error("COPY error", "table", item.tableName, "err", err)
+				return err
+			}
+		case cachedStmt:
+			_, err := base.Exec(ctx, item.sql, item.args...)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	tx.cache = nil
+	return base.Commit(ctx)
+}
+
+func (tx *DelayedTx) Rollback(ctx context.Context) error {
+	tx.cache = nil
+	return nil
+}

indexer/database/sql/mainnet_tests/indexer_test.go

@@ -0,0 +1,95 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mainnet_tests
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+var (
+	err       error
+	db        sql.Database
+	ind       interfaces.StateDiffIndexer
+	chainConf = params.MainnetChainConfig
+)
+
+func init() {
+	if os.Getenv("MODE") != "statediff" {
+		fmt.Println("Skipping statediff test")
+		os.Exit(0)
+	}
+}
+
+func TestMainnetIndexer(t *testing.T) {
+	conf := test_helpers.GetTestConfig()
+
+	for _, blockNumber := range test_helpers.ProblemBlocks {
+		conf.BlockNumber = big.NewInt(blockNumber)
+		tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
+		require.NoError(t, err)
+
+		testPushBlockAndState(t, tb, trs)
+	}
+
+	testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
+	require.NoError(t, err)
+
+	testPushBlockAndState(t, testBlock, testReceipts)
+}
+
+func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
+	t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
+		setupMainnetIndexer(t)
+		defer checkTxClosure(t, 0, 0, 0)
+		defer tearDown(t)
+
+		test.TestBlock(t, ind, block, receipts)
+	})
+}
+
+func setupMainnetIndexer(t *testing.T) {
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, db)
+}
+
+func checkTxClosure(t *testing.T, idle, inUse, open int64) {
+	require.Equal(t, idle, db.Stats().Idle())
+	require.Equal(t, inUse, db.Stats().InUse())
+	require.Equal(t, open, db.Stats().Open())
+}
+
+func tearDown(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+	require.NoError(t, ind.Close())
+}

indexer/database/sql/pgx_indexer_legacy_test.go

@@ -0,0 +1,52 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupLegacyPGXIndexer(t *testing.T) {
+	db, err = postgres.SetupPGXDB(postgres.TestConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db)
+	require.NoError(t, err)
+}
+
+func setupLegacyPGX(t *testing.T) {
+	setupLegacyPGXIndexer(t)
+	test.SetupLegacyTestData(t, ind)
+}
+
+func TestLegacyPGXIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs", func(t *testing.T) {
+		setupLegacyPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestLegacyIndexer(t, db)
+	})
+}

indexer/database/sql/pgx_indexer_test.go

@@ -0,0 +1,245 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupPGXIndexer(t *testing.T, config postgres.Config) {
+	db, err = postgres.SetupPGXDB(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db)
+	require.NoError(t, err)
+}
+
+func setupPGX(t *testing.T) {
+	setupPGXWithConfig(t, postgres.TestConfig)
+}
+
+func setupPGXWithConfig(t *testing.T, config postgres.Config) {
+	setupPGXIndexer(t, config)
+	test.SetupTestData(t, ind)
+}
+
+func setupPGXNonCanonical(t *testing.T) {
+	setupPGXIndexer(t, postgres.TestConfig)
+	test.SetupTestDataNonCanonical(t, ind)
+}
+
+// Test indexer for a canonical block
+func TestPGXIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexHeaderIPLDs(t, db)
+	})
+
+	t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexTransactionIPLDs(t, db)
+	})
+
+	t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexLogIPLDs(t, db)
+	})
+
+	t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexReceiptIPLDs(t, db)
+	})
+
+	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStateIPLDs(t, db)
+	})
+
+	t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStorageIPLDs(t, db)
+	})
+
+	t.Run("Publish and index with CopyFrom enabled.", func(t *testing.T) {
+		config := postgres.TestConfig
+		config.CopyFrom = true
+
+		setupPGXWithConfig(t, config)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStateIPLDs(t, db)
+		test.TestPublishAndIndexStorageIPLDs(t, db)
+		test.TestPublishAndIndexReceiptIPLDs(t, db)
+		test.TestPublishAndIndexLogIPLDs(t, db)
+	})
+}
+
+// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1
+func TestPGXIndexerNonCanonical(t *testing.T) {
+	t.Run("Publish and index header", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexHeaderNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index transactions", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexTransactionsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index receipts", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexReceiptsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index logs", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexLogsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index state nodes", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStateNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index storage nodes", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStorageNonCanonical(t, db)
+	})
+}
+
+func TestPGXWatchAddressMethods(t *testing.T) {
+	setupPGXIndexer(t, postgres.TestConfig)
+	defer tearDown(t)
+	defer checkTxClosure(t, 1, 0, 1)
+
+	t.Run("Load watched addresses (empty table)", func(t *testing.T) {
+		test.TestLoadEmptyWatchedAddresses(t, ind)
+	})
+
+	t.Run("Insert watched addresses", func(t *testing.T) {
+		args := mocks.GetInsertWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
+		require.NoError(t, err)
+
+		test.TestInsertWatchedAddresses(t, db)
+	})
+
+	t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetInsertAlreadyWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		test.TestInsertAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses", func(t *testing.T) {
+		args := mocks.GetRemoveWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		test.TestRemoveWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
+		args := mocks.GetRemoveNonWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		test.TestRemoveNonWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses", func(t *testing.T) {
+		args := mocks.GetSetWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		test.TestSetWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetSetAlreadyWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
+		require.NoError(t, err)
+
+		test.TestSetAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Load watched addresses", func(t *testing.T) {
+		test.TestLoadWatchedAddresses(t, ind)
+	})
+
+	t.Run("Clear watched addresses", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		test.TestClearWatchedAddresses(t, db)
+	})
+
+	t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		test.TestClearEmptyWatchedAddresses(t, db)
+	})
+}

indexer/database/sql/postgres/config.go

@@ -0,0 +1,140 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+)
+
+// DriverType to explicitly type the kind of sql driver we are using
+type DriverType string
+
+const (
+	PGX     DriverType = "PGX"
+	SQLX    DriverType = "SQLX"
+	Unknown DriverType = "Unknown"
+)
+
+// Env variables
+const (
+	DATABASE_NAME     = "DATABASE_NAME"
+	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
+	DATABASE_PORT     = "DATABASE_PORT"
+	DATABASE_USER     = "DATABASE_USER"
+	DATABASE_PASSWORD = "DATABASE_PASSWORD"
+)
+
+// ResolveDriverType resolves a DriverType from a provided string
+func ResolveDriverType(str string) (DriverType, error) {
+	switch strings.ToLower(str) {
+	case "pgx", "pgxpool":
+		return PGX, nil
+	case "sqlx":
+		return SQLX, nil
+	default:
+		return Unknown, fmt.Errorf("unrecognized driver type string: %s", str)
+	}
+}
+
+// TestConfig specifies default parameters for connecting to a testing DB
+var TestConfig = Config{
+	Hostname:     "localhost",
+	Port:         8077,
+	DatabaseName: "cerc_testing",
+	Username:     "vdbm",
+	Password:     "password",
+	Driver:       SQLX,
+}
+
+// Config holds params for a Postgres db
+type Config struct {
+	// conn string params
+	Hostname     string
+	Port         int
+	DatabaseName string
+	Username     string
+	Password     string
+
+	// conn settings
+	MaxConns        int
+	MaxIdle         int
+	MinConns        int
+	MaxConnIdleTime time.Duration
+	MaxConnLifetime time.Duration
+	ConnTimeout     time.Duration
+	LogStatements   bool
+
+	// node info params
+	ID         string
+	ClientName string
+
+	// driver type
+	Driver DriverType
+
+	// toggle on/off upserts
+	Upsert bool
+
+	// toggle on/off CopyFrom
+	CopyFrom bool
+}
+
+// Type satisfies interfaces.Config
+func (c Config) Type() shared.DBType {
+	return shared.POSTGRES
+}
+
+// DbConnectionString constructs and returns the connection string from the config
+func (c Config) DbConnectionString() string {
+	if len(c.Username) > 0 && len(c.Password) > 0 {
+		return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+			c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName)
+	}
+	if len(c.Username) > 0 && len(c.Password) == 0 {
+		return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
+			c.Username, c.Hostname, c.Port, c.DatabaseName)
+	}
+	return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
+}
+
+func (c Config) WithEnv() (Config, error) {
+	if val := os.Getenv(DATABASE_NAME); val != "" {
+		c.DatabaseName = val
+	}
+	if val := os.Getenv(DATABASE_HOSTNAME); val != "" {
+		c.Hostname = val
+	}
+	if val := os.Getenv(DATABASE_PORT); val != "" {
+		port, err := strconv.Atoi(val)
+		if err != nil {
+			return c, err
+		}
+		c.Port = port
+	}
+	if val := os.Getenv(DATABASE_USER); val != "" {
+		c.Username = val
+	}
+	if val := os.Getenv(DATABASE_PASSWORD); val != "" {
+		c.Password = val
+	}
+	return c, nil
+}

indexer/database/sql/postgres/database.go

@@ -0,0 +1,126 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
+)
+
+var _ sql.Database = &DB{}
+
+const (
+	createNodeStm = `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
+					 ON CONFLICT (node_id) DO NOTHING`
+)
+
+// NewPostgresDB returns a postgres.DB using the provided driver
+func NewPostgresDB(driver sql.Driver, upsert bool) *DB {
+	return &DB{upsert, driver}
+}
+
+// DB implements sql.Database using a configured driver and Postgres statement syntax
+type DB struct {
+	upsert bool
+	sql.Driver
+}
+
+// InsertHeaderStm satisfies the sql.Statements interface
+// Stm == Statement
+func (db *DB) InsertHeaderStm() string {
+	return schema.TableHeader.ToInsertStatement(db.upsert)
+}
+
+// InsertUncleStm satisfies the sql.Statements interface
+func (db *DB) InsertUncleStm() string {
+	return schema.TableUncle.ToInsertStatement(db.upsert)
+}
+
+// InsertTxStm satisfies the sql.Statements interface
+func (db *DB) InsertTxStm() string {
+	return schema.TableTransaction.ToInsertStatement(db.upsert)
+}
+
+// InsertRctStm satisfies the sql.Statements interface
+func (db *DB) InsertRctStm() string {
+	return schema.TableReceipt.ToInsertStatement(db.upsert)
+}
+
+// InsertLogStm satisfies the sql.Statements interface
+func (db *DB) InsertLogStm() string {
+	return schema.TableLog.ToInsertStatement(db.upsert)
+}
+
+// InsertStateStm satisfies the sql.Statements interface
+func (db *DB) InsertStateStm() string {
+	return schema.TableStateNode.ToInsertStatement(db.upsert)
+}
+
+// InsertStorageStm satisfies the sql.Statements interface
+func (db *DB) InsertStorageStm() string {
+	return schema.TableStorageNode.ToInsertStatement(db.upsert)
+}
+
+// InsertIPLDStm satisfies the sql.Statements interface
+func (db *DB) InsertIPLDStm() string {
+	return schema.TableIPLDBlock.ToInsertStatement(db.upsert)
+}
+
+// InsertIPLDsStm satisfies the sql.Statements interface
+func (db *DB) InsertIPLDsStm() string {
+	return `INSERT INTO ipld.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING`
+}
+
+func (db *DB) LogTableName() []string {
+	return []string{"eth", "log_cids"}
+}
+
+func (db *DB) LogColumnNames() []string {
+	return []string{"block_number", "header_id", "cid", "rct_id", "address", "index", "topic0", "topic1", "topic2", "topic3"}
+}
+
+func (db *DB) RctTableName() []string {
+	return []string{"eth", "receipt_cids"}
+}
+
+func (db *DB) RctColumnNames() []string {
+	return []string{"block_number", "header_id", "tx_id", "cid", "contract", "post_state", "post_status"}
+}
+
+func (db *DB) StateTableName() []string {
+	return []string{"eth", "state_cids"}
+}
+
+func (db *DB) StateColumnNames() []string {
+	return []string{"block_number", "header_id", "state_leaf_key", "cid", "diff", "balance", "nonce", "code_hash", "storage_root", "removed"}
+}
+
+func (db *DB) StorageTableName() []string {
+	return []string{"eth", "storage_cids"}
+}
+
+func (db *DB) StorageColumnNames() []string {
+	return []string{"block_number", "header_id", "state_leaf_key", "storage_leaf_key", "cid", "diff", "val", "removed"}
+}
+
+func (db *DB) TxTableName() []string {
+	return []string{"eth", "transaction_cids"}
+}
+
+func (db *DB) TxColumnNames() []string {
+	return []string{"block_number", "header_id", "tx_hash", "cid", "dst", "src", "index", "tx_type", "value"}
+}

indexer/database/sql/postgres/errors.go

@@ -0,0 +1,38 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"fmt"
+)
+
+const (
+	DbConnectionFailedMsg = "db connection failed"
+	SettingNodeFailedMsg  = "unable to set db node"
+)
+
+func ErrDBConnectionFailed(connectErr error) error {
+	return formatError(DbConnectionFailedMsg, connectErr)
+}
+
+func ErrUnableToSetNode(setErr error) error {
+	return formatError(SettingNodeFailedMsg, setErr)
+}
+
+func formatError(msg string, err error) error {
+	return fmt.Errorf("%s: %w", msg, err)
+}

indexer/database/sql/postgres/log_adapter.go

@@ -0,0 +1,61 @@
+// Copyright © 2023 Cerc
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/jackc/pgx/v4"
+)
+
+type LogAdapter struct {
+	l log.Logger
+}
+
+func NewLogAdapter(l log.Logger) *LogAdapter {
+	return &LogAdapter{l: l}
+}
+
+func (l *LogAdapter) Log(ctx context.Context, level pgx.LogLevel, msg string, data map[string]interface{}) {
+	var logger log.Logger
+	if data != nil {
+		var args = make([]interface{}, 0)
+		for key, value := range data {
+			if value != nil {
+				args = append(args, key, value)
+			}
+		}
+		logger = l.l.New(args...)
+	} else {
+		logger = l.l
+	}
+
+	switch level {
+	case pgx.LogLevelTrace:
+		logger.Trace(msg)
+	case pgx.LogLevelDebug:
+		logger.Debug(msg)
+	case pgx.LogLevelInfo:
+		logger.Info(msg)
+	case pgx.LogLevelWarn:
+		logger.Warn(msg)
+	case pgx.LogLevelError:
+		logger.Error(msg)
+	default:
+		logger.New("INVALID_PGX_LOG_LEVEL", level).Error(msg)
+	}
+}

indexer/database/sql/postgres/pgx.go

@@ -0,0 +1,256 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"context"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/georgysavva/scany/pgxscan"
+	"github.com/jackc/pgconn"
+	"github.com/jackc/pgx/v4"
+	"github.com/jackc/pgx/v4/pgxpool"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+// PGXDriver driver, implements sql.Driver
+type PGXDriver struct {
+	ctx      context.Context
+	pool     *pgxpool.Pool
+	nodeInfo node.Info
+	nodeID   string
+	config   Config
+}
+
+// ConnectPGX initializes and returns a PGX connection pool
+func ConnectPGX(ctx context.Context, config Config) (*pgxpool.Pool, error) {
+	pgConf, err := MakeConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return pgxpool.ConnectConfig(ctx, pgConf)
+}
+
+// NewPGXDriver returns a new pgx driver
+// it initializes the connection pool and creates the node info table
+func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
+	dbPool, err := ConnectPGX(ctx, config)
+	if err != nil {
+		return nil, ErrDBConnectionFailed(err)
+	}
+	pg := &PGXDriver{ctx: ctx, pool: dbPool, nodeInfo: node, config: config}
+	nodeErr := pg.createNode()
+	if nodeErr != nil {
+		return &PGXDriver{}, ErrUnableToSetNode(nodeErr)
+	}
+	return pg, nil
+}
+
+// MakeConfig creates a pgxpool.Config from the provided Config
+func MakeConfig(config Config) (*pgxpool.Config, error) {
+	conf, err := pgxpool.ParseConfig("")
+	if err != nil {
+		return nil, err
+	}
+
+	//conf.ConnConfig.BuildStatementCache = nil
+	conf.ConnConfig.Config.Host = config.Hostname
+	conf.ConnConfig.Config.Port = uint16(config.Port)
+	conf.ConnConfig.Config.Database = config.DatabaseName
+	conf.ConnConfig.Config.User = config.Username
+	conf.ConnConfig.Config.Password = config.Password
+
+	if config.ConnTimeout != 0 {
+		conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout
+	}
+	if config.MaxConns != 0 {
+		conf.MaxConns = int32(config.MaxConns)
+	}
+	if config.MinConns != 0 {
+		conf.MinConns = int32(config.MinConns)
+	}
+	if config.MaxConnLifetime != 0 {
+		conf.MaxConnLifetime = config.MaxConnLifetime
+	}
+	if config.MaxConnIdleTime != 0 {
+		conf.MaxConnIdleTime = config.MaxConnIdleTime
+	}
+
+	if config.LogStatements {
+		conf.ConnConfig.Logger = NewLogAdapter(log.New())
+	}
+
+	return conf, nil
+}
+
+func (pgx *PGXDriver) createNode() error {
+	_, err := pgx.pool.Exec(
+		pgx.ctx,
+		createNodeStm,
+		pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID,
+		pgx.nodeInfo.ID, pgx.nodeInfo.ClientName,
+		pgx.nodeInfo.ChainID)
+	if err != nil {
+		return ErrUnableToSetNode(err)
+	}
+	pgx.nodeID = pgx.nodeInfo.ID
+	return nil
+}
+
+// QueryRow satisfies sql.Database
+func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
+	return pgx.pool.QueryRow(ctx, sql, args...)
+}
+
+// Exec satisfies sql.Database
+func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
+	res, err := pgx.pool.Exec(ctx, sql, args...)
+	return resultWrapper{ct: res}, err
+}
+
+// Select satisfies sql.Database
+func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	return pgxscan.Select(ctx, pgx.pool, dest, query, args...)
+}
+
+// Get satisfies sql.Database
+func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	return pgxscan.Get(ctx, pgx.pool, dest, query, args...)
+}
+
+// Begin satisfies sql.Database
+func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) {
+	tx, err := pgx.pool.Begin(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return pgxTxWrapper{tx: tx}, nil
+}
+
+func (pgx *PGXDriver) Stats() metrics.DbStats {
+	stats := pgx.pool.Stat()
+	return pgxStatsWrapper{stats: stats}
+}
+
+// NodeID satisfies sql.Database
+func (pgx *PGXDriver) NodeID() string {
+	return pgx.nodeID
+}
+
+// Close satisfies sql.Database/io.Closer
+func (pgx *PGXDriver) Close() error {
+	pgx.pool.Close()
+	return nil
+}
+
+// Context satisfies sql.Database
+func (pgx *PGXDriver) Context() context.Context {
+	return pgx.ctx
+}
+
+// HasCopy satisfies sql.Database
+func (pgx *PGXDriver) UseCopyFrom() bool {
+	return pgx.config.CopyFrom
+}
+
+type resultWrapper struct {
+	ct pgconn.CommandTag
+}
+
+// RowsAffected satisfies sql.Result
+func (r resultWrapper) RowsAffected() (int64, error) {
+	return r.ct.RowsAffected(), nil
+}
+
+type pgxStatsWrapper struct {
+	stats *pgxpool.Stat
+}
+
+// MaxOpen satisfies metrics.DbStats
+func (s pgxStatsWrapper) MaxOpen() int64 {
+	return int64(s.stats.MaxConns())
+}
+
+// Open satisfies metrics.DbStats
+func (s pgxStatsWrapper) Open() int64 {
+	return int64(s.stats.TotalConns())
+}
+
+// InUse satisfies metrics.DbStats
+func (s pgxStatsWrapper) InUse() int64 {
+	return int64(s.stats.AcquiredConns())
+}
+
+// Idle satisfies metrics.DbStats
+func (s pgxStatsWrapper) Idle() int64 {
+	return int64(s.stats.IdleConns())
+}
+
+// WaitCount satisfies metrics.DbStats
+func (s pgxStatsWrapper) WaitCount() int64 {
+	return s.stats.EmptyAcquireCount()
+}
+
+// WaitDuration satisfies metrics.DbStats
+func (s pgxStatsWrapper) WaitDuration() time.Duration {
+	return s.stats.AcquireDuration()
+}
+
+// MaxIdleClosed satisfies metrics.DbStats
+func (s pgxStatsWrapper) MaxIdleClosed() int64 {
+	// this stat isn't supported by pgxpool, but we don't want to panic
+	return 0
+}
+
+// MaxLifetimeClosed satisfies metrics.DbStats
+func (s pgxStatsWrapper) MaxLifetimeClosed() int64 {
+	return s.stats.CanceledAcquireCount()
+}
+
+type pgxTxWrapper struct {
+	tx pgx.Tx
+}
+
+// QueryRow satisfies sql.Tx
+func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
+	return t.tx.QueryRow(ctx, sql, args...)
+}
+
+// Exec satisfies sql.Tx
+func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
+	res, err := t.tx.Exec(ctx, sql, args...)
+	return resultWrapper{ct: res}, err
+}
+
+// Commit satisfies sql.Tx
+func (t pgxTxWrapper) Commit(ctx context.Context) error {
+	return t.tx.Commit(ctx)
+}
+
+// Rollback satisfies sql.Tx
+func (t pgxTxWrapper) Rollback(ctx context.Context) error {
+	return t.tx.Rollback(ctx)
+}
+
+func (t pgxTxWrapper) CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error) {
+	return t.tx.CopyFrom(ctx, tableName, columnNames, pgx.CopyFromRows(rows))
+}

indexer/database/sql/postgres/pgx_test.go

@@ -0,0 +1,121 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"strings"
+	"testing"
+
+	"github.com/jackc/pgx/v4/pgxpool"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+var (
+	pgConfig, _ = postgres.MakeConfig(postgres.TestConfig)
+	ctx         = context.Background()
+)
+
+func expectContainsSubstring(t *testing.T, full string, sub string) {
+	if !strings.Contains(full, sub) {
+		t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub)
+	}
+}
+
+func TestPostgresPGX(t *testing.T) {
+	t.Run("connects to the sql", func(t *testing.T) {
+		dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig)
+		if err != nil {
+			t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err)
+		}
+		if dbPool == nil {
+			t.Fatal("DB pool is nil")
+		}
+		dbPool.Close()
+	})
+
+	t.Run("serializes big.Int to db", func(t *testing.T) {
+		// postgres driver doesn't support go big.Int type
+		// various casts in golang uint64, int64, overflow for
+		// transaction value (in wei) even though
+		// postgres numeric can handle an arbitrary
+		// sized int, so use string representation of big.Int
+		// and cast on insert
+
+		dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig)
+		if err != nil {
+			t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err)
+		}
+		defer dbPool.Close()
+
+		bi := new(big.Int)
+		bi.SetString("34940183920000000000", 10)
+		require.Equal(t, "34940183920000000000", bi.String())
+
+		defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`)
+		_, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		sqlStatement := `
+			INSERT INTO example (id, data)
+			VALUES (1, cast($1 AS NUMERIC))`
+		_, err = dbPool.Exec(ctx, sqlStatement, bi.String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var data string
+		err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		require.Equal(t, data, bi.String())
+		actual := new(big.Int)
+		actual.SetString(data, 10)
+		require.Equal(t, bi, actual)
+	})
+
+	t.Run("throws error when can't connect to the database", func(t *testing.T) {
+		goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
+		_, err := postgres.NewPGXDriver(ctx, postgres.Config{}, goodInfo)
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
+
+		expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
+	})
+
+	t.Run("throws error when can't create node", func(t *testing.T) {
+		badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
+		badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
+
+		_, err := postgres.NewPGXDriver(ctx, postgres.TestConfig, badInfo)
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
+
+		expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
+	})
+}

indexer/database/sql/postgres/postgres_suite_test.go

@@ -0,0 +1,33 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+func init() {
+	if os.Getenv("MODE") != "statediff" {
+		fmt.Println("Skipping statediff test")
+		os.Exit(0)
+	}
+
+	log.Root().SetHandler(log.DiscardHandler())
+}

indexer/database/sql/postgres/sqlx.go

@@ -0,0 +1,210 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"context"
+	coresql "database/sql"
+	"errors"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+// SQLXDriver driver, implements sql.Driver
+type SQLXDriver struct {
+	ctx      context.Context
+	db       *sqlx.DB
+	nodeInfo node.Info
+	nodeID   string
+}
+
+// ConnectSQLX initializes and returns a SQLX connection pool for postgres
+func ConnectSQLX(ctx context.Context, config Config) (*sqlx.DB, error) {
+	db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString())
+	if err != nil {
+		return nil, ErrDBConnectionFailed(err)
+	}
+	if config.MaxConns > 0 {
+		db.SetMaxOpenConns(config.MaxConns)
+	}
+	if config.MaxConnLifetime > 0 {
+		db.SetConnMaxLifetime(config.MaxConnLifetime)
+	}
+	db.SetMaxIdleConns(config.MaxIdle)
+	return db, nil
+}
+
+// NewSQLXDriver returns a new sqlx driver for Postgres
+// it initializes the connection pool and creates the node info table
+func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
+	db, err := ConnectSQLX(ctx, config)
+	if err != nil {
+		return nil, err
+	}
+	driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node}
+	if err := driver.createNode(); err != nil {
+		return nil, err
+	}
+	return driver, nil
+}
+
+func (driver *SQLXDriver) createNode() error {
+	_, err := driver.db.Exec(
+		createNodeStm,
+		driver.nodeInfo.GenesisBlock,
+		driver.nodeInfo.NetworkID,
+		driver.nodeInfo.ID,
+		driver.nodeInfo.ClientName,
+		driver.nodeInfo.ChainID)
+	if err != nil {
+		return ErrUnableToSetNode(err)
+	}
+	driver.nodeID = driver.nodeInfo.ID
+	return nil
+}
+
+// QueryRow satisfies sql.Database
+func (driver *SQLXDriver) QueryRow(_ context.Context, sql string, args ...interface{}) sql.ScannableRow {
+	return driver.db.QueryRowx(sql, args...)
+}
+
+// Exec satisfies sql.Database
+func (driver *SQLXDriver) Exec(_ context.Context, sql string, args ...interface{}) (sql.Result, error) {
+	return driver.db.Exec(sql, args...)
+}
+
+// Select satisfies sql.Database
+func (driver *SQLXDriver) Select(_ context.Context, dest interface{}, query string, args ...interface{}) error {
+	return driver.db.Select(dest, query, args...)
+}
+
+// Get satisfies sql.Database
+func (driver *SQLXDriver) Get(_ context.Context, dest interface{}, query string, args ...interface{}) error {
+	return driver.db.Get(dest, query, args...)
+}
+
+// Begin satisfies sql.Database
+func (driver *SQLXDriver) Begin(_ context.Context) (sql.Tx, error) {
+	tx, err := driver.db.Beginx()
+	if err != nil {
+		return nil, err
+	}
+	return sqlxTxWrapper{tx: tx}, nil
+}
+
+func (driver *SQLXDriver) Stats() metrics.DbStats {
+	stats := driver.db.Stats()
+	return sqlxStatsWrapper{stats: stats}
+}
+
+// NodeID satisfies sql.Database
+func (driver *SQLXDriver) NodeID() string {
+	return driver.nodeID
+}
+
+// Close satisfies sql.Database/io.Closer
+func (driver *SQLXDriver) Close() error {
+	return driver.db.Close()
+}
+
+// Context satisfies sql.Database
+func (driver *SQLXDriver) Context() context.Context {
+	return driver.ctx
+}
+
+// HasCopy satisfies sql.Database
+func (driver *SQLXDriver) UseCopyFrom() bool {
+	// sqlx does not currently support COPY.
+	return false
+}
+
+type sqlxStatsWrapper struct {
+	stats coresql.DBStats
+}
+
+// MaxOpen satisfies metrics.DbStats
+func (s sqlxStatsWrapper) MaxOpen() int64 {
+	return int64(s.stats.MaxOpenConnections)
+}
+
+// Open satisfies metrics.DbStats
+func (s sqlxStatsWrapper) Open() int64 {
+	return int64(s.stats.OpenConnections)
+}
+
+// InUse satisfies metrics.DbStats
+func (s sqlxStatsWrapper) InUse() int64 {
+	return int64(s.stats.InUse)
+}
+
+// Idle satisfies metrics.DbStats
+func (s sqlxStatsWrapper) Idle() int64 {
+	return int64(s.stats.Idle)
+}
+
+// WaitCount satisfies metrics.DbStats
+func (s sqlxStatsWrapper) WaitCount() int64 {
+	return s.stats.WaitCount
+}
+
+// WaitDuration satisfies metrics.DbStats
+func (s sqlxStatsWrapper) WaitDuration() time.Duration {
+	return s.stats.WaitDuration
+}
+
+// MaxIdleClosed satisfies metrics.DbStats
+func (s sqlxStatsWrapper) MaxIdleClosed() int64 {
+	return s.stats.MaxIdleClosed
+}
+
+// MaxLifetimeClosed satisfies metrics.DbStats
+func (s sqlxStatsWrapper) MaxLifetimeClosed() int64 {
+	return s.stats.MaxLifetimeClosed
+}
+
+type sqlxTxWrapper struct {
+	tx *sqlx.Tx
+}
+
+// QueryRow satisfies sql.Tx
+func (t sqlxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
+	return t.tx.QueryRowx(sql, args...)
+}
+
+// Exec satisfies sql.Tx
+func (t sqlxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
+	return t.tx.Exec(sql, args...)
+}
+
+// Commit satisfies sql.Tx
+func (t sqlxTxWrapper) Commit(ctx context.Context) error {
+	return t.tx.Commit()
+}
+
+// Rollback satisfies sql.Tx
+func (t sqlxTxWrapper) Rollback(ctx context.Context) error {
+	return t.tx.Rollback()
+}
+
+func (t sqlxTxWrapper) CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error) {
+	return 0, errors.New("Unsupported Operation")
+}

indexer/database/sql/postgres/sqlx_test.go

@@ -0,0 +1,119 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"fmt"
+	"math/big"
+	"strings"
+	"testing"
+
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+func TestPostgresSQLX(t *testing.T) {
+	var sqlxdb *sqlx.DB
+
+	t.Run("connects to the database", func(t *testing.T) {
+		var err error
+		connStr := postgres.TestConfig.DbConnectionString()
+
+		sqlxdb, err = sqlx.Connect("postgres", connStr)
+		if err != nil {
+			t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err)
+		}
+		if sqlxdb == nil {
+			t.Fatal("DB is nil")
+		}
+		err = sqlxdb.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("serializes big.Int to db", func(t *testing.T) {
+		// postgres driver doesn't support go big.Int type
+		// various casts in golang uint64, int64, overflow for
+		// transaction value (in wei) even though
+		// postgres numeric can handle an arbitrary
+		// sized int, so use string representation of big.Int
+		// and cast on insert
+
+		connStr := postgres.TestConfig.DbConnectionString()
+		db, err := sqlx.Connect("postgres", connStr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer db.Close()
+
+		bi := new(big.Int)
+		bi.SetString("34940183920000000000", 10)
+		require.Equal(t, "34940183920000000000", bi.String())
+
+		defer db.Exec(`DROP TABLE IF EXISTS example`)
+		_, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		sqlStatement := `
+			INSERT INTO example (id, data)
+			VALUES (1, cast($1 AS NUMERIC))`
+		_, err = db.Exec(sqlStatement, bi.String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var data string
+		err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		require.Equal(t, data, bi.String())
+		actual := new(big.Int)
+		actual.SetString(data, 10)
+		require.Equal(t, bi, actual)
+	})
+
+	t.Run("throws error when can't connect to the database", func(t *testing.T) {
+		goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
+		_, err := postgres.NewSQLXDriver(ctx, postgres.Config{}, goodInfo)
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
+
+		expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
+	})
+
+	t.Run("throws error when can't create node", func(t *testing.T) {
+		badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
+		badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
+
+		_, err := postgres.NewSQLXDriver(ctx, postgres.TestConfig, badInfo)
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
+
+		expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
+	})
+}

indexer/database/sql/postgres/test_helpers.go

@@ -0,0 +1,44 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+// SetupSQLXDB is used to setup a sqlx db for tests
+func SetupSQLXDB() (sql.Database, error) {
+	conf := TestConfig
+	conf.MaxIdle = 0
+	driver, err := NewSQLXDriver(context.Background(), conf, node.Info{})
+	if err != nil {
+		return nil, err
+	}
+	return NewPostgresDB(driver, false), nil
+}
+
+// SetupPGXDB is used to setup a pgx db for tests
+func SetupPGXDB(config Config) (sql.Database, error) {
+	driver, err := NewPGXDriver(context.Background(), config, node.Info{})
+	if err != nil {
+		return nil, err
+	}
+	return NewPostgresDB(driver, false), nil
+}

indexer/database/sql/sqlx_indexer_legacy_test.go

@@ -0,0 +1,52 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupLegacySQLXIndexer(t *testing.T) {
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db)
+	require.NoError(t, err)
+}
+
+func setupLegacySQLX(t *testing.T) {
+	setupLegacySQLXIndexer(t)
+	test.SetupLegacyTestData(t, ind)
+}
+
+func TestLegacySQLXIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs", func(t *testing.T) {
+		setupLegacySQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestLegacyIndexer(t, db)
+	})
+}

indexer/database/sql/sqlx_indexer_test.go

@@ -0,0 +1,227 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupSQLXIndexer(t *testing.T) {
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db)
+	require.NoError(t, err)
+}
+
+func setupSQLX(t *testing.T) {
+	setupSQLXIndexer(t)
+	test.SetupTestData(t, ind)
+}
+
+func setupSQLXNonCanonical(t *testing.T) {
+	setupSQLXIndexer(t)
+	test.SetupTestDataNonCanonical(t, ind)
+}
+
+// Test indexer for a canonical block
+func TestSQLXIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexHeaderIPLDs(t, db)
+	})
+
+	t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexTransactionIPLDs(t, db)
+	})
+
+	t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexLogIPLDs(t, db)
+	})
+
+	t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexReceiptIPLDs(t, db)
+	})
+
+	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexStateIPLDs(t, db)
+	})
+
+	t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexStorageIPLDs(t, db)
+	})
+}
+
+// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1
+func TestSQLXIndexerNonCanonical(t *testing.T) {
+	t.Run("Publish and index header", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexHeaderNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index transactions", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexTransactionsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index receipts", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexReceiptsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index logs", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexLogsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index state nodes", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexStateNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index storage nodes", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexStorageNonCanonical(t, db)
+	})
+}
+
+func TestSQLXWatchAddressMethods(t *testing.T) {
+	setupSQLXIndexer(t)
+	defer tearDown(t)
+	defer checkTxClosure(t, 0, 0, 0)
+
+	t.Run("Load watched addresses (empty table)", func(t *testing.T) {
+		test.TestLoadEmptyWatchedAddresses(t, ind)
+	})
+
+	t.Run("Insert watched addresses", func(t *testing.T) {
+		args := mocks.GetInsertWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
+		require.NoError(t, err)
+
+		test.TestInsertWatchedAddresses(t, db)
+	})
+
+	t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetInsertAlreadyWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		test.TestInsertAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses", func(t *testing.T) {
+		args := mocks.GetRemoveWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		test.TestRemoveWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
+		args := mocks.GetRemoveNonWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		test.TestRemoveNonWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses", func(t *testing.T) {
+		args := mocks.GetSetWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		test.TestSetWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetSetAlreadyWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
+		require.NoError(t, err)
+
+		test.TestSetAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Load watched addresses", func(t *testing.T) {
+		test.TestLoadWatchedAddresses(t, ind)
+	})
+
+	t.Run("Clear watched addresses", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		test.TestClearWatchedAddresses(t, db)
+	})
+
+	t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		test.TestClearEmptyWatchedAddresses(t, db)
+	})
+}

indexer/database/sql/writer.go

@@ -0,0 +1,345 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/jackc/pgtype"
+	shopspring "github.com/jackc/pgtype/ext/shopspring-numeric"
+	"github.com/lib/pq"
+	"github.com/shopspring/decimal"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+)
+
+// Writer handles processing and writing of indexed IPLD objects to Postgres
+type Writer struct {
+	db Database
+}
+
+// NewWriter creates a new pointer to a Writer
+func NewWriter(db Database) *Writer {
+	return &Writer{
+		db: db,
+	}
+}
+
+// Close satisfies io.Closer
+func (w *Writer) Close() error {
+	return w.db.Close()
+}
+
+/*
+INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
+ON CONFLICT (block_hash, block_number) DO NOTHING
+*/
+func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error {
+	nodeIDs := pq.StringArray([]string{w.db.NodeID()})
+	_, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(),
+		header.BlockNumber,
+		header.BlockHash,
+		header.ParentHash,
+		header.CID,
+		header.TotalDifficulty,
+		nodeIDs,
+		header.Reward,
+		header.StateRoot,
+		header.TxRoot,
+		header.RctRoot,
+		header.UnclesHash,
+		header.Bloom,
+		header.Timestamp,
+		header.Coinbase)
+	if err != nil {
+		return insertError{"eth.header_cids", err, w.db.InsertHeaderStm(), header}
+	}
+	metrics.IndexerMetrics.BlocksCounter.Inc(1)
+	return nil
+}
+
+/*
+INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, index) VALUES ($1, $2, $3, $4, $5, $6, $7)
+ON CONFLICT (block_hash, block_number) DO NOTHING
+*/
+func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error {
+	_, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(),
+		uncle.BlockNumber,
+		uncle.BlockHash,
+		uncle.HeaderID,
+		uncle.ParentHash,
+		uncle.CID,
+		uncle.Reward,
+		uncle.Index)
+	if err != nil {
+		return insertError{"eth.uncle_cids", err, w.db.InsertUncleStm(), uncle}
+	}
+	return nil
+}
+
+/*
+INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING
+*/
+func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
+	val := transaction.Value
+	if val == "" {
+		val = "0"
+	}
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseInt(transaction.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.transaction_cids", err, "COPY", transaction}
+		}
+
+		value, err := toNumeric(val)
+		if err != nil {
+			return insertError{"eth.transaction_cids", err, "COPY", transaction}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), w.db.TxTableName(), w.db.TxColumnNames(),
+			toRows(toRow(blockNum, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
+				transaction.Src, transaction.Index, int(transaction.Type), value)))
+		if err != nil {
+			return insertError{"eth.transaction_cids", err, "COPY", transaction}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(),
+			transaction.BlockNumber,
+			transaction.HeaderID,
+			transaction.TxHash,
+			transaction.CID,
+			transaction.Dst,
+			transaction.Src,
+			transaction.Index,
+			transaction.Type,
+			transaction.Value)
+		if err != nil {
+			return insertError{"eth.transaction_cids", err, w.db.InsertTxStm(), transaction}
+		}
+	}
+	metrics.IndexerMetrics.TransactionsCounter.Inc(1)
+	return nil
+}
+
+/*
+INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, post_status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+ON CONFLICT (tx_id, header_id, block_number) DO NOTHING
+*/
+func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseUint(rct.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.receipt_cids", err, "COPY", rct}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), w.db.RctTableName(), w.db.RctColumnNames(),
+			toRows(toRow(blockNum, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
+				rct.PostState, int(rct.PostStatus))))
+		if err != nil {
+			return insertError{"eth.receipt_cids", err, "COPY", rct}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(),
+			rct.BlockNumber,
+			rct.HeaderID,
+			rct.TxID,
+			rct.CID,
+			rct.Contract,
+			rct.PostState,
+			rct.PostStatus)
+		if err != nil {
+			return insertError{"eth.receipt_cids", err, w.db.InsertRctStm(), *rct}
+		}
+	}
+	metrics.IndexerMetrics.ReceiptsCounter.Inc(1)
+	return nil
+}
+
+/*
+INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, topic3) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING
+*/
+func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
+	if w.useCopyForTx(tx) {
+		var rows [][]interface{}
+		for _, log := range logs {
+			blockNum, err := strconv.ParseUint(log.BlockNumber, 10, 64)
+			if err != nil {
+				return insertError{"eth.log_cids", err, "COPY", log}
+			}
+
+			rows = append(rows, toRow(blockNum, log.HeaderID, log.CID, log.ReceiptID,
+				log.Address, log.Index, log.Topic0, log.Topic1, log.Topic2, log.Topic3))
+		}
+		if nil != rows && len(rows) >= 0 {
+			_, err := tx.CopyFrom(w.db.Context(), w.db.LogTableName(), w.db.LogColumnNames(), rows)
+			if err != nil {
+				return insertError{"eth.log_cids", err, "COPY", rows}
+			}
+			metrics.IndexerMetrics.LogsCounter.Inc(int64(len(rows)))
+		}
+	} else {
+		for _, log := range logs {
+			_, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(),
+				log.BlockNumber,
+				log.HeaderID,
+				log.CID,
+				log.ReceiptID,
+				log.Address,
+				log.Index,
+				log.Topic0,
+				log.Topic1,
+				log.Topic2,
+				log.Topic3)
+			if err != nil {
+				return insertError{"eth.log_cids", err, w.db.InsertLogStm(), *log}
+			}
+			metrics.IndexerMetrics.LogsCounter.Inc(1)
+		}
+	}
+	return nil
+}
+
+/*
+INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING
+*/
+func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
+	bal := stateNode.Balance
+	if stateNode.Removed {
+		bal = "0"
+	}
+
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseUint(stateNode.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.state_cids", err, "COPY", stateNode}
+		}
+
+		balance, err := toNumeric(bal)
+		if err != nil {
+			return insertError{"eth.state_cids", err, "COPY", stateNode}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), w.db.StateTableName(), w.db.StateColumnNames(),
+			toRows(toRow(blockNum, stateNode.HeaderID, stateNode.StateKey, stateNode.CID,
+				true, balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot, stateNode.Removed)))
+		if err != nil {
+			return insertError{"eth.state_cids", err, "COPY", stateNode}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
+			stateNode.BlockNumber,
+			stateNode.HeaderID,
+			stateNode.StateKey,
+			stateNode.CID,
+			true,
+			bal,
+			stateNode.Nonce,
+			stateNode.CodeHash,
+			stateNode.StorageRoot,
+			stateNode.Removed,
+		)
+		if err != nil {
+			return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
+		}
+	}
+	return nil
+}
+
+/*
+INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHING
+*/
+func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error {
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseUint(storageCID.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.storage_cids", err, "COPY", storageCID}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), w.db.StorageTableName(), w.db.StorageColumnNames(),
+			toRows(toRow(blockNum, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID,
+				true, storageCID.Value, storageCID.Removed)))
+		if err != nil {
+			return insertError{"eth.storage_cids", err, "COPY", storageCID}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(),
+			storageCID.BlockNumber,
+			storageCID.HeaderID,
+			storageCID.StateKey,
+			storageCID.StorageKey,
+			storageCID.CID,
+			true,
+			storageCID.Value,
+			storageCID.Removed,
+		)
+		if err != nil {
+			return insertError{"eth.storage_cids", err, w.db.InsertStorageStm(), storageCID}
+		}
+	}
+	return nil
+}
+
+func (w *Writer) useCopyForTx(tx Tx) bool {
+	// Using COPY instead of INSERT only makes much sense if also using a DelayedTx, so that operations
+	// can be collected over time and then all submitted within in a single TX.
+	if _, ok := tx.(*DelayedTx); ok {
+		return w.db.UseCopyFrom()
+	}
+	return false
+}
+
+// combine args into a row
+func toRow(args ...interface{}) []interface{} {
+	var row []interface{}
+	row = append(row, args...)
+	return row
+}
+
+func toNumeric(value string) (*shopspring.Numeric, error) {
+	decimalValue, err := decimal.NewFromString(value)
+	if nil != err {
+		return nil, err
+	}
+
+	return &shopspring.Numeric{Decimal: decimalValue, Status: pgtype.Present}, nil
+}
+
+// combine row (or rows) into a slice of rows for CopyFrom
+func toRows(rows ...[]interface{}) [][]interface{} {
+	return rows
+}
+
+type insertError struct {
+	table     string
+	err       error
+	stmt      string
+	arguments interface{}
+}
+
+var _ error = insertError{}
+
+func (dbe insertError) Error() string {
+	return fmt.Sprintf("error inserting %s entry: %v\r\nstatement: %s\r\narguments: %+v",
+		dbe.table, dbe.err, dbe.stmt, dbe.arguments)
+}