Copy package from geth fork

indexer/database/dump/batch_tx.go

@@ -0,0 +1,80 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dump
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+)
+
+// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration
+type BatchTx struct {
+	BlockNumber string
+	dump        io.Writer
+	quit        chan struct{}
+	iplds       chan models.IPLDModel
+	ipldCache   models.IPLDBatch
+
+	submit func(blockTx *BatchTx, err error) error
+}
+
+// Submit satisfies indexer.AtomicTx
+func (tx *BatchTx) Submit(err error) error {
+	return tx.submit(tx, err)
+}
+
+func (tx *BatchTx) flush() error {
+	if _, err := fmt.Fprintf(tx.dump, "%+v\r\n", tx.ipldCache); err != nil {
+		return err
+	}
+	tx.ipldCache = models.IPLDBatch{}
+	return nil
+}
+
+// run in background goroutine to synchronize concurrent appends to the ipldCache
+func (tx *BatchTx) cache() {
+	for {
+		select {
+		case i := <-tx.iplds:
+			tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
+			tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
+		case <-tx.quit:
+			tx.ipldCache = models.IPLDBatch{}
+			return
+		}
+	}
+}
+
+func (tx *BatchTx) cacheDirect(key string, value []byte) {
+	tx.iplds <- models.IPLDModel{
+		BlockNumber: tx.BlockNumber,
+		Key:         key,
+		Data:        value,
+	}
+}
+
+func (tx *BatchTx) cacheIPLD(i ipld.IPLD) {
+	tx.iplds <- models.IPLDModel{
+		BlockNumber: tx.BlockNumber,
+		Key:         i.Cid().String(),
+		Data:        i.RawData(),
+	}
+}

indexer/database/dump/config.go

@@ -0,0 +1,79 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dump
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+)
+
+// DumpType to explicitly type the dump destination
+type DumpType string
+
+const (
+	STDOUT  = "Stdout"
+	STDERR  = "Stderr"
+	DISCARD = "Discard"
+	UNKNOWN = "Unknown"
+)
+
+// ResolveDumpType resolves the dump type for the provided string
+func ResolveDumpType(str string) (DumpType, error) {
+	switch strings.ToLower(str) {
+	case "stdout", "out", "std out":
+		return STDOUT, nil
+	case "stderr", "err", "std err":
+		return STDERR, nil
+	case "discard", "void", "devnull", "dev null":
+		return DISCARD, nil
+	default:
+		return UNKNOWN, fmt.Errorf("unrecognized dump type: %s", str)
+	}
+}
+
+// Config for data dump
+type Config struct {
+	Dump io.WriteCloser
+}
+
+// Type satisfies interfaces.Config
+func (c Config) Type() shared.DBType {
+	return shared.DUMP
+}
+
+// NewDiscardWriterCloser returns a discardWrapper wrapping io.Discard
+func NewDiscardWriterCloser() io.WriteCloser {
+	return discardWrapper{blackhole: io.Discard}
+}
+
+// discardWrapper wraps io.Discard with io.Closer
+type discardWrapper struct {
+	blackhole io.Writer
+}
+
+// Write satisfies io.Writer
+func (dw discardWrapper) Write(b []byte) (int, error) {
+	return dw.blackhole.Write(b)
+}
+
+// Close satisfies io.Closer
+func (dw discardWrapper) Close() error {
+	return nil
+}

indexer/database/dump/indexer.go

@@ -0,0 +1,443 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dump
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math/big"
+	"time"
+
+	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
+
+// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of a void
+type StateDiffIndexer struct {
+	dump        io.WriteCloser
+	chainConfig *params.ChainConfig
+}
+
+// NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer
+func NewStateDiffIndexer(chainConfig *params.ChainConfig, config Config) *StateDiffIndexer {
+	return &StateDiffIndexer{
+		dump:        config.Dump,
+		chainConfig: chainConfig,
+	}
+}
+
+// ReportDBMetrics has nothing to report for dump
+func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {}
+
+// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
+// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
+func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
+	start, t := time.Now(), time.Now()
+	blockHash := block.Hash()
+	blockHashStr := blockHash.String()
+	height := block.NumberU64()
+	traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
+	transactions := block.Transactions()
+	// Derive any missing fields
+	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.BaseFee(), transactions); err != nil {
+		return nil, err
+	}
+
+	// Generate the block iplds
+	headerNode, txNodes, rctNodes, logNodes, err := ipld.FromBlockAndReceipts(block, receipts)
+	if err != nil {
+		return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
+	}
+
+	if len(txNodes) != len(rctNodes) {
+		return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d)", len(txNodes), len(rctNodes))
+	}
+
+	// Calculate reward
+	var reward *big.Int
+	// in PoA networks block reward is 0
+	if sdi.chainConfig.Clique != nil {
+		reward = big.NewInt(0)
+	} else {
+		reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
+	}
+	t = time.Now()
+
+	blockTx := &BatchTx{
+		BlockNumber: block.Number().String(),
+		dump:        sdi.dump,
+		iplds:       make(chan models.IPLDModel),
+		quit:        make(chan struct{}),
+		ipldCache:   models.IPLDBatch{},
+		submit: func(self *BatchTx, err error) error {
+			close(self.quit)
+			close(self.iplds)
+			tDiff := time.Since(t)
+			metrics.IndexerMetrics.StateStoreCodeProcessingTimer.Update(tDiff)
+			traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
+			t = time.Now()
+			if err := self.flush(); err != nil {
+				traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
+				log.Debug(traceMsg)
+				return err
+			}
+			tDiff = time.Since(t)
+			metrics.IndexerMetrics.PostgresCommitTimer.Update(tDiff)
+			traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
+			traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
+			log.Debug(traceMsg)
+			return err
+		},
+	}
+	go blockTx.cache()
+
+	tDiff := time.Since(t)
+	metrics.IndexerMetrics.FreePostgresTimer.Update(tDiff)
+
+	traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
+	t = time.Now()
+
+	// Publish and index header, collect headerID
+	var headerID string
+	headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty)
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics.IndexerMetrics.HeaderProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+	// Publish and index uncles
+	err = sdi.processUncles(blockTx, headerID, block.Number(), block.UncleHash(), block.Uncles())
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics.IndexerMetrics.UncleProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+	// Publish and index receipts and txs
+	err = sdi.processReceiptsAndTxs(blockTx, processArgs{
+		headerID:    headerID,
+		blockNumber: block.Number(),
+		receipts:    receipts,
+		txs:         transactions,
+		rctNodes:    rctNodes,
+		txNodes:     txNodes,
+		logNodes:    logNodes,
+	})
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics.IndexerMetrics.TxAndRecProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+
+	return blockTx, err
+}
+
+// processHeader publishes and indexes a header IPLD in Postgres
+// it returns the headerID
+func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode ipld.IPLD, reward, td *big.Int) (string, error) {
+	tx.cacheIPLD(headerNode)
+
+	headerID := header.Hash().String()
+	mod := models.HeaderModel{
+		CID:             headerNode.Cid().String(),
+		ParentHash:      header.ParentHash.String(),
+		BlockNumber:     header.Number.String(),
+		BlockHash:       headerID,
+		TotalDifficulty: td.String(),
+		Reward:          reward.String(),
+		Bloom:           header.Bloom.Bytes(),
+		StateRoot:       header.Root.String(),
+		RctRoot:         header.ReceiptHash.String(),
+		TxRoot:          header.TxHash.String(),
+		UnclesHash:      header.UncleHash.String(),
+		Timestamp:       header.Time,
+		Coinbase:        header.Coinbase.String(),
+	}
+	_, err := fmt.Fprintf(sdi.dump, "%+v\r\n", mod)
+	return headerID, err
+}
+
+// processUncles publishes and indexes uncle IPLDs in Postgres
+func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, unclesHash common.Hash, uncles []*types.Header) error {
+	// publish and index uncles
+	uncleEncoding, err := rlp.EncodeToBytes(uncles)
+	if err != nil {
+		return err
+	}
+	preparedHash := crypto.Keccak256Hash(uncleEncoding)
+	if !bytes.Equal(preparedHash.Bytes(), unclesHash.Bytes()) {
+		return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.Hex(), unclesHash.Hex())
+	}
+	unclesCID, err := ipld.RawdataToCid(ipld.MEthHeaderList, uncleEncoding, multihash.KECCAK_256)
+	if err != nil {
+		return err
+	}
+	tx.cacheDirect(unclesCID.String(), uncleEncoding)
+	for i, uncle := range uncles {
+		var uncleReward *big.Int
+		// in PoA networks uncle reward is 0
+		if sdi.chainConfig.Clique != nil {
+			uncleReward = big.NewInt(0)
+		} else {
+			uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncle.Number.Uint64())
+		}
+		uncle := models.UncleModel{
+			BlockNumber: blockNumber.String(),
+			HeaderID:    headerID,
+			CID:         unclesCID.String(),
+			ParentHash:  uncle.ParentHash.String(),
+			BlockHash:   uncle.Hash().String(),
+			Reward:      uncleReward.String(),
+			Index:       int64(i),
+		}
+		if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", uncle); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// processArgs bundles arguments to processReceiptsAndTxs
+type processArgs struct {
+	headerID    string
+	blockNumber *big.Int
+	receipts    types.Receipts
+	txs         types.Transactions
+	rctNodes    []*ipld.EthReceipt
+	txNodes     []*ipld.EthTx
+	logNodes    [][]*ipld.EthLog
+}
+
+// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
+func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error {
+	// Process receipts and txs
+	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
+	for i, receipt := range args.receipts {
+		txNode := args.txNodes[i]
+		tx.cacheIPLD(txNode)
+
+		// Indexing
+		// index tx
+		trx := args.txs[i]
+		trxID := trx.Hash().String()
+
+		var val string
+		if trx.Value() != nil {
+			val = trx.Value().String()
+		}
+
+		// derive sender for the tx that corresponds with this receipt
+		from, err := types.Sender(signer, trx)
+		if err != nil {
+			return fmt.Errorf("error deriving tx sender: %v", err)
+		}
+		txModel := models.TxModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			Dst:         shared.HandleZeroAddrPointer(trx.To()),
+			Src:         shared.HandleZeroAddr(from),
+			TxHash:      trxID,
+			Index:       int64(i),
+			CID:         txNode.Cid().String(),
+			Type:        trx.Type(),
+			Value:       val,
+		}
+		if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", txModel); err != nil {
+			return err
+		}
+
+		// this is the contract address if this receipt is for a contract creation tx
+		contract := shared.HandleZeroAddr(receipt.ContractAddress)
+
+		// index the receipt
+		rctModel := &models.ReceiptModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			TxID:        trxID,
+			Contract:    contract,
+			CID:         args.rctNodes[i].Cid().String(),
+		}
+		if len(receipt.PostState) == 0 {
+			rctModel.PostStatus = receipt.Status
+		} else {
+			rctModel.PostState = common.Bytes2Hex(receipt.PostState)
+		}
+
+		if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", rctModel); err != nil {
+			return err
+		}
+
+		logDataSet := make([]*models.LogsModel, len(receipt.Logs))
+		for idx, l := range receipt.Logs {
+			topicSet := make([]string, 4)
+			for ti, topic := range l.Topics {
+				topicSet[ti] = topic.Hex()
+			}
+
+			logDataSet[idx] = &models.LogsModel{
+				BlockNumber: args.blockNumber.String(),
+				HeaderID:    args.headerID,
+				ReceiptID:   trxID,
+				Address:     l.Address.String(),
+				Index:       int64(l.Index),
+				CID:         args.logNodes[i][idx].Cid().String(),
+				Topic0:      topicSet[0],
+				Topic1:      topicSet[1],
+				Topic2:      topicSet[2],
+				Topic3:      topicSet[3],
+			}
+		}
+
+		if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", logDataSet); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql
+func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateLeafNode, headerID string) error {
+	tx, ok := batch.(*BatchTx)
+	if !ok {
+		return fmt.Errorf("dump: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
+	}
+	// publish the state node
+	var stateModel models.StateNodeModel
+	if stateNode.Removed {
+		// short circuit if it is a Removed node
+		// this assumes the db has been initialized and a ipld.blocks entry for the Removed node is present
+		stateModel = models.StateNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			CID:         shared.RemovedNodeStateCID,
+			Removed:     true,
+		}
+	} else {
+		stateModel = models.StateNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			CID:         stateNode.AccountWrapper.CID,
+			Removed:     false,
+			Balance:     stateNode.AccountWrapper.Account.Balance.String(),
+			Nonce:       stateNode.AccountWrapper.Account.Nonce,
+			CodeHash:    common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
+			StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
+		}
+	}
+
+	// index the state node, collect the stateID to reference by FK
+	if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel); err != nil {
+		return err
+	}
+
+	// if there are any storage nodes associated with this node, publish and index them
+	for _, storageNode := range stateNode.StorageDiff {
+		if storageNode.Removed {
+			// short circuit if it is a Removed node
+			// this assumes the db has been initialized and a ipld.blocks entry for the Removed node is present
+			storageModel := models.StorageNodeModel{
+				BlockNumber: tx.BlockNumber,
+				HeaderID:    headerID,
+				StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+				StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
+				CID:         shared.RemovedNodeStorageCID,
+				Removed:     true,
+			}
+			if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", storageModel); err != nil {
+				return err
+			}
+			continue
+		}
+		storageModel := models.StorageNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
+			CID:         storageNode.CID,
+			Removed:     false,
+			Value:       storageNode.Value,
+		}
+		if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", storageModel); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PushIPLD publishes iplds to ipld.blocks
+func (sdi *StateDiffIndexer) PushIPLD(batch interfaces.Batch, ipld sdtypes.IPLD) error {
+	tx, ok := batch.(*BatchTx)
+	if !ok {
+		return fmt.Errorf("dump: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
+	}
+	tx.cacheDirect(ipld.CID, ipld.Content)
+	return nil
+}
+
+// Close satisfies io.Closer
+func (sdi *StateDiffIndexer) Close() error {
+	return sdi.dump.Close()
+}
+
+// LoadWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
+func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
+	return nil, nil
+}
+
+// InsertWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
+func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
+	return nil
+}
+
+// RemoveWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
+func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error {
+	return nil
+}
+
+// SetWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
+func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
+	return nil
+}
+
+// ClearWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
+func (sdi *StateDiffIndexer) ClearWatchedAddresses() error {
+	return nil
+}

indexer/database/file/batch_tx.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file
+
+// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration
+type BatchTx struct {
+	BlockNumber string
+
+	submit func(blockTx *BatchTx, err error) error
+}
+
+// Submit satisfies indexer.AtomicTx
+func (tx *BatchTx) Submit(err error) error {
+	return tx.submit(tx, err)
+}

indexer/database/file/config.go

@@ -0,0 +1,84 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+)
+
+// FileMode to explicitly type the mode of file writer we are using
+type FileMode string
+
+const (
+	CSV     FileMode = "CSV"
+	SQL     FileMode = "SQL"
+	Unknown FileMode = "Unknown"
+)
+
+// ResolveFileMode resolves a FileMode from a provided string
+func ResolveFileMode(str string) (FileMode, error) {
+	switch strings.ToLower(str) {
+	case "csv":
+		return CSV, nil
+	case "sql":
+		return SQL, nil
+	default:
+		return Unknown, fmt.Errorf("unrecognized file type string: %s", str)
+	}
+}
+
+// Config holds params for writing out CSV or SQL files
+type Config struct {
+	Mode                     FileMode
+	OutputDir                string
+	FilePath                 string
+	WatchedAddressesFilePath string
+	NodeInfo                 node.Info
+}
+
+// Type satisfies interfaces.Config
+func (c Config) Type() shared.DBType {
+	return shared.FILE
+}
+
+var nodeInfo = node.Info{
+	GenesisBlock: "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3",
+	NetworkID:    "1",
+	ChainID:      1,
+	ID:           "mockNodeID",
+	ClientName:   "go-ethereum",
+}
+
+// CSVTestConfig config for unit tests
+var CSVTestConfig = Config{
+	Mode:                     CSV,
+	OutputDir:                "./statediffing_test",
+	WatchedAddressesFilePath: "./statediffing_watched_addresses_test_file.csv",
+	NodeInfo:                 nodeInfo,
+}
+
+// SQLTestConfig config for unit tests
+var SQLTestConfig = Config{
+	Mode:                     SQL,
+	FilePath:                 "./statediffing_test_file.sql",
+	WatchedAddressesFilePath: "./statediffing_watched_addresses_test_file.sql",
+	NodeInfo:                 nodeInfo,
+}

indexer/database/file/csv_indexer_legacy_test.go

@@ -0,0 +1,118 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+const dbDirectory = "/file_indexer"
+const pgCopyStatement = `COPY %s FROM '%s' CSV`
+
+func setupLegacyCSVIndexer(t *testing.T) {
+	if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) {
+		err := os.RemoveAll(file.CSVTestConfig.OutputDir)
+		require.NoError(t, err)
+	}
+
+	ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.CSVTestConfig)
+	require.NoError(t, err)
+
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func setupLegacyCSV(t *testing.T) {
+	setupLegacyCSVIndexer(t)
+	test.SetupLegacyTestData(t, ind)
+}
+
+func dumpCSVFileData(t *testing.T) {
+	outputDir := filepath.Join(dbDirectory, file.CSVTestConfig.OutputDir)
+	workingDir, err := os.Getwd()
+	require.NoError(t, err)
+
+	localOutputDir := filepath.Join(workingDir, file.CSVTestConfig.OutputDir)
+
+	for _, tbl := range file.Tables {
+		err := test_helpers.DedupFile(file.TableFilePath(localOutputDir, tbl.Name))
+		require.NoError(t, err)
+
+		var stmt string
+		varcharColumns := tbl.VarcharColumns()
+		if len(varcharColumns) > 0 {
+			stmt = fmt.Sprintf(
+				pgCopyStatement+" FORCE NOT NULL %s",
+				tbl.Name,
+				file.TableFilePath(outputDir, tbl.Name),
+				strings.Join(varcharColumns, ", "),
+			)
+		} else {
+			stmt = fmt.Sprintf(pgCopyStatement, tbl.Name, file.TableFilePath(outputDir, tbl.Name))
+		}
+
+		_, err = db.Exec(context.Background(), stmt)
+		require.NoError(t, err)
+	}
+}
+
+func resetAndDumpWatchedAddressesCSVFileData(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+
+	outputFilePath := filepath.Join(dbDirectory, file.CSVTestConfig.WatchedAddressesFilePath)
+	stmt := fmt.Sprintf(pgCopyStatement, schema.TableWatchedAddresses.Name, outputFilePath)
+
+	_, err = db.Exec(context.Background(), stmt)
+	require.NoError(t, err)
+}
+
+func tearDownCSV(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+	require.NoError(t, db.Close())
+
+	require.NoError(t, os.RemoveAll(file.CSVTestConfig.OutputDir))
+
+	if err := os.Remove(file.CSVTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) {
+		require.NoError(t, err)
+	}
+}
+
+func TestLegacyCSVFileIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs", func(t *testing.T) {
+		setupLegacyCSV(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestLegacyIndexer(t, db)
+	})
+}

indexer/database/file/csv_indexer_test.go

@@ -0,0 +1,255 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file_test
+
+import (
+	"context"
+	"errors"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupCSVIndexer(t *testing.T) {
+	file.CSVTestConfig.OutputDir = "./statediffing_test"
+
+	if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) {
+		err := os.RemoveAll(file.CSVTestConfig.OutputDir)
+		require.NoError(t, err)
+	}
+
+	if _, err := os.Stat(file.CSVTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) {
+		err := os.Remove(file.CSVTestConfig.WatchedAddressesFilePath)
+		require.NoError(t, err)
+	}
+
+	ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.CSVTestConfig)
+	require.NoError(t, err)
+
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func setupCSV(t *testing.T) {
+	setupCSVIndexer(t)
+	test.SetupTestData(t, ind)
+}
+
+func setupCSVNonCanonical(t *testing.T) {
+	setupCSVIndexer(t)
+	test.SetupTestDataNonCanonical(t, ind)
+}
+
+func TestCSVFileIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
+		setupCSV(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexHeaderIPLDs(t, db)
+	})
+
+	t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
+		setupCSV(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexTransactionIPLDs(t, db)
+	})
+
+	t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
+		setupCSV(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexLogIPLDs(t, db)
+	})
+
+	t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
+		setupCSV(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexReceiptIPLDs(t, db)
+	})
+
+	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
+		setupCSV(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexStateIPLDs(t, db)
+	})
+
+	t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
+		setupCSV(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexStorageIPLDs(t, db)
+	})
+}
+
+func TestCSVFileIndexerNonCanonical(t *testing.T) {
+	t.Run("Publish and index header", func(t *testing.T) {
+		setupCSVNonCanonical(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexHeaderNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index transactions", func(t *testing.T) {
+		setupCSVNonCanonical(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexTransactionsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index receipts", func(t *testing.T) {
+		setupCSVNonCanonical(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexReceiptsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index logs", func(t *testing.T) {
+		setupCSVNonCanonical(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexLogsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index state nodes", func(t *testing.T) {
+		setupCSVNonCanonical(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexStateNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index storage nodes", func(t *testing.T) {
+		setupCSVNonCanonical(t)
+		dumpCSVFileData(t)
+		defer tearDownCSV(t)
+
+		test.TestPublishAndIndexStorageNonCanonical(t, db)
+	})
+}
+
+func TestCSVFileWatchAddressMethods(t *testing.T) {
+	setupCSVIndexer(t)
+	defer tearDownCSV(t)
+
+	t.Run("Load watched addresses (empty table)", func(t *testing.T) {
+		test.TestLoadEmptyWatchedAddresses(t, ind)
+	})
+
+	t.Run("Insert watched addresses", func(t *testing.T) {
+		args := mocks.GetInsertWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesCSVFileData(t)
+
+		test.TestInsertWatchedAddresses(t, db)
+	})
+
+	t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetInsertAlreadyWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesCSVFileData(t)
+
+		test.TestInsertAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses", func(t *testing.T) {
+		args := mocks.GetRemoveWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesCSVFileData(t)
+
+		test.TestRemoveWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
+		args := mocks.GetRemoveNonWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesCSVFileData(t)
+
+		test.TestRemoveNonWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses", func(t *testing.T) {
+		args := mocks.GetSetWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesCSVFileData(t)
+
+		test.TestSetWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetSetAlreadyWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesCSVFileData(t)
+
+		test.TestSetAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Load watched addresses", func(t *testing.T) {
+		test.TestLoadWatchedAddresses(t, ind)
+	})
+
+	t.Run("Clear watched addresses", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesCSVFileData(t)
+
+		test.TestClearWatchedAddresses(t, db)
+	})
+
+	t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesCSVFileData(t)
+
+		test.TestClearEmptyWatchedAddresses(t, db)
+	})
+}

indexer/database/file/csv_writer.go

@@ -0,0 +1,418 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file
+
+import (
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/thoas/go-funk"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+var (
+	Tables = []*schema.Table{
+		&schema.TableIPLDBlock,
+		&schema.TableNodeInfo,
+		&schema.TableHeader,
+		&schema.TableStateNode,
+		&schema.TableStorageNode,
+		&schema.TableUncle,
+		&schema.TableTransaction,
+		&schema.TableReceipt,
+		&schema.TableLog,
+	}
+)
+
+type tableRow struct {
+	table  schema.Table
+	values []interface{}
+}
+
+type CSVWriter struct {
+	// dir containing output files
+	dir string
+
+	writers                fileWriters
+	watchedAddressesWriter fileWriter
+
+	rows          chan tableRow
+	flushChan     chan struct{}
+	flushFinished chan struct{}
+	quitChan      chan struct{}
+	doneChan      chan struct{}
+}
+
+type fileWriter struct {
+	*csv.Writer
+	file *os.File
+}
+
+// fileWriters wraps the file writers for each output table
+type fileWriters map[string]fileWriter
+
+func newFileWriter(path string) (ret fileWriter, err error) {
+	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
+	if err != nil {
+		return
+	}
+
+	ret = fileWriter{
+		Writer: csv.NewWriter(file),
+		file:   file,
+	}
+
+	return
+}
+
+func makeFileWriters(dir string, tables []*schema.Table) (fileWriters, error) {
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return nil, err
+	}
+	writers := fileWriters{}
+	for _, tbl := range tables {
+		w, err := newFileWriter(TableFilePath(dir, tbl.Name))
+		if err != nil {
+			return nil, err
+		}
+		writers[tbl.Name] = w
+	}
+	return writers, nil
+}
+
+func (tx fileWriters) write(tbl *schema.Table, args ...interface{}) error {
+	row := tbl.ToCsvRow(args...)
+	return tx[tbl.Name].Write(row)
+}
+
+func (tx fileWriters) close() error {
+	for _, w := range tx {
+		err := w.file.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (tx fileWriters) flush() error {
+	for _, w := range tx {
+		w.Flush()
+		if err := w.Error(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func NewCSVWriter(path string, watchedAddressesFilePath string) (*CSVWriter, error) {
+	if err := os.MkdirAll(path, 0777); err != nil {
+		return nil, fmt.Errorf("unable to make MkdirAll for path: %s err: %s", path, err)
+	}
+
+	writers, err := makeFileWriters(path, Tables)
+	if err != nil {
+		return nil, err
+	}
+
+	watchedAddressesWriter, err := newFileWriter(watchedAddressesFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	csvWriter := &CSVWriter{
+		writers:                writers,
+		watchedAddressesWriter: watchedAddressesWriter,
+		dir:                    path,
+		rows:                   make(chan tableRow),
+		flushChan:              make(chan struct{}),
+		flushFinished:          make(chan struct{}),
+		quitChan:               make(chan struct{}),
+		doneChan:               make(chan struct{}),
+	}
+	return csvWriter, nil
+}
+
+func (csw *CSVWriter) Loop() {
+	go func() {
+		defer close(csw.doneChan)
+		for {
+			select {
+			case row := <-csw.rows:
+				err := csw.writers.write(&row.table, row.values...)
+				if err != nil {
+					panic(fmt.Sprintf("error writing csv buffer: %v", err))
+				}
+			case <-csw.quitChan:
+				if err := csw.writers.flush(); err != nil {
+					panic(fmt.Sprintf("error writing csv buffer to file: %v", err))
+				}
+				return
+			case <-csw.flushChan:
+				if err := csw.writers.flush(); err != nil {
+					panic(fmt.Sprintf("error writing csv buffer to file: %v", err))
+				}
+				csw.flushFinished <- struct{}{}
+			}
+		}
+	}()
+}
+
+// Flush sends a flush signal to the looping process
+func (csw *CSVWriter) Flush() {
+	csw.flushChan <- struct{}{}
+	<-csw.flushFinished
+}
+
+func TableFilePath(dir, name string) string { return filepath.Join(dir, name+".csv") }
+
+// Close satisfies io.Closer
+func (csw *CSVWriter) Close() error {
+	close(csw.quitChan)
+	<-csw.doneChan
+	close(csw.rows)
+	close(csw.flushChan)
+	close(csw.flushFinished)
+	return csw.writers.close()
+}
+
+func (csw *CSVWriter) upsertNode(node nodeinfo.Info) {
+	var values []interface{}
+	values = append(values, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID)
+	csw.rows <- tableRow{schema.TableNodeInfo, values}
+}
+
+func (csw *CSVWriter) upsertIPLD(ipld models.IPLDModel) {
+	var values []interface{}
+	values = append(values, ipld.BlockNumber, ipld.Key, ipld.Data)
+	csw.rows <- tableRow{schema.TableIPLDBlock, values}
+}
+
+func (csw *CSVWriter) upsertIPLDDirect(blockNumber, key string, value []byte) {
+	csw.upsertIPLD(models.IPLDModel{
+		BlockNumber: blockNumber,
+		Key:         key,
+		Data:        value,
+	})
+}
+
+func (csw *CSVWriter) upsertIPLDNode(blockNumber string, i ipld.IPLD) {
+	csw.upsertIPLD(models.IPLDModel{
+		BlockNumber: blockNumber,
+		Key:         i.Cid().String(),
+		Data:        i.RawData(),
+	})
+}
+
+func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) {
+	var values []interface{}
+	values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
+		header.TotalDifficulty, header.NodeIDs, header.Reward, header.StateRoot, header.TxRoot,
+		header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.Coinbase)
+	csw.rows <- tableRow{schema.TableHeader, values}
+	metrics.IndexerMetrics.BlocksCounter.Inc(1)
+}
+
+func (csw *CSVWriter) upsertUncleCID(uncle models.UncleModel) {
+	var values []interface{}
+	values = append(values, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
+		uncle.Reward, uncle.Index)
+	csw.rows <- tableRow{schema.TableUncle, values}
+}
+
+func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) {
+	var values []interface{}
+	values = append(values, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
+		transaction.Src, transaction.Index, transaction.Type, transaction.Value)
+	csw.rows <- tableRow{schema.TableTransaction, values}
+	metrics.IndexerMetrics.TransactionsCounter.Inc(1)
+}
+
+func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
+	var values []interface{}
+	values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
+		rct.PostState, rct.PostStatus)
+	csw.rows <- tableRow{schema.TableReceipt, values}
+	metrics.IndexerMetrics.ReceiptsCounter.Inc(1)
+}
+
+func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) {
+	for _, l := range logs {
+		var values []interface{}
+		values = append(values, l.BlockNumber, l.HeaderID, l.CID, l.ReceiptID, l.Address, l.Index, l.Topic0,
+			l.Topic1, l.Topic2, l.Topic3)
+		csw.rows <- tableRow{schema.TableLog, values}
+		metrics.IndexerMetrics.LogsCounter.Inc(1)
+	}
+}
+
+func (csw *CSVWriter) upsertStateCID(stateNode models.StateNodeModel) {
+	balance := stateNode.Balance
+	if stateNode.Removed {
+		balance = "0"
+	}
+
+	var values []interface{}
+	values = append(values, stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID,
+		true, balance, strconv.FormatUint(stateNode.Nonce, 10), stateNode.CodeHash, stateNode.StorageRoot, stateNode.Removed)
+	csw.rows <- tableRow{schema.TableStateNode, values}
+}
+
+func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
+	var values []interface{}
+	values = append(values, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID,
+		true, storageCID.Value, storageCID.Removed)
+	csw.rows <- tableRow{schema.TableStorageNode, values}
+}
+
+// LoadWatchedAddresses loads watched addresses from a file
+func (csw *CSVWriter) loadWatchedAddresses() ([]common.Address, error) {
+	watchedAddressesFilePath := csw.watchedAddressesWriter.file.Name()
+	// load csv rows from watched addresses file
+	rows, err := loadWatchedAddressesRows(watchedAddressesFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// extract addresses from the csv rows
+	watchedAddresses := funk.Map(rows, func(row []string) common.Address {
+		// first column is for address in eth_meta.watched_addresses
+		addressString := row[0]
+
+		return common.HexToAddress(addressString)
+	}).([]common.Address)
+
+	return watchedAddresses, nil
+}
+
+// InsertWatchedAddresses inserts the given addresses in a file
+func (csw *CSVWriter) insertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
+	// load csv rows from watched addresses file
+	watchedAddresses, err := csw.loadWatchedAddresses()
+	if err != nil {
+		return err
+	}
+
+	// append rows for new addresses to existing csv file
+	for _, arg := range args {
+		// ignore if already watched
+		if funk.Contains(watchedAddresses, common.HexToAddress(arg.Address)) {
+			continue
+		}
+
+		var values []interface{}
+		values = append(values, arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0")
+		row := schema.TableWatchedAddresses.ToCsvRow(values...)
+
+		// writing directly instead of using rows channel as it needs to be flushed immediately
+		err = csw.watchedAddressesWriter.Write(row)
+		if err != nil {
+			return err
+		}
+	}
+
+	// watched addresses need to be flushed immediately to the file to keep them in sync with in-memory watched addresses
+	csw.watchedAddressesWriter.Flush()
+	err = csw.watchedAddressesWriter.Error()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RemoveWatchedAddresses removes the given watched addresses from a file
+func (csw *CSVWriter) removeWatchedAddresses(args []sdtypes.WatchAddressArg) error {
+	// load csv rows from watched addresses file
+	watchedAddressesFilePath := csw.watchedAddressesWriter.file.Name()
+	rows, err := loadWatchedAddressesRows(watchedAddressesFilePath)
+	if err != nil {
+		return err
+	}
+
+	// get rid of rows having addresses to be removed
+	filteredRows := funk.Filter(rows, func(row []string) bool {
+		return !funk.Contains(args, func(arg sdtypes.WatchAddressArg) bool {
+			// Compare first column in table for address
+			return arg.Address == row[0]
+		})
+	}).([][]string)
+
+	return dumpWatchedAddressesRows(csw.watchedAddressesWriter, filteredRows)
+}
+
+// SetWatchedAddresses clears and inserts the given addresses in a file
+func (csw *CSVWriter) setWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
+	var rows [][]string
+	for _, arg := range args {
+		row := schema.TableWatchedAddresses.ToCsvRow(arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0")
+		rows = append(rows, row)
+	}
+
+	return dumpWatchedAddressesRows(csw.watchedAddressesWriter, rows)
+}
+
+// loadCSVWatchedAddresses loads csv rows from the given file
+func loadWatchedAddressesRows(filePath string) ([][]string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return [][]string{}, nil
+		}
+
+		return nil, fmt.Errorf("error opening watched addresses file: %v", err)
+	}
+
+	defer file.Close()
+	reader := csv.NewReader(file)
+
+	return reader.ReadAll()
+}
+
+// dumpWatchedAddressesRows dumps csv rows to the given file
+func dumpWatchedAddressesRows(watchedAddressesWriter fileWriter, filteredRows [][]string) error {
+	file := watchedAddressesWriter.file
+	file.Close()
+
+	file, err := os.Create(file.Name())
+	if err != nil {
+		return fmt.Errorf("error creating watched addresses file: %v", err)
+	}
+
+	watchedAddressesWriter.Writer = csv.NewWriter(file)
+	watchedAddressesWriter.file = file
+
+	for _, row := range filteredRows {
+		watchedAddressesWriter.Write(row)
+	}
+
+	watchedAddressesWriter.Flush()
+
+	return nil
+}

indexer/database/file/helpers.go

@@ -0,0 +1,60 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file
+
+import "bytes"
+
+// formatPostgresStringArray parses an array of strings into the proper Postgres string representation of that array
+func formatPostgresStringArray(a []string) string {
+	if a == nil {
+		return ""
+	}
+
+	if n := len(a); n > 0 {
+		// There will be at least two curly brackets, 2*N bytes of quotes,
+		// and N-1 bytes of delimiters.
+		b := make([]byte, 1, 1+3*n)
+		b[0] = '{'
+
+		b = appendArrayQuotedBytes(b, []byte(a[0]))
+		for i := 1; i < n; i++ {
+			b = append(b, ',')
+			b = appendArrayQuotedBytes(b, []byte(a[i]))
+		}
+
+		return string(append(b, '}'))
+	}
+
+	return "{}"
+}
+
+func appendArrayQuotedBytes(b, v []byte) []byte {
+	b = append(b, '"')
+	for {
+		i := bytes.IndexAny(v, `"\`)
+		if i < 0 {
+			b = append(b, v...)
+			break
+		}
+		if i > 0 {
+			b = append(b, v[:i]...)
+		}
+		b = append(b, '\\', v[i])
+		v = v[i+1:]
+	}
+	return append(b, '"')
+}

indexer/database/file/indexer.go

@@ -0,0 +1,492 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+const defaultCSVOutputDir = "./statediff_output"
+const defaultSQLFilePath = "./statediff.sql"
+const defaultWatchedAddressesCSVFilePath = "./statediff-watched-addresses.csv"
+const defaultWatchedAddressesSQLFilePath = "./statediff-watched-addresses.sql"
+
+const watchedAddressesInsert = "INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ('%s', '%d', '%d') ON CONFLICT (address) DO NOTHING;"
+
+var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
+
+// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of a void
+type StateDiffIndexer struct {
+	fileWriter       FileWriter
+	chainConfig      *params.ChainConfig
+	nodeID           string
+	wg               *sync.WaitGroup
+	removedCacheFlag *uint32
+}
+
+// NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer
+func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, config Config) (*StateDiffIndexer, error) {
+	var err error
+	var writer FileWriter
+
+	watchedAddressesFilePath := config.WatchedAddressesFilePath
+
+	switch config.Mode {
+	case CSV:
+		outputDir := config.OutputDir
+		if outputDir == "" {
+			outputDir = defaultCSVOutputDir
+		}
+
+		if _, err := os.Stat(outputDir); !errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("cannot create output directory, directory (%s) already exists", outputDir)
+		}
+		log.Info("Writing statediff CSV files to directory", "file", outputDir)
+
+		if watchedAddressesFilePath == "" {
+			watchedAddressesFilePath = defaultWatchedAddressesCSVFilePath
+		}
+		log.Info("Writing watched addresses to file", "file", watchedAddressesFilePath)
+
+		writer, err = NewCSVWriter(outputDir, watchedAddressesFilePath)
+		if err != nil {
+			return nil, err
+		}
+	case SQL:
+		filePath := config.FilePath
+		if filePath == "" {
+			filePath = defaultSQLFilePath
+		}
+		if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("cannot create file, file (%s) already exists", filePath)
+		}
+		file, err := os.Create(filePath)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create file (%s), err: %v", filePath, err)
+		}
+		log.Info("Writing statediff SQL statements to file", "file", filePath)
+
+		if watchedAddressesFilePath == "" {
+			watchedAddressesFilePath = defaultWatchedAddressesSQLFilePath
+		}
+		log.Info("Writing watched addresses to file", "file", watchedAddressesFilePath)
+
+		writer = NewSQLWriter(file, watchedAddressesFilePath)
+	default:
+		return nil, fmt.Errorf("unrecognized file mode: %s", config.Mode)
+	}
+
+	wg := new(sync.WaitGroup)
+	writer.Loop()
+	writer.upsertNode(config.NodeInfo)
+
+	return &StateDiffIndexer{
+		fileWriter:  writer,
+		chainConfig: chainConfig,
+		nodeID:      config.NodeInfo.ID,
+		wg:          wg,
+	}, nil
+}
+
+// ReportDBMetrics has nothing to report for dump
+func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {}
+
+// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
+// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
+func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
+	sdi.removedCacheFlag = new(uint32)
+	start, t := time.Now(), time.Now()
+	blockHash := block.Hash()
+	blockHashStr := blockHash.String()
+	height := block.NumberU64()
+	traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
+	transactions := block.Transactions()
+	// Derive any missing fields
+	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.BaseFee(), transactions); err != nil {
+		return nil, err
+	}
+
+	// Generate the block iplds
+	headerNode, txNodes, rctNodes, logNodes, err := ipld.FromBlockAndReceipts(block, receipts)
+	if err != nil {
+		return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
+	}
+
+	if len(txNodes) != len(rctNodes) {
+		return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d)", len(txNodes), len(rctNodes))
+	}
+
+	// Calculate reward
+	var reward *big.Int
+	// in PoA networks block reward is 0
+	if sdi.chainConfig.Clique != nil {
+		reward = big.NewInt(0)
+	} else {
+		reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
+	}
+	t = time.Now()
+
+	blockTx := &BatchTx{
+		BlockNumber: block.Number().String(),
+		submit: func(self *BatchTx, err error) error {
+			tDiff := time.Since(t)
+			metrics.IndexerMetrics.StateStoreCodeProcessingTimer.Update(tDiff)
+			traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
+			t = time.Now()
+			sdi.fileWriter.Flush()
+			tDiff = time.Since(t)
+			metrics.IndexerMetrics.PostgresCommitTimer.Update(tDiff)
+			traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
+			traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
+			log.Debug(traceMsg)
+			return err
+		},
+	}
+	tDiff := time.Since(t)
+	metrics.IndexerMetrics.FreePostgresTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
+	t = time.Now()
+
+	// write header, collect headerID
+	headerID := sdi.processHeader(block.Header(), headerNode, reward, totalDifficulty)
+	tDiff = time.Since(t)
+	metrics.IndexerMetrics.HeaderProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+
+	// write uncles
+	sdi.processUncles(headerID, block.Number(), block.UncleHash(), block.Uncles())
+	tDiff = time.Since(t)
+	metrics.IndexerMetrics.UncleProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+
+	// write receipts and txs
+	err = sdi.processReceiptsAndTxs(processArgs{
+		headerID:    headerID,
+		blockNumber: block.Number(),
+		receipts:    receipts,
+		txs:         transactions,
+		rctNodes:    rctNodes,
+		txNodes:     txNodes,
+		logNodes:    logNodes,
+	})
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics.IndexerMetrics.TxAndRecProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+
+	return blockTx, err
+}
+
+// processHeader write a header IPLD insert SQL stmt to a file
+// it returns the headerID
+func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode ipld.IPLD, reward, td *big.Int) string {
+	sdi.fileWriter.upsertIPLDNode(header.Number.String(), headerNode)
+
+	var baseFee *string
+	if header.BaseFee != nil {
+		baseFee = new(string)
+		*baseFee = header.BaseFee.String()
+	}
+	headerID := header.Hash().String()
+	sdi.fileWriter.upsertHeaderCID(models.HeaderModel{
+		NodeIDs:         pq.StringArray([]string{sdi.nodeID}),
+		CID:             headerNode.Cid().String(),
+		ParentHash:      header.ParentHash.String(),
+		BlockNumber:     header.Number.String(),
+		BlockHash:       headerID,
+		TotalDifficulty: td.String(),
+		Reward:          reward.String(),
+		Bloom:           header.Bloom.Bytes(),
+		StateRoot:       header.Root.String(),
+		RctRoot:         header.ReceiptHash.String(),
+		TxRoot:          header.TxHash.String(),
+		UnclesHash:      header.UncleHash.String(),
+		Timestamp:       header.Time,
+		Coinbase:        header.Coinbase.String(),
+	})
+	return headerID
+}
+
+// processUncles publishes and indexes uncle IPLDs in Postgres
+func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber *big.Int, unclesHash common.Hash, uncles []*types.Header) error {
+	// publish and index uncles
+	uncleEncoding, err := rlp.EncodeToBytes(uncles)
+	if err != nil {
+		return err
+	}
+	preparedHash := crypto.Keccak256Hash(uncleEncoding)
+	if !bytes.Equal(preparedHash.Bytes(), unclesHash.Bytes()) {
+		return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.Hex(), unclesHash.Hex())
+	}
+	unclesCID, err := ipld.RawdataToCid(ipld.MEthHeaderList, uncleEncoding, multihash.KECCAK_256)
+	if err != nil {
+		return err
+	}
+	sdi.fileWriter.upsertIPLDDirect(blockNumber.String(), unclesCID.String(), uncleEncoding)
+	for i, uncle := range uncles {
+		var uncleReward *big.Int
+		// in PoA networks uncle reward is 0
+		if sdi.chainConfig.Clique != nil {
+			uncleReward = big.NewInt(0)
+		} else {
+			uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncle.Number.Uint64())
+		}
+		sdi.fileWriter.upsertUncleCID(models.UncleModel{
+			BlockNumber: blockNumber.String(),
+			HeaderID:    headerID,
+			CID:         unclesCID.String(),
+			ParentHash:  uncle.ParentHash.String(),
+			BlockHash:   uncle.Hash().String(),
+			Reward:      uncleReward.String(),
+			Index:       int64(i),
+		})
+	}
+	return nil
+}
+
+// processArgs bundles arguments to processReceiptsAndTxs
+type processArgs struct {
+	headerID    string
+	blockNumber *big.Int
+	receipts    types.Receipts
+	txs         types.Transactions
+	rctNodes    []*ipld.EthReceipt
+	txNodes     []*ipld.EthTx
+	logNodes    [][]*ipld.EthLog
+}
+
+// processReceiptsAndTxs writes receipt and tx IPLD insert SQL stmts to a file
+func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
+	// Process receipts and txs
+	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
+	for i, receipt := range args.receipts {
+		txNode := args.txNodes[i]
+		sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), txNode)
+		sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), args.rctNodes[i])
+
+		// index tx
+		trx := args.txs[i]
+		txID := trx.Hash().String()
+
+		var val string
+		if trx.Value() != nil {
+			val = trx.Value().String()
+		}
+
+		// derive sender for the tx that corresponds with this receipt
+		from, err := types.Sender(signer, trx)
+		if err != nil {
+			return fmt.Errorf("error deriving tx sender: %v", err)
+		}
+		txModel := models.TxModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			Dst:         shared.HandleZeroAddrPointer(trx.To()),
+			Src:         shared.HandleZeroAddr(from),
+			TxHash:      txID,
+			Index:       int64(i),
+			CID:         txNode.Cid().String(),
+			Type:        trx.Type(),
+			Value:       val,
+		}
+		sdi.fileWriter.upsertTransactionCID(txModel)
+
+		// this is the contract address if this receipt is for a contract creation tx
+		contract := shared.HandleZeroAddr(receipt.ContractAddress)
+
+		// index receipt
+		rctModel := &models.ReceiptModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			TxID:        txID,
+			Contract:    contract,
+			CID:         args.rctNodes[i].Cid().String(),
+		}
+		if len(receipt.PostState) == 0 {
+			rctModel.PostStatus = receipt.Status
+		} else {
+			rctModel.PostState = common.BytesToHash(receipt.PostState).String()
+		}
+		sdi.fileWriter.upsertReceiptCID(rctModel)
+
+		// index logs
+		logDataSet := make([]*models.LogsModel, len(receipt.Logs))
+		for idx, l := range receipt.Logs {
+			sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), args.logNodes[i][idx])
+			topicSet := make([]string, 4)
+			for ti, topic := range l.Topics {
+				topicSet[ti] = topic.Hex()
+			}
+
+			logDataSet[idx] = &models.LogsModel{
+				BlockNumber: args.blockNumber.String(),
+				HeaderID:    args.headerID,
+				ReceiptID:   txID,
+				Address:     l.Address.String(),
+				Index:       int64(l.Index),
+				CID:         args.logNodes[i][idx].Cid().String(),
+				Topic0:      topicSet[0],
+				Topic1:      topicSet[1],
+				Topic2:      topicSet[2],
+				Topic3:      topicSet[3],
+			}
+		}
+		sdi.fileWriter.upsertLogCID(logDataSet)
+	}
+
+	return nil
+}
+
+// PushStateNode writes a state diff node object (including any child storage nodes) IPLD insert SQL stmt to a file
+func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateLeafNode, headerID string) error {
+	tx, ok := batch.(*BatchTx)
+	if !ok {
+		return fmt.Errorf("file: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
+	}
+	// publish the state node
+	var stateModel models.StateNodeModel
+	if stateNode.Removed {
+		if atomic.LoadUint32(sdi.removedCacheFlag) == 0 {
+			atomic.StoreUint32(sdi.removedCacheFlag, 1)
+			sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, shared.RemovedNodeStateCID, []byte{})
+		}
+		stateModel = models.StateNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			CID:         shared.RemovedNodeStateCID,
+			Removed:     true,
+		}
+	} else {
+		stateModel = models.StateNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			CID:         stateNode.AccountWrapper.CID,
+			Removed:     false,
+			Balance:     stateNode.AccountWrapper.Account.Balance.String(),
+			Nonce:       stateNode.AccountWrapper.Account.Nonce,
+			CodeHash:    common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
+			StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
+		}
+	}
+
+	// index the state node
+	sdi.fileWriter.upsertStateCID(stateModel)
+
+	// if there are any storage nodes associated with this node, publish and index them
+	for _, storageNode := range stateNode.StorageDiff {
+		if storageNode.Removed {
+			if atomic.LoadUint32(sdi.removedCacheFlag) == 0 {
+				atomic.StoreUint32(sdi.removedCacheFlag, 1)
+				sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, shared.RemovedNodeStorageCID, []byte{})
+			}
+			storageModel := models.StorageNodeModel{
+				BlockNumber: tx.BlockNumber,
+				HeaderID:    headerID,
+				StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+				StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
+				CID:         shared.RemovedNodeStorageCID,
+				Removed:     true,
+				Value:       []byte{},
+			}
+			sdi.fileWriter.upsertStorageCID(storageModel)
+			continue
+		}
+		storageModel := models.StorageNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
+			CID:         storageNode.CID,
+			Removed:     false,
+			Value:       storageNode.Value,
+		}
+		sdi.fileWriter.upsertStorageCID(storageModel)
+	}
+
+	return nil
+}
+
+// PushIPLD writes iplds to ipld.blocks
+func (sdi *StateDiffIndexer) PushIPLD(batch interfaces.Batch, ipld sdtypes.IPLD) error {
+	tx, ok := batch.(*BatchTx)
+	if !ok {
+		return fmt.Errorf("file: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
+	}
+	sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, ipld.CID, ipld.Content)
+	return nil
+}
+
+// Close satisfies io.Closer
+func (sdi *StateDiffIndexer) Close() error {
+	return sdi.fileWriter.Close()
+}
+
+// LoadWatchedAddresses loads watched addresses from a file
+func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
+	return sdi.fileWriter.loadWatchedAddresses()
+}
+
+// InsertWatchedAddresses inserts the given addresses in a file
+func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
+	return sdi.fileWriter.insertWatchedAddresses(args, currentBlockNumber)
+}
+
+// RemoveWatchedAddresses removes the given watched addresses from a file
+func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error {
+	return sdi.fileWriter.removeWatchedAddresses(args)
+}
+
+// SetWatchedAddresses clears and inserts the given addresses in a file
+func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
+	return sdi.fileWriter.setWatchedAddresses(args, currentBlockNumber)
+}
+
+// ClearWatchedAddresses clears all the watched addresses from a file
+func (sdi *StateDiffIndexer) ClearWatchedAddresses() error {
+	return sdi.SetWatchedAddresses([]sdtypes.WatchAddressArg{}, big.NewInt(0))
+}

indexer/database/file/interfaces.go

@@ -0,0 +1,57 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/types"
+)
+
+// Writer interface required by the file indexer
+type FileWriter interface {
+	// Methods used to control the writer
+	Loop()
+	Close() error
+	Flush()
+
+	// Methods to upsert ethereum data model objects
+	upsertNode(node nodeinfo.Info)
+	upsertHeaderCID(header models.HeaderModel)
+	upsertUncleCID(uncle models.UncleModel)
+	upsertTransactionCID(transaction models.TxModel)
+	upsertReceiptCID(rct *models.ReceiptModel)
+	upsertLogCID(logs []*models.LogsModel)
+	upsertStateCID(stateNode models.StateNodeModel)
+	upsertStorageCID(storageCID models.StorageNodeModel)
+	upsertIPLD(ipld models.IPLDModel)
+
+	// Methods to upsert IPLD in different ways
+	upsertIPLDDirect(blockNumber, key string, value []byte)
+	upsertIPLDNode(blockNumber string, i ipld.IPLD)
+
+	// Methods to read and write watched addresses
+	loadWatchedAddresses() ([]common.Address, error)
+	insertWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error
+	removeWatchedAddresses(args []types.WatchAddressArg) error
+	setWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error
+}

indexer/database/file/mainnet_tests/indexer_test.go

@@ -0,0 +1,112 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mainnet_tests
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+var (
+	err       error
+	db        sql.Database
+	ind       interfaces.StateDiffIndexer
+	chainConf = params.MainnetChainConfig
+)
+
+func init() {
+	if os.Getenv("MODE") != "statediff" {
+		fmt.Println("Skipping statediff test")
+		os.Exit(0)
+	}
+	if os.Getenv("STATEDIFF_DB") != "file" {
+		fmt.Println("Skipping statediff .sql file writing mode test")
+		os.Exit(0)
+	}
+}
+
+func TestPushBlockAndState(t *testing.T) {
+	conf := test_helpers.GetTestConfig()
+
+	for _, blockNumber := range test_helpers.ProblemBlocks {
+		conf.BlockNumber = big.NewInt(blockNumber)
+		tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
+		require.NoError(t, err)
+
+		testPushBlockAndState(t, tb, trs)
+	}
+
+	testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
+	require.NoError(t, err)
+
+	testPushBlockAndState(t, testBlock, testReceipts)
+}
+
+func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
+	t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
+		setupMainnetIndexer(t)
+		defer dumpData(t)
+		defer tearDown(t)
+
+		test.TestBlock(t, ind, block, receipts)
+	})
+}
+
+func setupMainnetIndexer(t *testing.T) {
+	if _, err := os.Stat(file.CSVTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
+		err := os.Remove(file.CSVTestConfig.FilePath)
+		require.NoError(t, err)
+	}
+
+	ind, err = file.NewStateDiffIndexer(context.Background(), chainConf, file.CSVTestConfig)
+	require.NoError(t, err)
+
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func dumpData(t *testing.T) {
+	sqlFileBytes, err := os.ReadFile(file.CSVTestConfig.FilePath)
+	require.NoError(t, err)
+
+	_, err = db.Exec(context.Background(), string(sqlFileBytes))
+	require.NoError(t, err)
+}
+
+func tearDown(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+	require.NoError(t, db.Close())
+
+	require.NoError(t, os.Remove(file.CSVTestConfig.FilePath))
+}

indexer/database/file/sql_indexer_legacy_test.go

@@ -0,0 +1,101 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file_test
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+var (
+	db  sql.Database
+	err error
+	ind interfaces.StateDiffIndexer
+)
+
+func setupLegacySQLIndexer(t *testing.T) {
+	if _, err := os.Stat(file.SQLTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
+		err := os.Remove(file.SQLTestConfig.FilePath)
+		require.NoError(t, err)
+	}
+
+	ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.SQLTestConfig)
+	require.NoError(t, err)
+
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func setupLegacySQL(t *testing.T) {
+	setupLegacySQLIndexer(t)
+	test.SetupLegacyTestData(t, ind)
+}
+
+func dumpFileData(t *testing.T) {
+	err := test_helpers.DedupFile(file.SQLTestConfig.FilePath)
+	require.NoError(t, err)
+
+	sqlFileBytes, err := os.ReadFile(file.SQLTestConfig.FilePath)
+	require.NoError(t, err)
+
+	_, err = db.Exec(context.Background(), string(sqlFileBytes))
+	require.NoError(t, err)
+}
+
+func resetAndDumpWatchedAddressesFileData(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+
+	sqlFileBytes, err := os.ReadFile(file.SQLTestConfig.WatchedAddressesFilePath)
+	require.NoError(t, err)
+
+	_, err = db.Exec(context.Background(), string(sqlFileBytes))
+	require.NoError(t, err)
+}
+
+func tearDown(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+	require.NoError(t, db.Close())
+
+	require.NoError(t, os.Remove(file.SQLTestConfig.FilePath))
+
+	if err := os.Remove(file.SQLTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) {
+		require.NoError(t, err)
+	}
+}
+
+func TestLegacySQLFileIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs", func(t *testing.T) {
+		setupLegacySQL(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestLegacyIndexer(t, db)
+	})
+}

indexer/database/file/sql_indexer_test.go

@@ -0,0 +1,253 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file_test
+
+import (
+	"context"
+	"errors"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupIndexer(t *testing.T) {
+	if _, err := os.Stat(file.SQLTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
+		err := os.Remove(file.SQLTestConfig.FilePath)
+		require.NoError(t, err)
+	}
+
+	if _, err := os.Stat(file.SQLTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) {
+		err := os.Remove(file.SQLTestConfig.WatchedAddressesFilePath)
+		require.NoError(t, err)
+	}
+
+	ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.SQLTestConfig)
+	require.NoError(t, err)
+
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func setup(t *testing.T) {
+	setupIndexer(t)
+	test.SetupTestData(t, ind)
+}
+
+func setupSQLNonCanonical(t *testing.T) {
+	setupIndexer(t)
+	test.SetupTestDataNonCanonical(t, ind)
+}
+
+func TestSQLFileIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
+		setup(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexHeaderIPLDs(t, db)
+	})
+
+	t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
+		setup(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexTransactionIPLDs(t, db)
+	})
+
+	t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
+		setup(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexLogIPLDs(t, db)
+	})
+
+	t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
+		setup(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexReceiptIPLDs(t, db)
+	})
+
+	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
+		setup(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexStateIPLDs(t, db)
+	})
+
+	t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
+		setup(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexStorageIPLDs(t, db)
+	})
+}
+
+func TestSQLFileIndexerNonCanonical(t *testing.T) {
+	t.Run("Publish and index header", func(t *testing.T) {
+		setupSQLNonCanonical(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexHeaderNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index transactions", func(t *testing.T) {
+		setupSQLNonCanonical(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexTransactionsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index receipts", func(t *testing.T) {
+		setupSQLNonCanonical(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexReceiptsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index logs", func(t *testing.T) {
+		setupSQLNonCanonical(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexLogsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index state nodes", func(t *testing.T) {
+		setupSQLNonCanonical(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexStateNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index storage nodes", func(t *testing.T) {
+		setupSQLNonCanonical(t)
+		dumpFileData(t)
+		defer tearDown(t)
+
+		test.TestPublishAndIndexStorageNonCanonical(t, db)
+	})
+}
+
+func TestSQLFileWatchAddressMethods(t *testing.T) {
+	setupIndexer(t)
+	defer tearDown(t)
+
+	t.Run("Load watched addresses (empty table)", func(t *testing.T) {
+		test.TestLoadEmptyWatchedAddresses(t, ind)
+	})
+
+	t.Run("Insert watched addresses", func(t *testing.T) {
+		args := mocks.GetInsertWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesFileData(t)
+
+		test.TestInsertWatchedAddresses(t, db)
+	})
+
+	t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetInsertAlreadyWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesFileData(t)
+
+		test.TestInsertAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses", func(t *testing.T) {
+		args := mocks.GetRemoveWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesFileData(t)
+
+		test.TestRemoveWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
+		args := mocks.GetRemoveNonWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesFileData(t)
+
+		test.TestRemoveNonWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses", func(t *testing.T) {
+		args := mocks.GetSetWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesFileData(t)
+
+		test.TestSetWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetSetAlreadyWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesFileData(t)
+
+		test.TestSetAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Load watched addresses", func(t *testing.T) {
+		test.TestLoadWatchedAddresses(t, ind)
+	})
+
+	t.Run("Clear watched addresses", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesFileData(t)
+
+		test.TestClearWatchedAddresses(t, db)
+	})
+
+	t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		resetAndDumpWatchedAddressesFileData(t)
+
+		test.TestClearEmptyWatchedAddresses(t, db)
+	})
+}

indexer/database/file/sql_writer.go

@@ -0,0 +1,391 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package file
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+	"os"
+
+	pg_query "github.com/pganalyze/pg_query_go/v2"
+	"github.com/thoas/go-funk"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/types"
+)
+
+var (
+	pipeSize        = 65336 // min(linuxPipeSize, macOSPipeSize)
+	writeBufferSize = pipeSize * 16 * 96
+)
+
+// SQLWriter writes sql statements to a file
+type SQLWriter struct {
+	wc             io.WriteCloser
+	stmts          chan []byte
+	collatedStmt   []byte
+	collationIndex int
+
+	flushChan     chan struct{}
+	flushFinished chan struct{}
+	quitChan      chan struct{}
+	doneChan      chan struct{}
+
+	watchedAddressesFilePath string
+}
+
+// NewSQLWriter creates a new pointer to a Writer
+func NewSQLWriter(wc io.WriteCloser, watchedAddressesFilePath string) *SQLWriter {
+	return &SQLWriter{
+		wc:                       wc,
+		stmts:                    make(chan []byte),
+		collatedStmt:             make([]byte, writeBufferSize),
+		flushChan:                make(chan struct{}),
+		flushFinished:            make(chan struct{}),
+		quitChan:                 make(chan struct{}),
+		doneChan:                 make(chan struct{}),
+		watchedAddressesFilePath: watchedAddressesFilePath,
+	}
+}
+
+// Loop enables concurrent writes to the underlying os.File
+// since os.File does not buffer, it utilizes an internal buffer that is the size of a unix pipe
+// by using copy() and tracking the index/size of the buffer, we require only the initial memory allocation
+func (sqw *SQLWriter) Loop() {
+	sqw.collationIndex = 0
+	go func() {
+		defer close(sqw.doneChan)
+		var l int
+		for {
+			select {
+			case stmt := <-sqw.stmts:
+				l = len(stmt)
+				if sqw.collationIndex+l > writeBufferSize {
+					if err := sqw.flush(); err != nil {
+						panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err))
+					}
+					if l > writeBufferSize {
+						if _, err := sqw.wc.Write(stmt); err != nil {
+							panic(fmt.Sprintf("error writing large sql stmt to file: %v", err))
+						}
+						continue
+					}
+				}
+				copy(sqw.collatedStmt[sqw.collationIndex:sqw.collationIndex+l], stmt)
+				sqw.collationIndex += l
+			case <-sqw.quitChan:
+				if err := sqw.flush(); err != nil {
+					panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err))
+				}
+				return
+			case <-sqw.flushChan:
+				if err := sqw.flush(); err != nil {
+					panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err))
+				}
+				sqw.flushFinished <- struct{}{}
+			}
+		}
+	}()
+}
+
+// Close satisfies io.Closer
+func (sqw *SQLWriter) Close() error {
+	close(sqw.quitChan)
+	<-sqw.doneChan
+	close(sqw.stmts)
+	close(sqw.flushChan)
+	close(sqw.flushFinished)
+	return sqw.wc.Close()
+}
+
+// Flush sends a flush signal to the looping process
+func (sqw *SQLWriter) Flush() {
+	sqw.flushChan <- struct{}{}
+	<-sqw.flushFinished
+}
+
+func (sqw *SQLWriter) flush() error {
+	if _, err := sqw.wc.Write(sqw.collatedStmt[0:sqw.collationIndex]); err != nil {
+		return err
+	}
+	sqw.collationIndex = 0
+	return nil
+}
+
+const (
+	nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES " +
+		"('%s', '%s', '%s', '%s', %d);\n"
+
+	ipldInsert = "INSERT INTO ipld.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n"
+
+	headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, " +
+		"state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase) VALUES " +
+		"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s');\n"
+
+	uncleInsert = "INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, index) VALUES " +
+		"('%s', '%s', '%s', '%s', '%s', '%s', %d);\n"
+
+	txInsert = "INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, tx_type, " +
+		"value) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s');\n"
+
+	rctInsert = "INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, " +
+		"post_status) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d);\n"
+
+	logInsert = "INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, " +
+		"topic3) VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s');\n"
+
+	stateInsert = "INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, " +
+		"balance, nonce, code_hash, storage_root) VALUES ('%s', '%s', '%s', '%s', %t, %t, '%s', %d, '%s', '%s');\n"
+
+	storageInsert = "INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, " +
+		"removed, diff, val) VALUES ('%s', '%s', '%s', '%s', '%s', %t, %t, '\\x%x');\n"
+)
+
+func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) {
+	sqw.stmts <- []byte(fmt.Sprintf(nodeInsert, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID))
+}
+
+func (sqw *SQLWriter) upsertIPLD(ipld models.IPLDModel) {
+	sqw.stmts <- []byte(fmt.Sprintf(ipldInsert, ipld.BlockNumber, ipld.Key, ipld.Data))
+}
+
+func (sqw *SQLWriter) upsertIPLDDirect(blockNumber, key string, value []byte) {
+	sqw.upsertIPLD(models.IPLDModel{
+		BlockNumber: blockNumber,
+		Key:         key,
+		Data:        value,
+	})
+}
+
+func (sqw *SQLWriter) upsertIPLDNode(blockNumber string, i ipld.IPLD) {
+	sqw.upsertIPLD(models.IPLDModel{
+		BlockNumber: blockNumber,
+		Key:         i.Cid().String(),
+		Data:        i.RawData(),
+	})
+}
+
+func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
+	stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
+		header.TotalDifficulty, formatPostgresStringArray(header.NodeIDs), header.Reward, header.StateRoot, header.TxRoot,
+		header.RctRoot, header.UnclesHash, header.Bloom, header.Timestamp, header.Coinbase)
+	sqw.stmts <- []byte(stmt)
+	metrics.IndexerMetrics.BlocksCounter.Inc(1)
+}
+
+func (sqw *SQLWriter) upsertUncleCID(uncle models.UncleModel) {
+	sqw.stmts <- []byte(fmt.Sprintf(uncleInsert, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
+		uncle.Reward, uncle.Index))
+}
+
+func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) {
+	sqw.stmts <- []byte(fmt.Sprintf(txInsert, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
+		transaction.Src, transaction.Index, transaction.Type, transaction.Value))
+	metrics.IndexerMetrics.TransactionsCounter.Inc(1)
+}
+
+func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) {
+	sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
+		rct.PostState, rct.PostStatus))
+	metrics.IndexerMetrics.ReceiptsCounter.Inc(1)
+}
+
+func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) {
+	for _, l := range logs {
+		sqw.stmts <- []byte(fmt.Sprintf(logInsert, l.BlockNumber, l.HeaderID, l.CID, l.ReceiptID, l.Address, l.Index, l.Topic0,
+			l.Topic1, l.Topic2, l.Topic3))
+		metrics.IndexerMetrics.LogsCounter.Inc(1)
+	}
+}
+
+func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) {
+	balance := stateNode.Balance
+	if stateNode.Removed {
+		balance = "0"
+	}
+	sqw.stmts <- []byte(fmt.Sprintf(stateInsert, stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID,
+		stateNode.Removed, true, balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot))
+}
+
+func (sqw *SQLWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
+	sqw.stmts <- []byte(fmt.Sprintf(storageInsert, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID,
+		storageCID.Removed, true, storageCID.Value))
+}
+
+// LoadWatchedAddresses loads watched addresses from a file
+func (sqw *SQLWriter) loadWatchedAddresses() ([]common.Address, error) {
+	// load sql statements from watched addresses file
+	stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// extract addresses from the sql statements
+	watchedAddresses := []common.Address{}
+	for _, stmt := range stmts {
+		addressString, err := parseWatchedAddressStatement(stmt)
+		if err != nil {
+			return nil, err
+		}
+		watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString))
+	}
+
+	return watchedAddresses, nil
+}
+
+// InsertWatchedAddresses inserts the given addresses in a file
+func (sqw *SQLWriter) insertWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error {
+	// load sql statements from watched addresses file
+	stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath)
+	if err != nil {
+		return err
+	}
+
+	// get already watched addresses
+	var watchedAddresses []string
+	for _, stmt := range stmts {
+		addressString, err := parseWatchedAddressStatement(stmt)
+		if err != nil {
+			return err
+		}
+
+		watchedAddresses = append(watchedAddresses, addressString)
+	}
+
+	// append statements for new addresses to existing statements
+	for _, arg := range args {
+		// ignore if already watched
+		if funk.Contains(watchedAddresses, arg.Address) {
+			continue
+		}
+
+		stmt := fmt.Sprintf(watchedAddressesInsert, arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
+		stmts = append(stmts, stmt)
+	}
+
+	return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, stmts)
+}
+
+// RemoveWatchedAddresses removes the given watched addresses from a file
+func (sqw *SQLWriter) removeWatchedAddresses(args []types.WatchAddressArg) error {
+	// load sql statements from watched addresses file
+	stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath)
+	if err != nil {
+		return err
+	}
+
+	// get rid of statements having addresses to be removed
+	var filteredStmts []string
+	for _, stmt := range stmts {
+		addressString, err := parseWatchedAddressStatement(stmt)
+		if err != nil {
+			return err
+		}
+
+		toRemove := funk.Contains(args, func(arg types.WatchAddressArg) bool {
+			return arg.Address == addressString
+		})
+
+		if !toRemove {
+			filteredStmts = append(filteredStmts, stmt)
+		}
+	}
+
+	return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, filteredStmts)
+}
+
+// SetWatchedAddresses clears and inserts the given addresses in a file
+func (sqw *SQLWriter) setWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error {
+	var stmts []string
+	for _, arg := range args {
+		stmt := fmt.Sprintf(watchedAddressesInsert, arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
+		stmts = append(stmts, stmt)
+	}
+
+	return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, stmts)
+}
+
+// loadWatchedAddressesStatements loads sql statements from the given file in a string slice
+func loadWatchedAddressesStatements(filePath string) ([]string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return []string{}, nil
+		}
+
+		return nil, fmt.Errorf("error opening watched addresses file: %v", err)
+	}
+	defer file.Close()
+
+	stmts := []string{}
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		stmts = append(stmts, scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error loading watched addresses: %v", err)
+	}
+
+	return stmts, nil
+}
+
+// dumpWatchedAddressesStatements dumps sql statements to the given file
+func dumpWatchedAddressesStatements(filePath string, stmts []string) error {
+	file, err := os.Create(filePath)
+	if err != nil {
+		return fmt.Errorf("error creating watched addresses file: %v", err)
+	}
+	defer file.Close()
+
+	for _, stmt := range stmts {
+		_, err := file.Write([]byte(stmt + "\n"))
+		if err != nil {
+			return fmt.Errorf("error inserting watched_addresses entry: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// parseWatchedAddressStatement parses given sql insert statement to extract the address argument
+func parseWatchedAddressStatement(stmt string) (string, error) {
+	parseResult, err := pg_query.Parse(stmt)
+	if err != nil {
+		return "", fmt.Errorf("error parsing sql stmt: %v", err)
+	}
+
+	// extract address argument from parse output for a SQL statement of form
+	// "INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at)
+	// VALUES ('0xabc', '123', '130') ON CONFLICT (address) DO NOTHING;"
+	addressString := parseResult.Stmts[0].Stmt.GetInsertStmt().
+		SelectStmt.GetSelectStmt().
+		ValuesLists[0].GetList().
+		Items[0].GetAConst().
+		GetVal().
+		GetString_().
+		Str
+
+	return addressString, nil
+}

indexer/database/metrics/metrics.go

@@ -0,0 +1,263 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package metrics
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/ethereum/go-ethereum/metrics"
+)
+
+const (
+	namespace = "statediff"
+)
+
+var (
+	IndexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry)
+	DBMetrics      = RegisterDBMetrics(metrics.DefaultRegistry)
+)
+
+// Build a fully qualified metric name
+func metricName(subsystem, name string) string {
+	if name == "" {
+		return ""
+	}
+	parts := []string{namespace, name}
+	if subsystem != "" {
+		parts = []string{namespace, subsystem, name}
+	}
+	// Prometheus uses _ but geth metrics uses / and replaces
+	return strings.Join(parts, "/")
+}
+
+type IndexerMetricsHandles struct {
+	// The total number of processed BlocksCounter
+	BlocksCounter metrics.Counter
+	// The total number of processed transactions
+	TransactionsCounter metrics.Counter
+	// The total number of processed receipts
+	ReceiptsCounter metrics.Counter
+	// The total number of processed logs
+	LogsCounter metrics.Counter
+	// The total number of access list entries processed
+	AccessListEntriesCounter metrics.Counter
+	// Time spent waiting for free postgres tx
+	FreePostgresTimer metrics.Timer
+	// Postgres transaction commit duration
+	PostgresCommitTimer metrics.Timer
+	// Header processing time
+	HeaderProcessingTimer metrics.Timer
+	// Uncle processing time
+	UncleProcessingTimer metrics.Timer
+	// Tx and receipt processing time
+	TxAndRecProcessingTimer metrics.Timer
+	// State, storage, and code combined processing time
+	StateStoreCodeProcessingTimer metrics.Timer
+
+	// Fine-grained code timers
+	BuildStateDiffWithIntermediateStateNodesTimer    metrics.Timer
+	BuildStateDiffWithoutIntermediateStateNodesTimer metrics.Timer
+	CreatedAndUpdatedStateWithIntermediateNodesTimer metrics.Timer
+	DeletedOrUpdatedStateTimer                       metrics.Timer
+	BuildAccountUpdatesTimer                         metrics.Timer
+	BuildAccountCreationsTimer                       metrics.Timer
+	ResolveNodeTimer                                 metrics.Timer
+	SortKeysTimer                                    metrics.Timer
+	FindIntersectionTimer                            metrics.Timer
+	OutputTimer                                      metrics.Timer
+	IPLDOutputTimer                                  metrics.Timer
+	DifferenceIteratorNextTimer                      metrics.Timer
+	DifferenceIteratorCounter                        metrics.Counter
+	DeletedOrUpdatedStorageTimer                     metrics.Timer
+	CreatedAndUpdatedStorageTimer                    metrics.Timer
+	BuildStorageNodesIncrementalTimer                metrics.Timer
+	BuildStateTrieObjectTimer                        metrics.Timer
+	BuildStateTrieTimer                              metrics.Timer
+	BuildStateDiffObjectTimer                        metrics.Timer
+	WriteStateDiffObjectTimer                        metrics.Timer
+	CreatedAndUpdatedStateTimer                      metrics.Timer
+	BuildStorageNodesEventualTimer                   metrics.Timer
+	BuildStorageNodesFromTrieTimer                   metrics.Timer
+	BuildRemovedAccountStorageNodesTimer             metrics.Timer
+	BuildRemovedStorageNodesFromTrieTimer            metrics.Timer
+	IsWatchedAddressTimer                            metrics.Timer
+}
+
+func RegisterIndexerMetrics(reg metrics.Registry) IndexerMetricsHandles {
+	ctx := IndexerMetricsHandles{
+		BlocksCounter:                                    metrics.NewCounter(),
+		TransactionsCounter:                              metrics.NewCounter(),
+		ReceiptsCounter:                                  metrics.NewCounter(),
+		LogsCounter:                                      metrics.NewCounter(),
+		AccessListEntriesCounter:                         metrics.NewCounter(),
+		FreePostgresTimer:                                metrics.NewTimer(),
+		PostgresCommitTimer:                              metrics.NewTimer(),
+		HeaderProcessingTimer:                            metrics.NewTimer(),
+		UncleProcessingTimer:                             metrics.NewTimer(),
+		TxAndRecProcessingTimer:                          metrics.NewTimer(),
+		StateStoreCodeProcessingTimer:                    metrics.NewTimer(),
+		BuildStateDiffWithIntermediateStateNodesTimer:    metrics.NewTimer(),
+		BuildStateDiffWithoutIntermediateStateNodesTimer: metrics.NewTimer(),
+		CreatedAndUpdatedStateWithIntermediateNodesTimer: metrics.NewTimer(),
+		DeletedOrUpdatedStateTimer:                       metrics.NewTimer(),
+		BuildAccountUpdatesTimer:                         metrics.NewTimer(),
+		BuildAccountCreationsTimer:                       metrics.NewTimer(),
+		ResolveNodeTimer:                                 metrics.NewTimer(),
+		SortKeysTimer:                                    metrics.NewTimer(),
+		FindIntersectionTimer:                            metrics.NewTimer(),
+		OutputTimer:                                      metrics.NewTimer(),
+		IPLDOutputTimer:                                  metrics.NewTimer(),
+		DifferenceIteratorNextTimer:                      metrics.NewTimer(),
+		DifferenceIteratorCounter:                        metrics.NewCounter(),
+		DeletedOrUpdatedStorageTimer:                     metrics.NewTimer(),
+		CreatedAndUpdatedStorageTimer:                    metrics.NewTimer(),
+		BuildStorageNodesIncrementalTimer:                metrics.NewTimer(),
+		BuildStateTrieObjectTimer:                        metrics.NewTimer(),
+		BuildStateTrieTimer:                              metrics.NewTimer(),
+		BuildStateDiffObjectTimer:                        metrics.NewTimer(),
+		WriteStateDiffObjectTimer:                        metrics.NewTimer(),
+		CreatedAndUpdatedStateTimer:                      metrics.NewTimer(),
+		BuildStorageNodesEventualTimer:                   metrics.NewTimer(),
+		BuildStorageNodesFromTrieTimer:                   metrics.NewTimer(),
+		BuildRemovedAccountStorageNodesTimer:             metrics.NewTimer(),
+		BuildRemovedStorageNodesFromTrieTimer:            metrics.NewTimer(),
+		IsWatchedAddressTimer:                            metrics.NewTimer(),
+	}
+	subsys := "indexer"
+	reg.Register(metricName(subsys, "blocks"), ctx.BlocksCounter)
+	reg.Register(metricName(subsys, "transactions"), ctx.TransactionsCounter)
+	reg.Register(metricName(subsys, "receipts"), ctx.ReceiptsCounter)
+	reg.Register(metricName(subsys, "logs"), ctx.LogsCounter)
+	reg.Register(metricName(subsys, "access_list_entries"), ctx.AccessListEntriesCounter)
+	reg.Register(metricName(subsys, "t_free_postgres"), ctx.FreePostgresTimer)
+	reg.Register(metricName(subsys, "t_postgres_commit"), ctx.PostgresCommitTimer)
+	reg.Register(metricName(subsys, "t_header_processing"), ctx.HeaderProcessingTimer)
+	reg.Register(metricName(subsys, "t_uncle_processing"), ctx.UncleProcessingTimer)
+	reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.TxAndRecProcessingTimer)
+	reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.StateStoreCodeProcessingTimer)
+	reg.Register(metricName(subsys, "t_build_statediff_with_intermediate_state_nodes"), ctx.BuildStateDiffWithIntermediateStateNodesTimer)
+	reg.Register(metricName(subsys, "t_build_statediff_without_intermediate_state_nodes"), ctx.BuildStateDiffWithoutIntermediateStateNodesTimer)
+	reg.Register(metricName(subsys, "t_created_and_update_state_with_intermediate_nodes"), ctx.CreatedAndUpdatedStateWithIntermediateNodesTimer)
+	reg.Register(metricName(subsys, "t_deleted_or_updated_state"), ctx.DeletedOrUpdatedStateTimer)
+	reg.Register(metricName(subsys, "t_build_account_updates"), ctx.BuildAccountUpdatesTimer)
+	reg.Register(metricName(subsys, "t_build_account_creations"), ctx.BuildAccountCreationsTimer)
+	reg.Register(metricName(subsys, "t_resolve_node"), ctx.ResolveNodeTimer)
+	reg.Register(metricName(subsys, "t_sort_keys"), ctx.SortKeysTimer)
+	reg.Register(metricName(subsys, "t_find_intersection"), ctx.FindIntersectionTimer)
+	reg.Register(metricName(subsys, "t_output_fn"), ctx.OutputTimer)
+	reg.Register(metricName(subsys, "t_ipld_output_fn"), ctx.IPLDOutputTimer)
+	reg.Register(metricName(subsys, "t_difference_iterator_next"), ctx.DifferenceIteratorNextTimer)
+	reg.Register(metricName(subsys, "difference_iterator_counter"), ctx.DifferenceIteratorCounter)
+	reg.Register(metricName(subsys, "t_created_and_updated_storage"), ctx.CreatedAndUpdatedStorageTimer)
+	reg.Register(metricName(subsys, "t_deleted_or_updated_storage"), ctx.DeletedOrUpdatedStorageTimer)
+	reg.Register(metricName(subsys, "t_build_storage_nodes_incremental"), ctx.BuildStorageNodesIncrementalTimer)
+	reg.Register(metricName(subsys, "t_build_state_trie_object"), ctx.BuildStateTrieObjectTimer)
+	reg.Register(metricName(subsys, "t_build_state_trie"), ctx.BuildStateTrieTimer)
+	reg.Register(metricName(subsys, "t_build_statediff_object"), ctx.BuildStateDiffObjectTimer)
+	reg.Register(metricName(subsys, "t_write_statediff_object"), ctx.WriteStateDiffObjectTimer)
+	reg.Register(metricName(subsys, "t_created_and_updated_state"), ctx.CreatedAndUpdatedStateTimer)
+	reg.Register(metricName(subsys, "t_build_storage_nodes_eventual"), ctx.BuildStorageNodesEventualTimer)
+	reg.Register(metricName(subsys, "t_build_storage_nodes_from_trie"), ctx.BuildStorageNodesFromTrieTimer)
+	reg.Register(metricName(subsys, "t_build_removed_accounts_storage_nodes"), ctx.BuildRemovedAccountStorageNodesTimer)
+	reg.Register(metricName(subsys, "t_build_removed_storage_nodes_from_trie"), ctx.BuildRemovedStorageNodesFromTrieTimer)
+	reg.Register(metricName(subsys, "t_is_watched_address"), ctx.IsWatchedAddressTimer)
+
+	log.Debug("Registering statediff indexer metrics.")
+	return ctx
+}
+
+type dbMetricsHandles struct {
+	// Maximum number of open connections to the sql
+	maxOpen metrics.Gauge
+	// The number of established connections both in use and idle
+	open metrics.Gauge
+	// The number of connections currently in use
+	inUse metrics.Gauge
+	// The number of idle connections
+	idle metrics.Gauge
+	// The total number of connections waited for
+	waitedFor metrics.Counter
+	// The total time blocked waiting for a new connection
+	blockedMilliseconds metrics.Counter
+	// The total number of connections closed due to SetMaxIdleConns
+	closedMaxIdle metrics.Counter
+	// The total number of connections closed due to SetConnMaxLifetime
+	closedMaxLifetime metrics.Counter
+}
+
+func RegisterDBMetrics(reg metrics.Registry) dbMetricsHandles {
+	ctx := dbMetricsHandles{
+		maxOpen:             metrics.NewGauge(),
+		open:                metrics.NewGauge(),
+		inUse:               metrics.NewGauge(),
+		idle:                metrics.NewGauge(),
+		waitedFor:           metrics.NewCounter(),
+		blockedMilliseconds: metrics.NewCounter(),
+		closedMaxIdle:       metrics.NewCounter(),
+		closedMaxLifetime:   metrics.NewCounter(),
+	}
+	subsys := "connections"
+	reg.Register(metricName(subsys, "max_open"), ctx.maxOpen)
+	reg.Register(metricName(subsys, "open"), ctx.open)
+	reg.Register(metricName(subsys, "in_use"), ctx.inUse)
+	reg.Register(metricName(subsys, "idle"), ctx.idle)
+	reg.Register(metricName(subsys, "waited_for"), ctx.waitedFor)
+	reg.Register(metricName(subsys, "blocked_milliseconds"), ctx.blockedMilliseconds)
+	reg.Register(metricName(subsys, "closed_max_idle"), ctx.closedMaxIdle)
+	reg.Register(metricName(subsys, "closed_max_lifetime"), ctx.closedMaxLifetime)
+
+	log.Debug("Registering statediff DB metrics.")
+	return ctx
+}
+
+// DbStats interface to accommodate different concrete sql stats types
+type DbStats interface {
+	MaxOpen() int64
+	Open() int64
+	InUse() int64
+	Idle() int64
+	WaitCount() int64
+	WaitDuration() time.Duration
+	MaxIdleClosed() int64
+	MaxLifetimeClosed() int64
+}
+
+func (met *dbMetricsHandles) Update(stats DbStats) {
+	met.maxOpen.Update(stats.MaxOpen())
+	met.open.Update(stats.Open())
+	met.inUse.Update(stats.InUse())
+	met.idle.Update(stats.Idle())
+	met.waitedFor.Inc(stats.WaitCount())
+	met.blockedMilliseconds.Inc(stats.WaitDuration().Milliseconds())
+	met.closedMaxIdle.Inc(stats.MaxIdleClosed())
+	met.closedMaxLifetime.Inc(stats.MaxLifetimeClosed())
+}
+
+func ReportAndUpdateDuration(msg string, start time.Time, logger log.Logger, timer metrics.Timer) {
+	since := UpdateDuration(start, timer)
+	logger.Trace(fmt.Sprintf("%s duration=%dms", msg, since.Milliseconds()))
+}
+
+func UpdateDuration(start time.Time, timer metrics.Timer) time.Duration {
+	since := time.Since(start)
+	timer.Update(since)
+	return since
+}

indexer/database/sql/batch_tx.go

@@ -0,0 +1,126 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+
+	"github.com/lib/pq"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+)
+
+const startingCacheCapacity = 1024 * 24
+
+// BatchTx wraps a sql tx with the state necessary for building the tx concurrently during trie difference iteration
+type BatchTx struct {
+	BlockNumber      string
+	ctx              context.Context
+	dbtx             Tx
+	stm              string
+	quit             chan struct{}
+	iplds            chan models.IPLDModel
+	ipldCache        models.IPLDBatch
+	removedCacheFlag *uint32
+	// Tracks expected cache size and ensures cache is caught up before flush
+	cacheWg sync.WaitGroup
+
+	submit func(blockTx *BatchTx, err error) error
+}
+
+// Submit satisfies indexer.AtomicTx
+func (tx *BatchTx) Submit(err error) error {
+	return tx.submit(tx, err)
+}
+
+func (tx *BatchTx) flush() error {
+	tx.cacheWg.Wait()
+	_, err := tx.dbtx.Exec(tx.ctx, tx.stm, pq.Array(tx.ipldCache.BlockNumbers), pq.Array(tx.ipldCache.Keys),
+		pq.Array(tx.ipldCache.Values))
+	if err != nil {
+		log.Debug(insertError{"ipld.blocks", err, tx.stm,
+			struct {
+				blockNumbers []string
+				keys         []string
+				values       [][]byte
+			}{
+				tx.ipldCache.BlockNumbers,
+				tx.ipldCache.Keys,
+				tx.ipldCache.Values,
+			}}.Error())
+		return insertError{"ipld.blocks", err, tx.stm, "too many arguments; use debug mode for full list"}
+	}
+	tx.ipldCache = models.IPLDBatch{}
+	return nil
+}
+
+// run in background goroutine to synchronize concurrent appends to the ipldCache
+func (tx *BatchTx) cache() {
+	for {
+		select {
+		case i := <-tx.iplds:
+			tx.ipldCache.BlockNumbers = append(tx.ipldCache.BlockNumbers, i.BlockNumber)
+			tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
+			tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
+			tx.cacheWg.Done()
+		case <-tx.quit:
+			tx.ipldCache = models.IPLDBatch{}
+			return
+		}
+	}
+}
+
+func (tx *BatchTx) cacheDirect(key string, value []byte) {
+	tx.cacheWg.Add(1)
+	tx.iplds <- models.IPLDModel{
+		BlockNumber: tx.BlockNumber,
+		Key:         key,
+		Data:        value,
+	}
+}
+
+func (tx *BatchTx) cacheIPLD(i ipld.IPLD) {
+	tx.cacheWg.Add(1)
+	tx.iplds <- models.IPLDModel{
+		BlockNumber: tx.BlockNumber,
+		Key:         i.Cid().String(),
+		Data:        i.RawData(),
+	}
+}
+
+func (tx *BatchTx) cacheRemoved(key string, value []byte) {
+	if atomic.LoadUint32(tx.removedCacheFlag) == 0 {
+		atomic.StoreUint32(tx.removedCacheFlag, 1)
+		tx.cacheWg.Add(1)
+		tx.iplds <- models.IPLDModel{
+			BlockNumber: tx.BlockNumber,
+			Key:         key,
+			Data:        value,
+		}
+	}
+}
+
+// rollback sql transaction and log any error
+func rollback(ctx context.Context, tx Tx) {
+	if err := tx.Rollback(ctx); err != nil {
+		log.Error(err.Error())
+	}
+}

indexer/database/sql/indexer.go

@@ -0,0 +1,583 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package sql provides an interface for pushing and indexing IPLD objects into a sql database
+// Metrics for reporting processing and connection stats are defined in ./metrics.go
+
+package sql
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math/big"
+	"time"
+
+	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
+
+// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of an SQL sql
+type StateDiffIndexer struct {
+	ctx         context.Context
+	chainConfig *params.ChainConfig
+	dbWriter    *Writer
+}
+
+// NewStateDiffIndexer creates a sql implementation of interfaces.StateDiffIndexer
+func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, db Database) (*StateDiffIndexer, error) {
+	return &StateDiffIndexer{
+		ctx:         ctx,
+		chainConfig: chainConfig,
+		dbWriter:    NewWriter(db),
+	}, nil
+}
+
+// ReportDBMetrics is a reporting function to run as goroutine
+func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) {
+	if !metrics.Enabled {
+		return
+	}
+	ticker := time.NewTicker(delay)
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				metrics2.DBMetrics.Update(sdi.dbWriter.db.Stats())
+			case <-quit:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+}
+
+// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
+// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
+func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
+	start, t := time.Now(), time.Now()
+	blockHash := block.Hash()
+	blockHashStr := blockHash.String()
+	height := block.NumberU64()
+	traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
+	transactions := block.Transactions()
+	// Derive any missing fields
+	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.BaseFee(), transactions); err != nil {
+		return nil, err
+	}
+
+	// Generate the block iplds
+	headerNode, txNodes, rctNodes, logNodes, err := ipld.FromBlockAndReceipts(block, receipts)
+	if err != nil {
+		return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
+	}
+
+	if len(txNodes) != len(rctNodes) {
+		return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d)", len(txNodes), len(rctNodes))
+	}
+
+	// Calculate reward
+	var reward *big.Int
+	// in PoA networks block reward is 0
+	if sdi.chainConfig.Clique != nil {
+		reward = big.NewInt(0)
+	} else {
+		reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
+	}
+	t = time.Now()
+
+	// Begin new DB tx for everything
+	tx := NewDelayedTx(sdi.dbWriter.db)
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(sdi.ctx, tx)
+			panic(p)
+		} else if err != nil {
+			rollback(sdi.ctx, tx)
+		}
+	}()
+	blockTx := &BatchTx{
+		removedCacheFlag: new(uint32),
+		ctx:              sdi.ctx,
+		BlockNumber:      block.Number().String(),
+		stm:              sdi.dbWriter.db.InsertIPLDsStm(),
+		iplds:            make(chan models.IPLDModel),
+		quit:             make(chan struct{}),
+		ipldCache: models.IPLDBatch{
+			BlockNumbers: make([]string, 0, startingCacheCapacity),
+			Keys:         make([]string, 0, startingCacheCapacity),
+			Values:       make([][]byte, 0, startingCacheCapacity),
+		},
+		dbtx: tx,
+		// handle transaction commit or rollback for any return case
+		submit: func(self *BatchTx, err error) error {
+			defer func() {
+				close(self.quit)
+				close(self.iplds)
+			}()
+			if p := recover(); p != nil {
+				log.Info("panic detected before tx submission, rolling back the tx", "panic", p)
+				rollback(sdi.ctx, tx)
+				panic(p)
+			} else if err != nil {
+				log.Info("error detected before tx submission, rolling back the tx", "error", err)
+				rollback(sdi.ctx, tx)
+			} else {
+				tDiff := time.Since(t)
+				metrics2.IndexerMetrics.StateStoreCodeProcessingTimer.Update(tDiff)
+				traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
+				t = time.Now()
+				if err := self.flush(); err != nil {
+					rollback(sdi.ctx, tx)
+					traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
+					log.Debug(traceMsg)
+					return err
+				}
+				err = tx.Commit(sdi.ctx)
+				tDiff = time.Since(t)
+				metrics2.IndexerMetrics.PostgresCommitTimer.Update(tDiff)
+				traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
+			}
+			traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
+			log.Debug(traceMsg)
+			return err
+		},
+	}
+	go blockTx.cache()
+
+	tDiff := time.Since(t)
+	metrics2.IndexerMetrics.FreePostgresTimer.Update(tDiff)
+
+	traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
+	t = time.Now()
+
+	// Publish and index header, collect headerID
+	var headerID string
+	headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty)
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics2.IndexerMetrics.HeaderProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+	// Publish and index uncles
+	err = sdi.processUncles(blockTx, headerID, block.Number(), block.UncleHash(), block.Uncles())
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics2.IndexerMetrics.UncleProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+	// Publish and index receipts and txs
+	err = sdi.processReceiptsAndTxs(blockTx, processArgs{
+		headerID:    headerID,
+		blockNumber: block.Number(),
+		receipts:    receipts,
+		txs:         transactions,
+		rctNodes:    rctNodes,
+		txNodes:     txNodes,
+		logNodes:    logNodes,
+	})
+	if err != nil {
+		return nil, err
+	}
+	tDiff = time.Since(t)
+	metrics2.IndexerMetrics.TxAndRecProcessingTimer.Update(tDiff)
+	traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
+	t = time.Now()
+
+	return blockTx, err
+}
+
+// processHeader publishes and indexes a header IPLD in Postgres
+// it returns the headerID
+func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode ipld.IPLD, reward, td *big.Int) (string, error) {
+	tx.cacheIPLD(headerNode)
+
+	var baseFee *string
+	if header.BaseFee != nil {
+		baseFee = new(string)
+		*baseFee = header.BaseFee.String()
+	}
+	headerID := header.Hash().String()
+	// index header
+	return headerID, sdi.dbWriter.upsertHeaderCID(tx.dbtx, models.HeaderModel{
+		CID:             headerNode.Cid().String(),
+		ParentHash:      header.ParentHash.String(),
+		BlockNumber:     header.Number.String(),
+		BlockHash:       headerID,
+		TotalDifficulty: td.String(),
+		Reward:          reward.String(),
+		Bloom:           header.Bloom.Bytes(),
+		StateRoot:       header.Root.String(),
+		RctRoot:         header.ReceiptHash.String(),
+		TxRoot:          header.TxHash.String(),
+		UnclesHash:      header.UncleHash.String(),
+		Timestamp:       header.Time,
+		Coinbase:        header.Coinbase.String(),
+	})
+}
+
+// processUncles publishes and indexes uncle IPLDs in Postgres
+func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, unclesHash common.Hash, uncles []*types.Header) error {
+	// publish and index uncles
+	uncleEncoding, err := rlp.EncodeToBytes(uncles)
+	if err != nil {
+		return err
+	}
+	preparedHash := crypto.Keccak256Hash(uncleEncoding)
+	if !bytes.Equal(preparedHash.Bytes(), unclesHash.Bytes()) {
+		return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.Hex(), unclesHash.Hex())
+	}
+	unclesCID, err := ipld.RawdataToCid(ipld.MEthHeaderList, uncleEncoding, multihash.KECCAK_256)
+	if err != nil {
+		return err
+	}
+	tx.cacheDirect(unclesCID.String(), uncleEncoding)
+	for i, uncle := range uncles {
+		var uncleReward *big.Int
+		// in PoA networks uncle reward is 0
+		if sdi.chainConfig.Clique != nil {
+			uncleReward = big.NewInt(0)
+		} else {
+			uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncle.Number.Uint64())
+		}
+		uncle := models.UncleModel{
+			BlockNumber: blockNumber.String(),
+			HeaderID:    headerID,
+			CID:         unclesCID.String(),
+			ParentHash:  uncle.ParentHash.String(),
+			BlockHash:   uncle.Hash().String(),
+			Reward:      uncleReward.String(),
+			Index:       int64(i),
+		}
+		if err := sdi.dbWriter.upsertUncleCID(tx.dbtx, uncle); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// processArgs bundles arguments to processReceiptsAndTxs
+type processArgs struct {
+	headerID    string
+	blockNumber *big.Int
+	receipts    types.Receipts
+	txs         types.Transactions
+	rctNodes    []*ipld.EthReceipt
+	txNodes     []*ipld.EthTx
+	logNodes    [][]*ipld.EthLog
+}
+
+// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
+func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error {
+	// Process receipts and txs
+	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
+	for i, receipt := range args.receipts {
+		txNode := args.txNodes[i]
+		tx.cacheIPLD(txNode)
+		tx.cacheIPLD(args.rctNodes[i])
+
+		// index tx
+		trx := args.txs[i]
+		txID := trx.Hash().String()
+
+		var val string
+		if trx.Value() != nil {
+			val = trx.Value().String()
+		}
+
+		// derive sender for the tx that corresponds with this receipt
+		from, err := types.Sender(signer, trx)
+		if err != nil {
+			return fmt.Errorf("error deriving tx sender: %v", err)
+		}
+		txModel := models.TxModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			Dst:         shared.HandleZeroAddrPointer(trx.To()),
+			Src:         shared.HandleZeroAddr(from),
+			TxHash:      txID,
+			Index:       int64(i),
+			CID:         txNode.Cid().String(),
+			Type:        trx.Type(),
+			Value:       val,
+		}
+		if err := sdi.dbWriter.upsertTransactionCID(tx.dbtx, txModel); err != nil {
+			return err
+		}
+
+		// this is the contract address if this receipt is for a contract creation tx
+		contract := shared.HandleZeroAddr(receipt.ContractAddress)
+
+		rctModel := &models.ReceiptModel{
+			BlockNumber: args.blockNumber.String(),
+			HeaderID:    args.headerID,
+			TxID:        txID,
+			Contract:    contract,
+			CID:         args.rctNodes[i].Cid().String(),
+		}
+		if len(receipt.PostState) == 0 {
+			rctModel.PostStatus = receipt.Status
+		} else {
+			rctModel.PostState = common.BytesToHash(receipt.PostState).String()
+		}
+
+		if err := sdi.dbWriter.upsertReceiptCID(tx.dbtx, rctModel); err != nil {
+			return err
+		}
+
+		// index logs
+		logDataSet := make([]*models.LogsModel, len(receipt.Logs))
+		for idx, l := range receipt.Logs {
+			tx.cacheIPLD(args.logNodes[i][idx])
+			topicSet := make([]string, 4)
+			for ti, topic := range l.Topics {
+				topicSet[ti] = topic.Hex()
+			}
+
+			logDataSet[idx] = &models.LogsModel{
+				BlockNumber: args.blockNumber.String(),
+				HeaderID:    args.headerID,
+				ReceiptID:   txID,
+				Address:     l.Address.String(),
+				Index:       int64(l.Index),
+				CID:         args.logNodes[i][idx].Cid().String(),
+				Topic0:      topicSet[0],
+				Topic1:      topicSet[1],
+				Topic2:      topicSet[2],
+				Topic3:      topicSet[3],
+			}
+		}
+
+		if err := sdi.dbWriter.upsertLogCID(tx.dbtx, logDataSet); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql
+func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateLeafNode, headerID string) error {
+	tx, ok := batch.(*BatchTx)
+	if !ok {
+		return fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
+	}
+	// publish the state node
+	var stateModel models.StateNodeModel
+	if stateNode.Removed {
+		tx.cacheRemoved(shared.RemovedNodeStateCID, []byte{})
+		stateModel = models.StateNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			CID:         shared.RemovedNodeStateCID,
+			Removed:     true,
+		}
+	} else {
+		stateModel = models.StateNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			CID:         stateNode.AccountWrapper.CID,
+			Removed:     false,
+			Balance:     stateNode.AccountWrapper.Account.Balance.String(),
+			Nonce:       stateNode.AccountWrapper.Account.Nonce,
+			CodeHash:    common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
+			StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
+		}
+	}
+
+	// index the state node
+	if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil {
+		return err
+	}
+
+	// if there are any storage nodes associated with this node, publish and index them
+	for _, storageNode := range stateNode.StorageDiff {
+		if storageNode.Removed {
+			tx.cacheRemoved(shared.RemovedNodeStorageCID, []byte{})
+			storageModel := models.StorageNodeModel{
+				BlockNumber: tx.BlockNumber,
+				HeaderID:    headerID,
+				StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+				StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
+				CID:         shared.RemovedNodeStorageCID,
+				Removed:     true,
+				Value:       []byte{},
+			}
+			if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
+				return err
+			}
+			continue
+		}
+		storageModel := models.StorageNodeModel{
+			BlockNumber: tx.BlockNumber,
+			HeaderID:    headerID,
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
+			StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
+			CID:         storageNode.CID,
+			Removed:     false,
+			Value:       storageNode.Value,
+		}
+		if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PushIPLD publishes iplds to ipld.blocks
+func (sdi *StateDiffIndexer) PushIPLD(batch interfaces.Batch, ipld sdtypes.IPLD) error {
+	tx, ok := batch.(*BatchTx)
+	if !ok {
+		return fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
+	}
+	tx.cacheDirect(ipld.CID, ipld.Content)
+	return nil
+}
+
+// Close satisfies io.Closer
+func (sdi *StateDiffIndexer) Close() error {
+	return sdi.dbWriter.Close()
+}
+
+// Update the known gaps table with the gap information.
+
+// LoadWatchedAddresses reads watched addresses from the database
+func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
+	addressStrings := make([]string, 0)
+	pgStr := "SELECT address FROM eth_meta.watched_addresses"
+	err := sdi.dbWriter.db.Select(sdi.ctx, &addressStrings, pgStr)
+	if err != nil {
+		return nil, fmt.Errorf("error loading watched addresses: %v", err)
+	}
+
+	watchedAddresses := []common.Address{}
+	for _, addressString := range addressStrings {
+		watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString))
+	}
+
+	return watchedAddresses, nil
+}
+
+// InsertWatchedAddresses inserts the given addresses in the database
+func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) (err error) {
+	tx := NewDelayedTx(sdi.dbWriter.db)
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(sdi.ctx, tx)
+			panic(p)
+		} else if err != nil {
+			rollback(sdi.ctx, tx)
+		} else {
+			err = tx.Commit(sdi.ctx)
+		}
+	}()
+
+	for _, arg := range args {
+		_, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`,
+			arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
+		if err != nil {
+			return fmt.Errorf("error inserting watched_addresses entry: %v", err)
+		}
+	}
+
+	return err
+}
+
+// RemoveWatchedAddresses removes the given watched addresses from the database
+func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) (err error) {
+	tx := NewDelayedTx(sdi.dbWriter.db)
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(sdi.ctx, tx)
+			panic(p)
+		} else if err != nil {
+			rollback(sdi.ctx, tx)
+		} else {
+			err = tx.Commit(sdi.ctx)
+		}
+	}()
+
+	for _, arg := range args {
+		_, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses WHERE address = $1`, arg.Address)
+		if err != nil {
+			return fmt.Errorf("error removing watched_addresses entry: %v", err)
+		}
+	}
+
+	return err
+}
+
+// SetWatchedAddresses clears and inserts the given addresses in the database
+func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) (err error) {
+	tx := NewDelayedTx(sdi.dbWriter.db)
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(sdi.ctx, tx)
+			panic(p)
+		} else if err != nil {
+			rollback(sdi.ctx, tx)
+		} else {
+			err = tx.Commit(sdi.ctx)
+		}
+	}()
+
+	_, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`)
+	if err != nil {
+		return fmt.Errorf("error setting watched_addresses table: %v", err)
+	}
+
+	for _, arg := range args {
+		_, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`,
+			arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
+		if err != nil {
+			return fmt.Errorf("error setting watched_addresses table: %v", err)
+		}
+	}
+
+	return err
+}
+
+// ClearWatchedAddresses clears all the watched addresses from the database
+func (sdi *StateDiffIndexer) ClearWatchedAddresses() error {
+	_, err := sdi.dbWriter.db.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`)
+	if err != nil {
+		return fmt.Errorf("error clearing watched_addresses table: %v", err)
+	}
+
+	return nil
+}

indexer/database/sql/indexer_shared_test.go

@@ -0,0 +1,28 @@
+package sql_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+var (
+	db  sql.Database
+	err error
+	ind interfaces.StateDiffIndexer
+)
+
+func checkTxClosure(t *testing.T, idle, inUse, open int64) {
+	require.Equal(t, idle, db.Stats().Idle())
+	require.Equal(t, inUse, db.Stats().InUse())
+	require.Equal(t, open, db.Stats().Open())
+}
+
+func tearDown(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+	require.NoError(t, ind.Close())
+}

indexer/database/sql/interfaces.go

@@ -0,0 +1,88 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"context"
+	"io"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+)
+
+// Database interfaces required by the sql indexer
+type Database interface {
+	Driver
+	Statements
+}
+
+// Driver interface has all the methods required by a driver implementation to support the sql indexer
+type Driver interface {
+	UseCopyFrom() bool
+	QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
+	Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
+	Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error
+	Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error
+	Begin(ctx context.Context) (Tx, error)
+	Stats() metrics.DbStats
+	NodeID() string
+	Context() context.Context
+	io.Closer
+}
+
+// Statements interface to accommodate different SQL query syntax
+type Statements interface {
+	InsertHeaderStm() string
+	InsertUncleStm() string
+	InsertTxStm() string
+	InsertRctStm() string
+	InsertLogStm() string
+	InsertStateStm() string
+	InsertStorageStm() string
+	InsertIPLDStm() string
+	InsertIPLDsStm() string
+
+	// Table/column descriptions for use with CopyFrom and similar commands.
+	LogTableName() []string
+	LogColumnNames() []string
+	RctTableName() []string
+	RctColumnNames() []string
+	StateTableName() []string
+	StateColumnNames() []string
+	StorageTableName() []string
+	StorageColumnNames() []string
+	TxTableName() []string
+	TxColumnNames() []string
+}
+
+// Tx interface to accommodate different concrete SQL transaction types
+type Tx interface {
+	QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
+	Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
+	CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error)
+	Commit(ctx context.Context) error
+	Rollback(ctx context.Context) error
+}
+
+// ScannableRow interface to accommodate different concrete row types
+type ScannableRow interface {
+	Scan(dest ...interface{}) error
+}
+
+// Result interface to accommodate different concrete result types
+type Result interface {
+	RowsAffected() (int64, error)
+}

indexer/database/sql/lazy_tx.go

@@ -0,0 +1,106 @@
+package sql
+
+import (
+	"context"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// Changing this to 1 would make sure only sequential COPYs were combined.
+const copyFromCheckLimit = 100
+
+type DelayedTx struct {
+	cache []interface{}
+	db    Database
+}
+type cachedStmt struct {
+	sql  string
+	args []interface{}
+}
+
+type copyFrom struct {
+	tableName   []string
+	columnNames []string
+	rows        [][]interface{}
+}
+
+func (cf *copyFrom) appendRows(rows [][]interface{}) {
+	cf.rows = append(cf.rows, rows...)
+}
+
+func (cf *copyFrom) matches(tableName []string, columnNames []string) bool {
+	return reflect.DeepEqual(cf.tableName, tableName) && reflect.DeepEqual(cf.columnNames, columnNames)
+}
+
+func NewDelayedTx(db Database) *DelayedTx {
+	return &DelayedTx{db: db}
+}
+
+func (tx *DelayedTx) QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow {
+	return tx.db.QueryRow(ctx, sql, args...)
+}
+
+func (tx *DelayedTx) findPrevCopyFrom(tableName []string, columnNames []string, limit int) (*copyFrom, int) {
+	for pos, count := len(tx.cache)-1, 0; pos >= 0 && count < limit; pos, count = pos-1, count+1 {
+		prevCopy, ok := tx.cache[pos].(*copyFrom)
+		if ok && prevCopy.matches(tableName, columnNames) {
+			return prevCopy, count
+		}
+	}
+	return nil, -1
+}
+
+func (tx *DelayedTx) CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error) {
+	if prevCopy, distance := tx.findPrevCopyFrom(tableName, columnNames, copyFromCheckLimit); nil != prevCopy {
+		log.Trace("statediff lazy_tx : Appending to COPY", "table", tableName,
+			"current", len(prevCopy.rows), "new", len(rows), "distance", distance)
+		prevCopy.appendRows(rows)
+	} else {
+		tx.cache = append(tx.cache, &copyFrom{tableName, columnNames, rows})
+	}
+
+	return 0, nil
+}
+
+func (tx *DelayedTx) Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) {
+	tx.cache = append(tx.cache, cachedStmt{sql, args})
+	return nil, nil
+}
+
+func (tx *DelayedTx) Commit(ctx context.Context) error {
+	base, err := tx.db.Begin(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			rollback(ctx, base)
+			panic(p)
+		} else if err != nil {
+			rollback(ctx, base)
+		}
+	}()
+	for _, item := range tx.cache {
+		switch item := item.(type) {
+		case *copyFrom:
+			_, err := base.CopyFrom(ctx, item.tableName, item.columnNames, item.rows)
+			if err != nil {
+				log.Error("COPY error", "table", item.tableName, "err", err)
+				return err
+			}
+		case cachedStmt:
+			_, err := base.Exec(ctx, item.sql, item.args...)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	tx.cache = nil
+	return base.Commit(ctx)
+}
+
+func (tx *DelayedTx) Rollback(ctx context.Context) error {
+	tx.cache = nil
+	return nil
+}

indexer/database/sql/mainnet_tests/indexer_test.go

@@ -0,0 +1,95 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mainnet_tests
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+var (
+	err       error
+	db        sql.Database
+	ind       interfaces.StateDiffIndexer
+	chainConf = params.MainnetChainConfig
+)
+
+func init() {
+	if os.Getenv("MODE") != "statediff" {
+		fmt.Println("Skipping statediff test")
+		os.Exit(0)
+	}
+}
+
+func TestMainnetIndexer(t *testing.T) {
+	conf := test_helpers.GetTestConfig()
+
+	for _, blockNumber := range test_helpers.ProblemBlocks {
+		conf.BlockNumber = big.NewInt(blockNumber)
+		tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
+		require.NoError(t, err)
+
+		testPushBlockAndState(t, tb, trs)
+	}
+
+	testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
+	require.NoError(t, err)
+
+	testPushBlockAndState(t, testBlock, testReceipts)
+}
+
+func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
+	t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
+		setupMainnetIndexer(t)
+		defer checkTxClosure(t, 0, 0, 0)
+		defer tearDown(t)
+
+		test.TestBlock(t, ind, block, receipts)
+	})
+}
+
+func setupMainnetIndexer(t *testing.T) {
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, db)
+}
+
+func checkTxClosure(t *testing.T, idle, inUse, open int64) {
+	require.Equal(t, idle, db.Stats().Idle())
+	require.Equal(t, inUse, db.Stats().InUse())
+	require.Equal(t, open, db.Stats().Open())
+}
+
+func tearDown(t *testing.T) {
+	test_helpers.TearDownDB(t, db)
+	require.NoError(t, ind.Close())
+}

indexer/database/sql/pgx_indexer_legacy_test.go

@@ -0,0 +1,52 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupLegacyPGXIndexer(t *testing.T) {
+	db, err = postgres.SetupPGXDB(postgres.TestConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db)
+	require.NoError(t, err)
+}
+
+func setupLegacyPGX(t *testing.T) {
+	setupLegacyPGXIndexer(t)
+	test.SetupLegacyTestData(t, ind)
+}
+
+func TestLegacyPGXIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs", func(t *testing.T) {
+		setupLegacyPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestLegacyIndexer(t, db)
+	})
+}

indexer/database/sql/pgx_indexer_test.go

@@ -0,0 +1,245 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupPGXIndexer(t *testing.T, config postgres.Config) {
+	db, err = postgres.SetupPGXDB(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db)
+	require.NoError(t, err)
+}
+
+func setupPGX(t *testing.T) {
+	setupPGXWithConfig(t, postgres.TestConfig)
+}
+
+func setupPGXWithConfig(t *testing.T, config postgres.Config) {
+	setupPGXIndexer(t, config)
+	test.SetupTestData(t, ind)
+}
+
+func setupPGXNonCanonical(t *testing.T) {
+	setupPGXIndexer(t, postgres.TestConfig)
+	test.SetupTestDataNonCanonical(t, ind)
+}
+
+// Test indexer for a canonical block
+func TestPGXIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexHeaderIPLDs(t, db)
+	})
+
+	t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexTransactionIPLDs(t, db)
+	})
+
+	t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexLogIPLDs(t, db)
+	})
+
+	t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexReceiptIPLDs(t, db)
+	})
+
+	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStateIPLDs(t, db)
+	})
+
+	t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
+		setupPGX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStorageIPLDs(t, db)
+	})
+
+	t.Run("Publish and index with CopyFrom enabled.", func(t *testing.T) {
+		config := postgres.TestConfig
+		config.CopyFrom = true
+
+		setupPGXWithConfig(t, config)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStateIPLDs(t, db)
+		test.TestPublishAndIndexStorageIPLDs(t, db)
+		test.TestPublishAndIndexReceiptIPLDs(t, db)
+		test.TestPublishAndIndexLogIPLDs(t, db)
+	})
+}
+
+// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1
+func TestPGXIndexerNonCanonical(t *testing.T) {
+	t.Run("Publish and index header", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexHeaderNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index transactions", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexTransactionsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index receipts", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexReceiptsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index logs", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexLogsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index state nodes", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStateNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index storage nodes", func(t *testing.T) {
+		setupPGXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		test.TestPublishAndIndexStorageNonCanonical(t, db)
+	})
+}
+
+func TestPGXWatchAddressMethods(t *testing.T) {
+	setupPGXIndexer(t, postgres.TestConfig)
+	defer tearDown(t)
+	defer checkTxClosure(t, 1, 0, 1)
+
+	t.Run("Load watched addresses (empty table)", func(t *testing.T) {
+		test.TestLoadEmptyWatchedAddresses(t, ind)
+	})
+
+	t.Run("Insert watched addresses", func(t *testing.T) {
+		args := mocks.GetInsertWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
+		require.NoError(t, err)
+
+		test.TestInsertWatchedAddresses(t, db)
+	})
+
+	t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetInsertAlreadyWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		test.TestInsertAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses", func(t *testing.T) {
+		args := mocks.GetRemoveWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		test.TestRemoveWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
+		args := mocks.GetRemoveNonWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		test.TestRemoveNonWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses", func(t *testing.T) {
+		args := mocks.GetSetWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		test.TestSetWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetSetAlreadyWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
+		require.NoError(t, err)
+
+		test.TestSetAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Load watched addresses", func(t *testing.T) {
+		test.TestLoadWatchedAddresses(t, ind)
+	})
+
+	t.Run("Clear watched addresses", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		test.TestClearWatchedAddresses(t, db)
+	})
+
+	t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		test.TestClearEmptyWatchedAddresses(t, db)
+	})
+}

indexer/database/sql/postgres/config.go

@@ -0,0 +1,140 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+)
+
+// DriverType to explicitly type the kind of sql driver we are using
+type DriverType string
+
+const (
+	PGX     DriverType = "PGX"
+	SQLX    DriverType = "SQLX"
+	Unknown DriverType = "Unknown"
+)
+
+// Env variables
+const (
+	DATABASE_NAME     = "DATABASE_NAME"
+	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
+	DATABASE_PORT     = "DATABASE_PORT"
+	DATABASE_USER     = "DATABASE_USER"
+	DATABASE_PASSWORD = "DATABASE_PASSWORD"
+)
+
+// ResolveDriverType resolves a DriverType from a provided string
+func ResolveDriverType(str string) (DriverType, error) {
+	switch strings.ToLower(str) {
+	case "pgx", "pgxpool":
+		return PGX, nil
+	case "sqlx":
+		return SQLX, nil
+	default:
+		return Unknown, fmt.Errorf("unrecognized driver type string: %s", str)
+	}
+}
+
+// TestConfig specifies default parameters for connecting to a testing DB
+var TestConfig = Config{
+	Hostname:     "localhost",
+	Port:         8077,
+	DatabaseName: "cerc_testing",
+	Username:     "vdbm",
+	Password:     "password",
+	Driver:       SQLX,
+}
+
+// Config holds params for a Postgres db
+type Config struct {
+	// conn string params
+	Hostname     string
+	Port         int
+	DatabaseName string
+	Username     string
+	Password     string
+
+	// conn settings
+	MaxConns        int
+	MaxIdle         int
+	MinConns        int
+	MaxConnIdleTime time.Duration
+	MaxConnLifetime time.Duration
+	ConnTimeout     time.Duration
+	LogStatements   bool
+
+	// node info params
+	ID         string
+	ClientName string
+
+	// driver type
+	Driver DriverType
+
+	// toggle on/off upserts
+	Upsert bool
+
+	// toggle on/off CopyFrom
+	CopyFrom bool
+}
+
+// Type satisfies interfaces.Config
+func (c Config) Type() shared.DBType {
+	return shared.POSTGRES
+}
+
+// DbConnectionString constructs and returns the connection string from the config
+func (c Config) DbConnectionString() string {
+	if len(c.Username) > 0 && len(c.Password) > 0 {
+		return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+			c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName)
+	}
+	if len(c.Username) > 0 && len(c.Password) == 0 {
+		return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
+			c.Username, c.Hostname, c.Port, c.DatabaseName)
+	}
+	return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
+}
+
+func (c Config) WithEnv() (Config, error) {
+	if val := os.Getenv(DATABASE_NAME); val != "" {
+		c.DatabaseName = val
+	}
+	if val := os.Getenv(DATABASE_HOSTNAME); val != "" {
+		c.Hostname = val
+	}
+	if val := os.Getenv(DATABASE_PORT); val != "" {
+		port, err := strconv.Atoi(val)
+		if err != nil {
+			return c, err
+		}
+		c.Port = port
+	}
+	if val := os.Getenv(DATABASE_USER); val != "" {
+		c.Username = val
+	}
+	if val := os.Getenv(DATABASE_PASSWORD); val != "" {
+		c.Password = val
+	}
+	return c, nil
+}

indexer/database/sql/postgres/database.go

@@ -0,0 +1,126 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
+)
+
+var _ sql.Database = &DB{}
+
+const (
+	createNodeStm = `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
+					 ON CONFLICT (node_id) DO NOTHING`
+)
+
+// NewPostgresDB returns a postgres.DB using the provided driver
+func NewPostgresDB(driver sql.Driver, upsert bool) *DB {
+	return &DB{upsert, driver}
+}
+
+// DB implements sql.Database using a configured driver and Postgres statement syntax
+type DB struct {
+	upsert bool
+	sql.Driver
+}
+
+// InsertHeaderStm satisfies the sql.Statements interface
+// Stm == Statement
+func (db *DB) InsertHeaderStm() string {
+	return schema.TableHeader.ToInsertStatement(db.upsert)
+}
+
+// InsertUncleStm satisfies the sql.Statements interface
+func (db *DB) InsertUncleStm() string {
+	return schema.TableUncle.ToInsertStatement(db.upsert)
+}
+
+// InsertTxStm satisfies the sql.Statements interface
+func (db *DB) InsertTxStm() string {
+	return schema.TableTransaction.ToInsertStatement(db.upsert)
+}
+
+// InsertRctStm satisfies the sql.Statements interface
+func (db *DB) InsertRctStm() string {
+	return schema.TableReceipt.ToInsertStatement(db.upsert)
+}
+
+// InsertLogStm satisfies the sql.Statements interface
+func (db *DB) InsertLogStm() string {
+	return schema.TableLog.ToInsertStatement(db.upsert)
+}
+
+// InsertStateStm satisfies the sql.Statements interface
+func (db *DB) InsertStateStm() string {
+	return schema.TableStateNode.ToInsertStatement(db.upsert)
+}
+
+// InsertStorageStm satisfies the sql.Statements interface
+func (db *DB) InsertStorageStm() string {
+	return schema.TableStorageNode.ToInsertStatement(db.upsert)
+}
+
+// InsertIPLDStm satisfies the sql.Statements interface
+func (db *DB) InsertIPLDStm() string {
+	return schema.TableIPLDBlock.ToInsertStatement(db.upsert)
+}
+
+// InsertIPLDsStm satisfies the sql.Statements interface
+func (db *DB) InsertIPLDsStm() string {
+	return `INSERT INTO ipld.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING`
+}
+
+func (db *DB) LogTableName() []string {
+	return []string{"eth", "log_cids"}
+}
+
+func (db *DB) LogColumnNames() []string {
+	return []string{"block_number", "header_id", "cid", "rct_id", "address", "index", "topic0", "topic1", "topic2", "topic3"}
+}
+
+func (db *DB) RctTableName() []string {
+	return []string{"eth", "receipt_cids"}
+}
+
+func (db *DB) RctColumnNames() []string {
+	return []string{"block_number", "header_id", "tx_id", "cid", "contract", "post_state", "post_status"}
+}
+
+func (db *DB) StateTableName() []string {
+	return []string{"eth", "state_cids"}
+}
+
+func (db *DB) StateColumnNames() []string {
+	return []string{"block_number", "header_id", "state_leaf_key", "cid", "diff", "balance", "nonce", "code_hash", "storage_root", "removed"}
+}
+
+func (db *DB) StorageTableName() []string {
+	return []string{"eth", "storage_cids"}
+}
+
+func (db *DB) StorageColumnNames() []string {
+	return []string{"block_number", "header_id", "state_leaf_key", "storage_leaf_key", "cid", "diff", "val", "removed"}
+}
+
+func (db *DB) TxTableName() []string {
+	return []string{"eth", "transaction_cids"}
+}
+
+func (db *DB) TxColumnNames() []string {
+	return []string{"block_number", "header_id", "tx_hash", "cid", "dst", "src", "index", "tx_type", "value"}
+}

indexer/database/sql/postgres/errors.go

@@ -0,0 +1,38 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"fmt"
+)
+
+const (
+	DbConnectionFailedMsg = "db connection failed"
+	SettingNodeFailedMsg  = "unable to set db node"
+)
+
+func ErrDBConnectionFailed(connectErr error) error {
+	return formatError(DbConnectionFailedMsg, connectErr)
+}
+
+func ErrUnableToSetNode(setErr error) error {
+	return formatError(SettingNodeFailedMsg, setErr)
+}
+
+func formatError(msg string, err error) error {
+	return fmt.Errorf("%s: %w", msg, err)
+}

indexer/database/sql/postgres/log_adapter.go

@@ -0,0 +1,61 @@
+// Copyright © 2023 Cerc
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/jackc/pgx/v4"
+)
+
+type LogAdapter struct {
+	l log.Logger
+}
+
+func NewLogAdapter(l log.Logger) *LogAdapter {
+	return &LogAdapter{l: l}
+}
+
+func (l *LogAdapter) Log(ctx context.Context, level pgx.LogLevel, msg string, data map[string]interface{}) {
+	var logger log.Logger
+	if data != nil {
+		var args = make([]interface{}, 0)
+		for key, value := range data {
+			if value != nil {
+				args = append(args, key, value)
+			}
+		}
+		logger = l.l.New(args...)
+	} else {
+		logger = l.l
+	}
+
+	switch level {
+	case pgx.LogLevelTrace:
+		logger.Trace(msg)
+	case pgx.LogLevelDebug:
+		logger.Debug(msg)
+	case pgx.LogLevelInfo:
+		logger.Info(msg)
+	case pgx.LogLevelWarn:
+		logger.Warn(msg)
+	case pgx.LogLevelError:
+		logger.Error(msg)
+	default:
+		logger.New("INVALID_PGX_LOG_LEVEL", level).Error(msg)
+	}
+}

indexer/database/sql/postgres/pgx.go

@@ -0,0 +1,256 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"context"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/georgysavva/scany/pgxscan"
+	"github.com/jackc/pgconn"
+	"github.com/jackc/pgx/v4"
+	"github.com/jackc/pgx/v4/pgxpool"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+// PGXDriver driver, implements sql.Driver
+type PGXDriver struct {
+	ctx      context.Context
+	pool     *pgxpool.Pool
+	nodeInfo node.Info
+	nodeID   string
+	config   Config
+}
+
+// ConnectPGX initializes and returns a PGX connection pool
+func ConnectPGX(ctx context.Context, config Config) (*pgxpool.Pool, error) {
+	pgConf, err := MakeConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return pgxpool.ConnectConfig(ctx, pgConf)
+}
+
+// NewPGXDriver returns a new pgx driver
+// it initializes the connection pool and creates the node info table
+func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
+	dbPool, err := ConnectPGX(ctx, config)
+	if err != nil {
+		return nil, ErrDBConnectionFailed(err)
+	}
+	pg := &PGXDriver{ctx: ctx, pool: dbPool, nodeInfo: node, config: config}
+	nodeErr := pg.createNode()
+	if nodeErr != nil {
+		return &PGXDriver{}, ErrUnableToSetNode(nodeErr)
+	}
+	return pg, nil
+}
+
+// MakeConfig creates a pgxpool.Config from the provided Config
+func MakeConfig(config Config) (*pgxpool.Config, error) {
+	conf, err := pgxpool.ParseConfig("")
+	if err != nil {
+		return nil, err
+	}
+
+	//conf.ConnConfig.BuildStatementCache = nil
+	conf.ConnConfig.Config.Host = config.Hostname
+	conf.ConnConfig.Config.Port = uint16(config.Port)
+	conf.ConnConfig.Config.Database = config.DatabaseName
+	conf.ConnConfig.Config.User = config.Username
+	conf.ConnConfig.Config.Password = config.Password
+
+	if config.ConnTimeout != 0 {
+		conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout
+	}
+	if config.MaxConns != 0 {
+		conf.MaxConns = int32(config.MaxConns)
+	}
+	if config.MinConns != 0 {
+		conf.MinConns = int32(config.MinConns)
+	}
+	if config.MaxConnLifetime != 0 {
+		conf.MaxConnLifetime = config.MaxConnLifetime
+	}
+	if config.MaxConnIdleTime != 0 {
+		conf.MaxConnIdleTime = config.MaxConnIdleTime
+	}
+
+	if config.LogStatements {
+		conf.ConnConfig.Logger = NewLogAdapter(log.New())
+	}
+
+	return conf, nil
+}
+
+func (pgx *PGXDriver) createNode() error {
+	_, err := pgx.pool.Exec(
+		pgx.ctx,
+		createNodeStm,
+		pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID,
+		pgx.nodeInfo.ID, pgx.nodeInfo.ClientName,
+		pgx.nodeInfo.ChainID)
+	if err != nil {
+		return ErrUnableToSetNode(err)
+	}
+	pgx.nodeID = pgx.nodeInfo.ID
+	return nil
+}
+
+// QueryRow satisfies sql.Database
+func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
+	return pgx.pool.QueryRow(ctx, sql, args...)
+}
+
+// Exec satisfies sql.Database
+func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
+	res, err := pgx.pool.Exec(ctx, sql, args...)
+	return resultWrapper{ct: res}, err
+}
+
+// Select satisfies sql.Database
+func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	return pgxscan.Select(ctx, pgx.pool, dest, query, args...)
+}
+
+// Get satisfies sql.Database
+func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	return pgxscan.Get(ctx, pgx.pool, dest, query, args...)
+}
+
+// Begin satisfies sql.Database
+func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) {
+	tx, err := pgx.pool.Begin(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return pgxTxWrapper{tx: tx}, nil
+}
+
+func (pgx *PGXDriver) Stats() metrics.DbStats {
+	stats := pgx.pool.Stat()
+	return pgxStatsWrapper{stats: stats}
+}
+
+// NodeID satisfies sql.Database
+func (pgx *PGXDriver) NodeID() string {
+	return pgx.nodeID
+}
+
+// Close satisfies sql.Database/io.Closer
+func (pgx *PGXDriver) Close() error {
+	pgx.pool.Close()
+	return nil
+}
+
+// Context satisfies sql.Database
+func (pgx *PGXDriver) Context() context.Context {
+	return pgx.ctx
+}
+
+// HasCopy satisfies sql.Database
+func (pgx *PGXDriver) UseCopyFrom() bool {
+	return pgx.config.CopyFrom
+}
+
+type resultWrapper struct {
+	ct pgconn.CommandTag
+}
+
+// RowsAffected satisfies sql.Result
+func (r resultWrapper) RowsAffected() (int64, error) {
+	return r.ct.RowsAffected(), nil
+}
+
+type pgxStatsWrapper struct {
+	stats *pgxpool.Stat
+}
+
+// MaxOpen satisfies metrics.DbStats
+func (s pgxStatsWrapper) MaxOpen() int64 {
+	return int64(s.stats.MaxConns())
+}
+
+// Open satisfies metrics.DbStats
+func (s pgxStatsWrapper) Open() int64 {
+	return int64(s.stats.TotalConns())
+}
+
+// InUse satisfies metrics.DbStats
+func (s pgxStatsWrapper) InUse() int64 {
+	return int64(s.stats.AcquiredConns())
+}
+
+// Idle satisfies metrics.DbStats
+func (s pgxStatsWrapper) Idle() int64 {
+	return int64(s.stats.IdleConns())
+}
+
+// WaitCount satisfies metrics.DbStats
+func (s pgxStatsWrapper) WaitCount() int64 {
+	return s.stats.EmptyAcquireCount()
+}
+
+// WaitDuration satisfies metrics.DbStats
+func (s pgxStatsWrapper) WaitDuration() time.Duration {
+	return s.stats.AcquireDuration()
+}
+
+// MaxIdleClosed satisfies metrics.DbStats
+func (s pgxStatsWrapper) MaxIdleClosed() int64 {
+	// this stat isn't supported by pgxpool, but we don't want to panic
+	return 0
+}
+
+// MaxLifetimeClosed satisfies metrics.DbStats
+func (s pgxStatsWrapper) MaxLifetimeClosed() int64 {
+	return s.stats.CanceledAcquireCount()
+}
+
+type pgxTxWrapper struct {
+	tx pgx.Tx
+}
+
+// QueryRow satisfies sql.Tx
+func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
+	return t.tx.QueryRow(ctx, sql, args...)
+}
+
+// Exec satisfies sql.Tx
+func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
+	res, err := t.tx.Exec(ctx, sql, args...)
+	return resultWrapper{ct: res}, err
+}
+
+// Commit satisfies sql.Tx
+func (t pgxTxWrapper) Commit(ctx context.Context) error {
+	return t.tx.Commit(ctx)
+}
+
+// Rollback satisfies sql.Tx
+func (t pgxTxWrapper) Rollback(ctx context.Context) error {
+	return t.tx.Rollback(ctx)
+}
+
+func (t pgxTxWrapper) CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error) {
+	return t.tx.CopyFrom(ctx, tableName, columnNames, pgx.CopyFromRows(rows))
+}

indexer/database/sql/postgres/pgx_test.go

@@ -0,0 +1,121 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"strings"
+	"testing"
+
+	"github.com/jackc/pgx/v4/pgxpool"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+var (
+	pgConfig, _ = postgres.MakeConfig(postgres.TestConfig)
+	ctx         = context.Background()
+)
+
+func expectContainsSubstring(t *testing.T, full string, sub string) {
+	if !strings.Contains(full, sub) {
+		t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub)
+	}
+}
+
+func TestPostgresPGX(t *testing.T) {
+	t.Run("connects to the sql", func(t *testing.T) {
+		dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig)
+		if err != nil {
+			t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err)
+		}
+		if dbPool == nil {
+			t.Fatal("DB pool is nil")
+		}
+		dbPool.Close()
+	})
+
+	t.Run("serializes big.Int to db", func(t *testing.T) {
+		// postgres driver doesn't support go big.Int type
+		// various casts in golang uint64, int64, overflow for
+		// transaction value (in wei) even though
+		// postgres numeric can handle an arbitrary
+		// sized int, so use string representation of big.Int
+		// and cast on insert
+
+		dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig)
+		if err != nil {
+			t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err)
+		}
+		defer dbPool.Close()
+
+		bi := new(big.Int)
+		bi.SetString("34940183920000000000", 10)
+		require.Equal(t, "34940183920000000000", bi.String())
+
+		defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`)
+		_, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		sqlStatement := `
+			INSERT INTO example (id, data)
+			VALUES (1, cast($1 AS NUMERIC))`
+		_, err = dbPool.Exec(ctx, sqlStatement, bi.String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var data string
+		err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		require.Equal(t, data, bi.String())
+		actual := new(big.Int)
+		actual.SetString(data, 10)
+		require.Equal(t, bi, actual)
+	})
+
+	t.Run("throws error when can't connect to the database", func(t *testing.T) {
+		goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
+		_, err := postgres.NewPGXDriver(ctx, postgres.Config{}, goodInfo)
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
+
+		expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
+	})
+
+	t.Run("throws error when can't create node", func(t *testing.T) {
+		badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
+		badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
+
+		_, err := postgres.NewPGXDriver(ctx, postgres.TestConfig, badInfo)
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
+
+		expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
+	})
+}

indexer/database/sql/postgres/postgres_suite_test.go

@@ -0,0 +1,33 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+func init() {
+	if os.Getenv("MODE") != "statediff" {
+		fmt.Println("Skipping statediff test")
+		os.Exit(0)
+	}
+
+	log.Root().SetHandler(log.DiscardHandler())
+}

indexer/database/sql/postgres/sqlx.go

@@ -0,0 +1,210 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"context"
+	coresql "database/sql"
+	"errors"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+// SQLXDriver driver, implements sql.Driver
+type SQLXDriver struct {
+	ctx      context.Context
+	db       *sqlx.DB
+	nodeInfo node.Info
+	nodeID   string
+}
+
+// ConnectSQLX initializes and returns a SQLX connection pool for postgres
+func ConnectSQLX(ctx context.Context, config Config) (*sqlx.DB, error) {
+	db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString())
+	if err != nil {
+		return nil, ErrDBConnectionFailed(err)
+	}
+	if config.MaxConns > 0 {
+		db.SetMaxOpenConns(config.MaxConns)
+	}
+	if config.MaxConnLifetime > 0 {
+		db.SetConnMaxLifetime(config.MaxConnLifetime)
+	}
+	db.SetMaxIdleConns(config.MaxIdle)
+	return db, nil
+}
+
+// NewSQLXDriver returns a new sqlx driver for Postgres
+// it initializes the connection pool and creates the node info table
+func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
+	db, err := ConnectSQLX(ctx, config)
+	if err != nil {
+		return nil, err
+	}
+	driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node}
+	if err := driver.createNode(); err != nil {
+		return nil, err
+	}
+	return driver, nil
+}
+
+func (driver *SQLXDriver) createNode() error {
+	_, err := driver.db.Exec(
+		createNodeStm,
+		driver.nodeInfo.GenesisBlock,
+		driver.nodeInfo.NetworkID,
+		driver.nodeInfo.ID,
+		driver.nodeInfo.ClientName,
+		driver.nodeInfo.ChainID)
+	if err != nil {
+		return ErrUnableToSetNode(err)
+	}
+	driver.nodeID = driver.nodeInfo.ID
+	return nil
+}
+
+// QueryRow satisfies sql.Database
+func (driver *SQLXDriver) QueryRow(_ context.Context, sql string, args ...interface{}) sql.ScannableRow {
+	return driver.db.QueryRowx(sql, args...)
+}
+
+// Exec satisfies sql.Database
+func (driver *SQLXDriver) Exec(_ context.Context, sql string, args ...interface{}) (sql.Result, error) {
+	return driver.db.Exec(sql, args...)
+}
+
+// Select satisfies sql.Database
+func (driver *SQLXDriver) Select(_ context.Context, dest interface{}, query string, args ...interface{}) error {
+	return driver.db.Select(dest, query, args...)
+}
+
+// Get satisfies sql.Database
+func (driver *SQLXDriver) Get(_ context.Context, dest interface{}, query string, args ...interface{}) error {
+	return driver.db.Get(dest, query, args...)
+}
+
+// Begin satisfies sql.Database
+func (driver *SQLXDriver) Begin(_ context.Context) (sql.Tx, error) {
+	tx, err := driver.db.Beginx()
+	if err != nil {
+		return nil, err
+	}
+	return sqlxTxWrapper{tx: tx}, nil
+}
+
+func (driver *SQLXDriver) Stats() metrics.DbStats {
+	stats := driver.db.Stats()
+	return sqlxStatsWrapper{stats: stats}
+}
+
+// NodeID satisfies sql.Database
+func (driver *SQLXDriver) NodeID() string {
+	return driver.nodeID
+}
+
+// Close satisfies sql.Database/io.Closer
+func (driver *SQLXDriver) Close() error {
+	return driver.db.Close()
+}
+
+// Context satisfies sql.Database
+func (driver *SQLXDriver) Context() context.Context {
+	return driver.ctx
+}
+
+// HasCopy satisfies sql.Database
+func (driver *SQLXDriver) UseCopyFrom() bool {
+	// sqlx does not currently support COPY.
+	return false
+}
+
+type sqlxStatsWrapper struct {
+	stats coresql.DBStats
+}
+
+// MaxOpen satisfies metrics.DbStats
+func (s sqlxStatsWrapper) MaxOpen() int64 {
+	return int64(s.stats.MaxOpenConnections)
+}
+
+// Open satisfies metrics.DbStats
+func (s sqlxStatsWrapper) Open() int64 {
+	return int64(s.stats.OpenConnections)
+}
+
+// InUse satisfies metrics.DbStats
+func (s sqlxStatsWrapper) InUse() int64 {
+	return int64(s.stats.InUse)
+}
+
+// Idle satisfies metrics.DbStats
+func (s sqlxStatsWrapper) Idle() int64 {
+	return int64(s.stats.Idle)
+}
+
+// WaitCount satisfies metrics.DbStats
+func (s sqlxStatsWrapper) WaitCount() int64 {
+	return s.stats.WaitCount
+}
+
+// WaitDuration satisfies metrics.DbStats
+func (s sqlxStatsWrapper) WaitDuration() time.Duration {
+	return s.stats.WaitDuration
+}
+
+// MaxIdleClosed satisfies metrics.DbStats
+func (s sqlxStatsWrapper) MaxIdleClosed() int64 {
+	return s.stats.MaxIdleClosed
+}
+
+// MaxLifetimeClosed satisfies metrics.DbStats
+func (s sqlxStatsWrapper) MaxLifetimeClosed() int64 {
+	return s.stats.MaxLifetimeClosed
+}
+
+type sqlxTxWrapper struct {
+	tx *sqlx.Tx
+}
+
+// QueryRow satisfies sql.Tx
+func (t sqlxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
+	return t.tx.QueryRowx(sql, args...)
+}
+
+// Exec satisfies sql.Tx
+func (t sqlxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
+	return t.tx.Exec(sql, args...)
+}
+
+// Commit satisfies sql.Tx
+func (t sqlxTxWrapper) Commit(ctx context.Context) error {
+	return t.tx.Commit()
+}
+
+// Rollback satisfies sql.Tx
+func (t sqlxTxWrapper) Rollback(ctx context.Context) error {
+	return t.tx.Rollback()
+}
+
+func (t sqlxTxWrapper) CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error) {
+	return 0, errors.New("Unsupported Operation")
+}

indexer/database/sql/postgres/sqlx_test.go

@@ -0,0 +1,119 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"fmt"
+	"math/big"
+	"strings"
+	"testing"
+
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+func TestPostgresSQLX(t *testing.T) {
+	var sqlxdb *sqlx.DB
+
+	t.Run("connects to the database", func(t *testing.T) {
+		var err error
+		connStr := postgres.TestConfig.DbConnectionString()
+
+		sqlxdb, err = sqlx.Connect("postgres", connStr)
+		if err != nil {
+			t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err)
+		}
+		if sqlxdb == nil {
+			t.Fatal("DB is nil")
+		}
+		err = sqlxdb.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("serializes big.Int to db", func(t *testing.T) {
+		// postgres driver doesn't support go big.Int type
+		// various casts in golang uint64, int64, overflow for
+		// transaction value (in wei) even though
+		// postgres numeric can handle an arbitrary
+		// sized int, so use string representation of big.Int
+		// and cast on insert
+
+		connStr := postgres.TestConfig.DbConnectionString()
+		db, err := sqlx.Connect("postgres", connStr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer db.Close()
+
+		bi := new(big.Int)
+		bi.SetString("34940183920000000000", 10)
+		require.Equal(t, "34940183920000000000", bi.String())
+
+		defer db.Exec(`DROP TABLE IF EXISTS example`)
+		_, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		sqlStatement := `
+			INSERT INTO example (id, data)
+			VALUES (1, cast($1 AS NUMERIC))`
+		_, err = db.Exec(sqlStatement, bi.String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var data string
+		err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		require.Equal(t, data, bi.String())
+		actual := new(big.Int)
+		actual.SetString(data, 10)
+		require.Equal(t, bi, actual)
+	})
+
+	t.Run("throws error when can't connect to the database", func(t *testing.T) {
+		goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
+		_, err := postgres.NewSQLXDriver(ctx, postgres.Config{}, goodInfo)
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
+
+		expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
+	})
+
+	t.Run("throws error when can't create node", func(t *testing.T) {
+		badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
+		badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
+
+		_, err := postgres.NewSQLXDriver(ctx, postgres.TestConfig, badInfo)
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
+
+		expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
+	})
+}

indexer/database/sql/postgres/test_helpers.go

@@ -0,0 +1,44 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+// SetupSQLXDB is used to setup a sqlx db for tests
+func SetupSQLXDB() (sql.Database, error) {
+	conf := TestConfig
+	conf.MaxIdle = 0
+	driver, err := NewSQLXDriver(context.Background(), conf, node.Info{})
+	if err != nil {
+		return nil, err
+	}
+	return NewPostgresDB(driver, false), nil
+}
+
+// SetupPGXDB is used to setup a pgx db for tests
+func SetupPGXDB(config Config) (sql.Database, error) {
+	driver, err := NewPGXDriver(context.Background(), config, node.Info{})
+	if err != nil {
+		return nil, err
+	}
+	return NewPostgresDB(driver, false), nil
+}

indexer/database/sql/sqlx_indexer_legacy_test.go

@@ -0,0 +1,52 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupLegacySQLXIndexer(t *testing.T) {
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db)
+	require.NoError(t, err)
+}
+
+func setupLegacySQLX(t *testing.T) {
+	setupLegacySQLXIndexer(t)
+	test.SetupLegacyTestData(t, ind)
+}
+
+func TestLegacySQLXIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs", func(t *testing.T) {
+		setupLegacySQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestLegacyIndexer(t, db)
+	})
+}

indexer/database/sql/sqlx_indexer_test.go

@@ -0,0 +1,227 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test"
+)
+
+func setupSQLXIndexer(t *testing.T) {
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db)
+	require.NoError(t, err)
+}
+
+func setupSQLX(t *testing.T) {
+	setupSQLXIndexer(t)
+	test.SetupTestData(t, ind)
+}
+
+func setupSQLXNonCanonical(t *testing.T) {
+	setupSQLXIndexer(t)
+	test.SetupTestDataNonCanonical(t, ind)
+}
+
+// Test indexer for a canonical block
+func TestSQLXIndexer(t *testing.T) {
+	t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexHeaderIPLDs(t, db)
+	})
+
+	t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexTransactionIPLDs(t, db)
+	})
+
+	t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexLogIPLDs(t, db)
+	})
+
+	t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexReceiptIPLDs(t, db)
+	})
+
+	t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexStateIPLDs(t, db)
+	})
+
+	t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
+		setupSQLX(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexStorageIPLDs(t, db)
+	})
+}
+
+// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1
+func TestSQLXIndexerNonCanonical(t *testing.T) {
+	t.Run("Publish and index header", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexHeaderNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index transactions", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexTransactionsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index receipts", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexReceiptsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index logs", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexLogsNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index state nodes", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexStateNonCanonical(t, db)
+	})
+
+	t.Run("Publish and index storage nodes", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 0, 0, 0)
+
+		test.TestPublishAndIndexStorageNonCanonical(t, db)
+	})
+}
+
+func TestSQLXWatchAddressMethods(t *testing.T) {
+	setupSQLXIndexer(t)
+	defer tearDown(t)
+	defer checkTxClosure(t, 0, 0, 0)
+
+	t.Run("Load watched addresses (empty table)", func(t *testing.T) {
+		test.TestLoadEmptyWatchedAddresses(t, ind)
+	})
+
+	t.Run("Insert watched addresses", func(t *testing.T) {
+		args := mocks.GetInsertWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
+		require.NoError(t, err)
+
+		test.TestInsertWatchedAddresses(t, db)
+	})
+
+	t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetInsertAlreadyWatchedAddressesArgs()
+		err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		test.TestInsertAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses", func(t *testing.T) {
+		args := mocks.GetRemoveWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		test.TestRemoveWatchedAddresses(t, db)
+	})
+
+	t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
+		args := mocks.GetRemoveNonWatchedAddressesArgs()
+		err = ind.RemoveWatchedAddresses(args)
+		require.NoError(t, err)
+
+		test.TestRemoveNonWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses", func(t *testing.T) {
+		args := mocks.GetSetWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
+		require.NoError(t, err)
+
+		test.TestSetWatchedAddresses(t, db)
+	})
+
+	t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
+		args := mocks.GetSetAlreadyWatchedAddressesArgs()
+		err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
+		require.NoError(t, err)
+
+		test.TestSetAlreadyWatchedAddresses(t, db)
+	})
+
+	t.Run("Load watched addresses", func(t *testing.T) {
+		test.TestLoadWatchedAddresses(t, ind)
+	})
+
+	t.Run("Clear watched addresses", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		test.TestClearWatchedAddresses(t, db)
+	})
+
+	t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
+		err = ind.ClearWatchedAddresses()
+		require.NoError(t, err)
+
+		test.TestClearEmptyWatchedAddresses(t, db)
+	})
+}

indexer/database/sql/writer.go

@@ -0,0 +1,345 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/jackc/pgtype"
+	shopspring "github.com/jackc/pgtype/ext/shopspring-numeric"
+	"github.com/lib/pq"
+	"github.com/shopspring/decimal"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+)
+
+// Writer handles processing and writing of indexed IPLD objects to Postgres
+type Writer struct {
+	db Database
+}
+
+// NewWriter creates a new pointer to a Writer
+func NewWriter(db Database) *Writer {
+	return &Writer{
+		db: db,
+	}
+}
+
+// Close satisfies io.Closer
+func (w *Writer) Close() error {
+	return w.db.Close()
+}
+
+/*
+INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
+ON CONFLICT (block_hash, block_number) DO NOTHING
+*/
+func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error {
+	nodeIDs := pq.StringArray([]string{w.db.NodeID()})
+	_, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(),
+		header.BlockNumber,
+		header.BlockHash,
+		header.ParentHash,
+		header.CID,
+		header.TotalDifficulty,
+		nodeIDs,
+		header.Reward,
+		header.StateRoot,
+		header.TxRoot,
+		header.RctRoot,
+		header.UnclesHash,
+		header.Bloom,
+		header.Timestamp,
+		header.Coinbase)
+	if err != nil {
+		return insertError{"eth.header_cids", err, w.db.InsertHeaderStm(), header}
+	}
+	metrics.IndexerMetrics.BlocksCounter.Inc(1)
+	return nil
+}
+
+/*
+INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, index) VALUES ($1, $2, $3, $4, $5, $6, $7)
+ON CONFLICT (block_hash, block_number) DO NOTHING
+*/
+func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error {
+	_, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(),
+		uncle.BlockNumber,
+		uncle.BlockHash,
+		uncle.HeaderID,
+		uncle.ParentHash,
+		uncle.CID,
+		uncle.Reward,
+		uncle.Index)
+	if err != nil {
+		return insertError{"eth.uncle_cids", err, w.db.InsertUncleStm(), uncle}
+	}
+	return nil
+}
+
+/*
+INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING
+*/
+func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
+	val := transaction.Value
+	if val == "" {
+		val = "0"
+	}
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseInt(transaction.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.transaction_cids", err, "COPY", transaction}
+		}
+
+		value, err := toNumeric(val)
+		if err != nil {
+			return insertError{"eth.transaction_cids", err, "COPY", transaction}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), w.db.TxTableName(), w.db.TxColumnNames(),
+			toRows(toRow(blockNum, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
+				transaction.Src, transaction.Index, int(transaction.Type), value)))
+		if err != nil {
+			return insertError{"eth.transaction_cids", err, "COPY", transaction}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(),
+			transaction.BlockNumber,
+			transaction.HeaderID,
+			transaction.TxHash,
+			transaction.CID,
+			transaction.Dst,
+			transaction.Src,
+			transaction.Index,
+			transaction.Type,
+			transaction.Value)
+		if err != nil {
+			return insertError{"eth.transaction_cids", err, w.db.InsertTxStm(), transaction}
+		}
+	}
+	metrics.IndexerMetrics.TransactionsCounter.Inc(1)
+	return nil
+}
+
+/*
+INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, post_status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+ON CONFLICT (tx_id, header_id, block_number) DO NOTHING
+*/
+func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseUint(rct.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.receipt_cids", err, "COPY", rct}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), w.db.RctTableName(), w.db.RctColumnNames(),
+			toRows(toRow(blockNum, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
+				rct.PostState, int(rct.PostStatus))))
+		if err != nil {
+			return insertError{"eth.receipt_cids", err, "COPY", rct}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(),
+			rct.BlockNumber,
+			rct.HeaderID,
+			rct.TxID,
+			rct.CID,
+			rct.Contract,
+			rct.PostState,
+			rct.PostStatus)
+		if err != nil {
+			return insertError{"eth.receipt_cids", err, w.db.InsertRctStm(), *rct}
+		}
+	}
+	metrics.IndexerMetrics.ReceiptsCounter.Inc(1)
+	return nil
+}
+
+/*
+INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, topic3) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING
+*/
+func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
+	if w.useCopyForTx(tx) {
+		var rows [][]interface{}
+		for _, log := range logs {
+			blockNum, err := strconv.ParseUint(log.BlockNumber, 10, 64)
+			if err != nil {
+				return insertError{"eth.log_cids", err, "COPY", log}
+			}
+
+			rows = append(rows, toRow(blockNum, log.HeaderID, log.CID, log.ReceiptID,
+				log.Address, log.Index, log.Topic0, log.Topic1, log.Topic2, log.Topic3))
+		}
+		if nil != rows && len(rows) >= 0 {
+			_, err := tx.CopyFrom(w.db.Context(), w.db.LogTableName(), w.db.LogColumnNames(), rows)
+			if err != nil {
+				return insertError{"eth.log_cids", err, "COPY", rows}
+			}
+			metrics.IndexerMetrics.LogsCounter.Inc(int64(len(rows)))
+		}
+	} else {
+		for _, log := range logs {
+			_, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(),
+				log.BlockNumber,
+				log.HeaderID,
+				log.CID,
+				log.ReceiptID,
+				log.Address,
+				log.Index,
+				log.Topic0,
+				log.Topic1,
+				log.Topic2,
+				log.Topic3)
+			if err != nil {
+				return insertError{"eth.log_cids", err, w.db.InsertLogStm(), *log}
+			}
+			metrics.IndexerMetrics.LogsCounter.Inc(1)
+		}
+	}
+	return nil
+}
+
+/*
+INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING
+*/
+func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
+	bal := stateNode.Balance
+	if stateNode.Removed {
+		bal = "0"
+	}
+
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseUint(stateNode.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.state_cids", err, "COPY", stateNode}
+		}
+
+		balance, err := toNumeric(bal)
+		if err != nil {
+			return insertError{"eth.state_cids", err, "COPY", stateNode}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), w.db.StateTableName(), w.db.StateColumnNames(),
+			toRows(toRow(blockNum, stateNode.HeaderID, stateNode.StateKey, stateNode.CID,
+				true, balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot, stateNode.Removed)))
+		if err != nil {
+			return insertError{"eth.state_cids", err, "COPY", stateNode}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
+			stateNode.BlockNumber,
+			stateNode.HeaderID,
+			stateNode.StateKey,
+			stateNode.CID,
+			true,
+			bal,
+			stateNode.Nonce,
+			stateNode.CodeHash,
+			stateNode.StorageRoot,
+			stateNode.Removed,
+		)
+		if err != nil {
+			return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
+		}
+	}
+	return nil
+}
+
+/*
+INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHING
+*/
+func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error {
+	if w.useCopyForTx(tx) {
+		blockNum, err := strconv.ParseUint(storageCID.BlockNumber, 10, 64)
+		if err != nil {
+			return insertError{"eth.storage_cids", err, "COPY", storageCID}
+		}
+
+		_, err = tx.CopyFrom(w.db.Context(), w.db.StorageTableName(), w.db.StorageColumnNames(),
+			toRows(toRow(blockNum, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID,
+				true, storageCID.Value, storageCID.Removed)))
+		if err != nil {
+			return insertError{"eth.storage_cids", err, "COPY", storageCID}
+		}
+	} else {
+		_, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(),
+			storageCID.BlockNumber,
+			storageCID.HeaderID,
+			storageCID.StateKey,
+			storageCID.StorageKey,
+			storageCID.CID,
+			true,
+			storageCID.Value,
+			storageCID.Removed,
+		)
+		if err != nil {
+			return insertError{"eth.storage_cids", err, w.db.InsertStorageStm(), storageCID}
+		}
+	}
+	return nil
+}
+
+func (w *Writer) useCopyForTx(tx Tx) bool {
+	// Using COPY instead of INSERT only makes much sense if also using a DelayedTx, so that operations
+	// can be collected over time and then all submitted within in a single TX.
+	if _, ok := tx.(*DelayedTx); ok {
+		return w.db.UseCopyFrom()
+	}
+	return false
+}
+
+// combine args into a row
+func toRow(args ...interface{}) []interface{} {
+	var row []interface{}
+	row = append(row, args...)
+	return row
+}
+
+func toNumeric(value string) (*shopspring.Numeric, error) {
+	decimalValue, err := decimal.NewFromString(value)
+	if nil != err {
+		return nil, err
+	}
+
+	return &shopspring.Numeric{Decimal: decimalValue, Status: pgtype.Present}, nil
+}
+
+// combine row (or rows) into a slice of rows for CopyFrom
+func toRows(rows ...[]interface{}) [][]interface{} {
+	return rows
+}
+
+type insertError struct {
+	table     string
+	err       error
+	stmt      string
+	arguments interface{}
+}
+
+var _ error = insertError{}
+
+func (dbe insertError) Error() string {
+	return fmt.Sprintf("error inserting %s entry: %v\r\nstatement: %s\r\narguments: %+v",
+		dbe.table, dbe.err, dbe.stmt, dbe.arguments)
+}