Merge pull request #40 from vulcanize/feature/update-go-geth-sharding

Update go and geth versions
Update geth versions
2022-06-01 09:22:04 -04:00 · 2022-06-01 09:20:45 -04:00 · 2022-06-01 09:19:18 -04:00 · 2022-06-01 09:10:35 -04:00 · 2022-06-01 09:09:19 -04:00 · 2022-05-25 18:38:09 +05:30
35 changed files with 2761 additions and 619 deletions
@@ -1,2 +1,2 @@
 .idea/
-eth-pg-ipfs-state-snapshot
+ipld-eth-state-snapshot
@@ -6,7 +6,11 @@ mockgen_cmd=mockgen
 mocks: mocks/snapshot/publisher.go

 mocks/snapshot/publisher.go: pkg/types/publisher.go
-	$(mockgen_cmd) -package snapshot_mock -destination $@ -source $< Publisher
+	$(mockgen_cmd) -package snapshot_mock -destination $@ -source $< Publisher Tx

 clean:
 	rm -f mocks/snapshot/publisher.go
+
+build:
+	go fmt ./...
+	go build
@@ -1,26 +1,54 @@
-# eth-pg-ipfs-state-snapshot
+# ipld-eth-state-snapshot

 > Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS

-[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-pg-ipfs-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/eth-pg-ipfs-state-snapshot)
+[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipld-eth-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-state-snapshot)

-## Usage 
+## Usage

-./eth-pg-ipfs-state-snapshot stateSnapshot --config={path to toml config file}
+./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}
+
+### Config

 Config format:

 ```toml
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-    user     = "postgres"
+[snapshot]
+    mode = "file" # indicates output mode ("postgres" or "file")
+    workers = 4 # degree of concurrency, the state trie is subdivided into sectiosn that are traversed and processed concurrently
+    blockHeight = -1 # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in leveldb)
+    recoveryFile = "recovery_file" # specifies a file to output recovery information on error or premature closure

 [leveldb]
-   path = "/Users/user/Library/Ethereum/geth/chaindata"
-   ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    path = "/Users/user/Library/Ethereum/geth/chaindata" # path to geth leveldb
+    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # path to geth ancient database

-[snapshot]
-   blockHeight = 0
+[database]
+    name     = "vulcanize_public" # postgres database name
+    hostname = "localhost" # postgres host
+    port     = 5432 # postgres port
+    user     = "postgres" # postgres user
+    password = "" # postgres password
+
+[file]
+    outputDir = "output_dir/" # when operating in 'file' output mode, this is the directory the files are written to
+
+[log]
+    level = "info" # log level (trace, debug, info, warn, error, fatal, panic) (default: info)
+    file = "log_file" # file path for logging
+
+[prom]
+    metrics = true # enable prometheus metrics (default: false)
+    http = true # enable prometheus http service (default: false)
+    httpAddr = "0.0.0.0" # prometheus http host (default: 127.0.0.1)
+    httpPort = 9101 # prometheus http port (default: 8086)
+    dbStats = true # enable prometheus db stats (default: false)
+
+# node info
+[ethereum]
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    nodeID = "arch1" # $ETH_NODE_ID
+    networkID = "1" # $ETH_NETWORK_ID
+    chainID = "1" # $ETH_CHAIN_ID
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
 ```
@@ -24,6 +24,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
+	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
 )

 var (
@@ -33,7 +36,7 @@ var (
 )

 var rootCmd = &cobra.Command{
-	Use:              "eth-pg-ipfs-state-snapshot",
+	Use:              "ipld-eth-state-snapshot",
 	PersistentPreRun: initFuncs,
 }

@@ -46,7 +49,7 @@ func Execute() {
 }

 func initFuncs(cmd *cobra.Command, args []string) {
-	logfile := viper.GetString("logfile")
+	logfile := viper.GetString(snapshot.LOGRUS_FILE_TOML)
 	if logfile != "" {
 		file, err := os.OpenFile(logfile,
 			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
@@ -63,10 +66,25 @@ func initFuncs(cmd *cobra.Command, args []string) {
 	if err := logLevel(); err != nil {
 		log.Fatal("Could not set log level: ", err)
 	}
+
+	if viper.GetBool(snapshot.PROM_METRICS_TOML) {
+		log.Info("initializing prometheus metrics")
+		prom.Init()
+	}
+
+	if viper.GetBool(snapshot.PROM_HTTP_TOML) {
+		addr := fmt.Sprintf(
+			"%s:%s",
+			viper.GetString(snapshot.PROM_HTTP_ADDR_TOML),
+			viper.GetString(snapshot.PROM_HTTP_PORT_TOML),
+		)
+		log.Info("starting prometheus server")
+		prom.Serve(addr)
+	}
 }

 func logLevel() error {
-	lvl, err := log.ParseLevel(viper.GetString("log.level"))
+	lvl, err := log.ParseLevel(viper.GetString(snapshot.LOGRUS_LEVEL_TOML))
 	if err != nil {
 		return err
 	}
@@ -75,6 +93,7 @@ func logLevel() error {
 		log.SetReportCaller(true)
 	}
 	log.Info("Log level set to ", lvl.String())
+
 	return nil
 }

@@ -84,21 +103,33 @@ func init() {
 	viper.AutomaticEnv()

 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
-	rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
-	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
-	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
-	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
-	rootCmd.PersistentFlags().String("database-user", "", "database user")
-	rootCmd.PersistentFlags().String("database-password", "", "database password")
-	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
+	rootCmd.PersistentFlags().String(snapshot.LOGRUS_FILE_CLI, "", "file path for logging")
+	rootCmd.PersistentFlags().String(snapshot.DATABASE_NAME_CLI, "vulcanize_public", "database name")
+	rootCmd.PersistentFlags().Int(snapshot.DATABASE_PORT_CLI, 5432, "database port")
+	rootCmd.PersistentFlags().String(snapshot.DATABASE_HOSTNAME_CLI, "localhost", "database hostname")
+	rootCmd.PersistentFlags().String(snapshot.DATABASE_USER_CLI, "", "database user")
+	rootCmd.PersistentFlags().String(snapshot.DATABASE_PASSWORD_CLI, "", "database password")
+	rootCmd.PersistentFlags().String(snapshot.LOGRUS_LEVEL_CLI, log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")

-	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
-	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
-	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
-	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
-	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
-	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
-	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+	rootCmd.PersistentFlags().Bool(snapshot.PROM_METRICS_CLI, false, "enable prometheus metrics")
+	rootCmd.PersistentFlags().Bool(snapshot.PROM_HTTP_CLI, false, "enable prometheus http service")
+	rootCmd.PersistentFlags().String(snapshot.PROM_HTTP_ADDR_CLI, "127.0.0.1", "prometheus http host")
+	rootCmd.PersistentFlags().String(snapshot.PROM_HTTP_PORT_CLI, "8086", "prometheus http port")
+	rootCmd.PersistentFlags().Bool(snapshot.PROM_DB_STATS_CLI, false, "enables prometheus db stats")
+
+	viper.BindPFlag(snapshot.LOGRUS_FILE_TOML, rootCmd.PersistentFlags().Lookup(snapshot.LOGRUS_FILE_CLI))
+	viper.BindPFlag(snapshot.DATABASE_NAME_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_NAME_CLI))
+	viper.BindPFlag(snapshot.DATABASE_PORT_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_PORT_CLI))
+	viper.BindPFlag(snapshot.DATABASE_HOSTNAME_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_HOSTNAME_CLI))
+	viper.BindPFlag(snapshot.DATABASE_USER_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_USER_CLI))
+	viper.BindPFlag(snapshot.DATABASE_PASSWORD_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_PASSWORD_CLI))
+	viper.BindPFlag(snapshot.LOGRUS_LEVEL_TOML, rootCmd.PersistentFlags().Lookup(snapshot.LOGRUS_LEVEL_CLI))
+
+	viper.BindPFlag(snapshot.PROM_METRICS_TOML, rootCmd.PersistentFlags().Lookup(snapshot.PROM_METRICS_CLI))
+	viper.BindPFlag(snapshot.PROM_HTTP_TOML, rootCmd.PersistentFlags().Lookup(snapshot.PROM_HTTP_CLI))
+	viper.BindPFlag(snapshot.PROM_HTTP_ADDR_TOML, rootCmd.PersistentFlags().Lookup(snapshot.PROM_HTTP_ADDR_CLI))
+	viper.BindPFlag(snapshot.PROM_HTTP_PORT_TOML, rootCmd.PersistentFlags().Lookup(snapshot.PROM_HTTP_PORT_CLI))
+	viper.BindPFlag(snapshot.PROM_DB_STATS_TOML, rootCmd.PersistentFlags().Lookup(snapshot.PROM_DB_STATS_CLI))
 }

 func initConfig() {
@@ -16,11 +16,13 @@
 package cmd

 import (
+	"fmt"
+
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

-	"github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot"
+	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
 )

 // stateSnapshotCmd represents the stateSnapshot command
@@ -29,7 +31,7 @@ var stateSnapshotCmd = &cobra.Command{
 	Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS",
 	Long: `Usage

-./eth-pg-ipfs-state-snapshot stateSnapshot --config={path to toml config file}`,
+./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}`,
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *logrus.WithField("SubCommand", subCommand)
@@ -38,22 +40,36 @@ var stateSnapshotCmd = &cobra.Command{
 }

 func stateSnapshot() {
-	snapConfig := snapshot.NewConfig()
-	pgDB, err := snapshot.NewPostgresDB(snapConfig.DB)
+	modeStr := viper.GetString(snapshot.SNAPSHOT_MODE_TOML)
+	mode := snapshot.SnapshotMode(modeStr)
+	config, err := snapshot.NewConfig(mode)
+	if err != nil {
+		logWithCommand.Fatal("unable to initialize config: %v", err)
+	}
+	logWithCommand.Infof("opening levelDB and ancient data at %s and %s",
+		config.Eth.LevelDBPath, config.Eth.AncientDBPath)
+	edb, err := snapshot.NewLevelDB(config.Eth)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
-	edb, err := snapshot.NewLevelDB(snapConfig.Eth)
+	height := viper.GetInt64(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML)
+	recoveryFile := viper.GetString(snapshot.SNAPSHOT_RECOVERY_FILE_TOML)
+	if recoveryFile == "" {
+		recoveryFile = fmt.Sprintf("./%d_snapshot_recovery", height)
+		logWithCommand.Infof("no recovery file set, creating default: %s", recoveryFile)
+	}
+
+	pub, err := snapshot.NewPublisher(mode, config)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}

-	snapshotService, err := snapshot.NewSnapshotService(edb, snapshot.NewPublisher(pgDB))
+	snapshotService, err := snapshot.NewSnapshotService(edb, pub, recoveryFile)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
-	height := viper.GetInt64("snapshot.blockHeight")
-	workers := viper.GetUint("snapshot.workers")
+	workers := viper.GetUint(snapshot.SNAPSHOT_WORKERS_TOML)
+
 	if height < 0 {
 		if err := snapshotService.CreateLatestSnapshot(workers); err != nil {
 			logWithCommand.Fatal(err)
@@ -70,13 +86,19 @@ func stateSnapshot() {
 func init() {
 	rootCmd.AddCommand(stateSnapshotCmd)

-	stateSnapshotCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
-	stateSnapshotCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
-	stateSnapshotCmd.PersistentFlags().String("block-height", "", "blockheight to extract state at")
-	stateSnapshotCmd.PersistentFlags().Int("workers", 0, "number of concurrent workers to use")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.LVL_DB_PATH_CLI, "", "path to primary datastore")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.ANCIENT_DB_PATH_CLI, "", "path to ancient datastore")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI, "", "block height to extract state at")
+	stateSnapshotCmd.PersistentFlags().Int(snapshot.SNAPSHOT_WORKERS_CLI, 1, "number of concurrent workers to use")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_RECOVERY_FILE_CLI, "", "file to recover from a previous iteration")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_MODE_CLI, "postgres", "output mode for snapshot ('file' or 'postgres')")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.FILE_OUTPUT_DIR_CLI, "", "directory for writing ouput to while operating in 'file' mode")

-	viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
-	viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
-	viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
-	viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
+	viper.BindPFlag(snapshot.LVL_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LVL_DB_PATH_CLI))
+	viper.BindPFlag(snapshot.ANCIENT_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ANCIENT_DB_PATH_CLI))
+	viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI))
+	viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_WORKERS_CLI))
+	viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_RECOVERY_FILE_CLI))
+	viper.BindPFlag(snapshot.SNAPSHOT_MODE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_MODE_CLI))
+	viper.BindPFlag(snapshot.FILE_OUTPUT_DIR_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.FILE_OUTPUT_DIR_CLI))
 }
@@ -5,8 +5,32 @@
    user     = "postgres"

 [leveldb]
-    path = "/Users/user/Library/Ethereum/geth/chaindata"
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    path = "/Users/iannorden/Library/Ethereum/geth/chaindata"
+    ancient = "/Users/iannorden/Library/Ethereum/geth/chaindata/ancient"
+
+[log]
+    level = "info"
+    file = "log_file"
+
+[prom]
+    metrics = true
+    http = true
+    httpAddr = "0.0.0.0"
+    httpPort = 9101
+    dbStats = true

 [snapshot]
-    blockHeight = -1
+    mode = "file"
+    workers = 4
+    blockHeight = -1
+    recoveryFile = "recovery_file"
+
+[file]
+    outputDir = "output_dir/"
+
+[ethereum]
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    nodeID = "arch1" # $ETH_NODE_ID
+    networkID = "1" # $ETH_NETWORK_ID
+    chainID = "1" # $ETH_CHAIN_ID
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
@@ -0,0 +1,27 @@
+package fixture
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+// TODO: embed some mainnet data
+// import "embed"
+//_go:embed mainnet_data.tar.gz
+
+var (
+	ChaindataPath, AncientdataPath string
+)
+
+func init() {
+	_, path, _, _ := runtime.Caller(0)
+	wd := filepath.Dir(path)
+
+	ChaindataPath = filepath.Join(wd, "..", "fixture", "chaindata")
+	AncientdataPath = filepath.Join(ChaindataPath, "ancient")
+
+	if _, err := os.Stat(ChaindataPath); err != nil {
+		panic("must populate chaindata at " + ChaindataPath)
+	}
+}
@@ -0,0 +1,6 @@
+*.log
+CURRENT*
+LOCK
+LOG
+MANIFEST-*
+ancient/FLOCK
@@ -0,0 +1,359 @@
+package fixture
+
+var Block1_StateNodePaths = [][]byte{
+	[]byte{},
+	[]byte{0},
+	[]byte{0, 0},
+	[]byte{0, 2},
+	[]byte{0, 2, 1},
+	[]byte{0, 2, 8},
+	[]byte{0, 2, 12},
+	[]byte{0, 3},
+	[]byte{0, 4},
+	[]byte{0, 6},
+	[]byte{0, 6, 3},
+	[]byte{0, 6, 13},
+	[]byte{0, 7},
+	[]byte{0, 8},
+	[]byte{0, 8, 7},
+	[]byte{0, 8, 11},
+	[]byte{0, 9},
+	[]byte{0, 9, 9},
+	[]byte{0, 9, 10},
+	[]byte{0, 12},
+	[]byte{0, 13},
+	[]byte{0, 14},
+	[]byte{1},
+	[]byte{1, 2},
+	[]byte{1, 2, 5},
+	[]byte{1, 2, 7},
+	[]byte{1, 3},
+	[]byte{1, 3, 1},
+	[]byte{1, 3, 11},
+	[]byte{1, 4},
+	[]byte{1, 5},
+	[]byte{1, 5, 11},
+	[]byte{1, 5, 12},
+	[]byte{1, 5, 15},
+	[]byte{1, 6},
+	[]byte{1, 8},
+	[]byte{1, 10},
+	[]byte{1, 13},
+	[]byte{1, 14},
+	[]byte{1, 14, 2},
+	[]byte{1, 14, 11},
+	[]byte{1, 15},
+	[]byte{1, 15, 9},
+	[]byte{1, 15, 15},
+	[]byte{2},
+	[]byte{2, 0},
+	[]byte{2, 0, 9},
+	[]byte{2, 0, 14},
+	[]byte{2, 1},
+	[]byte{2, 1, 1},
+	[]byte{2, 1, 3},
+	[]byte{2, 1, 14},
+	[]byte{2, 5},
+	[]byte{2, 6},
+	[]byte{2, 9},
+	[]byte{2, 9, 1},
+	[]byte{2, 9, 7},
+	[]byte{2, 11},
+	[]byte{2, 11, 7},
+	[]byte{2, 11, 13},
+	[]byte{2, 13},
+	[]byte{2, 13, 1},
+	[]byte{2, 13, 15},
+	[]byte{2, 15},
+	[]byte{3},
+	[]byte{3, 0},
+	[]byte{3, 0, 0},
+	[]byte{3, 0, 1},
+	[]byte{3, 2},
+	[]byte{3, 2, 3},
+	[]byte{3, 2, 15},
+	[]byte{3, 3},
+	[]byte{3, 4},
+	[]byte{3, 4, 2},
+	[]byte{3, 4, 4},
+	[]byte{3, 4, 5},
+	[]byte{3, 6},
+	[]byte{3, 8},
+	[]byte{3, 9},
+	[]byte{3, 10},
+	[]byte{3, 10, 2},
+	[]byte{3, 10, 8},
+	[]byte{3, 10, 12},
+	[]byte{3, 11},
+	[]byte{3, 12},
+	[]byte{3, 13},
+	[]byte{3, 14},
+	[]byte{3, 14, 4},
+	[]byte{3, 14, 9},
+	[]byte{3, 14, 14},
+	[]byte{3, 14, 14, 10},
+	[]byte{3, 14, 14, 15},
+	[]byte{4},
+	[]byte{4, 0},
+	[]byte{4, 0, 6},
+	[]byte{4, 0, 15},
+	[]byte{4, 1},
+	[]byte{4, 2},
+	[]byte{4, 2, 1},
+	[]byte{4, 2, 11},
+	[]byte{4, 3},
+	[]byte{4, 5},
+	[]byte{4, 6},
+	[]byte{4, 7},
+	[]byte{4, 8},
+	[]byte{4, 11},
+	[]byte{4, 11, 6},
+	[]byte{4, 11, 9},
+	[]byte{4, 11, 12},
+	[]byte{4, 14},
+	[]byte{5},
+	[]byte{5, 0},
+	[]byte{5, 0, 3},
+	[]byte{5, 0, 9},
+	[]byte{5, 0, 15},
+	[]byte{5, 1},
+	[]byte{5, 1, 14},
+	[]byte{5, 1, 15},
+	[]byte{5, 2},
+	[]byte{5, 2, 8},
+	[]byte{5, 2, 10},
+	[]byte{5, 3},
+	[]byte{5, 4},
+	[]byte{5, 4, 6},
+	[]byte{5, 4, 12},
+	[]byte{5, 6},
+	[]byte{5, 8},
+	[]byte{5, 8, 3},
+	[]byte{5, 8, 11},
+	[]byte{5, 10},
+	[]byte{5, 11},
+	[]byte{5, 12},
+	[]byte{5, 13},
+	[]byte{5, 15},
+	[]byte{6},
+	[]byte{6, 0},
+	[]byte{6, 2},
+	[]byte{6, 2, 3},
+	[]byte{6, 2, 9},
+	[]byte{6, 4},
+	[]byte{6, 4, 0},
+	[]byte{6, 4, 0, 0},
+	[]byte{6, 4, 0, 5},
+	[]byte{6, 5},
+	[]byte{6, 5, 4},
+	[]byte{6, 5, 10},
+	[]byte{6, 5, 12},
+	[]byte{6, 5, 13},
+	[]byte{6, 6},
+	[]byte{6, 6, 0},
+	[]byte{6, 6, 8},
+	[]byte{6, 8},
+	[]byte{6, 8, 4},
+	[]byte{6, 8, 4, 2},
+	[]byte{6, 8, 4, 9},
+	[]byte{6, 8, 9},
+	[]byte{6, 10},
+	[]byte{6, 10, 1},
+	[]byte{6, 10, 14},
+	[]byte{6, 11},
+	[]byte{6, 11, 2},
+	[]byte{6, 11, 12},
+	[]byte{6, 11, 14},
+	[]byte{6, 13},
+	[]byte{6, 13, 2},
+	[]byte{6, 13, 12},
+	[]byte{7},
+	[]byte{7, 1},
+	[]byte{7, 5},
+	[]byte{7, 7},
+	[]byte{7, 8},
+	[]byte{7, 8, 2},
+	[]byte{7, 8, 5},
+	[]byte{7, 9},
+	[]byte{7, 13},
+	[]byte{7, 13, 1},
+	[]byte{7, 13, 1, 0},
+	[]byte{7, 13, 1, 13},
+	[]byte{7, 13, 7},
+	[]byte{7, 14},
+	[]byte{7, 14, 8},
+	[]byte{7, 14, 11},
+	[]byte{8},
+	[]byte{8, 0},
+	[]byte{8, 0, 3},
+	[]byte{8, 0, 11},
+	[]byte{8, 2},
+	[]byte{8, 4},
+	[]byte{8, 8},
+	[]byte{8, 9},
+	[]byte{8, 9, 3},
+	[]byte{8, 9, 13},
+	[]byte{8, 10},
+	[]byte{8, 12},
+	[]byte{8, 12, 3},
+	[]byte{8, 12, 15},
+	[]byte{8, 13},
+	[]byte{8, 15},
+	[]byte{8, 15, 8},
+	[]byte{8, 15, 13},
+	[]byte{9},
+	[]byte{9, 0},
+	[]byte{9, 5},
+	[]byte{9, 6},
+	[]byte{9, 6, 10},
+	[]byte{9, 6, 14},
+	[]byte{9, 7},
+	[]byte{9, 9},
+	[]byte{9, 14},
+	[]byte{9, 15},
+	[]byte{9, 15, 0},
+	[]byte{9, 15, 4},
+	[]byte{9, 15, 10},
+	[]byte{10},
+	[]byte{10, 0},
+	[]byte{10, 0, 9},
+	[]byte{10, 0, 10},
+	[]byte{10, 0, 15},
+	[]byte{10, 2},
+	[]byte{10, 3},
+	[]byte{10, 6},
+	[]byte{10, 8},
+	[]byte{10, 9},
+	[]byte{10, 10},
+	[]byte{10, 10, 5},
+	[]byte{10, 10, 8},
+	[]byte{10, 13},
+	[]byte{10, 13, 0},
+	[]byte{10, 13, 13},
+	[]byte{10, 14},
+	[]byte{10, 14, 4},
+	[]byte{10, 14, 11},
+	[]byte{10, 14, 11, 8},
+	[]byte{10, 14, 11, 14},
+	[]byte{10, 15},
+	[]byte{11},
+	[]byte{11, 0},
+	[]byte{11, 0, 2},
+	[]byte{11, 0, 15},
+	[]byte{11, 1},
+	[]byte{11, 2},
+	[]byte{11, 3},
+	[]byte{11, 4},
+	[]byte{11, 5},
+	[]byte{11, 7},
+	[]byte{11, 7, 12},
+	[]byte{11, 7, 15},
+	[]byte{11, 8},
+	[]byte{11, 8, 8},
+	[]byte{11, 8, 15},
+	[]byte{11, 9},
+	[]byte{11, 11},
+	[]byte{11, 12},
+	[]byte{11, 13},
+	[]byte{11, 14},
+	[]byte{11, 14, 0},
+	[]byte{11, 14, 0, 1},
+	[]byte{11, 14, 0, 3},
+	[]byte{11, 14, 8},
+	[]byte{11, 14, 13},
+	[]byte{12},
+	[]byte{12, 0},
+	[]byte{12, 0, 0},
+	[]byte{12, 0, 1},
+	[]byte{12, 0, 1, 3},
+	[]byte{12, 0, 1, 11},
+	[]byte{12, 0, 15},
+	[]byte{12, 2},
+	[]byte{12, 2, 9},
+	[]byte{12, 2, 12},
+	[]byte{12, 4},
+	[]byte{12, 5},
+	[]byte{12, 6},
+	[]byte{12, 6, 0},
+	[]byte{12, 6, 4},
+	[]byte{12, 6, 14},
+	[]byte{12, 7},
+	[]byte{12, 7, 0},
+	[]byte{12, 7, 12},
+	[]byte{12, 7, 13},
+	[]byte{12, 9},
+	[]byte{12, 11},
+	[]byte{12, 12},
+	[]byte{13},
+	[]byte{13, 2},
+	[]byte{13, 2, 0},
+	[]byte{13, 2, 2},
+	[]byte{13, 2, 4},
+	[]byte{13, 3},
+	[]byte{13, 3, 7},
+	[]byte{13, 3, 10},
+	[]byte{13, 5},
+	[]byte{13, 8},
+	[]byte{13, 8, 1},
+	[]byte{13, 8, 15},
+	[]byte{13, 9},
+	[]byte{13, 9, 0},
+	[]byte{13, 9, 14},
+	[]byte{13, 10},
+	[]byte{13, 12},
+	[]byte{13, 12, 8},
+	[]byte{13, 12, 11},
+	[]byte{13, 13},
+	[]byte{13, 13, 7},
+	[]byte{13, 13, 12},
+	[]byte{13, 14},
+	[]byte{14},
+	[]byte{14, 0},
+	[]byte{14, 1},
+	[]byte{14, 2},
+	[]byte{14, 2, 2},
+	[]byte{14, 2, 12},
+	[]byte{14, 3},
+	[]byte{14, 4},
+	[]byte{14, 5},
+	[]byte{14, 6},
+	[]byte{14, 6, 9},
+	[]byte{14, 6, 12},
+	[]byte{14, 7},
+	[]byte{14, 7, 4},
+	[]byte{14, 7, 12},
+	[]byte{14, 8},
+	[]byte{14, 8, 3},
+	[]byte{14, 8, 12},
+	[]byte{14, 8, 12, 0},
+	[]byte{14, 8, 12, 6},
+	[]byte{14, 10},
+	[]byte{14, 10, 6},
+	[]byte{14, 10, 12},
+	[]byte{14, 11},
+	[]byte{14, 11, 8},
+	[]byte{14, 11, 13},
+	[]byte{14, 12},
+	[]byte{14, 14},
+	[]byte{14, 14, 3},
+	[]byte{14, 14, 9},
+	[]byte{15},
+	[]byte{15, 0},
+	[]byte{15, 5},
+	[]byte{15, 6},
+	[]byte{15, 9},
+	[]byte{15, 9, 0},
+	[]byte{15, 9, 2},
+	[]byte{15, 9, 3},
+	[]byte{15, 11},
+	[]byte{15, 11, 1},
+	[]byte{15, 11, 6},
+	[]byte{15, 12},
+	[]byte{15, 12, 3},
+	[]byte{15, 12, 14},
+	[]byte{15, 12, 14, 7},
+	[]byte{15, 12, 14, 13},
+	[]byte{15, 13},
+	[]byte{15, 14},
+	[]byte{15, 15},
+}
@@ -5,12 +5,11 @@ import (

 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	// "github.com/jmoiron/sqlx"

-	snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
+	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
 )

-var PublishHeader = &types.Header{
+var Block1_Header = types.Header{
 	ParentHash:  common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177"),
 	UncleHash:   common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"),
 	Coinbase:    common.HexToAddress("0x0000000000000000000000000000000000000000"),
@@ -29,7 +28,7 @@ var PublishHeader = &types.Header{
 	BaseFee:     nil,
 }

-var StateNode = snapt.Node{
+var Block1_StateNode0 = snapt.Node{
 	NodeType: 0,
 	Path:     []byte{12, 0},
 	Key:      common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"),
@@ -1,45 +1,117 @@
-module github.com/vulcanize/eth-pg-ipfs-state-snapshot
+module github.com/vulcanize/ipld-eth-state-snapshot

-go 1.15
+go 1.18

 require (
-	github.com/btcsuite/btcd v0.22.0-beta // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/ethereum/go-ethereum v1.10.15
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
-	github.com/go-kit/kit v0.10.0 // indirect
+	github.com/ethereum/go-ethereum v1.10.18
 	github.com/golang/mock v1.6.0
+	github.com/ipfs/go-cid v0.1.0
+	github.com/ipfs/go-ipfs-blockstore v1.1.2
+	github.com/ipfs/go-ipfs-ds-help v1.1.0
+	github.com/jackc/pgx/v4 v4.15.0
+	github.com/multiformats/go-multihash v0.1.0
+	github.com/prometheus/client_golang v1.3.0
+	github.com/sirupsen/logrus v1.6.0
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/viper v1.7.0
+	github.com/vulcanize/go-eth-state-node-iterator v1.0.3
+)
+
+require (
+	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
+	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/georgysavva/scany v0.2.9 // indirect
+	github.com/go-kit/kit v0.10.0 // indirect
+	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
-	github.com/ipfs/go-cid v0.1.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.0.3 // indirect
 	github.com/ipfs/go-datastore v0.5.1 // indirect
-	github.com/ipfs/go-ipfs-blockstore v1.1.2 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-format v0.2.0 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-log/v2 v2.4.0 // indirect
-	github.com/jmoiron/sqlx v1.2.0
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.11.0 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.10.0 // indirect
+	github.com/jackc/puddle v1.2.1 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jmoiron/sqlx v1.2.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
+	github.com/lib/pq v1.10.2 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.0.4 // indirect
-	github.com/multiformats/go-multihash v0.1.0
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.0.3 // indirect
+	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.13.0 // indirect
-	github.com/sirupsen/logrus v1.6.0
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pelletier/go-toml v1.2.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.1.0 // indirect
+	github.com/prometheus/common v0.7.0 // indirect
+	github.com/prometheus/procfs v0.0.8 // indirect
+	github.com/prometheus/tsdb v0.7.1 // indirect
+	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/smartystreets/assertions v1.0.0 // indirect
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.7.0
-	github.com/vulcanize/go-eth-state-node-iterator v1.0.0
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.1.2 // indirect
+	github.com/spf13/cast v1.3.0 // indirect
+	github.com/spf13/jwalterweatherman v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.3 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	github.com/tklauser/go-sysconf v0.3.5 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/goleak v1.1.11 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
 	golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
 	golang.org/x/sys v0.0.0-20211209171907-798191bca915 // indirect
-	golang.org/x/tools v0.1.8 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.51.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 )

-replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0
+replace github.com/ethereum/go-ethereum v1.10.18 => github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha
@@ -19,24 +19,15 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
-github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
-github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4=
-github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
-github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
-github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
-github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
-github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM=
+github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@@ -45,7 +36,6 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU
 github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o=
 github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
-github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -79,30 +69,21 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
-github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo=
-github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA=
-github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
-github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o=
-github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
-github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
-github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
-github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
-github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
-github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
-github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
+github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -113,6 +94,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304=
+github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
+github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
+github.com/cockroachdb/cockroach-go/v2 v2.0.3 h1:ZA346ACHIZctef6trOTwBAEvPVm1k0uLm/bb2Atc+S8=
+github.com/cockroachdb/cockroach-go/v2 v2.0.3/go.mod h1:hAuDgiVgDVkfirP9JnhXEfcXEPRKBpYdGz+l7mvYSzw=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ=
@@ -124,6 +109,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -132,21 +118,26 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
 github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
-github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0=
-github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
-github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
+github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4=
+github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
+github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
+github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
 github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
 github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
+github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
-github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
+github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
+github.com/docker/docker v1.6.2/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
 github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@@ -158,7 +149,9 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
@@ -167,7 +160,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8=
 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
+github.com/georgysavva/scany v0.2.9 h1:Xt6rjYpHnMClTm/g+oZTnoSxUwiln5GqMNU+QeLNHQU=
+github.com/georgysavva/scany v0.2.9/go.mod h1:yeOeC1BdIdl6hOwy8uefL2WNSlseFzbhlG/frrh65SA=
 github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -180,6 +176,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
@@ -195,7 +192,10 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
+github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -203,6 +203,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -241,6 +243,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
@@ -254,7 +257,7 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -270,7 +273,7 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -312,7 +315,7 @@ github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM
 github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
-github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM=
+github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y=
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -367,21 +370,95 @@ github.com/ipfs/go-log/v2 v2.4.0 h1:iR/2o9PGWanVJrBgIH5Ff8mPGOwpqLaPIAFqSnsdlzk=
 github.com/ipfs/go-log/v2 v2.4.0/go.mod h1:nPZnh7Cj7lwS3LpRU5Mwr2ol1c2gXIEXuF6aywqrtmo=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
-github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
+github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
+github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
+github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
+github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
+github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
+github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78=
+github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7JMeA=
+github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
+github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
+github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
+github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
+github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ=
+github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
+github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
+github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
+github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
+github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c=
+github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc=
+github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns=
+github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
+github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
+github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
+github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
+github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik=
+github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
+github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
+github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
+github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
+github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
+github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38=
+github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
+github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
+github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
+github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
+github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
+github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
+github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg=
+github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
+github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
+github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0=
+github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
+github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0=
+github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w=
+github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw=
+github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw=
+github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
 github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
 github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
-github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -393,14 +470,12 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
-github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio=
 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
@@ -408,6 +483,7 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
 github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -417,14 +493,21 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
 github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
@@ -440,13 +523,13 @@ github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIG
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
-github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -458,9 +541,10 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
-github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
+github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
@@ -484,6 +568,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
 github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
@@ -527,12 +612,12 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
@@ -559,6 +644,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
+github.com/pganalyze/pg_query_go/v2 v2.1.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -575,22 +661,26 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
 github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
@@ -602,17 +692,27 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
+github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
 github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v0.0.0-20200419222939-1884f454f8ea/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -647,6 +747,8 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -658,6 +760,7 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
+github.com/thoas/go-funk v0.9.2/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4=
 github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
@@ -673,10 +776,10 @@ github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/X
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
-github.com/vulcanize/go-eth-state-node-iterator v1.0.0 h1:FQ4s0K5TnRD44p3vO4uQtjfE8C1Gr2EDMT2vVGS3Lu8=
-github.com/vulcanize/go-eth-state-node-iterator v1.0.0/go.mod h1:uWhleTvUEZ+cEkNRIAmBpZ14KilTP71OxY5NZDrpNlo=
-github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0 h1:/BiYPUHnubh46YVtASGt4MPlFR96Rc+iJuTyOI8KZa4=
-github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0/go.mod h1:9L+QY31AnWnX2/2HDOySCjQoYUdWNGBRMezFJVfH73E=
+github.com/vulcanize/go-eth-state-node-iterator v1.0.3 h1:gtkU0aP7aW0tPBfzxVSymRhX9eyIQv7HiGkGVGwoh8I=
+github.com/vulcanize/go-eth-state-node-iterator v1.0.3/go.mod h1:a1aRW1/fEmcpY8AAU47p9TNL3QNjpeqCJd/hLtFKVL8=
+github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha h1:gTcfBHoaRpsRbP1/dnurg5LFUaqsQMTNqs4R4S2wm9U=
+github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha/go.mod h1:HelXH7UT1uWdb+St6UAj4pPf93GOggjIV7pVbrWIZ3o=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -686,6 +789,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
@@ -717,26 +821,32 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=
 go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
-golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -749,6 +859,7 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
 golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -760,8 +871,6 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -770,9 +879,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -799,14 +907,16 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -820,7 +930,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -835,6 +944,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -872,7 +982,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211209171907-798191bca915 h1:P+8mCzuEpyszAT6T42q0sxU+eveBAF/cJ2Kp0x6/8+0=
 golang.org/x/sys v0.0.0-20211209171907-798191bca915/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -906,12 +1015,14 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -921,20 +1032,20 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191126055441-b0650ceb63d9/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
-golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1007,11 +1118,11 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
+gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
-gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
@@ -1029,7 +1140,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -18,7 +18,7 @@ package main
 import (
 	"github.com/sirupsen/logrus"

-	"github.com/vulcanize/eth-pg-ipfs-state-snapshot/cmd"
+	"github.com/vulcanize/ipld-eth-state-snapshot/cmd"
 )

 func main() {
@@ -0,0 +1,157 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+)
+
+// DBStatsGetter is an interface that gets sql.DBStats.
+type DBStatsGetter interface {
+	Stats() sql.Stats
+}
+
+// DBStatsCollector implements the prometheus.Collector interface.
+type DBStatsCollector struct {
+	sg DBStatsGetter
+
+	// descriptions of exported metrics
+	maxOpenDesc           *prometheus.Desc
+	openDesc              *prometheus.Desc
+	inUseDesc             *prometheus.Desc
+	idleDesc              *prometheus.Desc
+	waitedForDesc         *prometheus.Desc
+	blockedSecondsDesc    *prometheus.Desc
+	closedMaxIdleDesc     *prometheus.Desc
+	closedMaxLifetimeDesc *prometheus.Desc
+}
+
+// NewDBStatsCollector creates a new DBStatsCollector.
+func NewDBStatsCollector(dbName string, sg DBStatsGetter) *DBStatsCollector {
+	labels := prometheus.Labels{"db_name": dbName}
+	return &DBStatsCollector{
+		sg: sg,
+		maxOpenDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, connSubsystem, "max_open"),
+			"Maximum number of open connections to the database.",
+			nil,
+			labels,
+		),
+		openDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, connSubsystem, "open"),
+			"The number of established connections both in use and idle.",
+			nil,
+			labels,
+		),
+		inUseDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, connSubsystem, "in_use"),
+			"The number of connections currently in use.",
+			nil,
+			labels,
+		),
+		idleDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, connSubsystem, "idle"),
+			"The number of idle connections.",
+			nil,
+			labels,
+		),
+		waitedForDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, connSubsystem, "waited_for"),
+			"The total number of connections waited for.",
+			nil,
+			labels,
+		),
+		blockedSecondsDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, connSubsystem, "blocked_seconds"),
+			"The total time blocked waiting for a new connection.",
+			nil,
+			labels,
+		),
+		closedMaxIdleDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, connSubsystem, "closed_max_idle"),
+			"The total number of connections closed due to SetMaxIdleConns.",
+			nil,
+			labels,
+		),
+		closedMaxLifetimeDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, connSubsystem, "closed_max_lifetime"),
+			"The total number of connections closed due to SetConnMaxLifetime.",
+			nil,
+			labels,
+		),
+	}
+}
+
+// Describe implements the prometheus.Collector interface.
+func (c DBStatsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.maxOpenDesc
+	ch <- c.openDesc
+	ch <- c.inUseDesc
+	ch <- c.idleDesc
+	ch <- c.waitedForDesc
+	ch <- c.blockedSecondsDesc
+	ch <- c.closedMaxIdleDesc
+	ch <- c.closedMaxLifetimeDesc
+}
+
+// Collect implements the prometheus.Collector interface.
+func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
+	stats := c.sg.Stats()
+
+	ch <- prometheus.MustNewConstMetric(
+		c.maxOpenDesc,
+		prometheus.GaugeValue,
+		float64(stats.MaxOpen()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.openDesc,
+		prometheus.GaugeValue,
+		float64(stats.Open()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.inUseDesc,
+		prometheus.GaugeValue,
+		float64(stats.InUse()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.idleDesc,
+		prometheus.GaugeValue,
+		float64(stats.Idle()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.waitedForDesc,
+		prometheus.CounterValue,
+		float64(stats.WaitCount()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.blockedSecondsDesc,
+		prometheus.CounterValue,
+		stats.WaitDuration().Seconds(),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.closedMaxIdleDesc,
+		prometheus.CounterValue,
+		float64(stats.MaxIdleClosed()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.closedMaxLifetimeDesc,
+		prometheus.CounterValue,
+		float64(stats.MaxLifetimeClosed()),
+	)
+}
@@ -0,0 +1,90 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	namespace = "ipld_eth_state_snapshot"
+
+	connSubsystem  = "connections"
+	statsSubsystem = "stats"
+)
+
+var (
+	metrics bool
+
+	stateNodeCount   prometheus.Counter
+	storageNodeCount prometheus.Counter
+	codeNodeCount    prometheus.Counter
+)
+
+func Init() {
+	metrics = true
+
+	stateNodeCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: statsSubsystem,
+		Name:      "state_node_count",
+		Help:      "Number of state nodes processed",
+	})
+
+	storageNodeCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: statsSubsystem,
+		Name:      "storage_node_count",
+		Help:      "Number of storage nodes processed",
+	})
+
+	codeNodeCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: statsSubsystem,
+		Name:      "code_node_count",
+		Help:      "Number of code nodes processed",
+	})
+}
+
+// RegisterDBCollector create metric collector for given connection
+func RegisterDBCollector(name string, db DBStatsGetter) {
+	if metrics {
+		prometheus.Register(NewDBStatsCollector(name, db))
+	}
+}
+
+// IncStateNodeCount increments the number of state nodes processed
+func IncStateNodeCount() {
+	if metrics {
+		stateNodeCount.Inc()
+	}
+}
+
+// IncStorageNodeCount increments the number of storage nodes processed
+func IncStorageNodeCount() {
+	if metrics {
+		storageNodeCount.Inc()
+	}
+}
+
+// IncCodeNodeCount increments the number of code nodes processed
+func IncCodeNodeCount() {
+	if metrics {
+		codeNodeCount.Inc()
+	}
+}
@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sirupsen/logrus"
+)
+
+var errPromHTTP = errors.New("can't start http server for prometheus")
+
+// Serve start listening http
+func Serve(addr string) *http.Server {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	srv := http.Server{
+		Addr:    addr,
+		Handler: mux,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			logrus.
+				WithError(err).
+				WithField("module", "prom").
+				WithField("addr", addr).
+				Fatal(errPromHTTP)
+		}
+	}()
+	return &srv
+}
@@ -16,89 +16,124 @@
 package snapshot

 import (
+	"fmt"
+	"time"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node"
-	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	"github.com/spf13/viper"
 )

+// SnapshotMode specifies the snapshot data output method
+type SnapshotMode string
+
 const (
-	ANCIENT_DB_PATH   = "ANCIENT_DB_PATH"
-	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
-	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
-	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
-	ETH_NODE_ID       = "ETH_NODE_ID"
-	LVL_DB_PATH       = "LVL_DB_PATH"
+	PgSnapshot   SnapshotMode = "postgres"
+	FileSnapshot SnapshotMode = "file"
+
+	defaultOutputDir = "./snapshot_output"
 )

 // Config contains params for both databases the service uses
 type Config struct {
-	DB  *DBConfig
-	Eth *EthConfig
-}
-
-// DBConfig is config parameters for DB.
-type DBConfig struct {
-	Node       ethNode.Info
-	URI        string
-	ConnConfig postgres.ConnectionConfig
+	Eth  *EthConfig
+	DB   *DBConfig
+	File *FileConfig
 }

 // EthConfig is config parameters for the chain.
 type EthConfig struct {
 	LevelDBPath   string
 	AncientDBPath string
+	NodeInfo      ethNode.Info
 }

-func NewConfig() *Config {
-	ret := &Config{&DBConfig{}, &EthConfig{}}
-	ret.Init()
-	return ret
+// DBConfig is config parameters for DB.
+type DBConfig struct {
+	URI        string
+	ConnConfig postgres.Config
+}
+
+type FileConfig struct {
+	OutputDir string
+}
+
+func NewConfig(mode SnapshotMode) (*Config, error) {
+	ret := &Config{
+		&EthConfig{},
+		&DBConfig{},
+		&FileConfig{},
+	}
+	return ret, ret.Init(mode)
 }

 // Init Initialises config
-func (c *Config) Init() {
-	c.DB.dbInit()
-	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
-	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
-	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
-	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
+func (c *Config) Init(mode SnapshotMode) error {
+	viper.BindEnv(ETH_NODE_ID_TOML, ETH_NODE_ID)
+	viper.BindEnv(ETH_CLIENT_NAME_TOML, ETH_CLIENT_NAME)
+	viper.BindEnv(ETH_GENESIS_BLOCK_TOML, ETH_GENESIS_BLOCK)
+	viper.BindEnv(ETH_NETWORK_ID_TOML, ETH_NETWORK_ID)
+	viper.BindEnv(ETH_CHAIN_ID_TOML, ETH_CHAIN_ID)

-	c.DB.Node = ethNode.Info{
-		ID:           viper.GetString("ethereum.nodeID"),
-		ClientName:   viper.GetString("ethereum.clientName"),
-		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
-		NetworkID:    viper.GetString("ethereum.networkID"),
-		ChainID:      viper.GetUint64("ethereum.chainID"),
+	c.Eth.NodeInfo = ethNode.Info{
+		ID:           viper.GetString(ETH_NODE_ID_TOML),
+		ClientName:   viper.GetString(ETH_CLIENT_NAME_TOML),
+		GenesisBlock: viper.GetString(ETH_GENESIS_BLOCK_TOML),
+		NetworkID:    viper.GetString(ETH_NETWORK_ID_TOML),
+		ChainID:      viper.GetUint64(ETH_CHAIN_ID_TOML),
 	}

-	viper.BindEnv("leveldb.ancient", ANCIENT_DB_PATH)
-	viper.BindEnv("leveldb.path", LVL_DB_PATH)
+	viper.BindEnv(ANCIENT_DB_PATH_TOML, ANCIENT_DB_PATH)
+	viper.BindEnv(LVL_DB_PATH_TOML, LVL_DB_PATH)

-	c.Eth.AncientDBPath = viper.GetString("leveldb.ancient")
-	c.Eth.LevelDBPath = viper.GetString("leveldb.path")
+	c.Eth.AncientDBPath = viper.GetString(ANCIENT_DB_PATH_TOML)
+	c.Eth.LevelDBPath = viper.GetString(LVL_DB_PATH_TOML)
+
+	switch mode {
+	case FileSnapshot:
+		c.File.Init()
+	case PgSnapshot:
+		c.DB.Init()
+	default:
+		return fmt.Errorf("no output mode specified")
+	}
+	return nil
 }

-func (c *DBConfig) dbInit() {
-	viper.BindEnv("database.name", postgres.DATABASE_NAME)
-	viper.BindEnv("database.hostname", postgres.DATABASE_HOSTNAME)
-	viper.BindEnv("database.port", postgres.DATABASE_PORT)
-	viper.BindEnv("database.user", postgres.DATABASE_USER)
-	viper.BindEnv("database.password", postgres.DATABASE_PASSWORD)
-	viper.BindEnv("database.maxIdle", postgres.DATABASE_MAX_IDLE_CONNECTIONS)
-	viper.BindEnv("database.maxOpen", postgres.DATABASE_MAX_OPEN_CONNECTIONS)
-	viper.BindEnv("database.maxLifetime", postgres.DATABASE_MAX_CONN_LIFETIME)
+func (c *DBConfig) Init() {
+	viper.BindEnv(DATABASE_NAME_TOML, DATABASE_NAME)
+	viper.BindEnv(DATABASE_HOSTNAME_TOML, DATABASE_HOSTNAME)
+	viper.BindEnv(DATABASE_PORT_TOML, DATABASE_PORT)
+	viper.BindEnv(DATABASE_USER_TOML, DATABASE_USER)
+	viper.BindEnv(DATABASE_PASSWORD_TOML, DATABASE_PASSWORD)
+	viper.BindEnv(DATABASE_MAX_IDLE_CONNECTIONS_TOML, DATABASE_MAX_IDLE_CONNECTIONS)
+	viper.BindEnv(DATABASE_MAX_OPEN_CONNECTIONS_TOML, DATABASE_MAX_OPEN_CONNECTIONS)
+	viper.BindEnv(DATABASE_MAX_CONN_LIFETIME_TOML, DATABASE_MAX_CONN_LIFETIME)

-	dbParams := postgres.ConnectionParams{}
+	dbParams := postgres.Config{}
 	// DB params
-	dbParams.Name = viper.GetString("database.name")
-	dbParams.Hostname = viper.GetString("database.hostname")
-	dbParams.Port = viper.GetInt("database.port")
-	dbParams.User = viper.GetString("database.user")
-	dbParams.Password = viper.GetString("database.password")
-
-	c.URI = postgres.DbConnectionString(dbParams)
+	dbParams.DatabaseName = viper.GetString(DATABASE_NAME_TOML)
+	dbParams.Hostname = viper.GetString(DATABASE_HOSTNAME_TOML)
+	dbParams.Port = viper.GetInt(DATABASE_PORT_TOML)
+	dbParams.Username = viper.GetString(DATABASE_USER_TOML)
+	dbParams.Password = viper.GetString(DATABASE_PASSWORD_TOML)
 	// Connection config
-	c.ConnConfig.MaxIdle = viper.GetInt("database.maxIdle")
-	c.ConnConfig.MaxOpen = viper.GetInt("database.maxOpen")
-	c.ConnConfig.MaxLifetime = viper.GetInt("database.maxLifetime")
+	dbParams.MaxIdle = viper.GetInt(DATABASE_MAX_IDLE_CONNECTIONS_TOML)
+	dbParams.MaxConns = viper.GetInt(DATABASE_MAX_OPEN_CONNECTIONS_TOML)
+	dbParams.MaxConnLifetime = time.Duration(viper.GetInt(DATABASE_MAX_CONN_LIFETIME_TOML)) * time.Second
+
+	c.ConnConfig = dbParams
+	c.URI = dbParams.DbConnectionString()
+}
+
+func (c *FileConfig) Init() error {
+	viper.BindEnv(FILE_OUTPUT_DIR_TOML, FILE_OUTPUT_DIR)
+	c.OutputDir = viper.GetString(FILE_OUTPUT_DIR_TOML)
+	if c.OutputDir == "" {
+		logrus.Infof("no output directory set, using default: %s", defaultOutputDir)
+		c.OutputDir = defaultOutputDir
+	}
+	return nil
 }
@@ -0,0 +1,127 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package snapshot
+
+// ENV variables
+const (
+	SNAPSHOT_BLOCK_HEIGHT  = "SNAPSHOT_BLOCK_HEIGHT"
+	SNAPSHOT_WORKERS       = "SNAPSHOT_WORKERS"
+	SNAPSHOT_RECOVERY_FILE = "SNAPSHOT_RECOVERY_FILE"
+	SNAPSHOT_MODE          = "SNAPSHOT_MODE"
+
+	LOGRUS_LEVEL = "LOGRUS_LEVEL"
+	LOGRUS_FILE  = "LOGRUS_FILE"
+
+	PROM_METRICS   = "PROM_METRICS"
+	PROM_HTTP      = "PROM_HTTP"
+	PROM_HTTP_ADDR = "PROM_HTTP_ADDR"
+	PROM_HTTP_PORT = "PROM_HTTP_PORT"
+	PROM_DB_STATS  = "PROM_DB_STATS"
+
+	FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
+
+	ANCIENT_DB_PATH = "ANCIENT_DB_PATH"
+	LVL_DB_PATH     = "LVL_DB_PATH"
+
+	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
+	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
+	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
+	ETH_NODE_ID       = "ETH_NODE_ID"
+	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+
+	DATABASE_NAME                 = "DATABASE_NAME"
+	DATABASE_HOSTNAME             = "DATABASE_HOSTNAME"
+	DATABASE_PORT                 = "DATABASE_PORT"
+	DATABASE_USER                 = "DATABASE_USER"
+	DATABASE_PASSWORD             = "DATABASE_PASSWORD"
+	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
+)
+
+// TOML bindings
+const (
+	SNAPSHOT_BLOCK_HEIGHT_TOML  = "snapshot.blockHeight"
+	SNAPSHOT_WORKERS_TOML       = "snapshot.workers"
+	SNAPSHOT_RECOVERY_FILE_TOML = "snapshot.recoveryFile"
+	SNAPSHOT_MODE_TOML          = "snapshot.mode"
+
+	LOGRUS_LEVEL_TOML = "log.level"
+	LOGRUS_FILE_TOML  = "log.file"
+
+	PROM_METRICS_TOML   = "prom.metrics"
+	PROM_HTTP_TOML      = "prom.http"
+	PROM_HTTP_ADDR_TOML = "prom.httpAddr"
+	PROM_HTTP_PORT_TOML = "prom.httpPort"
+	PROM_DB_STATS_TOML  = "prom.dbStats"
+
+	FILE_OUTPUT_DIR_TOML = "file.outputDir"
+
+	ANCIENT_DB_PATH_TOML = "leveldb.ancient"
+	LVL_DB_PATH_TOML     = "leveldb.path"
+
+	ETH_CLIENT_NAME_TOML   = "ethereum.clientName"
+	ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
+	ETH_NETWORK_ID_TOML    = "ethereum.networkID"
+	ETH_NODE_ID_TOML       = "ethereum.nodeID"
+	ETH_CHAIN_ID_TOML      = "ethereum.chainID"
+
+	DATABASE_NAME_TOML                 = "database.name"
+	DATABASE_HOSTNAME_TOML             = "database.hostname"
+	DATABASE_PORT_TOML                 = "database.port"
+	DATABASE_USER_TOML                 = "database.user"
+	DATABASE_PASSWORD_TOML             = "database.password"
+	DATABASE_MAX_IDLE_CONNECTIONS_TOML = "database.maxIdle"
+	DATABASE_MAX_OPEN_CONNECTIONS_TOML = "database.maxOpen"
+	DATABASE_MAX_CONN_LIFETIME_TOML    = "database.maxLifetime"
+)
+
+// CLI flags
+const (
+	SNAPSHOT_BLOCK_HEIGHT_CLI  = "block-height"
+	SNAPSHOT_WORKERS_CLI       = "workers"
+	SNAPSHOT_RECOVERY_FILE_CLI = "recovery-file"
+	SNAPSHOT_MODE_CLI          = "snapshot-mode"
+
+	LOGRUS_LEVEL_CLI = "log-level"
+	LOGRUS_FILE_CLI  = "log-file"
+
+	PROM_METRICS_CLI   = "prom-metrics"
+	PROM_HTTP_CLI      = "prom-http"
+	PROM_HTTP_ADDR_CLI = "prom-httpAddr"
+	PROM_HTTP_PORT_CLI = "prom-httpPort"
+	PROM_DB_STATS_CLI  = "prom-dbStats"
+
+	FILE_OUTPUT_DIR_CLI = "output-dir"
+
+	ANCIENT_DB_PATH_CLI = "ancient-path"
+	LVL_DB_PATH_CLI     = "leveldb-path"
+
+	ETH_CLIENT_NAME_CLI   = "ethereum-client-name"
+	ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block"
+	ETH_NETWORK_ID_CLI    = "ethereum-network-id"
+	ETH_NODE_ID_CLI       = "ethereum-node-id"
+	ETH_CHAIN_ID_CLI      = "ethereum-chain-id"
+
+	DATABASE_NAME_CLI                 = "database-name"
+	DATABASE_HOSTNAME_CLI             = "database-hostname"
+	DATABASE_PORT_CLI                 = "database-port"
+	DATABASE_USER_CLI                 = "database-user"
+	DATABASE_PASSWORD_CLI             = "database-password"
+	DATABASE_MAX_IDLE_CONNECTIONS_CLI = "database-max-idle"
+	DATABASE_MAX_OPEN_CONNECTIONS_CLI = "database-max-open"
+	DATABASE_MAX_CONN_LIFETIME_CLI    = "database-max-lifetime"
+)
@@ -0,0 +1,302 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package publisher
+
+import (
+	"encoding/csv"
+	"fmt"
+	"math/big"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	dshelp "github.com/ipfs/go-ipfs-ds-help"
+	"github.com/multiformats/go-multihash"
+	"github.com/sirupsen/logrus"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
+	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+)
+
+var _ snapt.Publisher = (*publisher)(nil)
+
+var (
+	// tables written once per block
+	perBlockTables = []*snapt.Table{
+		&snapt.TableIPLDBlock,
+		&snapt.TableNodeInfo,
+		&snapt.TableHeader,
+	}
+	// tables written during state iteration
+	perNodeTables = []*snapt.Table{
+		&snapt.TableIPLDBlock,
+		&snapt.TableStateNode,
+		&snapt.TableStorageNode,
+	}
+)
+
+const logInterval = 1 * time.Minute
+
+type publisher struct {
+	dir     string // dir containing output files
+	writers fileWriters
+
+	nodeInfo nodeinfo.Info
+
+	startTime          time.Time
+	currBatchSize      uint
+	stateNodeCounter   uint64
+	storageNodeCounter uint64
+	codeNodeCounter    uint64
+	txCounter          uint32
+}
+
+type fileWriter struct {
+	*csv.Writer
+}
+
+// fileWriters wraps the file writers for each output table
+type fileWriters map[string]fileWriter
+
+type fileTx struct{ fileWriters }
+
+func (tx fileWriters) Commit() error {
+	for _, w := range tx {
+		w.Flush()
+		if err := w.Error(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+func (fileWriters) Rollback() error { return nil } // TODO: delete the file?
+
+func newFileWriter(path string) (ret fileWriter, err error) {
+	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
+	if err != nil {
+		return
+	}
+	ret = fileWriter{csv.NewWriter(file)}
+	return
+}
+
+func (tx fileWriters) write(tbl *snapt.Table, args ...interface{}) error {
+	row := tbl.ToCsvRow(args...)
+	return tx[tbl.Name].Write(row)
+}
+
+func makeFileWriters(dir string, tables []*snapt.Table) (fileWriters, error) {
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return nil, err
+	}
+	writers := fileWriters{}
+	for _, tbl := range tables {
+		w, err := newFileWriter(TableFile(dir, tbl.Name))
+		if err != nil {
+			return nil, err
+		}
+		writers[tbl.Name] = w
+	}
+	return writers, nil
+}
+
+// NewPublisher creates a publisher which writes to per-table CSV files which can be imported
+// with the Postgres COPY command.
+// The output directory will be created if it does not exist.
+func NewPublisher(path string, node nodeinfo.Info) (*publisher, error) {
+	if err := os.MkdirAll(path, 0777); err != nil {
+		return nil, fmt.Errorf("unable to make MkdirAll for path: %s err: %s", path, err)
+	}
+	writers, err := makeFileWriters(path, perBlockTables)
+	if err != nil {
+		return nil, err
+	}
+	pub := &publisher{
+		writers:   writers,
+		dir:       path,
+		nodeInfo:  node,
+		startTime: time.Now(),
+	}
+	go pub.logNodeCounters()
+	return pub, nil
+}
+
+func TableFile(dir, name string) string { return filepath.Join(dir, name+".csv") }
+
+func (p *publisher) txDir(index uint32) string {
+	return filepath.Join(p.dir, fmt.Sprintf("%010d", index))
+}
+
+func (p *publisher) BeginTx() (snapt.Tx, error) {
+	index := atomic.AddUint32(&p.txCounter, 1) - 1
+	dir := p.txDir(index)
+	writers, err := makeFileWriters(dir, perNodeTables)
+	if err != nil {
+		return nil, err
+	}
+
+	return fileTx{writers}, nil
+}
+
+// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
+// returns the CID and blockstore prefixed multihash key
+func (tx fileWriters) publishRaw(codec uint64, raw []byte, height *big.Int) (cid, prefixedKey string, err error) {
+	c, err := ipld.RawdataToCid(codec, raw, multihash.KECCAK_256)
+	if err != nil {
+		return
+	}
+	cid = c.String()
+	prefixedKey, err = tx.publishIPLD(c, raw, height)
+	return
+}
+
+func (tx fileWriters) publishIPLD(c cid.Cid, raw []byte, height *big.Int) (string, error) {
+	dbKey := dshelp.MultihashToDsKey(c.Hash())
+	prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
+	return prefixedKey, tx.write(&snapt.TableIPLDBlock, height.String(), prefixedKey, raw)
+}
+
+// PublishHeader writes the header to the ipfs backing pg datastore and adds secondary
+// indexes in the header_cids table
+func (p *publisher) PublishHeader(header *types.Header) error {
+	headerNode, err := ipld.NewEthHeader(header)
+	if err != nil {
+		return err
+	}
+	if _, err = p.writers.publishIPLD(headerNode.Cid(), headerNode.RawData(), header.Number); err != nil {
+		return err
+	}
+
+	mhKey := shared.MultihashKeyFromCID(headerNode.Cid())
+	err = p.writers.write(&snapt.TableNodeInfo, p.nodeInfo.GenesisBlock, p.nodeInfo.NetworkID, p.nodeInfo.ID,
+		p.nodeInfo.ClientName, p.nodeInfo.ChainID)
+	if err != nil {
+		return err
+	}
+	err = p.writers.write(&snapt.TableHeader, header.Number.String(), header.Hash().Hex(), header.ParentHash.Hex(),
+		headerNode.Cid().String(), 0, p.nodeInfo.ID, 0, header.Root.Hex(), header.TxHash.Hex(),
+		header.ReceiptHash.Hex(), header.UncleHash.Hex(), header.Bloom.Bytes(), header.Time, mhKey,
+		0, header.Coinbase.String())
+	if err != nil {
+		return err
+	}
+	return p.writers.Commit()
+}
+
+// PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes
+// in the state_cids table
+func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, height *big.Int, snapTx snapt.Tx) error {
+	var stateKey string
+	if !snapt.IsNullHash(node.Key) {
+		stateKey = node.Key.Hex()
+	}
+
+	tx := snapTx.(fileTx)
+	stateCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStateTrie, node.Value, height)
+	if err != nil {
+		return err
+	}
+
+	err = tx.write(&snapt.TableStateNode, height.String(), headerID, stateKey, stateCIDStr, node.Path,
+		node.NodeType, false, mhKey)
+	if err != nil {
+		return err
+	}
+	// increment state node counter.
+	atomic.AddUint64(&p.stateNodeCounter, 1)
+	prom.IncStateNodeCount()
+
+	// increment current batch size counter
+	p.currBatchSize += 2
+	return err
+}
+
+// PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary
+// indexes in the storage_cids table
+func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, height *big.Int, statePath []byte, snapTx snapt.Tx) error {
+	var storageKey string
+	if !snapt.IsNullHash(node.Key) {
+		storageKey = node.Key.Hex()
+	}
+
+	tx := snapTx.(fileTx)
+	storageCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStorageTrie, node.Value, height)
+	if err != nil {
+		return err
+	}
+
+	err = tx.write(&snapt.TableStorageNode, height.String(), headerID, statePath, storageKey, storageCIDStr, node.Path,
+		node.NodeType, false, mhKey)
+	if err != nil {
+		return err
+	}
+	// increment storage node counter.
+	atomic.AddUint64(&p.storageNodeCounter, 1)
+	prom.IncStorageNodeCount()
+
+	// increment current batch size counter
+	p.currBatchSize += 2
+	return nil
+}
+
+// PublishCode writes code to the ipfs backing pg datastore
+func (p *publisher) PublishCode(height *big.Int, codeHash common.Hash, codeBytes []byte, snapTx snapt.Tx) error {
+	// no codec for code, doesn't matter though since blockstore key is multihash-derived
+	mhKey, err := shared.MultihashKeyFromKeccak256(codeHash)
+	if err != nil {
+		return fmt.Errorf("error deriving multihash key from codehash: %v", err)
+	}
+
+	tx := snapTx.(fileTx)
+	if err = tx.write(&snapt.TableIPLDBlock, height.String(), mhKey, codeBytes); err != nil {
+		return fmt.Errorf("error publishing code IPLD: %v", err)
+	}
+	// increment code node counter.
+	atomic.AddUint64(&p.codeNodeCounter, 1)
+	prom.IncCodeNodeCount()
+
+	p.currBatchSize++
+	return nil
+}
+
+func (p *publisher) PrepareTxForBatch(tx snapt.Tx, maxBatchSize uint) (snapt.Tx, error) {
+	return tx, nil
+}
+
+// logNodeCounters periodically logs the number of node processed.
+func (p *publisher) logNodeCounters() {
+	t := time.NewTicker(logInterval)
+	for range t.C {
+		p.printNodeCounters("progress")
+	}
+}
+
+func (p *publisher) printNodeCounters(msg string) {
+	logrus.WithFields(logrus.Fields{
+		"runtime":       time.Now().Sub(p.startTime).String(),
+		"state nodes":   atomic.LoadUint64(&p.stateNodeCounter),
+		"storage nodes": atomic.LoadUint64(&p.storageNodeCounter),
+		"code nodes":    atomic.LoadUint64(&p.codeNodeCounter),
+	}).Info(msg)
+}
@@ -0,0 +1,133 @@
+package publisher
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/jackc/pgx/v4"
+
+	fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
+	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+	"github.com/vulcanize/ipld-eth-state-snapshot/test"
+)
+
+var (
+	pgConfig = test.DefaultPgConfig
+	nodeInfo = test.DefaultNodeInfo
+	// tables ordered according to fkey depedencies
+	allTables = []*snapt.Table{
+		&snapt.TableIPLDBlock,
+		&snapt.TableNodeInfo,
+		&snapt.TableHeader,
+		&snapt.TableStateNode,
+		&snapt.TableStorageNode,
+	}
+)
+
+func writeFiles(t *testing.T, dir string) *publisher {
+	pub, err := NewPublisher(dir, nodeInfo)
+	test.NoError(t, err)
+	test.NoError(t, pub.PublishHeader(&fixt.Block1_Header))
+	tx, err := pub.BeginTx()
+	test.NoError(t, err)
+
+	headerID := fixt.Block1_Header.Hash().String()
+	test.NoError(t, pub.PublishStateNode(&fixt.Block1_StateNode0, headerID, fixt.Block1_Header.Number, tx))
+
+	test.NoError(t, tx.Commit())
+	return pub
+}
+
+// verify that we can parse the csvs
+// TODO check actual data
+func verifyFileData(t *testing.T, path string, tbl *snapt.Table) {
+	file, err := os.Open(path)
+	test.NoError(t, err)
+	r := csv.NewReader(file)
+	test.NoError(t, err)
+	r.FieldsPerRecord = len(tbl.Columns)
+
+	for {
+		_, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		test.NoError(t, err)
+	}
+}
+
+func TestWriting(t *testing.T) {
+	dir := t.TempDir()
+	// tempdir like /tmp/TempFoo/001/, TempFoo defaults to 0700
+	test.NoError(t, os.Chmod(filepath.Dir(dir), 0755))
+
+	pub := writeFiles(t, dir)
+
+	for _, tbl := range perBlockTables {
+		verifyFileData(t, TableFile(pub.dir, tbl.Name), tbl)
+	}
+	for i := uint32(0); i < pub.txCounter; i++ {
+		for _, tbl := range perNodeTables {
+			verifyFileData(t, TableFile(pub.txDir(i), tbl.Name), tbl)
+		}
+	}
+}
+
+// Note: DB user requires role membership "pg_read_server_files"
+func TestPgCopy(t *testing.T) {
+	test.NeedsDB(t)
+
+	dir := t.TempDir()
+	test.NoError(t, os.Chmod(filepath.Dir(dir), 0755))
+	pub := writeFiles(t, dir)
+
+	ctx := context.Background()
+	conn, err := pgx.Connect(ctx, pgConfig.DbConnectionString())
+	test.NoError(t, err)
+
+	// clear existing test data
+	pgDeleteTable := `DELETE FROM %s`
+	for _, tbl := range allTables {
+		_, err = conn.Exec(ctx, fmt.Sprintf(pgDeleteTable, tbl.Name))
+		test.NoError(t, err)
+	}
+
+	// copy from files
+	pgCopyStatement := `COPY %s FROM '%s' CSV`
+	for _, tbl := range perBlockTables {
+		stm := fmt.Sprintf(pgCopyStatement, tbl.Name, TableFile(pub.dir, tbl.Name))
+		_, err = conn.Exec(ctx, stm)
+		test.NoError(t, err)
+	}
+	for i := uint32(0); i < pub.txCounter; i++ {
+		for _, tbl := range perNodeTables {
+			stm := fmt.Sprintf(pgCopyStatement, tbl.Name, TableFile(pub.txDir(i), tbl.Name))
+			_, err = conn.Exec(ctx, stm)
+			test.NoError(t, err)
+		}
+	}
+
+	// check header was successfully committed
+	pgQueryHeader := `SELECT cid, block_hash
+					  FROM eth.header_cids
+				      WHERE block_number = $1`
+	type res struct {
+		CID       string
+		BlockHash string
+	}
+	var header res
+	err = conn.QueryRow(ctx, pgQueryHeader, fixt.Block1_Header.Number.Uint64()).Scan(
+		&header.CID, &header.BlockHash)
+	test.NoError(t, err)
+
+	headerNode, err := ipld.NewEthHeader(&fixt.Block1_Header)
+	test.NoError(t, err)
+	test.ExpectEqual(t, headerNode.Cid().String(), header.CID)
+	test.ExpectEqual(t, fixt.Block1_Header.Hash().String(), header.BlockHash)
+}
@@ -1,64 +0,0 @@
-package mock
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/jmoiron/sqlx"
-
-	"github.com/golang/mock/gomock"
-
-	mocks "github.com/vulcanize/eth-pg-ipfs-state-snapshot/mocks/snapshot"
-	snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
-)
-
-type MockPublisher struct {
-	*mocks.MockPublisher
-}
-
-func NewMockPublisher(t *testing.T) *MockPublisher {
-	ctl := gomock.NewController(t)
-	return &MockPublisher{mocks.NewMockPublisher(ctl)}
-}
-
-func dump(funcname string, xs ...interface{}) {
-	if true {
-		return
-	}
-	fmt.Printf(">> %s", funcname)
-	fmt.Printf(strings.Repeat(" %+v", len(xs))+"\n", xs...)
-}
-
-func (p *MockPublisher) PublishHeader(header *types.Header) (int64, error) {
-	// fmt.Printf("PublishHeader %+v\n", header)
-	dump("PublishHeader", header)
-	return p.MockPublisher.PublishHeader(header)
-}
-func (p *MockPublisher) PublishStateNode(node *snapt.Node, headerID int64, tx *sqlx.Tx) (int64, error) {
-	dump("PublishStateNode", node, headerID)
-	return p.MockPublisher.PublishStateNode(node, headerID, tx)
-
-}
-func (p *MockPublisher) PublishStorageNode(node *snapt.Node, stateID int64, tx *sqlx.Tx) error {
-	dump("PublishStorageNode", node, stateID)
-	return p.MockPublisher.PublishStorageNode(node, stateID, tx)
-}
-func (p *MockPublisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error {
-	dump("PublishCode", codeHash, codeBytes)
-	return p.MockPublisher.PublishCode(codeHash, codeBytes, tx)
-}
-func (p *MockPublisher) BeginTx() (*sqlx.Tx, error) {
-	dump("BeginTx")
-	return p.MockPublisher.BeginTx()
-}
-func (p *MockPublisher) CommitTx(tx *sqlx.Tx) error {
-	dump("CommitTx", tx)
-	return p.MockPublisher.CommitTx(tx)
-}
-func (p *MockPublisher) PrepareTxForBatch(tx *sqlx.Tx, batchSize uint) (*sqlx.Tx, error) {
-	dump("PrepareTxForBatch", tx, batchSize)
-	return p.MockPublisher.PrepareTxForBatch(tx, batchSize)
-}
@@ -1,7 +1,10 @@
 package mock

-import "fmt"
-import "github.com/golang/mock/gomock"
+import (
+	"fmt"
+
+	"github.com/golang/mock/gomock"
+)

 type anyOfMatcher struct {
 	values []interface{}
@@ -0,0 +1,245 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package pg
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"sync/atomic"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	dshelp "github.com/ipfs/go-ipfs-ds-help"
+	"github.com/multiformats/go-multihash"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
+	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+)
+
+var _ snapt.Publisher = (*publisher)(nil)
+
+const logInterval = 1 * time.Minute
+
+// Publisher is wrapper around DB.
+type publisher struct {
+	db                 *postgres.DB
+	currBatchSize      uint
+	stateNodeCounter   uint64
+	storageNodeCounter uint64
+	codeNodeCounter    uint64
+	startTime          time.Time
+}
+
+// NewPublisher creates Publisher
+func NewPublisher(db *postgres.DB) *publisher {
+	return &publisher{
+		db:        db,
+		startTime: time.Now(),
+	}
+}
+
+type pubTx struct {
+	sql.Tx
+	callback func()
+}
+
+func (tx pubTx) Rollback() error { return tx.Tx.Rollback(context.Background()) }
+func (tx pubTx) Commit() error {
+	if tx.callback != nil {
+		defer tx.callback()
+	}
+	return tx.Tx.Commit(context.Background())
+}
+func (tx pubTx) Exec(sql string, args ...interface{}) (sql.Result, error) {
+	return tx.Tx.Exec(context.Background(), sql, args...)
+}
+
+func (p *publisher) BeginTx() (snapt.Tx, error) {
+	tx, err := p.db.Begin(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	go p.logNodeCounters()
+	return pubTx{tx, func() {
+		p.printNodeCounters("final stats")
+	}}, nil
+}
+
+// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
+// returns the CID and blockstore prefixed multihash key
+func (tx pubTx) publishRaw(codec uint64, raw []byte, height *big.Int) (cid, prefixedKey string, err error) {
+	c, err := ipld.RawdataToCid(codec, raw, multihash.KECCAK_256)
+	if err != nil {
+		return
+	}
+	cid = c.String()
+	prefixedKey, err = tx.publishIPLD(c, raw, height)
+	return
+}
+
+func (tx pubTx) publishIPLD(c cid.Cid, raw []byte, height *big.Int) (string, error) {
+	dbKey := dshelp.MultihashToDsKey(c.Hash())
+	prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
+	_, err := tx.Exec(snapt.TableIPLDBlock.ToInsertStatement(), height.Uint64(), prefixedKey, raw)
+	return prefixedKey, err
+}
+
+// PublishHeader writes the header to the ipfs backing pg datastore and adds secondary indexes in the header_cids table
+func (p *publisher) PublishHeader(header *types.Header) (err error) {
+	headerNode, err := ipld.NewEthHeader(header)
+	if err != nil {
+		return err
+	}
+
+	snapTx, err := p.db.Begin(context.Background())
+	if err != nil {
+		return err
+	}
+	tx := pubTx{snapTx, nil}
+	defer func() { err = snapt.CommitOrRollback(tx, err) }()
+
+	if _, err = tx.publishIPLD(headerNode.Cid(), headerNode.RawData(), header.Number); err != nil {
+		return err
+	}
+
+	mhKey := shared.MultihashKeyFromCID(headerNode.Cid())
+	_, err = tx.Exec(snapt.TableHeader.ToInsertStatement(), header.Number.Uint64(), header.Hash().Hex(),
+		header.ParentHash.Hex(), headerNode.Cid().String(), "0", p.db.NodeID(), "0",
+		header.Root.Hex(), header.TxHash.Hex(), header.ReceiptHash.Hex(), header.UncleHash.Hex(),
+		header.Bloom.Bytes(), header.Time, mhKey, 0, header.Coinbase.String())
+	return err
+}
+
+// PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table
+func (p *publisher) PublishStateNode(node *snapt.Node, headerID string, height *big.Int, snapTx snapt.Tx) error {
+	var stateKey string
+	if !snapt.IsNullHash(node.Key) {
+		stateKey = node.Key.Hex()
+	}
+
+	tx := snapTx.(pubTx)
+	stateCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStateTrie, node.Value, height)
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec(snapt.TableStateNode.ToInsertStatement(),
+		height.Uint64(), headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey)
+	if err != nil {
+		return err
+	}
+	// increment state node counter.
+	atomic.AddUint64(&p.stateNodeCounter, 1)
+	prom.IncStateNodeCount()
+
+	// increment current batch size counter
+	p.currBatchSize += 2
+	return err
+}
+
+// PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table
+func (p *publisher) PublishStorageNode(node *snapt.Node, headerID string, height *big.Int, statePath []byte, snapTx snapt.Tx) error {
+	var storageKey string
+	if !snapt.IsNullHash(node.Key) {
+		storageKey = node.Key.Hex()
+	}
+
+	tx := snapTx.(pubTx)
+	storageCIDStr, mhKey, err := tx.publishRaw(ipld.MEthStorageTrie, node.Value, height)
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec(snapt.TableStorageNode.ToInsertStatement(),
+		height.Uint64(), headerID, statePath, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey)
+	if err != nil {
+		return err
+	}
+	// increment storage node counter.
+	atomic.AddUint64(&p.storageNodeCounter, 1)
+	prom.IncStorageNodeCount()
+
+	// increment current batch size counter
+	p.currBatchSize += 2
+	return err
+}
+
+// PublishCode writes code to the ipfs backing pg datastore
+func (p *publisher) PublishCode(height *big.Int, codeHash common.Hash, codeBytes []byte, snapTx snapt.Tx) error {
+	// no codec for code, doesn't matter though since blockstore key is multihash-derived
+	mhKey, err := shared.MultihashKeyFromKeccak256(codeHash)
+	if err != nil {
+		return fmt.Errorf("error deriving multihash key from codehash: %v", err)
+	}
+
+	tx := snapTx.(pubTx)
+	if _, err = tx.Exec(snapt.TableIPLDBlock.ToInsertStatement(), height.Uint64(), mhKey, codeBytes); err != nil {
+		return fmt.Errorf("error publishing code IPLD: %v", err)
+	}
+
+	// increment code node counter.
+	atomic.AddUint64(&p.codeNodeCounter, 1)
+	prom.IncCodeNodeCount()
+
+	p.currBatchSize++
+	return nil
+}
+
+func (p *publisher) PrepareTxForBatch(tx snapt.Tx, maxBatchSize uint) (snapt.Tx, error) {
+	var err error
+	// maximum batch size reached, commit the current transaction and begin a new transaction.
+	if maxBatchSize <= p.currBatchSize {
+		if err = tx.Commit(); err != nil {
+			return nil, err
+		}
+
+		snapTx, err := p.db.Begin(context.Background())
+		tx = pubTx{Tx: snapTx}
+		if err != nil {
+			return nil, err
+		}
+
+		p.currBatchSize = 0
+	}
+
+	return tx, nil
+}
+
+// logNodeCounters periodically logs the number of node processed.
+func (p *publisher) logNodeCounters() {
+	t := time.NewTicker(logInterval)
+	for range t.C {
+		p.printNodeCounters("progress")
+	}
+}
+
+func (p *publisher) printNodeCounters(msg string) {
+	log.WithFields(log.Fields{
+		"runtime":       time.Now().Sub(p.startTime).String(),
+		"state nodes":   atomic.LoadUint64(&p.stateNodeCounter),
+		"storage nodes": atomic.LoadUint64(&p.storageNodeCounter),
+		"code nodes":    atomic.LoadUint64(&p.codeNodeCounter),
+	}).Info(msg)
+}
@@ -0,0 +1,79 @@
+package pg
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/jackc/pgx/v4"
+
+	fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
+	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+	"github.com/vulcanize/ipld-eth-state-snapshot/test"
+)
+
+var (
+	pgConfig = test.DefaultPgConfig
+	nodeInfo = test.DefaultNodeInfo
+	// tables ordered according to fkey depedencies
+	allTables = []*snapt.Table{
+		&snapt.TableIPLDBlock,
+		&snapt.TableNodeInfo,
+		&snapt.TableHeader,
+		&snapt.TableStateNode,
+		&snapt.TableStorageNode,
+	}
+)
+
+func writeData(t *testing.T) *publisher {
+	driver, err := postgres.NewPGXDriver(context.Background(), pgConfig, nodeInfo)
+	test.NoError(t, err)
+	pub := NewPublisher(postgres.NewPostgresDB(driver))
+	test.NoError(t, pub.PublishHeader(&fixt.Block1_Header))
+	tx, err := pub.BeginTx()
+	test.NoError(t, err)
+
+	headerID := fixt.Block1_Header.Hash().String()
+	test.NoError(t, pub.PublishStateNode(&fixt.Block1_StateNode0, headerID, fixt.Block1_Header.Number, tx))
+
+	test.NoError(t, tx.Commit())
+	return pub
+}
+
+// Note: DB user requires role membership "pg_read_server_files"
+func TestBasic(t *testing.T) {
+	test.NeedsDB(t)
+
+	ctx := context.Background()
+	conn, err := pgx.Connect(ctx, pgConfig.DbConnectionString())
+	test.NoError(t, err)
+
+	// clear existing test data
+	pgDeleteTable := `DELETE FROM %s`
+	for _, tbl := range allTables {
+		_, err = conn.Exec(ctx, fmt.Sprintf(pgDeleteTable, tbl.Name))
+		test.NoError(t, err)
+	}
+
+	_ = writeData(t)
+
+	// check header was successfully committed
+	pgQueryHeader := `SELECT cid, block_hash
+					  FROM eth.header_cids
+				      WHERE block_number = $1`
+	type res struct {
+		CID       string
+		BlockHash string
+	}
+	var header res
+	err = conn.QueryRow(ctx, pgQueryHeader, fixt.Block1_Header.Number.Uint64()).Scan(
+		&header.CID, &header.BlockHash)
+	test.NoError(t, err)
+
+	headerNode, err := ipld.NewEthHeader(&fixt.Block1_Header)
+	test.NoError(t, err)
+	test.ExpectEqual(t, headerNode.Cid().String(), header.CID)
+	test.ExpectEqual(t, fixt.Block1_Header.Hash().String(), header.BlockHash)
+}
@@ -1,215 +0,0 @@
-// Copyright © 2020 Vulcanize, Inc
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-package snapshot
-
-import (
-	"bytes"
-	"fmt"
-	"sync/atomic"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/jmoiron/sqlx"
-	"github.com/multiformats/go-multihash"
-	"github.com/sirupsen/logrus"
-
-	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
-	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
-	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
-	snapt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
-)
-
-const logInterval = 1 * time.Minute
-
-// Publisher is wrapper around DB.
-type publisher struct {
-	db                 *postgres.DB
-	currBatchSize      uint
-	stateNodeCounter   uint64
-	storageNodeCounter uint64
-	codeNodeCounter    uint64
-	startTime          time.Time
-}
-
-// NewPublisher creates Publisher
-func NewPublisher(db *postgres.DB) *publisher {
-	return &publisher{
-		db:            db,
-		currBatchSize: 0,
-		startTime:     time.Now(),
-	}
-}
-
-func (p *publisher) BeginTx() (*sqlx.Tx, error) {
-	tx, err := p.db.Beginx()
-	if err != nil {
-		return nil, err
-	}
-	go p.logNodeCounters()
-	return tx, nil
-}
-
-func (p *publisher) CommitTx(tx *sqlx.Tx) error {
-	logrus.Info("----- final counts -----")
-	p.printNodeCounters()
-	return tx.Commit()
-}
-
-// PublishHeader writes the header to the ipfs backing pg datastore and adds secondary indexes in the header_cids table
-func (p *publisher) PublishHeader(header *types.Header) (int64, error) {
-	headerNode, err := ipld.NewEthHeader(header)
-	if err != nil {
-		return 0, err
-	}
-
-	tx, err := p.db.Beginx()
-	if err != nil {
-		return 0, err
-	}
-
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	if err = shared.PublishIPLD(tx, headerNode); err != nil {
-		return 0, err
-	}
-
-	mhKey, _ := shared.MultihashKeyFromCIDString(headerNode.Cid().String())
-	var headerID int64
-	err = tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated)
- 								VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
- 								ON CONFLICT (block_number, block_hash) DO UPDATE SET block_number = header_cids.block_number
- 								RETURNING id`,
-		header.Number.Uint64(), header.Hash().Hex(), header.ParentHash.Hex(), headerNode.Cid().String(), "0", p.db.NodeID, "0", header.Root.Hex(), header.TxHash.Hex(),
-		header.ReceiptHash.Hex(), header.UncleHash.Hex(), header.Bloom.Bytes(), header.Time, mhKey, 0).Scan(&headerID)
-
-	return headerID, err
-}
-
-// PublishStateNode writes the state node to the ipfs backing datastore and adds secondary indexes in the state_cids table
-func (p *publisher) PublishStateNode(node *snapt.Node, headerID int64, tx *sqlx.Tx) (int64, error) {
-	var stateID int64
-	var stateKey string
-	if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) {
-		stateKey = node.Key.Hex()
-	}
-
-	stateCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, node.Value)
-	if err != nil {
-		return 0, err
-	}
-
-	err = tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
- 									ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)
- 									RETURNING id`,
-		headerID, stateKey, stateCIDStr, node.Path, node.NodeType, false, mhKey).Scan(&stateID)
-
-	// increment state node counter.
-	atomic.AddUint64(&p.stateNodeCounter, 1)
-
-	// increment current batch size counter
-	p.currBatchSize += 2
-	return stateID, err
-}
-
-// PublishStorageNode writes the storage node to the ipfs backing pg datastore and adds secondary indexes in the storage_cids table
-func (p *publisher) PublishStorageNode(node *snapt.Node, stateID int64, tx *sqlx.Tx) error {
-	var storageKey string
-	if !bytes.Equal(node.Key.Bytes(), nullHash.Bytes()) {
-		storageKey = node.Key.Hex()
-	}
-
-	storageCIDStr, mhKey, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, node.Value)
-	if err != nil {
-		return err
-	}
-
-	_, err = tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
-                              	ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`,
-		stateID, storageKey, storageCIDStr, node.Path, node.NodeType, false, mhKey)
-	if err != nil {
-		return err
-	}
-
-	// increment storage node counter.
-	atomic.AddUint64(&p.storageNodeCounter, 1)
-
-	// increment current batch size counter
-	p.currBatchSize += 2
-	return nil
-}
-
-// PublishCode writes code to the ipfs backing pg datastore
-func (p *publisher) PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error {
-	// no codec for code, doesn't matter though since blockstore key is multihash-derived
-	mhKey, err := shared.MultihashKeyFromKeccak256(codeHash)
-	if err != nil {
-		return fmt.Errorf("error deriving multihash key from codehash: %v", err)
-	}
-
-	if err = shared.PublishDirect(tx, mhKey, codeBytes); err != nil {
-		return fmt.Errorf("error publishing code IPLD: %v", err)
-	}
-
-	// increment code node counter.
-	atomic.AddUint64(&p.codeNodeCounter, 1)
-
-	p.currBatchSize++
-	return nil
-}
-
-func (p *publisher) PrepareTxForBatch(tx *sqlx.Tx, maxBatchSize uint) (*sqlx.Tx, error) {
-	var err error
-	// maximum batch size reached, commit the current transaction and begin a new transaction.
-	if maxBatchSize <= p.currBatchSize {
-		if err = tx.Commit(); err != nil {
-			return nil, err
-		}
-
-		tx, err = p.db.Beginx()
-		if err != nil {
-			return nil, err
-		}
-
-		p.currBatchSize = 0
-	}
-
-	return tx, nil
-}
-
-// logNodeCounters periodically logs the number of node processed.
-func (p *publisher) logNodeCounters() {
-	t := time.NewTicker(logInterval)
-	for range t.C {
-		p.printNodeCounters()
-	}
-}
-
-func (p *publisher) printNodeCounters() {
-	logrus.Infof("runtime: %s", time.Now().Sub(p.startTime).String())
-	logrus.Infof("processed state nodes: %d", atomic.LoadUint64(&p.stateNodeCounter))
-	logrus.Infof("processed storage nodes: %d", atomic.LoadUint64(&p.storageNodeCounter))
-	logrus.Infof("processed code nodes: %d", atomic.LoadUint64(&p.codeNodeCounter))
-}
@@ -19,6 +19,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"math/big"
 	"sync"

 	"github.com/ethereum/go-ethereum/common"
@@ -28,19 +29,15 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
-	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/jmoiron/sqlx"
-	"github.com/sirupsen/logrus"
+	log "github.com/sirupsen/logrus"

-	. "github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/types"
 	iter "github.com/vulcanize/go-eth-state-node-iterator"
+	. "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
 )

 var (
-	nullHash          = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
-	emptyNode, _      = rlp.EncodeToBytes([]byte{})
+	emptyNode, _      = rlp.EncodeToBytes(&[]byte{})
 	emptyCodeHash     = crypto.Keccak256([]byte{})
 	emptyContractRoot = crypto.Keccak256Hash(emptyNode)

@@ -54,29 +51,28 @@ type Service struct {
 	stateDB       state.Database
 	ipfsPublisher Publisher
 	maxBatchSize  uint
-}
-
-func NewPostgresDB(con *DBConfig) (*postgres.DB, error) {
-	pgDB, err := postgres.NewDB(con.URI, con.ConnConfig, con.Node)
-	if err != nil {
-		return nil, err
-	}
-	return pgDB, nil
+	tracker       iteratorTracker
+	recoveryFile  string
 }

 func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
-	return rawdb.NewLevelDBDatabaseWithFreezer(
-		con.LevelDBPath, 1024, 256, con.AncientDBPath, "eth-pg-ipfs-state-snapshot", false,
+	edb, err := rawdb.NewLevelDBDatabaseWithFreezer(
+		con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true,
 	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create NewLevelDBDatabaseWithFreezer: %s", err)
+	}
+	return edb, nil
 }

 // NewSnapshotService creates Service.
-func NewSnapshotService(edb ethdb.Database, pub Publisher) (*Service, error) {
+func NewSnapshotService(edb ethdb.Database, pub Publisher, recoveryFile string) (*Service, error) {
 	return &Service{
 		ethDB:         edb,
 		stateDB:       state.NewDatabase(edb),
 		ipfsPublisher: pub,
 		maxBatchSize:  defaultBatchSize,
+		recoveryFile:  recoveryFile,
 	}, nil
 }

@@ -88,35 +84,70 @@ type SnapshotParams struct {
 func (s *Service) CreateSnapshot(params SnapshotParams) error {
 	// extract header from lvldb and publish to PG-IPFS
 	// hold onto the headerID so that we can link the state nodes to this header
-	logrus.Infof("Creating snapshot at height %d", params.Height)
+	log.Infof("Creating snapshot at height %d", params.Height)
 	hash := rawdb.ReadCanonicalHash(s.ethDB, params.Height)
 	header := rawdb.ReadHeader(s.ethDB, hash, params.Height)
 	if header == nil {
 		return fmt.Errorf("unable to read canonical header at height %d", params.Height)
 	}

-	logrus.Infof("head hash: %s head height: %d", hash.Hex(), params.Height)
+	log.Infof("head hash: %s head height: %d", hash.Hex(), params.Height)

-	headerID, err := s.ipfsPublisher.PublishHeader(header)
+	err := s.ipfsPublisher.PublishHeader(header)
 	if err != nil {
 		return err
 	}

-	t, err := s.stateDB.OpenTrie(header.Root)
+	tree, err := s.stateDB.OpenTrie(header.Root)
 	if err != nil {
 		return err
 	}
-	if params.Workers > 0 {
-		return s.createSnapshotAsync(t, headerID, params.Workers)
+	headerID := header.Hash().String()
+	s.tracker = newTracker(s.recoveryFile, int(params.Workers))
+	go s.tracker.run()
+	go s.tracker.captureSignal()
+
+	var iters []trie.NodeIterator
+	// attempt to restore from recovery file if it exists
+	iters, err = s.tracker.restore(tree)
+	if err != nil {
+		return err
+	}
+	if iters != nil {
+		if params.Workers < uint(len(iters)) {
+			return fmt.Errorf(
+				"number of recovered workers (%d) is greater than number configured (%d)",
+				len(iters), params.Workers,
+			)
+		}
+	} else { // nothing to restore
+		if params.Workers > 1 {
+			iters = iter.SubtrieIterators(tree, params.Workers)
+		} else {
+			iters = []trie.NodeIterator{tree.NodeIterator(nil)}
+		}
+		for i, it := range iters {
+			iters[i] = s.tracker.tracked(it)
+		}
+	}
+
+	defer func() {
+		err := s.tracker.haltAndDump()
+		if err != nil {
+			log.Error("failed to write recovery file: ", err)
+		}
+	}()
+
+	if len(iters) > 0 {
+		return s.createSnapshotAsync(iters, headerID, new(big.Int).SetUint64(params.Height))
 	} else {
-		return s.createSnapshot(t.NodeIterator(nil), headerID)
+		return s.createSnapshot(iters[0], headerID, new(big.Int).SetUint64(params.Height))
 	}
-	return nil
 }

 // Create snapshot up to head (ignores height param)
 func (s *Service) CreateLatestSnapshot(workers uint) error {
-	logrus.Info("Creating snapshot at head")
+	log.Info("Creating snapshot at head")
 	hash := rawdb.ReadHeadHeaderHash(s.ethDB)
 	height := rawdb.ReadHeaderNumber(s.ethDB, hash)
 	if height == nil {
@@ -135,7 +166,7 @@ func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (*nodeResult, erro
 	if it.Leaf() {
 		return nil, nil
 	}
-	if bytes.Equal(nullHash.Bytes(), it.Hash().Bytes()) {
+	if IsNullHash(it.Hash()) {
 		return nil, nil
 	}

@@ -163,22 +194,12 @@ func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (*nodeResult, erro
 	}, nil
 }

-func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error {
+func (s *Service) createSnapshot(it trie.NodeIterator, headerID string, height *big.Int) error {
 	tx, err := s.ipfsPublisher.BeginTx()
 	if err != nil {
 		return err
 	}
-
-	defer func() {
-		if rec := recover(); rec != nil {
-			shared.Rollback(tx)
-			panic(rec)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = s.ipfsPublisher.CommitTx(tx)
-		}
-	}()
+	defer func() { err = CommitOrRollback(tx, err) }()

 	for it.Next(true) {
 		res, err := resolveNode(it, s.stateDB.TrieDB())
@@ -196,7 +217,8 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error {

 		switch res.node.NodeType {
 		case Leaf:
-			// if the node is a leaf, decode the account and publish the associated storage trie nodes if there are any
+			// if the node is a leaf, decode the account and publish the associated storage trie
+			// nodes if there are any
 			var account types.StateAccount
 			if err := rlp.DecodeBytes(res.elements[1].([]byte), &account); err != nil {
 				return fmt.Errorf(
@@ -207,7 +229,7 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error {
 			encodedPath := trie.HexToCompact(valueNodePath)
 			leafKey := encodedPath[1:]
 			res.node.Key = common.BytesToHash(leafKey)
-			stateID, err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx)
+			err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, height, tx)
 			if err != nil {
 				return err
 			}
@@ -217,57 +239,60 @@ func (s *Service) createSnapshot(it trie.NodeIterator, headerID int64) error {
 				codeHash := common.BytesToHash(account.CodeHash)
 				codeBytes := rawdb.ReadCode(s.ethDB, codeHash)
 				if len(codeBytes) == 0 {
-					logrus.Error("Code is missing", "account", common.BytesToHash(it.LeafKey()))
+					log.Error("Code is missing", "account", common.BytesToHash(it.LeafKey()))
 					return errors.New("missing code")
 				}

-				if err = s.ipfsPublisher.PublishCode(codeHash, codeBytes, tx); err != nil {
+				if err = s.ipfsPublisher.PublishCode(height, codeHash, codeBytes, tx); err != nil {
 					return err
 				}
 			}

-			if tx, err = s.storageSnapshot(account.Root, stateID, tx); err != nil {
+			if tx, err = s.storageSnapshot(account.Root, headerID, height, res.node.Path, tx); err != nil {
 				return fmt.Errorf("failed building storage snapshot for account %+v\r\nerror: %w", account, err)
 			}
 		case Extension, Branch:
 			res.node.Key = common.BytesToHash([]byte{})
-			if _, err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, tx); err != nil {
+			if err := s.ipfsPublisher.PublishStateNode(&res.node, headerID, height, tx); err != nil {
 				return err
 			}
 		default:
 			return errors.New("unexpected node type")
 		}
-		return nil
 	}
 	return it.Error()
 }

 // Full-trie concurrent snapshot
-func (s *Service) createSnapshotAsync(tree state.Trie, headerID int64, workers uint) error {
+func (s *Service) createSnapshotAsync(iters []trie.NodeIterator, headerID string, height *big.Int) error {
 	errors := make(chan error)
 	var wg sync.WaitGroup
-	for _, it := range iter.SubtrieIterators(tree, workers) {
+	for _, it := range iters {
 		wg.Add(1)
-		go func() {
+		go func(it trie.NodeIterator) {
 			defer wg.Done()
-			if err := s.createSnapshot(it, headerID); err != nil {
+			if err := s.createSnapshot(it, headerID, height); err != nil {
 				errors <- err
 			}
-		}()
+		}(it)
 	}
+
+	done := make(chan struct{})
 	go func() {
-		defer close(errors)
 		wg.Wait()
+		done <- struct{}{}
 	}()

+	var err error
 	select {
-	case err := <-errors:
-		return err
+	case err = <-errors:
+	case <-done:
+		close(errors)
 	}
-	return nil
+	return err
 }

-func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (*sqlx.Tx, error) {
+func (s *Service) storageSnapshot(sr common.Hash, headerID string, height *big.Int, statePath []byte, tx Tx) (Tx, error) {
 	if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
 		return tx, nil
 	}
@@ -311,7 +336,7 @@ func (s *Service) storageSnapshot(sr common.Hash, stateID int64, tx *sqlx.Tx) (*
 		default:
 			return nil, errors.New("unexpected node type")
 		}
-		if err = s.ipfsPublisher.PublishStorageNode(&res.node, stateID, tx); err != nil {
+		if err = s.ipfsPublisher.PublishStorageNode(&res.node, headerID, height, statePath, tx); err != nil {
 			return nil, err
 		}
 	}
@@ -1,99 +1,141 @@
 package snapshot

 import (
-	"fmt"
+	"errors"
+	"math/big"
 	"os"
 	"path/filepath"
 	"testing"

 	"github.com/golang/mock/gomock"

-	ethNode "github.com/ethereum/go-ethereum/statediff/indexer/node"
-	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
-
-	fixt "github.com/vulcanize/eth-pg-ipfs-state-snapshot/fixture"
-	"github.com/vulcanize/eth-pg-ipfs-state-snapshot/pkg/snapshot/mock"
+	fixt "github.com/vulcanize/ipld-eth-state-snapshot/fixture"
+	mock "github.com/vulcanize/ipld-eth-state-snapshot/mocks/snapshot"
+	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+	"github.com/vulcanize/ipld-eth-state-snapshot/test"
 )

 func testConfig(leveldbpath, ancientdbpath string) *Config {
-	dbParams := postgres.ConnectionParams{
-		Name:     "snapshot_test",
-		Hostname: "localhost",
-		Port:     5432,
-		User:     "tester",
-		Password: "test_pw",
-	}
-	connconfig := postgres.ConnectionConfig{
-		MaxIdle:     0,
-		MaxLifetime: 0,
-		MaxOpen:     4,
-	}
-	nodeinfo := ethNode.Info{
-		ID:           "eth_node_id",
-		ClientName:   "eth_client",
-		GenesisBlock: "X",
-		NetworkID:    "eth_network",
-		ChainID:      0,
-	}
-
 	return &Config{
-		DB: &DBConfig{
-			Node:       nodeinfo,
-			URI:        postgres.DbConnectionString(dbParams),
-			ConnConfig: connconfig,
-		},
 		Eth: &EthConfig{
 			LevelDBPath:   leveldbpath,
 			AncientDBPath: ancientdbpath,
+			NodeInfo:      test.DefaultNodeInfo,
+		},
+		DB: &DBConfig{
+			URI:        test.DefaultPgConfig.DbConnectionString(),
+			ConnConfig: test.DefaultPgConfig,
 		},
 	}
 }

+func makeMocks(t *testing.T) (*mock.MockPublisher, *mock.MockTx) {
+	ctl := gomock.NewController(t)
+	pub := mock.NewMockPublisher(ctl)
+	tx := mock.NewMockTx(ctl)
+	return pub, tx
+}
+
 func TestCreateSnapshot(t *testing.T) {
-	wd, err := os.Getwd()
-	if err != nil {
-		panic(err)
+	runCase := func(t *testing.T, workers int) {
+		pub, tx := makeMocks(t)
+		pub.EXPECT().PublishHeader(gomock.Eq(&fixt.Block1_Header))
+		pub.EXPECT().BeginTx().Return(tx, nil).
+			Times(workers)
+		pub.EXPECT().PrepareTxForBatch(gomock.Any(), gomock.Any()).Return(tx, nil).
+			AnyTimes()
+		pub.EXPECT().PublishStateNode(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+			Times(len(fixt.Block1_StateNodePaths))
+
+		// TODO: fixtures for storage node
+		// pub.EXPECT().PublishStorageNode(gomock.Eq(fixt.StorageNode), gomock.Eq(int64(0)), gomock.Any())
+
+		tx.EXPECT().Commit().
+			Times(workers)
+
+		config := testConfig(fixt.ChaindataPath, fixt.AncientdataPath)
+		edb, err := NewLevelDB(config.Eth)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer edb.Close()
+
+		recovery := filepath.Join(t.TempDir(), "recover.csv")
+		service, err := NewSnapshotService(edb, pub, recovery)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		params := SnapshotParams{Height: 1, Workers: uint(workers)}
+		err = service.CreateSnapshot(params)
+		if err != nil {
+			t.Fatal(err)
+		}
 	}
-	fixture_path := filepath.Join(wd, "..", "..", "fixture")
-	datadir := filepath.Join(fixture_path, "chaindata")
-	if _, err := os.Stat(datadir); err != nil {
-		t.Fatal("no chaindata:", err)
+
+	testCases := []int{1, 4, 16, 32}
+	for _, tc := range testCases {
+		t.Run("case", func(t *testing.T) { runCase(t, tc) })
 	}
-	config := testConfig(datadir, filepath.Join(datadir, "ancient"))
-	fmt.Printf("config: %+v %+v\n", config.DB, config.Eth)
+}
+
+func failingPublishStateNode(_ *snapt.Node, _ string, _ *big.Int, _ snapt.Tx) error {
+	return errors.New("failingPublishStateNode")
+}

-	edb, err := NewLevelDB(config.Eth)
-	if err != nil {
-		t.Fatal(err)
-	}
-	workers := 8
-
-	pub := mock.NewMockPublisher(t)
-	pub.EXPECT().PublishHeader(gomock.Eq(fixt.PublishHeader))
-	pub.EXPECT().BeginTx().
-		Times(workers)
-	pub.EXPECT().PrepareTxForBatch(gomock.Any(), gomock.Any()).
-		Times(workers)
-	pub.EXPECT().PublishStateNode(gomock.Any(), mock.AnyOf(int64(0), int64(1)), gomock.Any()).
-		Times(workers)
-	// TODO: fixtures for storage node
-	// pub.EXPECT().PublishStorageNode(gomock.Eq(fixt.StorageNode), gomock.Eq(int64(0)), gomock.Any())
-	pub.EXPECT().CommitTx(gomock.Any()).
-		Times(workers)
-
-	service, err := NewSnapshotService(edb, pub)
-	if err != nil {
-		t.Fatal(err)
+func TestRecovery(t *testing.T) {
+	runCase := func(t *testing.T, workers int) {
+		pub, tx := makeMocks(t)
+		pub.EXPECT().PublishHeader(gomock.Any()).AnyTimes()
+		pub.EXPECT().BeginTx().Return(tx, nil).AnyTimes()
+		pub.EXPECT().PrepareTxForBatch(gomock.Any(), gomock.Any()).Return(tx, nil).AnyTimes()
+		pub.EXPECT().PublishStateNode(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+			Times(workers).
+			DoAndReturn(failingPublishStateNode)
+		tx.EXPECT().Commit().AnyTimes()
+
+		config := testConfig(fixt.ChaindataPath, fixt.AncientdataPath)
+		edb, err := NewLevelDB(config.Eth)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer edb.Close()
+
+		recovery := filepath.Join(t.TempDir(), "recover.csv")
+		service, err := NewSnapshotService(edb, pub, recovery)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		params := SnapshotParams{Height: 1, Workers: uint(workers)}
+		err = service.CreateSnapshot(params)
+		if err == nil {
+			t.Fatal("expected an error")
+		}
+
+		if _, err = os.Stat(recovery); err != nil {
+			t.Fatal("cannot stat recovery file:", err)
+		}
+
+		pub.EXPECT().PublishStateNode(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
+		err = service.CreateSnapshot(params)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = os.Stat(recovery)
+		if err == nil {
+			t.Fatal("recovery file still present")
+		} else {
+			if !os.IsNotExist(err) {
+				t.Fatal(err)
+			}
+		}
 	}

-	params := SnapshotParams{Height: 1, Workers: uint(workers)}
-	err = service.CreateSnapshot(params)
-	if err != nil {
-		t.Fatal(err)
+	testCases := []int{1, 4, 32}
+	for _, tc := range testCases {
+		t.Run("case", func(t *testing.T) { runCase(t, tc) })
 	}

-	// err = service.CreateLatestSnapshot(0)
-	// if err != nil {
-	// 	t.Fatal(err)
-	// }
 }
@@ -0,0 +1,173 @@
+package snapshot
+
+import (
+	"encoding/csv"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/trie"
+	log "github.com/sirupsen/logrus"
+
+	iter "github.com/vulcanize/go-eth-state-node-iterator"
+)
+
+type trackedIter struct {
+	trie.NodeIterator
+	tracker *iteratorTracker
+}
+
+func (it *trackedIter) Next(descend bool) bool {
+	ret := it.NodeIterator.Next(descend)
+	if !ret {
+		it.tracker.stopChan <- it
+	}
+	return ret
+}
+
+type iteratorTracker struct {
+	recoveryFile string
+
+	startChan chan *trackedIter
+	stopChan  chan *trackedIter
+	started   map[*trackedIter]struct{}
+	stopped   []*trackedIter
+
+	haltChan chan struct{}
+	done     chan struct{}
+}
+
+func newTracker(file string, buf int) iteratorTracker {
+	return iteratorTracker{
+		recoveryFile: file,
+		startChan:    make(chan *trackedIter, buf),
+		stopChan:     make(chan *trackedIter, buf),
+		started:      map[*trackedIter]struct{}{},
+		haltChan:     make(chan struct{}),
+		done:         make(chan struct{}),
+	}
+}
+
+func (tr *iteratorTracker) captureSignal() {
+	sigChan := make(chan os.Signal, 1)
+
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		sig := <-sigChan
+		log.Errorf("Signal received (%v), stopping", sig)
+		tr.haltAndDump()
+		os.Exit(1)
+	}()
+}
+
+// listens for starts/stops and manages current state
+func (tr *iteratorTracker) run() {
+loop:
+	for {
+		select {
+		case start := <-tr.startChan:
+			tr.started[start] = struct{}{}
+		case stop := <-tr.stopChan:
+			tr.stopped = append(tr.stopped, stop)
+		case <-tr.haltChan:
+			break loop
+		default:
+		}
+	}
+	tr.done <- struct{}{}
+}
+
+func (tr *iteratorTracker) tracked(it trie.NodeIterator) (ret *trackedIter) {
+	ret = &trackedIter{it, tr}
+	tr.startChan <- ret
+	return
+}
+
+// dumps iterator path and bounds to a text file so it can be restored later
+func (tr *iteratorTracker) dump() error {
+	log.Info("Dumping recovery state to: ", tr.recoveryFile)
+	var rows [][]string
+	for it, _ := range tr.started {
+		var endPath []byte
+		if impl, ok := it.NodeIterator.(*iter.PrefixBoundIterator); ok {
+			endPath = impl.EndPath
+		}
+		rows = append(rows, []string{
+			fmt.Sprintf("%x", it.Path()),
+			fmt.Sprintf("%x", endPath),
+		})
+	}
+	file, err := os.Create(tr.recoveryFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	out := csv.NewWriter(file)
+	return out.WriteAll(rows)
+}
+
+// attempts to read iterator state from file
+// if file doesn't exist, returns an empty slice with no error
+func (tr *iteratorTracker) restore(tree state.Trie) ([]trie.NodeIterator, error) {
+	file, err := os.Open(tr.recoveryFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer file.Close()
+	in := csv.NewReader(file)
+	in.FieldsPerRecord = 2
+	rows, err := in.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+	var ret []trie.NodeIterator
+	for _, row := range rows {
+		// pick up where each interval left off
+		var paths [2][]byte
+		for i, val := range row {
+			if len(val) != 0 {
+				if _, err = fmt.Sscanf(val, "%x", &paths[i]); err != nil {
+					return nil, err
+				}
+			}
+		}
+
+		it := iter.NewPrefixBoundIterator(tree, paths[0], paths[1])
+		ret = append(ret, tr.tracked(it))
+	}
+	return ret, nil
+}
+
+func (tr *iteratorTracker) haltAndDump() error {
+	tr.haltChan <- struct{}{}
+	<-tr.done
+
+	// drain any pending events
+	close(tr.startChan)
+	for start := range tr.startChan {
+		tr.started[start] = struct{}{}
+	}
+	close(tr.stopChan)
+	for stop := range tr.stopChan {
+		tr.stopped = append(tr.stopped, stop)
+	}
+
+	for _, stop := range tr.stopped {
+		delete(tr.started, stop)
+	}
+
+	if len(tr.started) == 0 {
+		// if the tracker state is empty, erase any existing recovery file
+		err := os.Remove(tr.recoveryFile)
+		if os.IsNotExist(err) {
+			err = nil
+		}
+		return err
+	}
+	return tr.dump()
+}
@@ -0,0 +1,30 @@
+package snapshot
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+
+	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
+	file "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/file"
+	pg "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg"
+	snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
+)
+
+func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {
+	switch mode {
+	case PgSnapshot:
+		driver, err := postgres.NewPGXDriver(context.Background(), config.DB.ConnConfig, config.Eth.NodeInfo)
+		if err != nil {
+			return nil, err
+		}
+
+		prom.RegisterDBCollector(config.DB.ConnConfig.DatabaseName, driver)
+
+		return pg.NewPublisher(postgres.NewPostgresDB(driver)), nil
+	case FileSnapshot:
+		return file.NewPublisher(config.File.OutputDir, config.Eth.NodeInfo)
+	}
+	return nil, fmt.Errorf("invalid snapshot mode: %s", mode)
+}
@@ -22,13 +22,12 @@ import (
 )

 // node for holding trie node information
-type node struct {
+type Node struct {
 	NodeType nodeType
 	Path     []byte
 	Key      common.Hash
 	Value    []byte
 }
-type Node = node

 // nodeType for explicitly setting type of node
 type nodeType int
@@ -1,17 +1,22 @@
 package types

 import (
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/jmoiron/sqlx"
 )

 type Publisher interface {
-	PublishHeader(header *types.Header) (int64, error)
-	PublishStateNode(node *Node, headerID int64, tx *sqlx.Tx) (int64, error)
-	PublishStorageNode(node *Node, stateID int64, tx *sqlx.Tx) error
-	PublishCode(codeHash common.Hash, codeBytes []byte, tx *sqlx.Tx) error
-	BeginTx() (*sqlx.Tx, error)
-	CommitTx(*sqlx.Tx) error
-	PrepareTxForBatch(tx *sqlx.Tx, batchSize uint) (*sqlx.Tx, error)
+	PublishHeader(header *types.Header) error
+	PublishStateNode(node *Node, headerID string, height *big.Int, tx Tx) error
+	PublishStorageNode(node *Node, headerID string, height *big.Int, statePath []byte, tx Tx) error
+	PublishCode(height *big.Int, codeHash common.Hash, codeBytes []byte, tx Tx) error
+	BeginTx() (Tx, error)
+	PrepareTxForBatch(tx Tx, batchSize uint) (Tx, error)
+}
+
+type Tx interface {
+	Rollback() error
+	Commit() error
 }
@@ -0,0 +1,76 @@
+package types
+
+var TableIPLDBlock = Table{
+	`public.blocks`,
+	[]column{
+		{"block_number", bigint},
+		{"key", text},
+		{"data", bytea},
+	},
+	"ON CONFLICT (key, block_number) DO NOTHING",
+}
+
+var TableNodeInfo = Table{
+	Name: `public.nodes`,
+	Columns: []column{
+		{"genesis_block", varchar},
+		{"network_id", varchar},
+		{"node_id", varchar},
+		{"client_name", varchar},
+		{"chain_id", integer},
+	},
+}
+
+var TableHeader = Table{
+	"eth.header_cids",
+	[]column{
+		{"block_number", bigint},
+		{"block_hash", varchar},
+		{"parent_hash", varchar},
+		{"cid", text},
+		{"td", numeric},
+		{"node_id", varchar},
+		{"reward", numeric},
+		{"state_root", varchar},
+		{"tx_root", varchar},
+		{"receipt_root", varchar},
+		{"uncle_root", varchar},
+		{"bloom", bytea},
+		{"timestamp", numeric},
+		{"mh_key", text},
+		{"times_validated", integer},
+		{"coinbase", varchar},
+	},
+	"ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = (EXCLUDED.parent_hash, EXCLUDED.cid, EXCLUDED.td, EXCLUDED.node_id, EXCLUDED.reward, EXCLUDED.state_root, EXCLUDED.tx_root, EXCLUDED.receipt_root, EXCLUDED.uncle_root, EXCLUDED.bloom, EXCLUDED.timestamp, EXCLUDED.mh_key, eth.header_cids.times_validated + 1, EXCLUDED.coinbase)",
+}
+
+var TableStateNode = Table{
+	"eth.state_cids",
+	[]column{
+		{"block_number", bigint},
+		{"header_id", varchar},
+		{"state_leaf_key", varchar},
+		{"cid", text},
+		{"state_path", bytea},
+		{"node_type", integer},
+		{"diff", boolean},
+		{"mh_key", text},
+	},
+	"ON CONFLICT (header_id, state_path, block_number) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = (EXCLUDED.state_leaf_key, EXCLUDED.cid, EXCLUDED.node_type, EXCLUDED.diff, EXCLUDED.mh_key)",
+}
+
+var TableStorageNode = Table{
+	"eth.storage_cids",
+	[]column{
+		{"block_number", bigint},
+		{"header_id", varchar},
+		{"state_path", bytea},
+		{"storage_leaf_key", varchar},
+		{"cid", text},
+		{"storage_path", bytea},
+		{"node_type", integer},
+		{"diff", boolean},
+		{"mh_key", text},
+	},
+	"ON CONFLICT (header_id, state_path, storage_path, block_number) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = (EXCLUDED.storage_leaf_key, EXCLUDED.cid, EXCLUDED.node_type, EXCLUDED.diff, EXCLUDED.mh_key)",
+}
@@ -0,0 +1,79 @@
+package types
+
+import (
+	"fmt"
+	"strings"
+)
+
+type colType int
+
+const (
+	integer colType = iota
+	boolean
+	bigint
+	numeric
+	bytea
+	varchar
+	text
+)
+
+type column struct {
+	name string
+	typ  colType
+}
+type Table struct {
+	Name           string
+	Columns        []column
+	conflictClause string
+}
+
+func (tbl *Table) ToCsvRow(args ...interface{}) []string {
+	var row []string
+	for i, col := range tbl.Columns {
+		row = append(row, col.typ.formatter()(args[i]))
+	}
+	return row
+}
+
+func (tbl *Table) ToInsertStatement() string {
+	var colnames, placeholders []string
+	for i, col := range tbl.Columns {
+		colnames = append(colnames, col.name)
+		placeholders = append(placeholders, fmt.Sprintf("$%d", i+1))
+	}
+	return fmt.Sprintf(
+		"INSERT INTO %s (%s) VALUES (%s) %s",
+		tbl.Name, strings.Join(colnames, ", "), strings.Join(placeholders, ", "), tbl.conflictClause,
+	)
+}
+
+type colfmt = func(interface{}) string
+
+func sprintf(f string) colfmt {
+	return func(x interface{}) string { return fmt.Sprintf(f, x) }
+}
+
+func (typ colType) formatter() colfmt {
+	switch typ {
+	case integer:
+		return sprintf("%d")
+	case boolean:
+		return func(x interface{}) string {
+			if x.(bool) {
+				return "t"
+			}
+			return "f"
+		}
+	case bigint:
+		return sprintf("%s")
+	case numeric:
+		return sprintf("%d")
+	case bytea:
+		return sprintf(`\x%x`)
+	case varchar:
+		return sprintf("%s")
+	case text:
+		return sprintf("%s")
+	}
+	panic("unreachable")
+}
@@ -0,0 +1,33 @@
+package types
+
+import (
+	"bytes"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+var nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
+
+func IsNullHash(hash common.Hash) bool {
+	return bytes.Equal(hash.Bytes(), nullHash.Bytes())
+}
+
+func CommitOrRollback(tx Tx, err error) error {
+	var rberr error
+	defer func() {
+		if rberr != nil {
+			logrus.Errorf("rollback failed: %s", rberr)
+		}
+	}()
+	if rec := recover(); rec != nil {
+		rberr = tx.Rollback()
+		panic(rec)
+	} else if err != nil {
+		rberr = tx.Rollback()
+	} else {
+		err = tx.Commit()
+	}
+	return err
+}
@@ -0,0 +1,59 @@
+package test
+
+import (
+	"bytes"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	ethnode "github.com/ethereum/go-ethereum/statediff/indexer/node"
+)
+
+var (
+	DefaultNodeInfo = ethnode.Info{
+		ID:           "test_nodeid",
+		ClientName:   "test_client",
+		GenesisBlock: "TEST_GENESIS",
+		NetworkID:    "test_network",
+		ChainID:      0,
+	}
+	DefaultPgConfig = postgres.Config{
+		Hostname:     "localhost",
+		Port:         5432,
+		DatabaseName: "vulcanize_test",
+		Username:     "vulcanize",
+		Password:     "vulcanize_password",
+
+		MaxIdle:         0,
+		MaxConnLifetime: 0,
+		MaxConns:        4,
+	}
+)
+
+func NeedsDB(t *testing.T) {
+	t.Helper()
+	if os.Getenv("TEST_WITH_DB") == "" {
+		t.Skip("set TEST_WITH_DB to enable test")
+	}
+}
+
+func NoError(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// ExpectEqual asserts the provided interfaces are deep equal
+func ExpectEqual(t *testing.T, want, got interface{}) {
+	if !reflect.DeepEqual(want, got) {
+		t.Fatalf("Values not equal:\nExpected:\t%v\nActual:\t\t%v", want, got)
+	}
+}
+
+func ExpectEqualBytes(t *testing.T, want, got []byte) {
+	if !bytes.Equal(want, got) {
+		t.Fatalf("Bytes not equal:\nExpected:\t%v\nActual:\t\t%v", want, got)
+	}
+}
Author	SHA1	Message	Date
Abdul Rabbani	c9d398b5af	Merge pull request #40 from vulcanize/feature/update-go-geth-sharding Update go and geth versions	2022-06-01 09:22:04 -04:00
Abdul Rabbani	8aab6a5137	Update geth versions	2022-06-01 09:20:45 -04:00
Abdul Rabbani	cd2559e434	Merge branch 'master' into feature/update-go-geth-sharding	2022-06-01 09:19:18 -04:00
Abdul Rabbani	d213644ff4	Merge pull request #39 from vulcanize/feature/update-go-geth Updating Go and Geth Version	2022-06-01 09:10:35 -04:00
Abdul Rabbani	a04777cb78	Updating Go and Geth Version Ran Tests: ``` 09:07:50:~/GitHub/cerc/ipld-eth-state-snapshot go test ./... -v # github.com/vulcanize/ipld-eth-state-snapshot/cmd cmd/stateSnapshot.go:47:3: (*github.com/sirupsen/logrus.Entry).Fatal call has possible formatting directive %v ? github.com/vulcanize/ipld-eth-state-snapshot [no test files] === RUN TestCreateSnapshot === RUN TestCreateSnapshot/case time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" === RUN TestCreateSnapshot/case#01 time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" === RUN TestCreateSnapshot/case#02 time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" === RUN TestCreateSnapshot/case#03 time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" --- PASS: TestCreateSnapshot (0.20s) --- PASS: TestCreateSnapshot/case (0.15s) --- PASS: TestCreateSnapshot/case#01 (0.01s) --- PASS: TestCreateSnapshot/case#02 (0.02s) --- PASS: TestCreateSnapshot/case#03 (0.03s) === RUN TestRecovery === RUN TestRecovery/case time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" time="2022-06-01T09:08:03-04:00" level=info msg="Dumping recovery state to: /var/folders/xf/cg688d9j7f39ktmd_4ttwxqh0000gn/T/TestRecoverycase3574218035/001/recover.csv" time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" === RUN TestRecovery/case#01 time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" time="2022-06-01T09:08:03-04:00" level=info msg="Dumping recovery state to: /var/folders/xf/cg688d9j7f39ktmd_4ttwxqh0000gn/T/TestRecoverycase#011814952517/001/recover.csv" time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" === RUN TestRecovery/case#02 time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" time="2022-06-01T09:08:03-04:00" level=info msg="Dumping recovery state to: /var/folders/xf/cg688d9j7f39ktmd_4ttwxqh0000gn/T/TestRecoverycase#021650290358/001/recover.csv" time="2022-06-01T09:08:03-04:00" level=info msg="Creating snapshot at height 1" time="2022-06-01T09:08:03-04:00" level=info msg="head hash: 0xa7684ac44d48494670b2e0d9085b7750e7341620f0a271db146ed5e70c1db854 head height: 1" --- PASS: TestRecovery (0.40s) --- PASS: TestRecovery/case (0.12s) --- PASS: TestRecovery/case#01 (0.19s) --- PASS: TestRecovery/case#02 (0.09s) PASS ok github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot 1.109s === RUN TestWriting --- PASS: TestWriting (0.00s) === RUN TestPgCopy publisher_test.go:84: set TEST_WITH_DB to enable test --- SKIP: TestPgCopy (0.00s) PASS ok github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/file 0.279s === RUN TestBasic publisher_test.go:47: set TEST_WITH_DB to enable test --- SKIP: TestBasic (0.00s) PASS ok github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg 0.647s ```	2022-06-01 09:09:19 -04:00
Ashwin Phatak	c785c12274	Merge pull request #35 from vulcanize/ng-file-ethereum-nodeId Add ethereum config to fix db import in file mode	2022-05-25 18:38:09 +05:30
nabarun	23c4d22885	Add ethereum config to fix db import in file mode	2022-05-25 18:15:08 +05:30
Ashwin Phatak	d3a149899b	Merge pull request #34 from deep-stack/pm-merge-metrics Merge changes to add env bindings and prometheus metrics	2022-05-23 18:14:06 +05:30
prathamesh	dc27487b66	Merge changes to add env bindings and prometheus metrics	2022-05-23 17:11:51 +05:30
prathamesh0	cce45bda57	Add prometheus metrics collection (#33 ) * Upgrade geth * Add prometheus metrics collection * Update README	2022-05-23 16:56:48 +05:30
Ian Norden	751e064f8f	Merge pull request #28 from vulcanize/release-v3.0.2 add env bindings	2022-05-17 22:36:08 -05:00
i-norden	004f3e6665	constants for CLI binding strings	2022-05-17 22:33:20 -05:00
prathamesh0	0a1ea3ae4a	Fix insert statements and upgrade dependencies (#31 )	2022-05-17 19:19:52 +05:30
Ashwin Phatak	b3af4e9e75	Merge pull request #30 from deep-stack/pm-upgrade-geth Upgrade vulcanize-geth to v4	2022-05-16 14:05:28 +05:30
prathamesh	67ec456e88	Upgrade vulcanize-geth to v4	2022-05-16 13:48:26 +05:30
Ashwin Phatak	04b57ca186	Merge pull request #29 from deep-stack/pm-db-sharding Updates to use v4 schema	2022-05-13 16:06:09 +05:30
prathamesh	fc9e82f717	Index block number using string in file mode	2022-05-13 14:00:40 +05:30
prathamesh	44d48b8644	Change ON CONFLICT clauses to do nothing and upgrade geth	2022-05-13 09:28:36 +05:30
prathamesh	26ae3ab008	Updates to use v4 schema	2022-04-19 18:14:25 +05:30
i-norden	31105a009c	add env bindings	2022-04-04 14:35:18 -05:00
Ian Norden	560199f930	Merge pull request #25 from vulcanize/release-v3.0.1 fix minor bug	2022-03-31 10:34:36 -05:00
i-norden	060eb5cf67	add some logs and guards, update module name, update readme	2022-03-30 18:57:30 -05:00
i-norden	de39842049	fix minor bug	2022-03-30 14:09:32 -05:00
Ian Norden	9e768dc2ba	Merge pull request #22 from vulcanize/recovery Service state recovery	2022-03-17 07:09:25 -05:00
roysc	8d1e79d22e	handle sigterm	2022-03-16 18:48:51 +08:00
roysc	b732ffc5f2	fix race condition	2022-03-14 23:34:26 +08:00
roysc	2481d0a28d	recovery - handle signals	2022-03-14 23:34:26 +08:00
roysc	02488e2b79	logging tweaks	2022-03-14 23:32:00 +08:00
roysc	7a2ef4f542	fix fixtures	2022-03-14 23:32:00 +08:00
roysc	56acb77829	update iterator mod	2022-03-14 23:31:48 +08:00
roysc	7a31f6f007	add chaindata/.gitignore	2022-03-03 17:12:29 +08:00
roysc	b5d0010581	implement job recovery; fix traversal & iterator	2022-03-03 17:12:29 +08:00
Ian Norden	a99d0a5ad6	Merge pull request #21 from vulcanize/file-output Create CSV file publisher with updated schema	2022-03-01 06:35:10 -06:00
roysc	e0df3399e1	clanup	2022-02-17 14:35:20 +08:00
roysc	1a65d9e022	string, not ptr, as map index	2022-02-17 14:34:54 +08:00
roysc	ef4ae3a0d4	revisions	2022-02-16 19:27:02 +08:00
roysc	3fda647ee9	clean up	2022-02-16 11:13:51 +08:00
roysc	73cc0f191b	clean up - can rm type here	2022-02-15 18:41:31 +08:00
roysc	c4b4f232e9	create csv file publisher; update geth and schema	2022-02-14 07:22:31 +08:00