Merge pull request #1 from vulcanize/watch

ipfs chain watcher cleanup

.gitignore

@@ -7,7 +7,7 @@ Vagrantfile
 vagrant*.sh
 .vagrant
 test_scripts/
-vulcanizedb
+ipfs-chain-watcher
 postgraphile/build/
 postgraphile/node_modules/
 postgraphile/package-lock.json

.travis.yml

@@ -7,7 +7,7 @@ services:
 addons:
   ssh_known_hosts: arch1.vdb.to
   postgresql: '11.2'
-go_import_path: github.com/vulcanize/vulcanizedb
+go_import_path: github.com/vulcanize/ipfs-chain-watcher
 before_install:
 - openssl aes-256-cbc -K $encrypted_e1db309e8776_key -iv $encrypted_e1db309e8776_iv
   -in temp_rsa.enc -out temp_rsa -d

Dockerfile

@@ -2,9 +2,9 @@ FROM golang:alpine as builder
 RUN apk --update --no-cache add make git g++
 
 # Build statically linked vDB binary (wonky path because of Dep)
-RUN mkdir -p /go/src/github.com/vulcanize/vulcanizedb
+RUN mkdir -p /go/src/github.com/vulcanize/ipfs-chain-watcher
-ADD . /go/src/github.com/vulcanize/vulcanizedb
+ADD . /go/src/github.com/vulcanize/ipfs-chain-watcher
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+WORKDIR /go/src/github.com/vulcanize/ipfs-chain-watcher
 RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' .
 
 # Build migration tool
@@ -14,10 +14,10 @@ RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflag
 
 # Second stage
 FROM alpine
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb /app/vulcanizedb
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/vulcanizedb /app/vulcanizedb
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments/staging.toml /app/environments/
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/environments/staging.toml /app/environments/
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/startup_script.sh /app/
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/dockerfiles/startup_script.sh /app/
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations/* /app/
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/db/migrations/* /app/
 COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose /app/goose
 
 WORKDIR /app

README.md

@@ -1,7 +1,7 @@
 # Vulcanize DB
 
 [![Build Status](https://travis-ci.org/vulcanize/vulcanizedb.svg?branch=master)](https://travis-ci.org/vulcanize/vulcanizedb)
-[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/vulcanizedb)](https://goreportcard.com/report/github.com/vulcanize/vulcanizedb)
+[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipfs-chain-watcher)](https://goreportcard.com/report/github.com/vulcanize/ipfs-chain-watcher)
 
 > Vulcanize DB is a set of tools that make it easier for developers to write application-specific indexes and caches for dapps built on Ethereum.
 
@@ -44,11 +44,11 @@ data from VulcanizeDB's underlying Postgres database and making it accessible.
 ### Building the project
 Download the codebase to your local `GOPATH` via:
 
-`go get github.com/vulcanize/vulcanizedb`
+`go get github.com/vulcanize/ipfs-chain-watcher`
 
 Move to the project directory:
 
-`cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
+`cd $GOPATH/src/github.com/vulcanize/ipfs-chain-watcher`
 
 Be sure you have enabled Go Modules (`export GO111MODULE=on`), and build the executable with:
 
@@ -65,7 +65,7 @@ It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
 1. Install Postgres
 1. Create a superuser for yourself and make sure `psql --list` works without prompting for a password.
 1. `createdb vulcanize_public`
-1. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
+1. `cd $GOPATH/src/github.com/vulcanize/ipfs-chain-watcher`
 1.  Run the migrations: `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432`
     - There is an optional var `USER=username` if the database user is not the default user `postgres`
     - To rollback a single step: `make rollback NAME=vulcanize_public`

cmd/compose.go

@@ -1,193 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"errors"
-	"fmt"
-	"strconv"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
-)
-
-// composeCmd represents the compose command
-var composeCmd = &cobra.Command{
-	Use:   "compose",
-	Short: "Composes transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "exampleTransformerExporter"
-    save     = false
-    transformerNames = [
-        "transformer1",
-        "transformer2",
-        "transformer3",
-        "transformer4",
-    ]
-    [exporter.transformer1]
-        path = "path/to/transformer1"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer2]
-        path = "path/to/transformer2"
-        type = "eth_contract"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer3]
-        path = "path/to/transformer3"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer4]
-        path = "path/to/transformer4"
-        type = "eth_storage"
-        repository = "github.com/account2/repo2"
-        migrations = "to/db/migrations"
-        rank = "1"
-
-
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-
-This information is used to write and build a go plugin with a transformer 
-set composed from the transformer imports specified in the config file
-This plugin is loaded and the set of transformer initializers is exported
-from it and loaded into and executed over by the appropriate watcher. 
-
-The type of watcher that the transformer works with is specified using the 
-type variable for each transformer in the config. Currently there are watchers 
-of event data from an eth node (eth_event) and storage data from an eth node 
-(eth_storage), and a more generic interface for accepting contract_watcher pkg
-based transformers which can perform both event watching and public method 
-polling (eth_contract).
-
-Transformers of different types can be ran together in the same command using a 
-single config file or in separate command instances using different config files
-
-Specify config location when executing the command:
-./vulcanizedb compose --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		compose()
-	},
-}
-
-func compose() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Generate code to build the plugin according to the config file
-	logWithCommand.Info("generating plugin")
-	generator, err := p2.NewGenerator(genConfig, databaseConfig)
-	if err != nil {
-		logWithCommand.Debug("initializing plugin generator failed")
-		logWithCommand.Fatal(err)
-	}
-	err = generator.GenerateExporterPlugin()
-	if err != nil {
-		logWithCommand.Debug("generating plugin failed")
-		logWithCommand.Fatal(err)
-	}
-	// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		logWithCommand.Debug("getting plugin path failed")
-		logWithCommand.Fatal(err)
-	}
-	fmt.Printf("Composed plugin %s", pluginPath)
-	logWithCommand.Info("plugin .so file output to ", pluginPath)
-}
-
-func init() {
-	rootCmd.AddCommand(composeCmd)
-}
-
-func prepConfig() {
-	logWithCommand.Info("configuring plugin")
-	names := viper.GetStringSlice("exporter.transformerNames")
-	transformers := make(map[string]config.Transformer)
-	for _, name := range names {
-		logWithCommand.Debug("Configuring " + name + " transformer")
-		transformer := viper.GetStringMapString("exporter." + name)
-		p, pOK := transformer["path"]
-		if !pOK || p == "" {
-			logWithCommand.Fatal(name, " transformer config is missing `path` value")
-		}
-		r, rOK := transformer["repository"]
-		if !rOK || r == "" {
-			logWithCommand.Fatal(name, " transformer config is missing `repository` value")
-		}
-		m, mOK := transformer["migrations"]
-		if !mOK || m == "" {
-			logWithCommand.Fatal(name, " transformer config is missing `migrations` value")
-		}
-		mr, mrOK := transformer["rank"]
-		if !mrOK || mr == "" {
-			logWithCommand.Fatal(name, " transformer config is missing `rank` value")
-		}
-		rank, err := strconv.ParseUint(mr, 10, 64)
-		if err != nil {
-			logWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
-		}
-		t, tOK := transformer["type"]
-		if !tOK {
-			logWithCommand.Fatal(name, " transformer config is missing `type` value")
-		}
-		transformerType := config.GetTransformerType(t)
-		if transformerType == config.UnknownTransformerType {
-			logWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
-		}
-
-		transformers[name] = config.Transformer{
-			Path:           p,
-			Type:           transformerType,
-			RepositoryPath: r,
-			MigrationPath:  m,
-			MigrationRank:  rank,
-		}
-	}
-
-	genConfig = config.Plugin{
-		Transformers: transformers,
-		FilePath:     "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
-		FileName:     viper.GetString("exporter.name"),
-		Save:         viper.GetBool("exporter.save"),
-		Home:         viper.GetString("exporter.home"),
-	}
-}

cmd/composeAndExecute.go

@@ -1,217 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"os"
-	"plugin"
-	syn "sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
-	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// composeAndExecuteCmd represents the composeAndExecute command
-var composeAndExecuteCmd = &cobra.Command{
-	Use:   "composeAndExecute",
-	Short: "Composes, loads, and executes transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "exampleTransformerExporter"
-    save     = false
-    transformerNames = [
-        "transformer1",
-        "transformer2",
-        "transformer3",
-        "transformer4",
-    ]
-    [exporter.transformer1]
-        path = "path/to/transformer1"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer2]
-        path = "path/to/transformer2"
-        type = "eth_contract"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "2"
-    [exporter.transformer3]
-        path = "path/to/transformer3"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer4]
-        path = "path/to/transformer4"
-        type = "eth_storage"
-        repository = "github.com/account2/repo2"
-        migrations = "to/db/migrations"
-        rank = "1"
-
-
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-
-This information is used to write and build a go plugin with a transformer 
-set composed from the transformer imports specified in the config file
-This plugin is loaded and the set of transformer initializers is exported
-from it and loaded into and executed over by the appropriate watcher. 
-
-The type of watcher that the transformer works with is specified using the 
-type variable for each transformer in the config. Currently there are watchers 
-of event data from an eth node (eth_event) and storage data from an eth node 
-(eth_storage), and a more generic interface for accepting contract_watcher pkg
-based transformers which can perform both event watching and public method 
-polling (eth_contract).
-
-Transformers of different types can be ran together in the same command using a 
-single config file or in separate command instances using different config files
-
-Specify config location when executing the command:
-./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		composeAndExecute()
-	},
-}
-
-func composeAndExecute() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Generate code to build the plugin according to the config file
-	logWithCommand.Info("generating plugin")
-	generator, err := p2.NewGenerator(genConfig, databaseConfig)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	err = generator.GenerateExporterPlugin()
-	if err != nil {
-		logWithCommand.Debug("generating plugin failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Get the plugin path and load the plugin
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	if !genConfig.Save {
-		defer helpers.ClearFiles(pluginPath)
-	}
-	logWithCommand.Info("linking plugin ", pluginPath)
-	plug, err := plugin.Open(pluginPath)
-	if err != nil {
-		logWithCommand.Debug("linking plugin failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Load the `Exporter` symbol from the plugin
-	logWithCommand.Info("loading transformers from plugin")
-	symExporter, err := plug.Lookup("Exporter")
-	if err != nil {
-		logWithCommand.Debug("loading Exporter symbol failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Assert that the symbol is of type Exporter
-	exporter, ok := symExporter.(Exporter)
-	if !ok {
-		logWithCommand.Debug("plugged-in symbol not of type Exporter")
-		os.Exit(1)
-	}
-
-	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
-	ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export()
-
-	// Setup bc and db objects
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	// Execute over transformer sets returned by the exporter
-	// Use WaitGroup to wait on both goroutines
-	var wg syn.WaitGroup
-	if len(ethEventInitializers) > 0 {
-		ew := watcher.NewEventWatcher(&db, blockChain)
-		err := ew.AddTransformers(ethEventInitializers)
-		if err != nil {
-			logWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
-		}
-		wg.Add(1)
-		go watchEthEvents(&ew, &wg)
-	}
-
-	if len(ethStorageInitializers) > 0 {
-		switch storageDiffsSource {
-		case "geth":
-			log.Debug("fetching storage diffs from geth pub sub")
-			rpcClient, _ := getClients()
-			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
-			storageFetcher := fetcher.NewGethRPCStorageFetcher(stateDiffStreamer)
-			sw := watcher.NewStorageWatcher(storageFetcher, &db)
-			sw.AddTransformers(ethStorageInitializers)
-			wg.Add(1)
-			go watchEthStorage(sw, &wg)
-		default:
-			log.Debug("fetching storage diffs from csv")
-			tailer := fs.FileTailer{Path: storageDiffsPath}
-			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-			sw := watcher.NewStorageWatcher(storageFetcher, &db)
-			sw.AddTransformers(ethStorageInitializers)
-			wg.Add(1)
-			go watchEthStorage(sw, &wg)
-		}
-	}
-
-	if len(ethContractInitializers) > 0 {
-		gw := watcher.NewContractWatcher(&db, blockChain)
-		gw.AddTransformers(ethContractInitializers)
-		wg.Add(1)
-		go watchEthContract(&gw, &wg)
-	}
-	wg.Wait()
-}
-
-func init() {
-	rootCmd.AddCommand(composeAndExecuteCmd)
-	composeAndExecuteCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
-}

cmd/contractWatcher.go

@@ -1,118 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	ht "github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/transformer"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// contractWatcherCmd represents the contractWatcher command
-var contractWatcherCmd = &cobra.Command{
-	Use:   "contractWatcher",
-	Short: "Watches events at the provided contract address using fully synced vDB",
-	Long: `Uses input contract address and event filters to watch events
-
-Expects an ethereum node to be running
-Expects an archival node synced into vulcanizeDB
-Requires a .toml config file:
-
-  [database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-  [client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-  [contract]
-    network  = ""
-    addresses  = [
-        "contractAddress1",
-        "contractAddress2"
-    ]
-    [contract.contractAddress1]
-        abi    = 'ABI for contract 1'
-        startingBlock = 982463
-    [contract.contractAddress2]
-        abi    = 'ABI for contract 2'
-        events = [
-            "event1",
-            "event2"
-        ]
-		eventArgs = [
-			"arg1",
-			"arg2"
-		]
-        methods = [
-            "method1",
-			"method2"
-        ]
-		methodArgs = [
-			"arg1",
-			"arg2"
-		]
-        startingBlock = 4448566
-        piping = true
-`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		contractWatcher()
-	},
-}
-
-var (
-	mode string
-)
-
-func contractWatcher() {
-	ticker := time.NewTicker(5 * time.Second)
-	defer ticker.Stop()
-
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	var t st.ContractTransformer
-	con := config.ContractConfig{}
-	con.PrepConfig()
-	t = ht.NewTransformer(con, blockChain, &db)
-
-	err := t.Init()
-	if err != nil {
-		logWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
-	}
-
-	for range ticker.C {
-		err = t.Execute()
-		if err != nil {
-			logWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
-		}
-	}
-}
-
-func init() {
-	rootCmd.AddCommand(contractWatcherCmd)
-}

cmd/execute.go

@@ -1,207 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"plugin"
-	syn "sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// executeCmd represents the execute command
-var executeCmd = &cobra.Command{
-	Use:   "execute",
-	Short: "executes a precomposed transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-[exporter]
-    name     = "exampleTransformerExporter"
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-The exporter.name is the name (without extension) of the plugin to be loaded.
-The plugin file needs to be located in the /plugins directory and this command assumes 
-the db migrations remain from when the plugin was composed. Additionally, the plugin 
-must have been composed by the same version of vulcanizedb or else it will not be compatible.
-Specify config location when executing the command:
-./vulcanizedb execute --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		execute()
-	},
-}
-
-func execute() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Get the plugin path and load the plugin
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
-	fmt.Printf("Executing plugin %s", pluginPath)
-	logWithCommand.Info("linking plugin ", pluginPath)
-	plug, err := plugin.Open(pluginPath)
-	if err != nil {
-		logWithCommand.Warn("linking plugin failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Load the `Exporter` symbol from the plugin
-	logWithCommand.Info("loading transformers from plugin")
-	symExporter, err := plug.Lookup("Exporter")
-	if err != nil {
-		logWithCommand.Warn("loading Exporter symbol failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Assert that the symbol is of type Exporter
-	exporter, ok := symExporter.(Exporter)
-	if !ok {
-		logWithCommand.Fatal("plugged-in symbol not of type Exporter")
-	}
-
-	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
-	ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export()
-
-	// Setup bc and db objects
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	// Execute over transformer sets returned by the exporter
-	// Use WaitGroup to wait on both goroutines
-	var wg syn.WaitGroup
-	if len(ethEventInitializers) > 0 {
-		ew := watcher.NewEventWatcher(&db, blockChain)
-		err = ew.AddTransformers(ethEventInitializers)
-		if err != nil {
-			logWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
-		}
-		wg.Add(1)
-		go watchEthEvents(&ew, &wg)
-	}
-
-	if len(ethStorageInitializers) > 0 {
-		switch storageDiffsSource {
-		case "geth":
-			log.Debug("fetching storage diffs from geth pub sub")
-			wsClient := getWSClient()
-			stateDiffStreamer := streamer.NewStateDiffStreamer(wsClient)
-			storageFetcher := fetcher.NewGethRPCStorageFetcher(stateDiffStreamer)
-			sw := watcher.NewStorageWatcher(storageFetcher, &db)
-			sw.AddTransformers(ethStorageInitializers)
-			wg.Add(1)
-			go watchEthStorage(sw, &wg)
-		default:
-			log.Debug("fetching storage diffs from csv")
-			tailer := fs.FileTailer{Path: storageDiffsPath}
-			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-			sw := watcher.NewStorageWatcher(storageFetcher, &db)
-			sw.AddTransformers(ethStorageInitializers)
-			wg.Add(1)
-			go watchEthStorage(sw, &wg)
-		}
-	}
-
-	if len(ethContractInitializers) > 0 {
-		gw := watcher.NewContractWatcher(&db, blockChain)
-		gw.AddTransformers(ethContractInitializers)
-		wg.Add(1)
-		go watchEthContract(&gw, &wg)
-	}
-	wg.Wait()
-}
-
-func init() {
-	rootCmd.AddCommand(executeCmd)
-	executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
-}
-
-type Exporter interface {
-	Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer, []transformer.ContractTransformerInitializer)
-}
-
-func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the EventTransformerInitializer set using the watcher
-	logWithCommand.Info("executing event transformers")
-	var recheck constants.TransformerExecution
-	if recheckHeadersArg {
-		recheck = constants.HeaderRecheck
-	} else {
-		recheck = constants.HeaderUnchecked
-	}
-	err := w.Execute(recheck)
-	if err != nil {
-		logWithCommand.Fatalf("error executing event watcher: %s", err.Error())
-	}
-}
-
-func watchEthStorage(w watcher.IStorageWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the StorageTransformerInitializer set using the storage watcher
-	logWithCommand.Info("executing storage transformers")
-	on := viper.GetBool("storageBackFill.on")
-	if on {
-		backFillStorage(w)
-	}
-	w.Execute(queueRecheckInterval, on)
-}
-
-func backFillStorage(w watcher.IStorageWatcher) {
-	rpcClient, _ := getClients()
-	// find min deployment block
-	minDeploymentBlock := constants.GetMinDeploymentBlock()
-	stateDiffFetcher := fetcher.NewStateDiffFetcher(rpcClient)
-	backFiller := storage.NewStorageBackFiller(stateDiffFetcher, storage.DefaultMaxBatchSize)
-	go w.BackFill(minDeploymentBlock, backFiller)
-}
-
-func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the ContractTransformerInitializer set using the contract watcher
-	logWithCommand.Info("executing contract_watcher transformers")
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-	for range ticker.C {
-		w.Execute()
-	}
-}

cmd/headerSync.go

@@ -1,114 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// headerSyncCmd represents the headerSync command
-var headerSyncCmd = &cobra.Command{
-	Use:   "headerSync",
-	Short: "Syncs VulcanizeDB with local ethereum node's block headers",
-	Long: `Syncs VulcanizeDB with local ethereum node. Populates
-Postgres with block headers.
-
-./vulcanizedb headerSync --starting-block-number 0 --config public.toml
-
-Expects ethereum node to be running and requires a .toml config:
-
-  [database]
-  name = "vulcanize_public"
-  hostname = "localhost"
-  port = 5432
-
-  [client]
-  ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
-`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		headerSync()
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(headerSyncCmd)
-	headerSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
-}
-
-func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
-	populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
-	if err != nil {
-		// TODO Lots of possible errors in the call stack above. If errors occur, we still put
-		// 0 in the channel, triggering another round
-		logWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
-	}
-	missingBlocksPopulated <- populated
-}
-
-func headerSync() {
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-	blockChain := getBlockChain()
-	validateArgs(blockChain)
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	headerRepository := repositories.NewHeaderRepository(&db)
-	validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow)
-	missingBlocksPopulated := make(chan int)
-	go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)
-
-	for {
-		select {
-		case <-ticker.C:
-			window, err := validator.ValidateHeaders()
-			if err != nil {
-				logWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
-			}
-			logWithCommand.Debug(window.GetString())
-		case n := <-missingBlocksPopulated:
-			if n == 0 {
-				time.Sleep(3 * time.Second)
-			}
-			go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)
-		}
-	}
-}
-
-func validateArgs(blockChain *eth.BlockChain) {
-	lastBlock, err := blockChain.LastBlock()
-	if err != nil {
-		logWithCommand.Error("validateArgs: Error getting last block: ", err)
-	}
-	if lastBlock.Int64() == 0 {
-		logWithCommand.Fatal("geth initial: state sync not finished")
-	}
-	if startingBlockNumber > lastBlock.Int64() {
-		logWithCommand.Fatal("starting block number > current block number")
-	}
-}

cmd/resync.go

@@ -19,11 +19,11 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/resync"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/resync"
-	v "github.com/vulcanize/vulcanizedb/version"
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
 )
 
 // resyncCmd represents the resync command

cmd/root.go

@@ -17,38 +17,24 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 )
 
 var (
 	cfgFile        string
 	databaseConfig config.Database
-	genConfig            config.Plugin
 	ipc            string
-	queueRecheckInterval time.Duration
-	startingBlockNumber  int64
-	storageDiffsPath     string
-	recheckHeadersArg    bool
 	subCommand     string
 	logWithCommand log.Entry
-	storageDiffsSource   string
 )
 
 const (
@@ -91,8 +77,6 @@ func initFuncs(cmd *cobra.Command, args []string) {
 
 func setViperConfigs() {
 	ipc = viper.GetString("client.ipcpath")
-	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
-	storageDiffsSource = viper.GetString("storageDiffs.source")
 	databaseConfig = config.Database{
 		Name:     viper.GetString("database.name"),
 		Hostname: viper.GetString("database.hostname"),
@@ -130,10 +114,6 @@ func init() {
 	rootCmd.PersistentFlags().String("database-user", "", "database user")
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
-	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
-	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
-	rootCmd.PersistentFlags().String("storageDiffs-source", "csv", "where to get the state diffs: csv or geth")
-	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
 
 	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
@@ -143,10 +123,6 @@ func init() {
 	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
-	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
-	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
-	viper.BindPFlag("storageDiffs.source", rootCmd.PersistentFlags().Lookup("storageDiffs-source"))
-	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }
 
@@ -162,35 +138,3 @@ func initConfig() {
 		log.Warn("No config file passed with --config flag")
 	}
 }
-
-func getBlockChain() *eth.BlockChain {
-	rpcClient, ethClient := getClients()
-	vdbEthClient := client.NewEthClient(ethClient)
-	vdbNode := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRPCTransactionConverter(ethClient)
-	return eth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
-}
-
-func getClients() (client.RPCClient, *ethclient.Client) {
-	rawRPCClient, err := rpc.Dial(ipc)
-
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	rpcClient := client.NewRPCClient(rawRPCClient, ipc)
-	ethClient := ethclient.NewClient(rawRPCClient)
-
-	return rpcClient, ethClient
-}
-
-func getWSClient() core.RPCClient {
-	wsRPCpath := viper.GetString("client.wsPath")
-	if wsRPCpath == "" {
-		logWithCommand.Fatal(errors.New("getWSClient() was called but no ws rpc path is provided"))
-	}
-	wsRPCClient, dialErr := rpc.Dial(wsRPCpath)
-	if dialErr != nil {
-		logWithCommand.Fatal(dialErr)
-	}
-	return client.NewRPCClient(wsRPCClient, wsRPCpath)
-}

cmd/streamEthSubscribe.go

@@ -28,11 +28,11 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/client"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/streamer"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/watcher"
 )
 
 // streamEthSubscriptionCmd represents the streamEthSubscription command
@@ -64,7 +64,7 @@ func streamEthSubscription() {
 	str := streamer.NewSuperNodeStreamer(rpcClient)
 
 	// Buffered channel for reading subscription payloads
-	payloadChan := make(chan super_node.SubscriptionPayload, 20000)
+	payloadChan := make(chan watcher.SubscriptionPayload, 20000)
 
 	// Subscribe to the super node service with the given config/filter parameters
 	rlpParams, err := rlp.EncodeToBytes(ethSubConfig)

cmd/version.go

@@ -19,20 +19,20 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	v "github.com/vulcanize/vulcanizedb/version"
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
 )
 
 // versionCmd represents the version command
 var versionCmd = &cobra.Command{
 	Use:   "version",
-	Short: "Prints the version of vulcanizeDB",
+	Short: "Prints the version of ipfs-chain-watcher",
-	Long: `Use this command to fetch the version of vulcanizeDB
+	Long: `Use this command to fetch the version of ipfs-chain-watcher
 
-Usage: ./vulcanizedb version`,
+Usage: ./ipfs-chain-watcher version`,
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *log.WithField("SubCommand", subCommand)
-		logWithCommand.Infof("VulcanizeDB version: %s", v.VersionWithMeta)
+		logWithCommand.Infof("ipfs-chain-watcher version: %s", v.VersionWithMeta)
 	},
 }
 

cmd/watch.go

@@ -25,17 +25,17 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/watcher"
-	v "github.com/vulcanize/vulcanizedb/version"
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
 )
 
 // superNodeCmd represents the superNode command
 var superNodeCmd = &cobra.Command{
 	Use:   "superNode",
-	Short: "VulcanizeDB SuperNode",
+	Short: "sync chain data into PG-IPFS",
-	Long: `This command configures a VulcanizeDB SuperNode.
+	Long: `This command configures a VulcanizeDB ipfs-chain-watcher.
 
 The Sync process streams all chain data from the appropriate chain, processes this data into IPLD objects
 and publishes them to IPFS. It then indexes the CIDs against useful data fields/metadata in Postgres. 
@@ -56,7 +56,7 @@ and fill in gaps in the data
 func superNode() {
 	logWithCommand.Infof("running vdb version: %s", v.VersionWithMeta)
 	logWithCommand.Debug("loading super node configuration variables")
-	superNodeConfig, err := super_node.NewSuperNodeConfig()
+	superNodeConfig, err := watcher.NewSuperNodeConfig()
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -68,14 +68,14 @@ func superNode() {
 	}
 	wg := &sync.WaitGroup{}
 	logWithCommand.Debug("initializing new super node service")
-	superNode, err := super_node.NewSuperNode(superNodeConfig)
+	superNode, err := watcher.NewSuperNode(superNodeConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
 	var forwardPayloadChan chan shared.ConvertedData
 	if superNodeConfig.Serve {
 		logWithCommand.Info("starting up super node servers")
-		forwardPayloadChan = make(chan shared.ConvertedData, super_node.PayloadChanBufferSize)
+		forwardPayloadChan = make(chan shared.ConvertedData, watcher.PayloadChanBufferSize)
 		superNode.Serve(wg, forwardPayloadChan)
 		if err := startServers(superNode, superNodeConfig); err != nil {
 			logWithCommand.Fatal(err)
@@ -87,10 +87,10 @@ func superNode() {
 			logWithCommand.Fatal(err)
 		}
 	}
-	var backFiller super_node.BackFillInterface
+	var backFiller watcher.BackFillInterface
 	if superNodeConfig.BackFill {
 		logWithCommand.Debug("initializing new super node backfill service")
-		backFiller, err = super_node.NewBackFillService(superNodeConfig, forwardPayloadChan)
+		backFiller, err = watcher.NewBackFillService(superNodeConfig, forwardPayloadChan)
 		if err != nil {
 			logWithCommand.Fatal(err)
 		}
@@ -107,7 +107,7 @@ func superNode() {
 	wg.Wait()
 }
 
-func startServers(superNode super_node.SuperNode, settings *super_node.Config) error {
+func startServers(superNode watcher.SuperNode, settings *watcher.Config) error {
 	logWithCommand.Debug("starting up IPC server")
 	_, _, err := rpc.StartIPCEndpoint(settings.IPCEndpoint, superNode.APIs())
 	if err != nil {

db/migrations/00002_create_addresses_table.sql

@@ -1,11 +0,0 @@
--- +goose Up
-CREATE TABLE public.addresses
-(
-    id             SERIAL PRIMARY KEY,
-    address        character varying(42),
-    hashed_address character varying(66),
-    UNIQUE (address)
-);
-
--- +goose Down
-DROP TABLE public.addresses;

db/migrations/00003_create_headers_table.sql

@@ -1,25 +0,0 @@
--- +goose Up
-CREATE TABLE public.headers
-(
-    id                   SERIAL PRIMARY KEY,
-    hash                 VARCHAR(66),
-    block_number         BIGINT,
-    raw                  JSONB,
-    block_timestamp      NUMERIC,
-    check_count          INTEGER NOT NULL DEFAULT 0,
-    node_id              INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
-    eth_node_fingerprint VARCHAR(128),
-    UNIQUE (block_number, hash, eth_node_fingerprint)
-);
-
-CREATE INDEX headers_block_number
-    ON public.headers (block_number);
-
-CREATE INDEX headers_block_timestamp
-    ON public.headers (block_timestamp);
-
--- +goose Down
-DROP INDEX public.headers_block_number;
-DROP INDEX public.headers_block_timestamp;
-
-DROP TABLE public.headers;

db/migrations/00004_create_checked_headers_table.sql

@@ -1,8 +0,0 @@
--- +goose Up
-CREATE TABLE public.checked_headers (
-  id                  SERIAL PRIMARY KEY,
-  header_id           INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE
-);
-
--- +goose Down
-DROP TABLE public.checked_headers;

db/migrations/00005_create_storage_diffs_table.sql

@@ -1,14 +0,0 @@
--- +goose Up
-CREATE TABLE public.storage_diff
-(
-    id             SERIAL PRIMARY KEY,
-    block_height   BIGINT,
-    block_hash     BYTEA,
-    hashed_address BYTEA,
-    storage_key    BYTEA,
-    storage_value  BYTEA,
-    UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value)
-);
-
--- +goose Down
-DROP TABLE public.storage_diff;

db/migrations/00006_create_queued_storage_diffs_table.sql

@@ -1,9 +0,0 @@
--- +goose Up
-CREATE TABLE public.queued_storage
-(
-    id      SERIAL PRIMARY KEY,
-    diff_id BIGINT UNIQUE NOT NULL REFERENCES public.storage_diff (id)
-);
-
--- +goose Down
-DROP TABLE public.queued_storage;

db/migrations/00007_create_header_sync_transactions_table.sql

@@ -1,29 +0,0 @@
--- +goose Up
-CREATE TABLE public.header_sync_transactions
-(
-    id         SERIAL PRIMARY KEY,
-    header_id  INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-    hash       VARCHAR(66),
-    gas_limit  NUMERIC,
-    gas_price  NUMERIC,
-    input_data BYTEA,
-    nonce      NUMERIC,
-    raw        BYTEA,
-    tx_from    VARCHAR(44),
-    tx_index   INTEGER,
-    tx_to      VARCHAR(44),
-    "value"    NUMERIC,
-    UNIQUE (header_id, hash)
-);
-
-CREATE INDEX header_sync_transactions_header
-    ON public.header_sync_transactions (header_id);
-
-CREATE INDEX header_sync_transactions_tx_index
-    ON public.header_sync_transactions (tx_index);
-
--- +goose Down
-DROP INDEX public.header_sync_transactions_header;
-DROP INDEX public.header_sync_transactions_tx_index;
-
-DROP TABLE header_sync_transactions;

db/migrations/00008_create_header_sync_receipts_table.sql

@@ -1,27 +0,0 @@
--- +goose Up
-CREATE TABLE header_sync_receipts
-(
-    id                  SERIAL PRIMARY KEY,
-    transaction_id      INTEGER NOT NULL REFERENCES header_sync_transactions (id) ON DELETE CASCADE,
-    header_id           INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-    contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
-    cumulative_gas_used NUMERIC,
-    gas_used            NUMERIC,
-    state_root          VARCHAR(66),
-    status              INTEGER,
-    tx_hash             VARCHAR(66),
-    rlp                 BYTEA,
-    UNIQUE (header_id, transaction_id)
-);
-
-CREATE INDEX header_sync_receipts_header
-    ON public.header_sync_receipts (header_id);
-
-CREATE INDEX header_sync_receipts_transaction
-    ON public.header_sync_receipts (transaction_id);
-
--- +goose Down
-DROP INDEX public.header_sync_receipts_header;
-DROP INDEX public.header_sync_receipts_transaction;
-
-DROP TABLE header_sync_receipts;

db/migrations/00009_create_header_sync_logs_table.sql

@@ -1,22 +0,0 @@
--- +goose Up
--- SQL in this section is executed when the migration is applied.
-CREATE TABLE header_sync_logs
-(
-    id           SERIAL PRIMARY KEY,
-    header_id    INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-    address      INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
-    topics       BYTEA[],
-    data         BYTEA,
-    block_number BIGINT,
-    block_hash   VARCHAR(66),
-    tx_hash      VARCHAR(66),
-    tx_index     INTEGER,
-    log_index    INTEGER,
-    raw          JSONB,
-    transformed  BOOL    NOT NULL DEFAULT FALSE,
-    UNIQUE (header_id, tx_index, log_index)
-);
-
--- +goose Down
--- SQL in this section is executed when the migration is rolled back.
-DROP TABLE header_sync_logs;

db/migrations/00010_create_watched_logs_table.sql

@@ -1,12 +0,0 @@
--- +goose Up
--- SQL in this section is executed when the migration is applied.
-CREATE TABLE public.watched_logs
-(
-    id               SERIAL PRIMARY KEY,
-    contract_address VARCHAR(42),
-    topic_zero       VARCHAR(66)
-);
-
--- +goose Down
--- SQL in this section is executed when the migration is rolled back.
-DROP TABLE public.watched_logs;

db/migrations/00017_create_postgraphile_comments.sql

@@ -6,8 +6,6 @@ COMMENT ON TABLE btc.queue_data IS E'@name BtcQueueData';
 COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
 COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
 COMMENT ON TABLE eth.queue_data IS E'@name EthQueueData';
-COMMENT ON TABLE public.headers IS E'@name EthHeaders';
-COMMENT ON COLUMN public.headers.node_id IS E'@name EthNodeID';
 COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
 COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';
 COMMENT ON COLUMN btc.header_cids.node_id IS E'@name BtcNodeID';

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 10.10
+-- Dumped from database version 12.1
--- Dumped by pg_dump version 10.10
+-- Dumped by pg_dump version 12.1
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -30,23 +30,9 @@ CREATE SCHEMA btc;
 CREATE SCHEMA eth;
 
 
---
--- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
---
-
-CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
-
-
---
--- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
---
-
-COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-
-
 SET default_tablespace = '';
 
-SET default_with_oids = false;
+SET default_table_access_method = heap;
 
 --
 -- Name: header_cids; Type: TABLE; Schema: btc; Owner: -
@@ -561,37 +547,6 @@ CREATE SEQUENCE eth.uncle_cids_id_seq
 ALTER SEQUENCE eth.uncle_cids_id_seq OWNED BY eth.uncle_cids.id;
 
 
---
--- Name: addresses; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.addresses (
-    id integer NOT NULL,
-    address character varying(42),
-    hashed_address character varying(66)
-);
-
-
---
--- Name: addresses_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.addresses_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: addresses_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.addresses_id_seq OWNED BY public.addresses.id;
-
-
 --
 -- Name: blocks; Type: TABLE; Schema: public; Owner: -
 --
@@ -602,36 +557,6 @@ CREATE TABLE public.blocks (
 );
 
 
---
--- Name: checked_headers; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.checked_headers (
-    id integer NOT NULL,
-    header_id integer NOT NULL
-);
-
-
---
--- Name: checked_headers_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.checked_headers_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: checked_headers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.checked_headers_id_seq OWNED BY public.checked_headers.id;
-
-
 --
 -- Name: goose_db_version; Type: TABLE; Schema: public; Owner: -
 --
@@ -664,174 +589,6 @@ CREATE SEQUENCE public.goose_db_version_id_seq
 ALTER SEQUENCE public.goose_db_version_id_seq OWNED BY public.goose_db_version.id;
 
 
---
--- Name: header_sync_logs; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.header_sync_logs (
-    id integer NOT NULL,
-    header_id integer NOT NULL,
-    address integer NOT NULL,
-    topics bytea[],
-    data bytea,
-    block_number bigint,
-    block_hash character varying(66),
-    tx_hash character varying(66),
-    tx_index integer,
-    log_index integer,
-    raw jsonb,
-    transformed boolean DEFAULT false NOT NULL
-);
-
-
---
--- Name: header_sync_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.header_sync_logs_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: header_sync_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.header_sync_logs_id_seq OWNED BY public.header_sync_logs.id;
-
-
---
--- Name: header_sync_receipts; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.header_sync_receipts (
-    id integer NOT NULL,
-    transaction_id integer NOT NULL,
-    header_id integer NOT NULL,
-    contract_address_id integer NOT NULL,
-    cumulative_gas_used numeric,
-    gas_used numeric,
-    state_root character varying(66),
-    status integer,
-    tx_hash character varying(66),
-    rlp bytea
-);
-
-
---
--- Name: header_sync_receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.header_sync_receipts_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: header_sync_receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.header_sync_receipts_id_seq OWNED BY public.header_sync_receipts.id;
-
-
---
--- Name: header_sync_transactions; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.header_sync_transactions (
-    id integer NOT NULL,
-    header_id integer NOT NULL,
-    hash character varying(66),
-    gas_limit numeric,
-    gas_price numeric,
-    input_data bytea,
-    nonce numeric,
-    raw bytea,
-    tx_from character varying(44),
-    tx_index integer,
-    tx_to character varying(44),
-    value numeric
-);
-
-
---
--- Name: header_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.header_sync_transactions_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: header_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.header_sync_transactions_id_seq OWNED BY public.header_sync_transactions.id;
-
-
---
--- Name: headers; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.headers (
-    id integer NOT NULL,
-    hash character varying(66),
-    block_number bigint,
-    raw jsonb,
-    block_timestamp numeric,
-    check_count integer DEFAULT 0 NOT NULL,
-    node_id integer NOT NULL,
-    eth_node_fingerprint character varying(128)
-);
-
-
---
--- Name: TABLE headers; Type: COMMENT; Schema: public; Owner: -
---
-
-COMMENT ON TABLE public.headers IS '@name EthHeaders';
-
-
---
--- Name: COLUMN headers.node_id; Type: COMMENT; Schema: public; Owner: -
---
-
-COMMENT ON COLUMN public.headers.node_id IS '@name EthNodeID';
-
-
---
--- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.headers_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: headers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id;
-
-
 --
 -- Name: nodes; Type: TABLE; Schema: public; Owner: -
 --
@@ -879,101 +636,6 @@ CREATE SEQUENCE public.nodes_id_seq
 ALTER SEQUENCE public.nodes_id_seq OWNED BY public.nodes.id;
 
 
---
--- Name: queued_storage; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.queued_storage (
-    id integer NOT NULL,
-    diff_id bigint NOT NULL
-);
-
-
---
--- Name: queued_storage_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.queued_storage_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: queued_storage_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.queued_storage_id_seq OWNED BY public.queued_storage.id;
-
-
---
--- Name: storage_diff; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.storage_diff (
-    id integer NOT NULL,
-    block_height bigint,
-    block_hash bytea,
-    hashed_address bytea,
-    storage_key bytea,
-    storage_value bytea
-);
-
-
---
--- Name: storage_diff_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.storage_diff_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: storage_diff_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.storage_diff_id_seq OWNED BY public.storage_diff.id;
-
-
---
--- Name: watched_logs; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.watched_logs (
-    id integer NOT NULL,
-    contract_address character varying(42),
-    topic_zero character varying(66)
-);
-
-
---
--- Name: watched_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.watched_logs_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: watched_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.watched_logs_id_seq OWNED BY public.watched_logs.id;
-
-
 --
 -- Name: header_cids id; Type: DEFAULT; Schema: btc; Owner: -
 --
@@ -1065,20 +727,6 @@ ALTER TABLE ONLY eth.transaction_cids ALTER COLUMN id SET DEFAULT nextval('eth.t
 ALTER TABLE ONLY eth.uncle_cids ALTER COLUMN id SET DEFAULT nextval('eth.uncle_cids_id_seq'::regclass);
 
 
---
--- Name: addresses id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.addresses ALTER COLUMN id SET DEFAULT nextval('public.addresses_id_seq'::regclass);
-
-
---
--- Name: checked_headers id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.checked_headers ALTER COLUMN id SET DEFAULT nextval('public.checked_headers_id_seq'::regclass);
-
-
 --
 -- Name: goose_db_version id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -1086,34 +734,6 @@ ALTER TABLE ONLY public.checked_headers ALTER COLUMN id SET DEFAULT nextval('pub
 ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('public.goose_db_version_id_seq'::regclass);
 
 
---
--- Name: header_sync_logs id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_logs ALTER COLUMN id SET DEFAULT nextval('public.header_sync_logs_id_seq'::regclass);
-
-
---
--- Name: header_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_receipts ALTER COLUMN id SET DEFAULT nextval('public.header_sync_receipts_id_seq'::regclass);
-
-
---
--- Name: header_sync_transactions id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.header_sync_transactions_id_seq'::regclass);
-
-
---
--- Name: headers id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass);
-
-
 --
 -- Name: nodes id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -1121,27 +741,6 @@ ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.head
 ALTER TABLE ONLY public.nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
 
 
---
--- Name: queued_storage id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.queued_storage ALTER COLUMN id SET DEFAULT nextval('public.queued_storage_id_seq'::regclass);
-
-
---
--- Name: storage_diff id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.storage_diff ALTER COLUMN id SET DEFAULT nextval('public.storage_diff_id_seq'::regclass);
-
-
---
--- Name: watched_logs id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.watched_logs ALTER COLUMN id SET DEFAULT nextval('public.watched_logs_id_seq'::regclass);
-
-
 --
 -- Name: header_cids header_cids_block_number_block_hash_key; Type: CONSTRAINT; Schema: btc; Owner: -
 --
@@ -1350,22 +949,6 @@ ALTER TABLE ONLY eth.uncle_cids
     ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (id);
 
 
---
--- Name: addresses addresses_address_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.addresses
-    ADD CONSTRAINT addresses_address_key UNIQUE (address);
-
-
---
--- Name: addresses addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.addresses
-    ADD CONSTRAINT addresses_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: blocks blocks_key_key; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -1374,22 +957,6 @@ ALTER TABLE ONLY public.blocks
     ADD CONSTRAINT blocks_key_key UNIQUE (key);
 
 
---
--- Name: checked_headers checked_headers_header_id_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.checked_headers
-    ADD CONSTRAINT checked_headers_header_id_key UNIQUE (header_id);
-
-
---
--- Name: checked_headers checked_headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.checked_headers
-    ADD CONSTRAINT checked_headers_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: goose_db_version goose_db_version_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -1398,70 +965,6 @@ ALTER TABLE ONLY public.goose_db_version
     ADD CONSTRAINT goose_db_version_pkey PRIMARY KEY (id);
 
 
---
--- Name: header_sync_logs header_sync_logs_header_id_tx_index_log_index_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_logs
-    ADD CONSTRAINT header_sync_logs_header_id_tx_index_log_index_key UNIQUE (header_id, tx_index, log_index);
-
-
---
--- Name: header_sync_logs header_sync_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_logs
-    ADD CONSTRAINT header_sync_logs_pkey PRIMARY KEY (id);
-
-
---
--- Name: header_sync_receipts header_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_receipts
-    ADD CONSTRAINT header_sync_receipts_header_id_transaction_id_key UNIQUE (header_id, transaction_id);
-
-
---
--- Name: header_sync_receipts header_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_receipts
-    ADD CONSTRAINT header_sync_receipts_pkey PRIMARY KEY (id);
-
-
---
--- Name: header_sync_transactions header_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_transactions
-    ADD CONSTRAINT header_sync_transactions_header_id_hash_key UNIQUE (header_id, hash);
-
-
---
--- Name: header_sync_transactions header_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_transactions
-    ADD CONSTRAINT header_sync_transactions_pkey PRIMARY KEY (id);
-
-
---
--- Name: headers headers_block_number_hash_eth_node_fingerprint_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.headers
-    ADD CONSTRAINT headers_block_number_hash_eth_node_fingerprint_key UNIQUE (block_number, hash, eth_node_fingerprint);
-
-
---
--- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.headers
-    ADD CONSTRAINT headers_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: nodes node_uc; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -1478,88 +981,6 @@ ALTER TABLE ONLY public.nodes
     ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
 
 
---
--- Name: queued_storage queued_storage_diff_id_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.queued_storage
-    ADD CONSTRAINT queued_storage_diff_id_key UNIQUE (diff_id);
-
-
---
--- Name: queued_storage queued_storage_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.queued_storage
-    ADD CONSTRAINT queued_storage_pkey PRIMARY KEY (id);
-
-
---
--- Name: storage_diff storage_diff_block_height_block_hash_hashed_address_storage_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.storage_diff
-    ADD CONSTRAINT storage_diff_block_height_block_hash_hashed_address_storage_key UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value);
-
-
---
--- Name: storage_diff storage_diff_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.storage_diff
-    ADD CONSTRAINT storage_diff_pkey PRIMARY KEY (id);
-
-
---
--- Name: watched_logs watched_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.watched_logs
-    ADD CONSTRAINT watched_logs_pkey PRIMARY KEY (id);
-
-
---
--- Name: header_sync_receipts_header; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX header_sync_receipts_header ON public.header_sync_receipts USING btree (header_id);
-
-
---
--- Name: header_sync_receipts_transaction; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX header_sync_receipts_transaction ON public.header_sync_receipts USING btree (transaction_id);
-
-
---
--- Name: header_sync_transactions_header; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX header_sync_transactions_header ON public.header_sync_transactions USING btree (header_id);
-
-
---
--- Name: header_sync_transactions_tx_index; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX header_sync_transactions_tx_index ON public.header_sync_transactions USING btree (tx_index);
-
-
---
--- Name: headers_block_number; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX headers_block_number ON public.headers USING btree (block_number);
-
-
---
--- Name: headers_block_timestamp; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX headers_block_timestamp ON public.headers USING btree (block_timestamp);
-
-
 --
 -- Name: header_cids header_cids_node_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: -
 --
@@ -1648,78 +1069,6 @@ ALTER TABLE ONLY eth.uncle_cids
     ADD CONSTRAINT uncle_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
 
 
---
--- Name: checked_headers checked_headers_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.checked_headers
-    ADD CONSTRAINT checked_headers_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
-
-
---
--- Name: header_sync_logs header_sync_logs_address_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_logs
-    ADD CONSTRAINT header_sync_logs_address_fkey FOREIGN KEY (address) REFERENCES public.addresses(id) ON DELETE CASCADE;
-
-
---
--- Name: header_sync_logs header_sync_logs_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_logs
-    ADD CONSTRAINT header_sync_logs_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
-
-
---
--- Name: header_sync_receipts header_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_receipts
-    ADD CONSTRAINT header_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE;
-
-
---
--- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_receipts
-    ADD CONSTRAINT header_sync_receipts_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
-
-
---
--- Name: header_sync_receipts header_sync_receipts_transaction_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_receipts
-    ADD CONSTRAINT header_sync_receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.header_sync_transactions(id) ON DELETE CASCADE;
-
-
---
--- Name: header_sync_transactions header_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.header_sync_transactions
-    ADD CONSTRAINT header_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
-
-
---
--- Name: headers headers_node_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.headers
-    ADD CONSTRAINT headers_node_id_fkey FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
-
-
---
--- Name: queued_storage queued_storage_diff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.queued_storage
-    ADD CONSTRAINT queued_storage_diff_id_fkey FOREIGN KEY (diff_id) REFERENCES public.storage_diff(id);
-
-
 --
 -- PostgreSQL database dump complete
 --

dockerfiles/migrations/Dockerfile

@@ -8,14 +8,14 @@ RUN apk add busybox-extras
 FROM golang:1.12.4 as builder
 
 # Get and build vulcanizedb
-ADD . /go/src/github.com/vulcanize/vulcanizedb
+ADD . /go/src/github.com/vulcanize/ipfs-chain-watcher
 
 # Build migration tool
 RUN go get -u -d github.com/pressly/goose/cmd/goose
 WORKDIR /go/src/github.com/pressly/goose/cmd/goose
 RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
 
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+WORKDIR /go/src/github.com/vulcanize/ipfs-chain-watcher
 
 # app container
 FROM alpine
@@ -29,12 +29,12 @@ USER $USER
 
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/migrations/startup_script.sh .
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/dockerfiles/migrations/startup_script.sh .
 
 
 # keep binaries immutable
 COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/db/migrations migrations/vulcanizedb
 # XXX dir is already writeable RUN touch vulcanizedb.log
 
 CMD ["./startup_script.sh"]

dockerfiles/super_node/Dockerfile

@@ -5,8 +5,8 @@ RUN apk --update --no-cache add make git g++ linux-headers
 RUN apk add busybox-extras
 
 # Get and build vulcanizedb
-ADD . /go/src/github.com/vulcanize/vulcanizedb
+ADD . /go/src/github.com/vulcanize/ipfs-chain-watcher
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+WORKDIR /go/src/github.com/vulcanize/ipfs-chain-watcher
 RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o vulcanizedb .
 
 # Build migration tool
@@ -14,7 +14,7 @@ RUN go get -u -d github.com/pressly/goose/cmd/goose
 WORKDIR /go/src/github.com/pressly/goose/cmd/goose
 RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
 
-WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+WORKDIR /go/src/github.com/vulcanize/ipfs-chain-watcher
 
 # app container
 FROM alpine
@@ -31,16 +31,16 @@ USER $USER
 
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/$CONFIG_FILE config.toml
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/super_node/startup_script.sh .
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/dockerfiles/super_node/startup_script.sh .
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/super_node/entrypoint.sh .
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/dockerfiles/super_node/entrypoint.sh .
 
 
 # keep binaries immutable
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcanizedb
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/vulcanizedb vulcanizedb
 COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/db/migrations migrations/vulcanizedb
-COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/environments environments
+COPY --from=builder /go/src/github.com/vulcanize/ipfs-chain-watcher/environments environments
 
 EXPOSE $EXPOSE_PORT_1
 EXPOSE $EXPOSE_PORT_2

documentation/custom-transformers.md

@@ -63,7 +63,7 @@ these are run independently, instead of using `composeAndExecute`, a couple of t
 
 * The `execute` command does not require the plugin transformer dependencies be located in their `$GOPATH` directories,
 instead it expects a .so file (of the name specified in the config file) to be in
-`$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin db migrations to
+`$GOPATH/src/github.com/vulcanize/ipfs-chain-watcher/plugins/` and, as noted above, also expects the plugin db migrations to
  have already been ran against the database.
 
  * Usage:
@@ -103,7 +103,7 @@ The config provides information for composing a set of transformers from externa
     wsPath   = "ws://127.0.0.1:8546"
 
 [exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
+    home     = "github.com/vulcanize/ipfs-chain-watcher"
     name     = "exampleTransformerExporter"
     save     = false
     transformerNames = [
@@ -137,7 +137,7 @@ The config provides information for composing a set of transformers from externa
         migrations = "to/db/migrations"
         rank = "1"
 ```
-- `home` is the name of the package you are building the plugin for, in most cases this is github.com/vulcanize/vulcanizedb
+- `home` is the name of the package you are building the plugin for, in most cases this is github.com/vulcanize/ipfs-chain-watcher
 - `name` is the name used for the plugin files (.so and .go)   
 - `save` indicates whether or not the user wants to save the .go file instead of removing it after .so compilation. Sometimes useful for debugging/trouble-shooting purposes.
 - `transformerNames` is the list of the names of the transformers we are composing together, so we know how to access their submaps in the exporter map
@@ -174,7 +174,7 @@ The general structure of a plugin .go file, and what we would see built with the
 package main
 
 import (
-	interface1 "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	interface1 "github.com/vulcanize/ipfs-chain-watcher/libraries/shared/transformer"
 	transformer1 "github.com/account/repo/path/to/transformer1"
 	transformer2 "github.com/account/repo/path/to/transformer2"
 	transformer3 "github.com/account/repo/path/to/transformer3"

documentation/super_node/apis.md

@@ -40,10 +40,10 @@ An example of how to subscribe to a real-time Ethereum data feed from the super
     	"github.com/ethereum/go-ethereum/rpc"
     	"github.com/spf13/viper"
     	
-    	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+    	"github.com/vulcanize/ipfs-chain-watcher/libraries/shared/streamer"
-    	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth/client"
-    	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/super_node"
-    	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/super_node/eth"
     )
 
     config, _ := eth.NewEthSubscriptionConfig()
@@ -162,10 +162,10 @@ An example of how to subscribe to a real-time Bitcoin data feed from the super n
     	"github.com/ethereum/go-ethereum/rpc"
     	"github.com/spf13/viper"
     	
-    	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+    	"github.com/vulcanize/ipfs-chain-watcher/libraries/shared/streamer"
-    	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth/client"
-    	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/super_node"
-    	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
+    	"github.com/vulcanize/ipfs-chain-watcher/pkg/super_node/btc"
     )
 
     config, _ := btc.NewBtcSubscriptionConfig()

documentation/super_node/setup.md

@@ -138,9 +138,9 @@ Finally, we can begin the vulcanizeDB process itself.
 
 Start by downloading vulcanizedb and moving into the repo:
 
-`go get github.com/vulcanize/vulcanizedb`
+`go get github.com/vulcanize/ipfs-chain-watcher`
 
-`cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
+`cd $GOPATH/src/github.com/vulcanize/ipfs-chain-watcher`
 
 Run the db migrations against the Postgres database we created for vulcanizeDB:
 

environments/composeAndExecuteAccountTransformer.toml

@@ -1,38 +0,0 @@
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-[client]
-    ipcPath = ""
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "accountTransformerExporter"
-    save     = false
-    transformerNames = [
-        "account"
-    ]
-    [exporter.account]
-        path = "transformers/account/light/initializer"
-        type = "eth_contract"
-        repository = "github.com/vulcanize/account_transformers"
-        migrations = "db/migrations"
-        rank = "0"
-
-[token]
-    addresses = [
-        "0x58b6A8A3302369DAEc383334672404Ee733aB239",
-        "0x862Da0A691bb0b74038377295f8fF523D0493eB4",
-    ]
-    [token.equivalents]
-        0x0000000000085d4780B73119b644AE5ecd22b376 = [
-           "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
-        ]
-        0x58b6A8A3302369DAEc383334672404Ee733aB239 = [
-           "0x8e306b005773bee6bA6A6e8972Bc79D766cC15c8"
-        ]
-
-[account]
-    start = 0
-    addresses = []

environments/example.toml

@@ -1,26 +0,0 @@
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-[client]
-    ipcPath  = ""
-
-[contract]
-    network  = ""
-    addresses  = [
-        "0x314159265dD8dbb310642f98f50C066173C1259b",
-        "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
-    ]
-    [contract.0x314159265dD8dbb310642f98f50C066173C1259b]
-        abi    = '[{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"resolver","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"label","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setSubnodeOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"ttl","type":"uint64"}],"name":"setTTL","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"ttl","outputs":[{"name":"","type":"uint64"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"resolver","type":"address"}],"name":"setResolver","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":true,"name":"label","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"NewOwner","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"resolver","type":"address"}],"name":"NewResolver","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"ttl","type":"uint64"}],"name":"NewTTL","type":"event"}]'
-        startingBlock = 3327417
-    [contract.0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E]
-        events = [
-            "Transfer",
-            "Issue"
-        ]
-        methods = [
-            "balanceOf"
-        ]
-        startingBlock = 5197514

environments/public.toml.example

@@ -1,8 +0,0 @@
-[database]
-    name = "vulcanize_public"
-    hostname = "localhost"
-    port = 5432
-
-[client]
-    ipcPath = <local node's IPC filepath>
-    levelDbPath = <local node's LevelDB chaindata filepath>

go.mod

@@ -1,4 +1,4 @@
-module github.com/vulcanize/vulcanizedb
+module github.com/vulcanize/ipfs-chain-watcher
 
 go 1.12
 

go.sum

@@ -83,7 +83,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0=
 github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
-github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk=
 github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -611,7 +610,6 @@ github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXx
 github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/polydawn/refmt v0.0.0-20190731040541-eff0b363297a h1:TdavzKWkPcC2G+6rKJclm/JfrWC6WZFfLUR7EJJX8MA=
 github.com/polydawn/refmt v0.0.0-20190731040541-eff0b363297a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
-github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k=
 github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
@@ -690,8 +688,6 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290 h1:uMWt+x6JhVT7GyL983weZSxv1zDBxvGlI9HNkcTnUeg=
-github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.2 h1:ebv2bWocCmNKGnpHtRjSWoTpkgyEbRBb028PanH43H8=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.2/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
 github.com/vulcanize/go-ipfs v0.4.22-alpha h1:W+6njT14KWllMhABRFtPndqHw8SHCt5SqD4YX528kxM=
@@ -788,7 +784,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

integration_test/block_rewards_test.go

@@ -1,62 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package integration
-
-import (
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Rewards calculations", func() {
-
-	It("calculates a block reward for a real block", func() {
-		rawRPCClient, err := rpc.Dial(test_config.TestClient.IPCPath)
-		Expect(err).NotTo(HaveOccurred())
-		rpcClient := client.NewRPCClient(rawRPCClient, test_config.TestClient.IPCPath)
-		ethClient := ethclient.NewClient(rawRPCClient)
-		blockChainClient := client.NewEthClient(ethClient)
-		node := node.MakeNode(rpcClient)
-		transactionConverter := vRpc.NewRPCTransactionConverter(ethClient)
-		blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
-		block, err := blockChain.GetBlockByNumber(1071819)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(block.Reward).To(Equal("5313550000000000000"))
-	})
-
-	It("calculates an uncle reward for a real block", func() {
-		rawRPCClient, err := rpc.Dial(test_config.TestClient.IPCPath)
-		Expect(err).NotTo(HaveOccurred())
-		rpcClient := client.NewRPCClient(rawRPCClient, test_config.TestClient.IPCPath)
-		ethClient := ethclient.NewClient(rawRPCClient)
-		blockChainClient := client.NewEthClient(ethClient)
-		node := node.MakeNode(rpcClient)
-		transactionConverter := vRpc.NewRPCTransactionConverter(ethClient)
-		blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
-		block, err := blockChain.GetBlockByNumber(1071819)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(block.UnclesReward).To(Equal("6875000000000000000"))
-	})
-
-})

integration_test/contract_test.go

@@ -1,110 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package integration
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/testing"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Reading contracts", func() {
-
-	Describe("Getting a contract attribute", func() {
-		It("retrieves the event log for a specific block and contract", func() {
-			expectedLogZero := core.FullSyncLog{
-				BlockNumber: 4703824,
-				TxHash:      "0xf896bfd1eb539d881a1a31102b78de9f25cd591bf1fe1924b86148c0b205fd5d",
-				Address:     "0xd26114cd6ee289accf82350c8d8487fedb8a0c07",
-				Topics: core.Topics{
-					0: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
-					1: "0x000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98",
-					2: "0x000000000000000000000000d26114cd6ee289accf82350c8d8487fedb8a0c07",
-				},
-				Index: 19,
-				Data:  "0x0000000000000000000000000000000000000000000000000c7d713b49da0000"}
-			rawRPCClient, err := rpc.Dial(test_config.TestClient.IPCPath)
-			Expect(err).NotTo(HaveOccurred())
-			rpcClient := client.NewRPCClient(rawRPCClient, test_config.TestClient.IPCPath)
-			ethClient := ethclient.NewClient(rawRPCClient)
-			blockChainClient := client.NewEthClient(ethClient)
-			node := node.MakeNode(rpcClient)
-			transactionConverter := rpc2.NewRPCTransactionConverter(ethClient)
-			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
-			contract := testing.SampleContract()
-
-			logs, err := blockChain.GetFullSyncLogs(contract, big.NewInt(4703824), nil)
-
-			Expect(err).To(BeNil())
-			Expect(len(logs)).To(Equal(3))
-			Expect(logs[0]).To(Equal(expectedLogZero))
-		})
-
-		It("returns and empty log array when no events for a given block / contract combo", func() {
-			rawRPCClient, err := rpc.Dial(test_config.TestClient.IPCPath)
-			Expect(err).NotTo(HaveOccurred())
-			rpcClient := client.NewRPCClient(rawRPCClient, test_config.TestClient.IPCPath)
-			ethClient := ethclient.NewClient(rawRPCClient)
-			blockChainClient := client.NewEthClient(ethClient)
-			node := node.MakeNode(rpcClient)
-			transactionConverter := rpc2.NewRPCTransactionConverter(ethClient)
-			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
-
-			logs, err := blockChain.GetFullSyncLogs(core.Contract{Hash: "0x123"}, big.NewInt(4703824), nil)
-
-			Expect(err).To(BeNil())
-			Expect(len(logs)).To(Equal(0))
-		})
-	})
-
-	Describe("Fetching Contract data", func() {
-		It("returns the correct attribute for a real contract", func() {
-			rawRPCClient, err := rpc.Dial(test_config.TestClient.IPCPath)
-			Expect(err).NotTo(HaveOccurred())
-			rpcClient := client.NewRPCClient(rawRPCClient, test_config.TestClient.IPCPath)
-			ethClient := ethclient.NewClient(rawRPCClient)
-			blockChainClient := client.NewEthClient(ethClient)
-			node := node.MakeNode(rpcClient)
-			transactionConverter := rpc2.NewRPCTransactionConverter(ethClient)
-			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
-
-			contract := testing.SampleContract()
-			var balance = new(big.Int)
-
-			args := make([]interface{}, 1)
-			args[0] = common.HexToHash("0xd26114cd6ee289accf82350c8d8487fedb8a0c07")
-
-			err = blockChain.FetchContractData(contract.Abi, "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", "balanceOf", args, &balance, 5167471)
-			Expect(err).NotTo(HaveOccurred())
-			expected := new(big.Int)
-			expected.SetString("10897295492887612977137", 10)
-			Expect(balance).To(Equal(expected))
-		})
-	})
-})

integration_test/contract_watcher_header_sync_transformer_test.go

@@ -1,496 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package integration
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-var _ = Describe("contractWatcher headerSync transformer", func() {
-	var db *postgres.DB
-	var err error
-	var blockChain core.BlockChain
-	var headerRepository repositories.HeaderRepository
-	var headerID int64
-	var ensAddr = strings.ToLower(constants.EnsContractAddress)   // 0x314159265dd8dbb310642f98f50c066173c1259b
-	var tusdAddr = strings.ToLower(constants.TusdContractAddress) // 0x8dd5fbce2f6a956c3022ba3663759011dd51e73e
-
-	BeforeEach(func() {
-		db, blockChain = test_helpers.SetupDBandBC()
-		headerRepository = repositories.NewHeaderRepository(db)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
-	Describe("Init", func() {
-		It("Initializes transformer's contract objects", func() {
-			_, insertErr := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-			Expect(insertErr).NotTo(HaveOccurred())
-			_, insertErrTwo := headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-			t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			c, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-
-			// TODO: Fix this
-			// This test sometimes randomly fails because
-			// for some reason the starting block number is not updated from
-			// its original value (5197514) to the block number (6194632)
-			// of the earliest header (mocks.MockHeader1) in the repository
-			// It is not clear how this happens without one of the above insertErrs
-			// having been thrown and without any errors thrown during the Init() call
-			Expect(c.StartingBlock).To(Equal(int64(6194632)))
-			Expect(c.Abi).To(Equal(constants.TusdAbiString))
-			Expect(c.Name).To(Equal("TrueUSD"))
-			Expect(c.Address).To(Equal(tusdAddr))
-		})
-
-		It("initializes when no headers available in db", func() {
-			t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-		})
-
-		It("Does nothing if nothing if no addresses are configured", func() {
-			_, insertErr := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-			Expect(insertErr).NotTo(HaveOccurred())
-			_, insertErrTwo := headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-			var testConf config.ContractConfig
-			testConf = test_helpers.TusdConfig
-			testConf.Addresses = nil
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			_, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(false))
-		})
-	})
-
-	Describe("Execute- against TrueUSD contract", func() {
-		BeforeEach(func() {
-			header1, err := blockChain.GetHeaderByNumber(6791668)
-			Expect(err).ToNot(HaveOccurred())
-			header2, err := blockChain.GetHeaderByNumber(6791669)
-			Expect(err).ToNot(HaveOccurred())
-			header3, err := blockChain.GetHeaderByNumber(6791670)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header1)
-			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header3)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.HeaderSyncTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&log)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(log.HeaderID).To(Equal(headerID))
-			Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
-			Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
-			Expect(log.Value).To(Equal("9998940000000000000000"))
-		})
-
-		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
-			var testConf config.ContractConfig
-			testConf = test_helpers.TusdConfig
-			testConf.Methods = map[string][]string{
-				tusdAddr: {"balanceOf"},
-			}
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			c, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(c.EmittedAddrs)).To(Equal(4))
-			Expect(len(c.EmittedHashes)).To(Equal(0))
-
-			b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[""]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given methods using generated token holder address", func() {
-			var testConf config.ContractConfig
-			testConf = test_helpers.TusdConfig
-			testConf.Methods = map[string][]string{
-				tusdAddr: {"balanceOf"},
-			}
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Balance).To(Equal("55849938025000000000000"))
-			Expect(res.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
-		})
-
-		It("Fails if initialization has not been done", func() {
-			t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db)
-			err = t.Execute()
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("transformer has no initialized contracts"))
-		})
-	})
-
-	Describe("Execute- against ENS registry contract", func() {
-		BeforeEach(func() {
-			header1, err := blockChain.GetHeaderByNumber(6885695)
-			Expect(err).ToNot(HaveOccurred())
-			header2, err := blockChain.GetHeaderByNumber(6885696)
-			Expect(err).ToNot(HaveOccurred())
-			header3, err := blockChain.GetHeaderByNumber(6885697)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header1)
-			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header3)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer(test_helpers.ENSConfig, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(t.Start).To(Equal(int64(6885698)))
-
-			log := test_helpers.HeaderSyncNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&log)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(log.HeaderID).To(Equal(headerID))
-			Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
-			Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
-			Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-		})
-
-		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
-			var testConf config.ContractConfig
-			testConf = test_helpers.ENSConfig
-			testConf.Methods = map[string][]string{
-				ensAddr: {"owner"},
-			}
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			c, ok := t.Contracts[ensAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(c.EmittedHashes)).To(Equal(2))
-			Expect(len(c.EmittedAddrs)).To(Equal(0))
-
-			b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given method using list of collected hashes", func() {
-			var testConf config.ContractConfig
-			testConf = test_helpers.ENSConfig
-			testConf.Methods = map[string][]string{
-				ensAddr: {"owner"},
-			}
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
-		})
-
-		It("It does not persist events if they do not pass the emitted arg filter", func() {
-			var testConf config.ContractConfig
-			testConf = test_helpers.ENSConfig
-			testConf.EventArgs = map[string][]string{
-				ensAddr: {"fake_filter_value"},
-			}
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.HeaderSyncNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&log)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("does not exist"))
-		})
-
-		It("If a method arg filter is applied, only those arguments are used in polling", func() {
-			var testConf config.ContractConfig
-			testConf = test_helpers.ENSConfig
-			testConf.MethodArgs = map[string][]string{
-				ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"},
-			}
-			testConf.Methods = map[string][]string{
-				ensAddr: {"owner"},
-			}
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
-		})
-	})
-
-	Describe("Execute- against both ENS and TrueUSD", func() {
-		BeforeEach(func() {
-			for i := 6885692; i <= 6885701; i++ {
-				header, err := blockChain.GetHeaderByNumber(int64(i))
-				Expect(err).ToNot(HaveOccurred())
-				_, err = headerRepository.CreateOrUpdateHeader(header)
-				Expect(err).ToNot(HaveOccurred())
-			}
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(t.Start).To(Equal(int64(6885702)))
-
-			newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
-			Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
-			Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-
-			transferLog := test_helpers.HeaderSyncTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(transferLog.From).To(Equal("0x8cA465764873E71CEa525F5EB6AE973d650c22C2"))
-			Expect(transferLog.To).To(Equal("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e"))
-			Expect(transferLog.Value).To(Equal("2800000000000000000000"))
-		})
-
-		It("Marks header checked for a contract that has no logs at that header", func() {
-			t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(t.Start).To(Equal(int64(6885702)))
-
-			newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
-			Expect(err).ToNot(HaveOccurred())
-			transferLog := test_helpers.HeaderSyncTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(transferLog.HeaderID).ToNot(Equal(newOwnerLog.HeaderID))
-
-			type checkedHeader struct {
-				ID       int64 `db:"id"`
-				HeaderID int64 `db:"header_id"`
-				NewOwner int64 `db:"newowner_0x314159265dd8dbb310642f98f50c066173c1259b"`
-				Transfer int64 `db:"transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e"`
-			}
-
-			transferCheckedHeader := new(checkedHeader)
-			err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", transferLog.HeaderID).StructScan(transferCheckedHeader)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(transferCheckedHeader.Transfer).To(Equal(int64(1)))
-			Expect(transferCheckedHeader.NewOwner).To(Equal(int64(1)))
-
-			newOwnerCheckedHeader := new(checkedHeader)
-			err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", newOwnerLog.HeaderID).StructScan(newOwnerCheckedHeader)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(newOwnerCheckedHeader.NewOwner).To(Equal(int64(1)))
-			Expect(newOwnerCheckedHeader.Transfer).To(Equal(int64(1)))
-		})
-
-		It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
-			var testConf config.ContractConfig
-			testConf = test_helpers.ENSandTusdConfig
-			testConf.Methods = map[string][]string{
-				ensAddr:  {"owner"},
-				tusdAddr: {"balanceOf"},
-			}
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			ens, ok := t.Contracts[ensAddr]
-			Expect(ok).To(Equal(true))
-			tusd, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(ens.EmittedHashes)).To(Equal(2))
-			Expect(len(ens.EmittedAddrs)).To(Equal(0))
-			Expect(len(tusd.EmittedAddrs)).To(Equal(2))
-			Expect(len(tusd.EmittedHashes)).To(Equal(0))
-
-			b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = tusd.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given methods for each contract, using list of collected values", func() {
-			var testConf config.ContractConfig
-			testConf = test_helpers.ENSandTusdConfig
-			testConf.Methods = map[string][]string{
-				ensAddr:  {"owner"},
-				tusdAddr: {"balanceOf"},
-			}
-			t := transformer.NewTransformer(testConf, blockChain, db)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			owner := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(owner.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(owner.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
-
-			bal := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal.Balance).To(Equal("1954436000000000000000"))
-			Expect(bal.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
-		})
-	})
-})

integration_test/geth_blockchain_test.go

@@ -1,121 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package integration_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Reading from the Geth blockchain", func() {
-	var blockChain *eth.BlockChain
-
-	BeforeEach(func() {
-		rawRPCClient, err := rpc.Dial(test_config.TestClient.IPCPath)
-		Expect(err).NotTo(HaveOccurred())
-		rpcClient := client.NewRPCClient(rawRPCClient, test_config.TestClient.IPCPath)
-		ethClient := ethclient.NewClient(rawRPCClient)
-		blockChainClient := client.NewEthClient(ethClient)
-		node := node.MakeNode(rpcClient)
-		transactionConverter := rpc2.NewRPCTransactionConverter(ethClient)
-		blockChain = eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
-	})
-
-	It("retrieves the genesis block and first block", func(done Done) {
-		genesisBlock, err := blockChain.GetBlockByNumber(int64(0))
-		Expect(err).ToNot(HaveOccurred())
-		firstBlock, err := blockChain.GetBlockByNumber(int64(1))
-		Expect(err).ToNot(HaveOccurred())
-		lastBlockNumber, err := blockChain.LastBlock()
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(genesisBlock.Number).To(Equal(int64(0)))
-		Expect(firstBlock.Number).To(Equal(int64(1)))
-		Expect(lastBlockNumber.Int64()).To(BeNumerically(">", 0))
-		close(done)
-	}, 15)
-
-	It("retrieves the node info", func(done Done) {
-		node := blockChain.Node()
-
-		Expect(node.GenesisBlock).ToNot(BeNil())
-		Expect(node.NetworkID).To(Equal("1.000000"))
-		Expect(len(node.ID)).ToNot(BeZero())
-		Expect(node.ClientName).ToNot(BeZero())
-
-		close(done)
-	}, 15)
-
-	It("retrieves transaction", func() {
-		// actual transaction: https://etherscan.io/tx/0x44d462f2a19ad267e276b234a62c542fc91c974d2e4754a325ca405f95440255
-		txHash := common.HexToHash("0x44d462f2a19ad267e276b234a62c542fc91c974d2e4754a325ca405f95440255")
-		transactions, err := blockChain.GetTransactions([]common.Hash{txHash})
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(len(transactions)).To(Equal(1))
-		expectedData := []byte{149, 227, 197, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-			1, 160, 85, 105, 13, 157, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
-			241, 202, 218, 90, 30, 178, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-			0, 0, 0, 92, 155, 193, 43}
-		expectedRaw := []byte{248, 201, 9, 132, 59, 154, 202, 0, 131, 1, 102, 93, 148, 44, 75, 208, 100, 185, 152, 131,
-			128, 118, 250, 52, 26, 131, 208, 7, 252, 47, 165, 9, 87, 128, 184, 100, 149, 227, 197, 11, 0, 0, 0, 0, 0, 0, 0,
-			0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 160, 85, 105, 13, 157, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-			0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 241, 202, 218, 90, 30, 178, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-			0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 155, 193, 43, 37, 160, 237, 184, 236, 248, 23, 152,
-			53, 238, 44, 215, 181, 234, 229, 157, 246, 212, 178, 88, 25, 116, 134, 163, 124, 64, 2, 66, 25, 118, 1, 253, 27,
-			101, 160, 36, 226, 116, 43, 147, 236, 124, 76, 227, 250, 228, 168, 22, 19, 248, 155, 248, 151, 219, 14, 1, 186,
-			159, 35, 154, 22, 222, 123, 254, 147, 63, 221}
-		expectedModel := core.TransactionModel{
-			Data:     expectedData,
-			From:     "0x3b08b99441086edd66f36f9f9aee733280698378",
-			GasLimit: 91741,
-			GasPrice: 1000000000,
-			Hash:     "0x44d462f2a19ad267e276b234a62c542fc91c974d2e4754a325ca405f95440255",
-			Nonce:    9,
-			Raw:      expectedRaw,
-			Receipt:  core.Receipt{},
-			To:       "0x2c4bd064b998838076fa341a83d007fc2fa50957",
-			TxIndex:  30,
-			Value:    "0",
-		}
-		Expect(transactions[0]).To(Equal(expectedModel))
-	})
-
-	//Benchmarking test: remove skip to test performance of block retrieval
-	XMeasure("retrieving n blocks", func(b Benchmarker) {
-		b.Time("runtime", func() {
-			var blocks []core.Block
-			n := 10
-			for i := 5327459; i > 5327459-n; i-- {
-				block, err := blockChain.GetBlockByNumber(int64(i))
-				Expect(err).ToNot(HaveOccurred())
-				blocks = append(blocks, block)
-			}
-			Expect(len(blocks)).To(Equal(n))
-		})
-	}, 10)
-})

integration_test/getter_test.go

@@ -1,57 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package integration_test
-
-import (
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/getter"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Interface Getter", func() {
-	Describe("GetAbi", func() {
-		It("Constructs and returns a custom abi based on results from supportsInterface calls", func() {
-			expectedABI := `[` + constants.AddrChangeInterface + `,` + constants.NameChangeInterface + `,` + constants.ContentChangeInterface + `,` + constants.AbiChangeInterface + `,` + constants.PubkeyChangeInterface + `]`
-			con := test_config.TestClient
-			testIPC := con.IPCPath
-			blockNumber := int64(6885696)
-			rawRpcClient, err := rpc.Dial(testIPC)
-			Expect(err).NotTo(HaveOccurred())
-			rpcClient := client.NewRPCClient(rawRpcClient, testIPC)
-			ethClient := ethclient.NewClient(rawRpcClient)
-			blockChainClient := client.NewEthClient(ethClient)
-			node := node.MakeNode(rpcClient)
-			transactionConverter := rpc2.NewRPCTransactionConverter(ethClient)
-			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
-			interfaceGetter := getter.NewInterfaceGetter(blockChain)
-			abi, err := interfaceGetter.GetABI(constants.PublicResolverAddress, blockNumber)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(abi).To(Equal(expectedABI))
-			_, err = eth.ParseAbi(abi)
-			Expect(err).ToNot(HaveOccurred())
-		})
-	})
-})

integration_test/poller_test.go

@@ -1,252 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package integration_test
-
-import (
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/poller"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-var _ = Describe("Poller", func() {
-
-	var contractPoller poller.Poller
-	var con *contract.Contract
-	var db *postgres.DB
-	var bc core.BlockChain
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
-	Describe("Full sync mode", func() {
-		BeforeEach(func() {
-			db, bc = test_helpers.SetupDBandBC()
-			contractPoller = poller.NewPoller(bc, db, types.FullSync)
-		})
-
-		Describe("PollContract", func() {
-			It("Polls specified contract methods using contract's argument list", func() {
-				con = test_helpers.SetupTusdContract(nil, []string{"balanceOf"})
-				Expect(con.Abi).To(Equal(constants.TusdAbiString))
-				con.StartingBlock = 6707322
-				con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE"))
-
-				err := contractPoller.PollContract(*con, 6707323)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.BalanceOf{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
-				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
-				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
-				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
-				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
-			})
-
-			It("Polls specified contract methods using contract's hash list", func() {
-				con = test_helpers.SetupENSContract(nil, []string{"owner"})
-				Expect(con.Abi).To(Equal(constants.ENSAbiString))
-				Expect(len(con.Methods)).To(Equal(1))
-				con.AddEmittedHash(common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"), common.HexToHash("0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86"))
-
-				err := contractPoller.PollContractAt(*con, 6885877)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.Owner{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Address).To(Equal("0x546aA2EaE2514494EeaDb7bbb35243348983C59d"))
-				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
-			})
-
-			It("Does not poll and persist any methods if none are specified", func() {
-				con = test_helpers.SetupTusdContract(nil, nil)
-				Expect(con.Abi).To(Equal(constants.TusdAbiString))
-				con.StartingBlock = 6707322
-				con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE"))
-
-				err := contractPoller.PollContract(*con, 6707323)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.BalanceOf{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).To(HaveOccurred())
-			})
-		})
-
-		Describe("FetchContractData", func() {
-			It("Calls a single contract method", func() {
-				var name = new(string)
-				err := contractPoller.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(*name).To(Equal("TrueUSD"))
-			})
-		})
-	})
-
-	Describe("Header sync mode", func() {
-		BeforeEach(func() {
-			db, bc = test_helpers.SetupDBandBC()
-			contractPoller = poller.NewPoller(bc, db, types.HeaderSync)
-		})
-
-		Describe("PollContract", func() {
-			It("Polls specified contract methods using contract's token holder address list", func() {
-				con = test_helpers.SetupTusdContract(nil, []string{"balanceOf"})
-				Expect(con.Abi).To(Equal(constants.TusdAbiString))
-				con.StartingBlock = 6707322
-				con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE"))
-
-				err := contractPoller.PollContract(*con, 6707323)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.BalanceOf{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
-				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
-				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
-				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
-				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
-			})
-
-			It("Polls specified contract methods using contract's hash list", func() {
-				con = test_helpers.SetupENSContract(nil, []string{"owner"})
-				Expect(con.Abi).To(Equal(constants.ENSAbiString))
-				Expect(len(con.Methods)).To(Equal(1))
-				con.AddEmittedHash(common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"), common.HexToHash("0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86"))
-
-				err := contractPoller.PollContractAt(*con, 6885877)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.Owner{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Address).To(Equal("0x546aA2EaE2514494EeaDb7bbb35243348983C59d"))
-				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
-			})
-
-			It("Does not poll and persist any methods if none are specified", func() {
-				con = test_helpers.SetupTusdContract(nil, nil)
-				Expect(con.Abi).To(Equal(constants.TusdAbiString))
-				con.StartingBlock = 6707322
-				con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE"))
-
-				err := contractPoller.PollContract(*con, 6707323)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.BalanceOf{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
-				Expect(err).To(HaveOccurred())
-			})
-
-			It("Caches returned values of the appropriate types for downstream method polling if method piping is turned on", func() {
-				con = test_helpers.SetupENSContract(nil, []string{"resolver"})
-				Expect(con.Abi).To(Equal(constants.ENSAbiString))
-				con.StartingBlock = 6921967
-				con.EmittedAddrs = map[interface{}]bool{}
-				con.Piping = false
-				con.AddEmittedHash(common.HexToHash("0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8"))
-				err := contractPoller.PollContract(*con, 6921968)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.Resolver{}
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Address).To(Equal("0x5FfC014343cd971B7eb70732021E26C35B744cc4"))
-				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
-				Expect(len(con.EmittedAddrs)).To(Equal(0)) // With piping off the address is not saved
-
-				test_helpers.TearDown(db)
-				db, bc = test_helpers.SetupDBandBC()
-				contractPoller = poller.NewPoller(bc, db, types.HeaderSync)
-
-				con.Piping = true
-				err = contractPoller.PollContract(*con, 6921968)
-				Expect(err).ToNot(HaveOccurred())
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanStruct.Address).To(Equal("0x5FfC014343cd971B7eb70732021E26C35B744cc4"))
-				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
-				Expect(len(con.EmittedAddrs)).To(Equal(1)) // With piping on it is saved
-				Expect(con.EmittedAddrs[common.HexToAddress("0x5FfC014343cd971B7eb70732021E26C35B744cc4")]).To(Equal(true))
-			})
-		})
-
-		Describe("FetchContractData", func() {
-			It("Calls a single contract method", func() {
-				var name = new(string)
-				err := contractPoller.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(*name).To(Equal("TrueUSD"))
-			})
-		})
-	})
-})

libraries/shared/README.md

@@ -1,8 +0,0 @@
-# Shared Tools
-
-## Description
-Code that is useful for or used by plugins written on top of VulcanizeDB.
-
-## Note
-Much code in this directory may not be used outside of the tests, but don't delete it - it could be used by a plugin.
-Renaming and/or deleting functions in this namespace requires a version bump to avoid breaking plugins.

libraries/shared/chunker/chunker_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package chunker_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	log "github.com/sirupsen/logrus"
-)
-
-func TestFactories(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Shared Chunker Suite")
-}
-
-var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
-})

libraries/shared/chunker/log_chunker.go

@@ -1,69 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package chunker
-
-import (
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-type Chunker interface {
-	AddConfig(transformerConfig transformer.EventTransformerConfig)
-	ChunkLogs(logs []core.HeaderSyncLog) map[string][]core.HeaderSyncLog
-}
-
-type LogChunker struct {
-	AddressToNames map[string][]string
-	NameToTopic0   map[string]common.Hash
-}
-
-// Returns a new log chunker with initialised maps.
-// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer.
-func NewLogChunker() *LogChunker {
-	return &LogChunker{
-		AddressToNames: map[string][]string{},
-		NameToTopic0:   map[string]common.Hash{},
-	}
-}
-
-// Configures the chunker by adding one config with more addresses and topics to consider.
-func (chunker *LogChunker) AddConfig(transformerConfig transformer.EventTransformerConfig) {
-	for _, address := range transformerConfig.ContractAddresses {
-		var lowerCaseAddress = strings.ToLower(address)
-		chunker.AddressToNames[lowerCaseAddress] = append(chunker.AddressToNames[lowerCaseAddress], transformerConfig.TransformerName)
-		chunker.NameToTopic0[transformerConfig.TransformerName] = common.HexToHash(transformerConfig.Topic)
-	}
-}
-
-// Goes through a slice of logs, associating relevant logs (matching addresses and topic) with transformers
-func (chunker *LogChunker) ChunkLogs(logs []core.HeaderSyncLog) map[string][]core.HeaderSyncLog {
-	chunks := map[string][]core.HeaderSyncLog{}
-	for _, log := range logs {
-		// Topic0 is not unique to each transformer, also need to consider the contract address
-		relevantTransformers := chunker.AddressToNames[strings.ToLower(log.Log.Address.Hex())]
-
-		for _, t := range relevantTransformers {
-			if chunker.NameToTopic0[t] == log.Log.Topics[0] {
-				chunks[t] = append(chunks[t], log)
-			}
-		}
-	}
-	return chunks
-}

libraries/shared/chunker/log_chunker_test.go

@@ -1,163 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package chunker_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	chunk "github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-var _ = Describe("Log chunker", func() {
-	var (
-		chunker *chunk.LogChunker
-	)
-
-	BeforeEach(func() {
-		chunker = chunk.NewLogChunker()
-
-		configA := transformer.EventTransformerConfig{
-			TransformerName:   "TransformerA",
-			ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"},
-			Topic:             "0xA",
-		}
-		chunker.AddConfig(configA)
-
-		configB := transformer.EventTransformerConfig{
-			TransformerName:   "TransformerB",
-			ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"},
-			Topic:             "0xB",
-		}
-		chunker.AddConfig(configB)
-
-		configC := transformer.EventTransformerConfig{
-			TransformerName:   "TransformerC",
-			ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"},
-			Topic:             "0xC",
-		}
-		chunker.AddConfig(configC)
-	})
-
-	Describe("initialisation", func() {
-		It("creates lookup maps correctly", func() {
-			Expect(chunker.AddressToNames).To(Equal(map[string][]string{
-				"0x00000000000000000000000000000000000000a1": {"TransformerA"},
-				"0x00000000000000000000000000000000000000a2": {"TransformerA", "TransformerC"},
-				"0x00000000000000000000000000000000000000b1": {"TransformerB"},
-			}))
-
-			Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{
-				"TransformerA": common.HexToHash("0xA"),
-				"TransformerB": common.HexToHash("0xB"),
-				"TransformerC": common.HexToHash("0xC"),
-			}))
-		})
-	})
-
-	Describe("AddConfig", func() {
-		It("can add more configs later", func() {
-			configD := transformer.EventTransformerConfig{
-				TransformerName:   "TransformerD",
-				ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
-				Topic:             "0xD",
-			}
-			chunker.AddConfig(configD)
-
-			Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"}))
-			Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD")))
-		})
-
-		It("lower cases address", func() {
-			configD := transformer.EventTransformerConfig{
-				TransformerName:   "TransformerD",
-				ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
-				Topic:             "0xD",
-			}
-			chunker.AddConfig(configD)
-
-			Expect(chunker.AddressToNames["0x000000000000000000000000000000000000000d"]).To(Equal([]string{"TransformerD"}))
-		})
-	})
-
-	Describe("ChunkLogs", func() {
-		It("only associates logs with relevant topic0 and address to transformers", func() {
-			logs := []core.HeaderSyncLog{log1, log2, log3, log4, log5}
-			chunks := chunker.ChunkLogs(logs)
-
-			Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4)))
-			Expect(chunks["TransformerB"]).To(BeEmpty())
-			Expect(chunks["TransformerC"]).To(ContainElement(log5))
-		})
-	})
-})
-
-var (
-	// Match TransformerA
-	log1 = core.HeaderSyncLog{
-		Log: types.Log{
-			Address: common.HexToAddress("0xA1"),
-			Topics: []common.Hash{
-				common.HexToHash("0xA"),
-				common.HexToHash("0xLogTopic1"),
-			},
-		},
-	}
-	// Match TransformerA address, but not topic0
-	log2 = core.HeaderSyncLog{
-		Log: types.Log{
-			Address: common.HexToAddress("0xA1"),
-			Topics: []common.Hash{
-				common.HexToHash("0xB"),
-				common.HexToHash("0xLogTopic2"),
-			},
-		},
-	}
-	// Match TransformerA topic, but TransformerB address
-	log3 = core.HeaderSyncLog{
-		Log: types.Log{
-			Address: common.HexToAddress("0xB1"),
-			Topics: []common.Hash{
-				common.HexToHash("0xA"),
-				common.HexToHash("0xLogTopic3"),
-			},
-		},
-	}
-	// Match TransformerA, with the other address
-	log4 = core.HeaderSyncLog{
-		Log: types.Log{
-			Address: common.HexToAddress("0xA2"),
-			Topics: []common.Hash{
-				common.HexToHash("0xA"),
-				common.HexToHash("0xLogTopic4"),
-			},
-		},
-	}
-	// Match TransformerC, which shares address with TransformerA
-	log5 = core.HeaderSyncLog{
-		Log: types.Log{
-			Address: common.HexToAddress("0xA2"),
-			Topics: []common.Hash{
-				common.HexToHash("0xC"),
-				common.HexToHash("0xLogTopic5"),
-			},
-		},
-	}
-)

libraries/shared/constants/checked_headers.go

@@ -1,25 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package constants
-
-type TransformerExecution bool
-
-const (
-	HeaderRecheck    TransformerExecution = true
-	HeaderUnchecked  TransformerExecution = false
-	RecheckHeaderCap                      = int64(5)
-)

libraries/shared/constants/data.go

@@ -1,19 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package constants
-
-var DataItemLength = 32

libraries/shared/constants/external.go

@@ -1,80 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package constants
-
-import (
-	"fmt"
-	"math"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
-)
-
-var initialized = false
-
-func initConfig() {
-	if initialized {
-		return
-	}
-
-	if err := viper.ReadInConfig(); err == nil {
-		log.Info("Using config file:", viper.ConfigFileUsed())
-	} else {
-		panic(fmt.Sprintf("Could not find environment file: %v", err))
-	}
-	initialized = true
-}
-
-// GetMinDeploymentBlock gets the minimum deployment block for multiple contracts from config
-func GetMinDeploymentBlock() uint64 {
-	initConfig()
-	contractNames := getContractNames()
-	if len(contractNames) < 1 {
-		log.Fatalf("No contracts supplied")
-	}
-	minBlock := uint64(math.MaxUint64)
-	for c := range contractNames {
-		deployed := getDeploymentBlock(c)
-		if deployed < minBlock {
-			minBlock = deployed
-		}
-	}
-	return minBlock
-}
-
-func getContractNames() map[string]bool {
-	transformerNames := viper.GetStringSlice("exporter.transformerNames")
-	contractNames := make(map[string]bool)
-	for _, transformerName := range transformerNames {
-		configKey := "exporter." + transformerName + ".contracts"
-		names := viper.GetStringSlice(configKey)
-		for _, name := range names {
-			contractNames[name] = true
-		}
-	}
-	return contractNames
-}
-
-func getDeploymentBlock(contractName string) uint64 {
-	configKey := "contract." + contractName + ".deployed"
-	value := viper.GetInt64(configKey)
-	if value < 0 {
-		log.Infof("No deployment block configured for contract \"%v\", defaulting to 0.", contractName)
-		return 0
-	}
-	return uint64(value)
-}

libraries/shared/factories/event/README.md

@@ -1,388 +0,0 @@
-# Watching Contract Events
-	
-One approach VulcanizeDB takes to caching and indexing smart contracts is to watch contract events emitted in receipt logs.
-
-With a header synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
-fetch and verify relevant event logs from a full Ethereum node, keeping track of which headers we have checked for which events 
-with our `checked_headers` table.
-
-## Assumptions
-
-This approach assumes you are running a vDB header sync which is run against a light Ethereum node;
-this approach also assumes there is a full node available. 
-
-Looking forward, we will be building fetchers that enable sourcing data from IPFS instead of an ETH node.
-
-## Shared Code
-
-VulcanizeDB has shared code built out for building and plugging in event transformers
-
-### [Event Watcher (header sync)](../staging/libraries/shared/watcher/event_watcher.go)
-
-The event watcher is responsible for continuously fetching and delegating chunks of logs and their associated header to the appropriate transformers.
-
-Using the `compose` or `composeAndExecute` command, event watchers can be loaded with plugin event transformers and execute over them.
-
-### [Event Transformer](../staging/libraries/shared/transformer/event_transformer.go)
-
-The event transformer is responsible for converting event logs into more useful data objects and storing them in Postgres.
-The event transformer is composed of converter and repository interfaces and a config struct:
-```go
-type EventTransformer struct {
-	Config     transformer.EventTransformerConfig
-	Converter  Converter
-	Repository Repository
-}
-```
-
-The event transformer executes over provided event logs at a given header.
-
-In this process, the converter unpacks these logs into entities and then converts these entities 
-to their final db models. These models are then written to the Postgres db by the repository.
-
-```go
-func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
-	transformerName := transformer.Config.TransformerName
-	config := transformer.Config
-
-	if len(logs) < 1 {
-		err := transformer.Repository.MarkHeaderChecked(header.Id)
-		if err != nil {
-			log.Printf("Error marking header as checked in %v: %v", transformerName, err)
-			return err
-		}
-		return nil
-	}
-
-	entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
-	if err != nil {
-		log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
-		return err
-	}
-
-	models, err := transformer.Converter.ToModels(entities)
-	if err != nil {
-		log.Printf("Error converting entities to models in %v: %v", transformerName, err)
-		return err
-	}
-
-	err = transformer.Repository.Create(header.Id, models)
-	if err != nil {
-		log.Printf("Error persisting %v record: %v", transformerName, err)
-		return err
-	}
-
-	return nil
-}
-```
-
-
-## Custom Code
-
-In order to watch events at a smart contract, for those events the developer must create:
-
-1. Config - struct to hold configuration information (contract address, starting block, event name and signature).
-1. Entity - struct to unpack the event log into.
-1. Model - struct representing the final data model we want to write to Postgres.
-1. Converter - an interface which can unpack event logs into our entities and convert those entities to our models. 
-1. Repository - an interface to write our models to Postgres.
-1. EventTransformerInitializer - a public variable which exports our configured transformer to be loaded as part of a plugin.
-1. DB migrations - migrations to generate the Postgres schema, tables, views, function, etc that are needed to store and interface with the transformed data models.
-
-The example event we will use looks like: 
-```
-event ExampleEvent(bytes32 indexed arg1, address indexed arg2, bytes32 arg3, uint256 arg4, uint256 arg5);
-```
-
-### Config
-
-The config holds configuration variables for the event transformer, including a name for the transformer, the contract address
-it is working at, the contract's ABI, the topic (e.g. event signature; topic0) that it is filtering for, and starting
-and ending block numbers.
-
-```go
-type EventTransformerConfig struct {
-	TransformerName     string
-	ContractAddresses   []string
-	ContractAbi         string
-	Topic               string
-	StartingBlockNumber int64
-	EndingBlockNumber   int64 // Set -1 for indefinite transformer
-}
-```
-
-### Entity
-
-Entity field names for event arguments need to be exported and match the argument's name and type. LogIndex, 
-TransactionIndex, and the Raw log are retained in order to link the data to it's source for downstream validation.
-
-```go
-type ExampleEntity struct {
-	Arg1             common.Hash
-	Arg2             common.Address
-	Arg3             common.Hash
-	Arg4             *big.Int
-	Arg5             *big.Int
-	LogIndex         uint
-	TransactionIndex uint
-	Raw              types.Log
-}
-```
-
-### Model
-
-Model fields are not constrained by the event log structure.
-This allows us to rename our fields, decode or convert our log values into more useful types, and perform operations 
-with or on the values before persisting the data to Postgres.
-
-```go
-type ExampleModel struct {
-	EventHash         string
-	UserAddress       string
-	FractionSkimmed   string 
-	Surplus           string 
-	Deficit           string
-	FinalPosition     string
-	LogIndex          uint
-	TransactionIndex  uint
-	Raw               types.Log
-}
-```
-
-### Converter
-
-The converter needs to satisfy the interface. One for unpacking logs into the custom defined entities, and
-another for converting those entities to their final db models.
-
-```go
-type Converter interface {
-	ToEntities(contractAbi string, ethLog []types.Log) ([]interface{}, error)
-	ToModels([]interface{}) ([]interface{}, error)
-}
-```
-
-For the example event, this might look like:
-```go
-type ExampleConverter struct{}
-
-func (ExampleConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
-	var entities []interface{}
-	for _, ethLog := range ethLogs {
-		entity := &ExampleEntity{}
-		address := ethLog.Address
-		abi, err := geth.ParseAbi(contractAbi)
-		if err != nil {
-			return nil, err
-		}
-
-		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
-
-		err = contract.UnpackLog(entity, "ExampleEvent", ethLog)
-		if err != nil {
-			return nil, err
-		}
-
-		entity.Raw = ethLog
-		entity.LogIndex = ethLog.Index
-		entity.TransactionIndex = ethLog.TxIndex
-
-		entities = append(entities, *entity)
-	}
-
-	return entities, nil
-}
-
-func (converter ExampleConverter) ToModels(entities []interface{}) ([]interface{}, error) {
-	var models []interface{}
-	for _, entity := range entities {
-        entity, ok := entity.(ExampleModel)
-        if !ok {
-            return nil, fmt.Errorf("entity of type %T, not %T", entity, ExampleModel{})
-        }
-        
-        fractionSkimmed, err := hexutil.DecodeBig(entity.Arg3.Hex())
-        if err != nil {
-        	reuturn nil, err
-        }
-        position := new(big.Int)
-        position.Sub(entity.Arg4, entity.Arg5)
-        finalPosition := new(big.Int)
-        if preTaxPosition.Sign() < 0 {
-            finalPosition = position
-        } else {
-            skim := new(big.Int)
-            skim.Div(position, fractionSkimmed)
-            finalPosition = position.Sub(position, skim)
-        }
-        
-        rawLog, err := json.Marshal(entity.Raw)
-        if err != nil {
-            return nil, err
-        }
-        
-        model := ExampleModel{
-            EventHash:        entity.Arg1.Hex(),
-            UserAddress:      entity.Arg2.Hex(),
-            FractionSkimmed:  fractionSkimmed.String(), 
-            Surplus:          entity.Arg4.String(),
-            Deficit:          entity.Arg5.String(),
-            FinalPosition:    finalPosition,
-            LogIndex:         entity.LogIndex,
-            TransactionIndex: entity.TransactionIndex,
-            Raw:              rawLog,
-        }
-        models = append(models, model)
-	}
-	return models, nil
-}
-```
-Notice that in this example we have a bytes32 argument in the event that needs to be decoded to an integer before it can be worked with
-to produce our hypothetical `FinalPosition` field. This is to highlight the fact that contracts can and sometimes do encode the
-data types we want to work with into raw bytes. Writing custom transformers with these converters allows us to account for this.
-
-### Repository
-
-The repository needs to satisfy the interface and use the `Create` method to write the model to Postgres.
-```go
-type Repository interface {
-	Create(headerID int64, models []interface{}) error
-	MarkHeaderChecked(headerID int64) error
-	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
-	RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
-	SetDB(db *postgres.DB)
-}
-```
-
-For the example event, this might look like:
-```go
-type ExampleRepository struct {
-	db *postgres.DB
-}
-
-func (repository *ExampleRepository) SetDB(db *postgres.DB) {
-	repository.db = db
-}
-
-func (repository ExampleRepository) Create(headerID int64, models []interface{}) error {
-	tx, dBaseErr := repository.db.Beginx()
-	if dBaseErr != nil {
-		return dBaseErr
-	}
-	for _, model := range models {
-		model, ok := model.(ExampleModel)
-		if !ok {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				log.Error("failed to rollback ", rollbackErr)
-			}
-			return fmt.Errorf("model of type %T, not %T", model, ExampleModel{})
-		}
-		
-		_, execErr := tx.Exec(
-			`INSERT into example_schema.example_event (header_id, event_hash, user_address, fraction_skimmed, surplus, deficit, final_position, log_idx, tx_idx, raw_log)
-        			VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
-					ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET event_hash = $2, user_address = $3, fraction_skimmed = $4, surplus = $5, deficit = $6, final_position = $7, raw_log = $10;`,
-			headerID, model.EventHash, model.UserAddress, model.FractonSkimmed, model.Surplus, model.Deficit, model.FinalPosition, model.LogIndex, model.TransactionIndex, model.Raw,
-		)
-		if execErr != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				log.Error("failed to rollback ", rollbackErr)
-			}
-			return execErr
-		}
-	}
-
-	checkHeaderErr := repo.MarkHeaderCheckedInTransaction(headerID, tx, "example_event_checked")
-	if checkHeaderErr != nil {
-		rollbackErr := tx.Rollback()
-		if rollbackErr != nil {
-			log.Error("failed to rollback ", rollbackErr)
-		}
-		return checkHeaderErr
-	}
-
-	return tx.Commit()
-}
-
-func (repository ExampleRepository) MarkHeaderChecked(headerID int64) error {
-	return repo.MarkHeaderChecked(headerID, repository.db, "example_event_checked")
-}
-
-func (repository ExampleRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
-	return repo.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db,"example_event_checked")
-}
-
-func (repository ExampleRepository) RecheckHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
-	return repo.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, "example_event_checked")
-}
-```
-
-### EventTransformerInitializer
-
-A transformer initializer variable needs to be exported from somewhere within the transformer repository so that the transformer can be
-loaded as part of a plugin in the `compose` or `composeAndExecute` commands. It is important that this variable is named `EventTransformerInitializer` and
-it must be of `type EventTransformerInitializer func(db *postgres.DB) EventTransformer`.
-
-```go
-var EventTransformerInitializer transformer.EventTransformerInitializer = factories.Transformer{
-	Config:     exampleEventConfig,
-	Converter:  ExampleConverter{},
-	Repository: &ExampleRepository{},
-}.NewTransformer
-```
-
-### DB migrations
-
-We use `goose` as our migration management tool. Any Go data model that needs to be written to Postgres by the
-repository needs a db migration for the corresponding Postgres data model.
-
-Each contract or set of transformers being watched should define its own namespace with a db schema:
-```postgresql
--- +goose Up
-CREATE SCHEMA example_schema;
-
-
--- +goose Down
-DROP SCHEMA example_schema;
-
-```
-
-For the example event and its resulting model, the table we write to would look like:
-```postgresql
--- +goose Up
-CREATE TABLE example_schema.example_event (
-  id               SERIAL PRIMARY KEY,
-  header_id        INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-  event_hash       CHARACTER VARYING(66) NOT NULL,
-  user_address     CHARACTER VARYING(66) NOT NULL,
-  fraction_skimmed NUMERIC NOT NULL,
-  surplus          NUMERIC NOT NULL,
-  deficit          NUMERIC NOT NULL,
-  final_position   NUMERIC NOT NULL,
-  tx_idx           INTEGER NOT NUll,
-  log_idx          INTEGER NOT NUll,
-  raw_log          JSONB,
-  UNIQUE (header_id, tx_idx, log_idx)
-);
-
-ALTER TABLE public.checked_headers
-  ADD COLUMN example_event_checked INTEGER NOT NULL DEFAULT 0;
-
-
--- +goose Down
-DROP TABLE example_schema.example_event;
-
-ALTER TABLE public.checked_headers
-  DROP COLUMN example_event_checked;
-``` 
-
-Notice that we have also added a column to the `checked_headers` table for this event so that we can keep track
-of which headers we have already filtered through for this event.
-
-## Summary
-
-To create a transformer for a contract event we need to create entities for unpacking the raw log, models to represent
-the final data structure, a converter to mediate this unpacking and conversion between entities to models, a repository to write
-these models to Postgres, db migrations to accommodate these models in Postgres, and a EventTransformerInitializer to export the
-configured transformer and load it as a plugin to the `compose` or `composeAndExecute` commands as described in the main readme.

libraries/shared/factories/event/converter.go

@@ -1,28 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event
-
-import (
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-// Converter transforms log data into general InsertionModels the Repository can persist__
-type Converter interface {
-	ToModels(contractAbi string, ethLog []core.HeaderSyncLog) ([]InsertionModel, error)
-	SetDB(db *postgres.DB)
-}

libraries/shared/factories/event/factories_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	log "github.com/sirupsen/logrus"
-)
-
-func TestFactories(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Factories Suite")
-}
-
-var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
-})

libraries/shared/factories/event/repository.go

@@ -1,179 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event
-
-import (
-	"database/sql/driver"
-	"fmt"
-	"strings"
-
-	"github.com/vulcanize/vulcanizedb/utils"
-
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-const SetLogTransformedQuery = `UPDATE public.header_sync_logs SET transformed = true WHERE id = $1`
-
-// Repository persists transformed values to the DB
-type Repository interface {
-	Create(models []InsertionModel) error
-	SetDB(db *postgres.DB)
-}
-
-// LogFK is the name of log foreign key columns
-const LogFK ColumnName = "log_id"
-
-// AddressFK is the name of address foreign key columns
-const AddressFK ColumnName = "address_id"
-
-// HeaderFK is the name of header foreign key columns
-const HeaderFK ColumnName = "header_id"
-
-// SchemaName is the schema to work with
-type SchemaName string
-
-// TableName identifies the table for inserting the data
-type TableName string
-
-// ColumnName identifies columns on the given table
-type ColumnName string
-
-// ColumnValues maps a column to the value for insertion. This is restricted to []byte, bool, float64, int64, string, time.Time
-type ColumnValues map[ColumnName]interface{}
-
-// ErrUnsupportedValue is thrown when a model supplies a type of value the postgres driver cannot handle.
-var ErrUnsupportedValue = func(value interface{}) error {
-	return fmt.Errorf("unsupported type of value supplied in model: %v (%T)", value, value)
-}
-
-// InsertionModel is the generalised data structure a converter returns, and contains everything the repository needs to
-// persist the converted data.
-type InsertionModel struct {
-	SchemaName     SchemaName
-	TableName      TableName
-	OrderedColumns []ColumnName // Defines the fields to insert, and in which order the table expects them
-	ColumnValues   ColumnValues // Associated values for columns, restricted to []byte, bool, float64, int64, string, time.Time
-}
-
-// ModelToQuery stores memoised insertion queries to minimise computation
-var ModelToQuery = map[string]string{}
-
-// GetMemoizedQuery gets/creates a DB insertion query for the model
-func GetMemoizedQuery(model InsertionModel) string {
-	// The schema and table name uniquely determines the insertion query, use that for memoization
-	queryKey := string(model.SchemaName) + string(model.TableName)
-	query, queryMemoized := ModelToQuery[queryKey]
-	if !queryMemoized {
-		query = GenerateInsertionQuery(model)
-		ModelToQuery[queryKey] = query
-	}
-	return query
-}
-
-// GenerateInsertionQuery creates an SQL insertion query from an insertion model.
-// Should be called through GetMemoizedQuery, so the query is not generated on each call to Create.
-func GenerateInsertionQuery(model InsertionModel) string {
-	var valuePlaceholders []string
-	var updateOnConflict []string
-	for i := 0; i < len(model.OrderedColumns); i++ {
-		valuePlaceholder := fmt.Sprintf("$%d", 1+i)
-		valuePlaceholders = append(valuePlaceholders, valuePlaceholder)
-		updateOnConflict = append(updateOnConflict,
-			fmt.Sprintf("%s = %s", model.OrderedColumns[i], valuePlaceholder))
-	}
-
-	baseQuery := `INSERT INTO %v.%v (%v) VALUES(%v)
-		ON CONFLICT (header_id, log_id) DO UPDATE SET %v;`
-
-	return fmt.Sprintf(baseQuery,
-		model.SchemaName,
-		model.TableName,
-		joinOrderedColumns(model.OrderedColumns),
-		strings.Join(valuePlaceholders, ", "),
-		strings.Join(updateOnConflict, ", "))
-}
-
-/*
-Create generates an insertion query and persists to the DB, given a slice of InsertionModels.
-ColumnValues are restricted to []byte, bool, float64, int64, string, time.Time.
-
-testModel = shared.InsertionModel{
-	SchemaName:     "public"
-	TableName:      "testEvent",
-	OrderedColumns: []string{"header_id", "log_id", "variable1"},
-	ColumnValues: ColumnValues{
-		"header_id": 303
-		"log_id":   "808",
-		"variable1": "value1",
-	},
-}
-*/
-func Create(models []InsertionModel, db *postgres.DB) error {
-	if len(models) == 0 {
-		return fmt.Errorf("repository got empty model slice")
-	}
-
-	tx, dbErr := db.Beginx()
-	if dbErr != nil {
-		return dbErr
-	}
-
-	for _, model := range models {
-		// Maps can't be iterated over in a reliable manner, so we rely on OrderedColumns to define the order to insert
-		// tx.Exec is variadically typed in the args, so if we wrap in []interface{} we can apply them all automatically
-		var args []interface{}
-		for _, col := range model.OrderedColumns {
-			value := model.ColumnValues[col]
-			// Check whether or not PG can accept the type of value in the model
-			okPgValue := driver.IsValue(value)
-			if !okPgValue {
-				logrus.WithField("model", model).Errorf("PG cannot handle value of this type: %T", value)
-				return ErrUnsupportedValue(value)
-			}
-			args = append(args, value)
-		}
-
-		insertionQuery := GetMemoizedQuery(model)
-		_, execErr := tx.Exec(insertionQuery, args...) // couldn't pass varying types in bulk with args :: []string
-
-		if execErr != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				logrus.Error("failed to rollback ", rollbackErr)
-			}
-			return execErr
-		}
-
-		_, logErr := tx.Exec(SetLogTransformedQuery, model.ColumnValues[LogFK])
-
-		if logErr != nil {
-			utils.RollbackAndLogFailure(tx, logErr, "header_sync_logs.transformed")
-			return logErr
-		}
-	}
-
-	return tx.Commit()
-}
-
-func joinOrderedColumns(columns []ColumnName) string {
-	var stringColumns []string
-	for _, columnName := range columns {
-		stringColumns = append(stringColumns, string(columnName))
-	}
-	return strings.Join(stringColumns, ", ")
-}

libraries/shared/factories/event/repository_test.go

@@ -1,205 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event_test
-
-import (
-	"fmt"
-	"math/big"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Repository", func() {
-	var db *postgres.DB
-
-	BeforeEach(func() {
-		db = test_config.NewTestDB(test_config.NewTestNode())
-		test_config.CleanTestDB(db)
-	})
-
-	Describe("Create", func() {
-		const createTestEventTableQuery = `CREATE TABLE public.testEvent(
-		id        SERIAL PRIMARY KEY,
-		header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-		log_id    BIGINT  NOT NULL REFERENCES header_sync_logs (id) ON DELETE CASCADE,
-		variable1 TEXT,
-		UNIQUE (header_id, log_id)
-		);`
-
-		var (
-			headerID, logID  int64
-			headerRepository repositories.HeaderRepository
-			testModel        event.InsertionModel
-		)
-
-		BeforeEach(func() {
-			_, tableErr := db.Exec(createTestEventTableQuery)
-			Expect(tableErr).NotTo(HaveOccurred())
-			headerRepository = repositories.NewHeaderRepository(db)
-			var insertHeaderErr error
-			headerID, insertHeaderErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
-			Expect(insertHeaderErr).NotTo(HaveOccurred())
-			headerSyncLog := test_data.CreateTestLog(headerID, db)
-			logID = headerSyncLog.ID
-
-			testModel = event.InsertionModel{
-				SchemaName: "public",
-				TableName:  "testEvent",
-				OrderedColumns: []event.ColumnName{
-					event.HeaderFK, event.LogFK, "variable1",
-				},
-				ColumnValues: event.ColumnValues{
-					event.HeaderFK: headerID,
-					event.LogFK:    logID,
-					"variable1":    "value1",
-				},
-			}
-		})
-
-		AfterEach(func() {
-			db.MustExec(`DROP TABLE public.testEvent;`)
-		})
-
-		// Needs to run before the other tests, since those insert keys in map
-		It("memoizes queries", func() {
-			Expect(len(event.ModelToQuery)).To(Equal(0))
-			event.GetMemoizedQuery(testModel)
-			Expect(len(event.ModelToQuery)).To(Equal(1))
-			event.GetMemoizedQuery(testModel)
-			Expect(len(event.ModelToQuery)).To(Equal(1))
-		})
-
-		It("persists a model to postgres", func() {
-			createErr := event.Create([]event.InsertionModel{testModel}, db)
-			Expect(createErr).NotTo(HaveOccurred())
-
-			var res TestEvent
-			dbErr := db.Get(&res, `SELECT log_id, variable1 FROM public.testEvent;`)
-			Expect(dbErr).NotTo(HaveOccurred())
-
-			Expect(res.LogID).To(Equal(fmt.Sprint(testModel.ColumnValues[event.LogFK])))
-			Expect(res.Variable1).To(Equal(testModel.ColumnValues["variable1"]))
-		})
-
-		Describe("returns errors", func() {
-			It("for empty model slice", func() {
-				err := event.Create([]event.InsertionModel{}, db)
-				Expect(err).To(MatchError("repository got empty model slice"))
-			})
-
-			It("for failed SQL inserts", func() {
-				header := fakes.GetFakeHeader(1)
-				headerID, headerErr := headerRepository.CreateOrUpdateHeader(header)
-				Expect(headerErr).NotTo(HaveOccurred())
-
-				brokenModel := event.InsertionModel{
-					SchemaName: "public",
-					TableName:  "testEvent",
-					// Wrong name of last column compared to DB, will generate incorrect query
-					OrderedColumns: []event.ColumnName{
-						event.HeaderFK, event.LogFK, "variable2",
-					},
-					ColumnValues: event.ColumnValues{
-						event.HeaderFK: headerID,
-						event.LogFK:    logID,
-						"variable1":    "value1",
-					},
-				}
-
-				// Remove cached queries, or we won't generate a new (incorrect) one
-				delete(event.ModelToQuery, "publictestEvent")
-
-				createErr := event.Create([]event.InsertionModel{brokenModel}, db)
-				// Remove incorrect query, so other tests won't get it
-				delete(event.ModelToQuery, "publictestEvent")
-
-				Expect(createErr).To(HaveOccurred())
-			})
-
-			It("for unsupported types in ColumnValue", func() {
-				unsupportedValue := big.NewInt(5)
-				testModel = event.InsertionModel{
-					SchemaName: "public",
-					TableName:  "testEvent",
-					OrderedColumns: []event.ColumnName{
-						event.HeaderFK, event.LogFK, "variable1",
-					},
-					ColumnValues: event.ColumnValues{
-						event.HeaderFK: headerID,
-						event.LogFK:    logID,
-						"variable1":    unsupportedValue,
-					},
-				}
-
-				createErr := event.Create([]event.InsertionModel{testModel}, db)
-				Expect(createErr).To(MatchError(event.ErrUnsupportedValue(unsupportedValue)))
-			})
-		})
-
-		It("upserts queries with conflicting source", func() {
-			conflictingModel := event.InsertionModel{
-				SchemaName: "public",
-				TableName:  "testEvent",
-				OrderedColumns: []event.ColumnName{
-					event.HeaderFK, event.LogFK, "variable1",
-				},
-				ColumnValues: event.ColumnValues{
-					event.HeaderFK: headerID,
-					event.LogFK:    logID,
-					"variable1":    "conflictingValue",
-				},
-			}
-
-			createErr := event.Create([]event.InsertionModel{testModel, conflictingModel}, db)
-			Expect(createErr).NotTo(HaveOccurred())
-
-			var res TestEvent
-			dbErr := db.Get(&res, `SELECT log_id, variable1 FROM public.testEvent;`)
-			Expect(dbErr).NotTo(HaveOccurred())
-			Expect(res.Variable1).To(Equal(conflictingModel.ColumnValues["variable1"]))
-		})
-
-		It("generates correct queries", func() {
-			actualQuery := event.GenerateInsertionQuery(testModel)
-			expectedQuery := `INSERT INTO public.testEvent (header_id, log_id, variable1) VALUES($1, $2, $3)
-		ON CONFLICT (header_id, log_id) DO UPDATE SET header_id = $1, log_id = $2, variable1 = $3;`
-			Expect(actualQuery).To(Equal(expectedQuery))
-		})
-
-		It("marks log transformed", func() {
-			createErr := event.Create([]event.InsertionModel{testModel}, db)
-			Expect(createErr).NotTo(HaveOccurred())
-
-			var logTransformed bool
-			getErr := db.Get(&logTransformed, `SELECT transformed FROM public.header_sync_logs WHERE id = $1`, logID)
-			Expect(getErr).NotTo(HaveOccurred())
-			Expect(logTransformed).To(BeTrue())
-		})
-	})
-})
-
-type TestEvent struct {
-	LogID     string `db:"log_id"`
-	Variable1 string
-}

libraries/shared/factories/event/transformer.go

@@ -1,68 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event
-
-import (
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-type Transformer struct {
-	Config     transformer.EventTransformerConfig
-	Converter  Converter
-	Repository Repository
-}
-
-func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.EventTransformer {
-	transformer.Converter.SetDB(db)
-	transformer.Repository.SetDB(db)
-	return transformer
-}
-
-func (transformer Transformer) Execute(logs []core.HeaderSyncLog) error {
-	transformerName := transformer.Config.TransformerName
-	config := transformer.Config
-
-	if len(logs) < 1 {
-		return nil
-	}
-
-	models, err := transformer.Converter.ToModels(config.ContractAbi, logs)
-	if err != nil {
-		logrus.Errorf("error converting entities to models in %v: %v", transformerName, err)
-		return err
-	}
-
-	err = transformer.Repository.Create(models)
-	if err != nil {
-		logrus.Errorf("error persisting %v record: %v", transformerName, err)
-		return err
-	}
-	logrus.Debug("Persisted log for " + transformerName)
-
-	return nil
-}
-
-func (transformer Transformer) GetName() string {
-	return transformer.Config.TransformerName
-}
-
-func (transformer Transformer) GetConfig() transformer.EventTransformerConfig {
-	return transformer.Config
-}

libraries/shared/factories/event/transformer_test.go

@@ -1,107 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event_test
-
-import (
-	"math/rand"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Transformer", func() {
-	var (
-		repository mocks.MockEventRepository
-		converter  mocks.MockConverter
-		t          transformer.EventTransformer
-		headerOne  core.Header
-		config     = test_data.GenericTestConfig
-		logs       []core.HeaderSyncLog
-	)
-
-	BeforeEach(func() {
-		repository = mocks.MockEventRepository{}
-		converter = mocks.MockConverter{}
-
-		t = event.Transformer{
-			Repository: &repository,
-			Converter:  &converter,
-			Config:     config,
-		}.NewTransformer(nil)
-
-		headerOne = core.Header{ID: rand.Int63(), BlockNumber: rand.Int63()}
-
-		logs = []core.HeaderSyncLog{{
-			ID:          0,
-			HeaderID:    headerOne.ID,
-			Log:         test_data.GenericTestLog(),
-			Transformed: false,
-		}}
-	})
-
-	It("sets the db", func() {
-		Expect(repository.SetDbCalled).To(BeTrue())
-	})
-
-	It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
-		err := t.Execute([]core.HeaderSyncLog{})
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.ToModelsCalledCounter).To(Equal(0))
-		Expect(repository.CreateCalledCounter).To(Equal(0))
-	})
-
-	It("converts an eth log to a model", func() {
-		err := t.Execute(logs)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
-		Expect(converter.LogsToConvert).To(Equal(logs))
-	})
-
-	It("returns an error if converting to models fails", func() {
-		converter.ToModelsError = fakes.FakeError
-
-		err := t.Execute(logs)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("persists the record", func() {
-		converter.ModelsToReturn = []event.InsertionModel{test_data.GenericModel}
-
-		err := t.Execute(logs)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(repository.PassedModels[0]).To(Equal(test_data.GenericModel))
-	})
-
-	It("returns error if persisting the record fails", func() {
-		repository.SetCreateError(fakes.FakeError)
-		err := t.Execute(logs)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-})

libraries/shared/factories/storage/EXAMPLE.md

@@ -1,167 +0,0 @@
-# Storage Transformer Example
-
-In the Storage Transformer README, we went over code that needs to be written to add a new storage transformer to VulcanizeDB.
-In this document, we'll go over an example contract and discuss how one would go about watching its storage.
-
-## Example Contract
-
-For the purposes of this document, we'll be assuming that we're interested in watching the following contract:
-
-```solidity
-pragma solidity ^0.5.1;
-
-contract Contract {
-  uint256 public num_addresses;
-  mapping(address => uint) public addresses;
-  
-  event AddressAdded(
-    address addr,
-    uint256 num_addrs
-  );
-  
-  constructor() public {
-    addresses[msg.sender] = 1;
-    num_addresses = 1;
-  }
-  
-  function add_address(address addr) public {
-    bool exists = addresses[addr] > 0;
-    addresses[addr]++;
-    if (!exists) {
-      emit AddressAdded(addr, ++num_addresses);
-    }
-  }
-}
-```
-
-Disclaimer: this contract has not been audited and is not intended to be modeled or used in production. :)
-
-This contract persists two values in its storage:
-
-1. `num_addresses`: the total number of unique addresses known to the contract.
-2. `addresses`: a mapping that records the number of times an address has been added to the contract.
-
-It also emits an event each time a new address is added into the contract's storage.
-
-## Custom Code
-
-In order to monitor the state of this smart contract, we'd need to implement: an event transformer, a mappings namespace, and a repository.
-We will go through each of these in turn.
-
-### Event Transformer
-
-Given that the contract's storage includes a mapping, `addresses`, we will need to be able to identify the keys to that mapping that exist in the system so that we can recognize contract storage keys that correspond to non-zero values in that mapping.
-
-The simplest way to be aware of keys used in a contract's mapping is to listen for contract events that emit the keys that are used in its mapping(s).
-Since this contract includes an event, `AddressAdded`, that is emitted each time a new address is added to the `addresses` mapping, we will want to listen for those events and cache the adddresses that map to non-zero values.
-
-Please see the event transformer README for detailed instructions about developing this code.
-In short, it should be feasible to recognize `AddressAdded` events on the blockchain and parse them to keep a record of addresses that have been added to the system.
-
-### Mappings
-
-If we point an ethereum node at a blockchain hosting this contract and our node is equipped to write out storage changes happening on this contract, we will expect such changes to appear each time `add_address` (which modifies the `addresses` mapping) is called.
-
-In order for those changes - which include raw hex versions of storage keys and storage values, to be useful for us - we need to know how to recognize and parse them.
-Our mappings file should assist us with both of these tasks: the `Lookup` function should recognize raw storage keys and return known metadata about the storage value.
-
-In order to perform this lookup, the mappings file should maintain its own mapping of known storage keys to the corresponding storage value metadata.
-This internal mapping should contain the storage key for `num_addresses` as well as a storage key for each `addresses` key known to be associated with a non-zero value.
-
-#### num_addresses
-
-`num_addresses` is the first variable declared on the contract, and it is a simple (non-array, non-mapping) type.
-Therefore, we know that its storage key is `0000000000000000000000000000000000000000000000000000000000000000`.
-The storage key for non-array and non-mapping variables is (usually*) the index of the variable on the contract's storage.
-If we see a storage diff being emitted from this contract with this storage key, we know that the `num_addresses` variable has been modified.
-
-In this case, we would expect that the call `mappings.Lookup("0000000000000000000000000000000000000000000000000000000000000000")` would return metadata corresponding to the `num_addresses` variable.
-This metadata would probably look something like:
-
-```golang
-shared.StorageValueMetadata{
-    Name: "num_addresses",
-    Keys: nil,
-    Type: shared.Uint256,
-}
-```
-
-<sup>*</sup> Occasionally, multiple variables may be packed into one storage slot, which complicates a direct translation of the index of the variable on the contract to its storage key.
-
-#### addresses
-
-`addresses` is the second variable declared on the contract, but it is a mapping.
-Since it is a mapping, the storage key is more complex than `0000000000000000000000000000000000000000000000000000000000000001` (which would be the key for the variable if it were not an array or mapping).
-Having a single storage slot for an entire mapping would not work, since there can be an arbitrary number of entries in a mapping, and a single storage value slot is constrained to 32 bytes.
-
-The way that smart contract mappings are maintained in storage (in Solidity) is by creating a new storage key/value pair for each entry in the mapping, where the storage key is a hash of the occupied slot's key concatenated with the mapping's index on the contract.
-Given an occupied slot's key, `k`, and a mapping's index on the contract, `i`, we can generate the storage key with the following code:
-
-```golang
-func GetMappingStorageKey(k, i string) string {
-    return common.BytesToHash(crypto.Keccak256(common.FromHex(k + i))).Hex()
-}
-```
-
-If we were to call the contract's `add_address` function with `0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe`, we would expect to see an `AddressAdded` event emitted, with `0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe` in its payload.
-From that event, we would know that there exists in the contract's storage a storage key of:
-
-```golang
-GetMappingStorageKey("0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe", "0000000000000000000000000000000000000000000000000000000000000001")
-```
-
-Executing the above code results in: `0x0f96a1133cfd5b94c329aa0526b5962bd791dbbfc481ca82f7d4a439e1e9bc40`.
-
-Therefore, the first time `add_address` was called for this address, we would also expect to see a storage diff with a key of `0x0f96a1133cfd5b94c329aa0526b5962bd791dbbfc481ca82f7d4a439e1e9bc40` and a value of `0000000000000000000000000000000000000000000000000000000000000001`.
-This would be the indication that in contract storage, the address `0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe` maps to the value 1.
-
-Given that we knew this address was a key in the mapping from our event transformer, we would expect a call to `mappings.Lookup("0x0f96a1133cfd5b94c329aa0526b5962bd791dbbfc481ca82f7d4a439e1e9bc40")` to return metadata corresponding to _this slot_ in the addresses mapping:
-
-```golang
-shared.StorageValueMetadata{
-    Name: "addresses,
-    Keys: map[Key]string{Address: "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"},
-    Type: shared.Uint256,
-}
-```
-
-### Repository
-
-Once we have recognized a storage diff, we can decode the storage value to the data's known type.
-Since the metadata tells us that the above values are `uint256`, we can decode a value like `0000000000000000000000000000000000000000000000000000000000000001` to `1`.
-
-The purpose of the contract-specific repository is to write that value to the database in a way that makes it useful for future queries.
-Typically, this involves writing the block hash, block number, decoded value, and any keys in the metadata to a table.
-
-The current repository interface has a generalized `Create` function that can accept any arbitrary storage row along with its metadata.
-This is deliberate, to facilitate shared use of the common storage transformer.
-An implication of this decision is that the `Create` function typically includes a `switch` statement that selects which table to write to, as well as what data to include, based on the name of the variable as defined in the metadata.
-
-An example implementation of `Create` for our example contract above might look like:
-
-```golang
-func (repository AddressStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error {
-    switch metadata.Name {
-    case "num_addresses":
-        _, err := repository.db.Exec(`INSERT INTO storage.num_addresses (block_hash, block_number, n) VALUES ($1, $2, $3)`,
-            blockHash, blockNumber, value)
-        return err
-    case "addresses":
-        _, err := repository.db.Exec(`INSERT INTO storage.addresses (block_hash, block_number, address, n) VALUES ($1, $2, $3, $4)`,
-            blockHash, blockNumber, metadata.Keys[Address], value)
-        return err
-    default:
-        panic(fmt.Sprintf("unrecognized contract storage name: %s", metadata.Name))
-    }
-}
-```
-
-## Summary
-
-With our very simple address storing contract, we would be able to read its storage diffs by implementing an event transformer, a mappings, and a repository.
-
-The mappings would be able to lookup storage keys reflecting `num_addresses` or any slot in `addresses`, using addresses derived from watching the `AddressAdded` event for the latter.
-
-The repository would be able to persist the value or `num_addresses` or any slot in `addresses`, using metadata returned from the mappings.
-
-The mappings and repository could be plugged into the common storage transformer, enabling us to know the contract's state as it is changing.

libraries/shared/factories/storage/README.md

@@ -1,140 +0,0 @@
-# Watching Contract Storage
-
-One approach VulcanizeDB takes to caching and indexing smart contracts is to ingest raw contract storage values.
-Assuming that you are running an ethereum node that is writing contract storage changes to a CSV file, VulcanizeDB can parse them and persist the results to postgres.
-
-## Assumptions
-
-The current approach for caching smart contract storage diffs assumes that you are running a node that is writing contract storage diffs to a CSV file.
-The CSV file is expected to have 5 columns: contract address, block hash, block number, storage key, storage value.
-
-We have [a branch on vulcanize/parity-ethereum](https://github.com/vulcanize/parity-ethereum/tree/watch-storage-diffs) that enables running a node that writes storage diffs this way.
-
-Looking forward, we would like to isolate this assumption as much as possible.
-We may end up needing to read CSV data that is formatted differently, or reading data from a non-CSV source, and we do not want resulting changes to cascade throughout the codebase.
-
-## Shared Code
-
-VulcanizeDB has shared code for continuously reading from the CSV file written by the ethereum node and writing a parsed version of each row to postgres.
-
-### Storage Watcher
-
-The storage watcher is responsible for continuously delegating CSV rows to the appropriate transformer as they are being written by the ethereum node.
-It maintains a mapping of contract addresses to transformers, and will ignore storage diff rows for contract addresses that do not have a corresponding transformer.
-
-Storage watchers can be loaded with plugin storage transformers and executed using the `composeAndExecute` command.
-
-### Storage Transformer
-
-The storage transformer is responsible for converting raw contract storage hex values into useful data and writing them to postgres.
-The storage transformer depends on contract-specific implementations of code capable of recognizing storage keys and writing the matching (decoded) storage value to disk.
-
-```golang
-func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
-	metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
-	if lookupErr != nil {
-		return lookupErr
-	}
-	value, decodeErr := utils.Decode(diff, metadata)
-	if decodeErr != nil {
-		return decodeErr
-	}
-	return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
-}
-```
-
-## Custom Code
-
-In order to watch an additional smart contract, a developer must create three things:
-
-1. StorageKeysLoader - identify keys in the contract's storage trie, providing metadata to describe how associated values should be decoded.
-1. Repository - specify how to persist a parsed version of the storage value matching the recognized storage key.
-1. Instance - create an instance of the storage transformer that uses your mappings and repository.
-
-### StorageKeysLoader
-
-A `StorageKeysLoader` is used by the `StorageKeysLookup` object on a storage transformer.
-
-```golang
-type KeysLoader interface {
-	LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
-	SetDB(db *postgres.DB)
-}
-```
-
-When a key is not found, the lookup object refreshes its known keys by calling the loader.
-
-```golang
-func (lookup *keysLookup) refreshMappings() error {
-	var err error
-	lookup.mappings, err = lookup.loader.LoadMappings()
-	if err != nil {
-		return err
-	}
-	lookup.mappings = utils.AddHashedKeys(lookup.mappings)
-	return nil
-}
-```
-
-A contract-specific implementation of the loader enables the storage transformer to fetch metadata associated with a storage key.
-
-Storage metadata contains: the name of the variable matching the storage key, a raw version of any keys associated with the variable (if the variable is a mapping), and the variable's type.
-
-```golang
-type StorageValueMetadata struct {
-	Name string
-	Keys map[Key]string
-	Type ValueType
-}
-```
-
-The `Keys` field on the metadata is only relevant if the variable is a mapping. For example, in the following Solidity code:
-
-```solidity
-pragma solidity ^0.4.0;
-
-contract Contract {
-  uint x;
-  mapping(address => uint) y;
-}
-```
-
-The metadata for variable `x` would not have any associated keys, but the metadata for a storage key associated with `y` would include the address used to specify that key's index in the mapping.
-
-The `SetDB` function is required for the storage key loader to connect to the database.
-A database connection may be desired when keys in a mapping variable need to be read from log events (e.g. to lookup what addresses may exist in `y`, above).
-
-### Repository
-
-```golang
-type Repository interface {
-	Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error
-	SetDB(db *postgres.DB)
-}
-```
-
-A contract-specific implementation of the repository interface enables the transformer to write the decoded storage value to the appropriate table in postgres.
-
-The `Create` function is expected to recognize and persist a given storage value by the variable's name, as indicated on the row's metadata.
-Note: we advise silently discarding duplicates in `Create` - as it's possible that you may read the same diff several times, and an error will trigger the storage watcher to queue that diff for later processing.
-
-The `SetDB` function is required for the repository to connect to the database.
-
-### Instance
-
-```golang
-type Transformer struct {
-	Address    common.Address
-	Mappings   storage_diffs.Mappings
-	Repository storage_diffs.Repository
-}
-```
-
-A new instance of the storage transformer is initialized with the contract-specific mappings and repository, as well as the contract's address.
-The contract's address is included so that the watcher can query that value from the transformer in order to build up its mapping of addresses to transformers.
-
-## Summary
-
-To begin watching an additional smart contract, create a new mappings file for looking up storage keys on that contract, a repository for writing storage values from the contract, and initialize a new storage transformer instance with the mappings, repository, and contract address.
-
-The new instance, wrapped in an initializer that calls `SetDB` on the mappings and repository, should be passed to the `AddTransformers` function on the storage watcher.

libraries/shared/factories/storage/keys_loader.go

@@ -1,28 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package storage
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-type KeysLoader interface {
-	LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
-	SetDB(db *postgres.DB)
-}

libraries/shared/factories/storage/keys_lookup.go

@@ -1,66 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package storage
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-type KeysLookup interface {
-	Lookup(key common.Hash) (utils.StorageValueMetadata, error)
-	SetDB(db *postgres.DB)
-}
-
-type keysLookup struct {
-	loader   KeysLoader
-	mappings map[common.Hash]utils.StorageValueMetadata
-}
-
-func NewKeysLookup(loader KeysLoader) KeysLookup {
-	return &keysLookup{loader: loader, mappings: make(map[common.Hash]utils.StorageValueMetadata)}
-}
-
-func (lookup *keysLookup) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
-	metadata, ok := lookup.mappings[key]
-	if !ok {
-		refreshErr := lookup.refreshMappings()
-		if refreshErr != nil {
-			return metadata, refreshErr
-		}
-		metadata, ok = lookup.mappings[key]
-		if !ok {
-			return metadata, utils.ErrStorageKeyNotFound{Key: key.Hex()}
-		}
-	}
-	return metadata, nil
-}
-
-func (lookup *keysLookup) refreshMappings() error {
-	var err error
-	lookup.mappings, err = lookup.loader.LoadMappings()
-	if err != nil {
-		return err
-	}
-	lookup.mappings = utils.AddHashedKeys(lookup.mappings)
-	return nil
-}
-
-func (lookup *keysLookup) SetDB(db *postgres.DB) {
-	lookup.loader.SetDB(db)
-}

libraries/shared/factories/storage/keys_lookup_test.go

@@ -1,113 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package storage_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Storage keys lookup", func() {
-	var (
-		fakeMetadata = utils.GetStorageValueMetadata("name", map[utils.Key]string{}, utils.Uint256)
-		lookup       storage.KeysLookup
-		loader       *mocks.MockStorageKeysLoader
-	)
-
-	BeforeEach(func() {
-		loader = &mocks.MockStorageKeysLoader{}
-		lookup = storage.NewKeysLookup(loader)
-	})
-
-	Describe("Lookup", func() {
-		Describe("when key not found", func() {
-			It("refreshes keys", func() {
-				loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
-				_, err := lookup.Lookup(fakes.FakeHash)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(loader.LoadMappingsCallCount).To(Equal(1))
-			})
-
-			It("returns error if refreshing keys fails", func() {
-				loader.LoadMappingsError = fakes.FakeError
-
-				_, err := lookup.Lookup(fakes.FakeHash)
-
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(fakes.FakeError))
-			})
-		})
-
-		Describe("when key found", func() {
-			BeforeEach(func() {
-				loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
-				_, err := lookup.Lookup(fakes.FakeHash)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(loader.LoadMappingsCallCount).To(Equal(1))
-			})
-
-			It("does not refresh keys", func() {
-				_, err := lookup.Lookup(fakes.FakeHash)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(loader.LoadMappingsCallCount).To(Equal(1))
-			})
-		})
-
-		It("returns metadata for loaded static key", func() {
-			loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
-
-			metadata, err := lookup.Lookup(fakes.FakeHash)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(metadata).To(Equal(fakeMetadata))
-		})
-
-		It("returns metadata for hashed version of key (accommodates keys emitted from Geth)", func() {
-			loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
-
-			hashedKey := common.BytesToHash(crypto.Keccak256(fakes.FakeHash.Bytes()))
-			metadata, err := lookup.Lookup(hashedKey)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(metadata).To(Equal(fakeMetadata))
-		})
-
-		It("returns key not found error if key not found", func() {
-			_, err := lookup.Lookup(fakes.FakeHash)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(utils.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
-		})
-	})
-
-	Describe("SetDB", func() {
-		It("sets the db on the loader", func() {
-			lookup.SetDB(test_config.NewTestDB(test_config.NewTestNode()))
-
-			Expect(loader.SetDBCalled).To(BeTrue())
-		})
-	})
-})

libraries/shared/factories/storage/repository.go

@@ -1,27 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package storage
-
-import (
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-type Repository interface {
-	Create(diffID int64, metadata utils.StorageValueMetadata, value interface{}) error
-	SetDB(db *postgres.DB)
-}

libraries/shared/factories/storage/storage_suite_test.go

@@ -1,29 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package storage_test
-
-import (
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestStorage(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Storage Factories Suite")
-}

libraries/shared/factories/storage/transformer.go

@@ -1,52 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package storage
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-type Transformer struct {
-	HashedAddress     common.Hash
-	StorageKeysLookup KeysLookup
-	Repository        Repository
-}
-
-func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {
-	transformer.StorageKeysLookup.SetDB(db)
-	transformer.Repository.SetDB(db)
-	return transformer
-}
-
-func (transformer Transformer) KeccakContractAddress() common.Hash {
-	return transformer.HashedAddress
-}
-
-func (transformer Transformer) Execute(diff utils.PersistedStorageDiff) error {
-	metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
-	if lookupErr != nil {
-		return lookupErr
-	}
-	value, decodeErr := utils.Decode(diff, metadata)
-	if decodeErr != nil {
-		return decodeErr
-	}
-	return transformer.Repository.Create(diff.ID, metadata, value)
-}

libraries/shared/factories/storage/transformer_test.go

@@ -1,160 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package storage_test
-
-import (
-	"math/rand"
-
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Storage transformer", func() {
-	var (
-		storageKeysLookup *mocks.MockStorageKeysLookup
-		repository        *mocks.MockStorageRepository
-		t                 storage.Transformer
-	)
-
-	BeforeEach(func() {
-		storageKeysLookup = &mocks.MockStorageKeysLookup{}
-		repository = &mocks.MockStorageRepository{}
-		t = storage.Transformer{
-			HashedAddress:     common.Hash{},
-			StorageKeysLookup: storageKeysLookup,
-			Repository:        repository,
-		}
-	})
-
-	It("returns the contract address being watched", func() {
-		fakeAddress := utils.HexToKeccak256Hash("0x12345")
-		t.HashedAddress = fakeAddress
-
-		Expect(t.KeccakContractAddress()).To(Equal(fakeAddress))
-	})
-
-	It("looks up metadata for storage key", func() {
-		t.Execute(utils.PersistedStorageDiff{})
-
-		Expect(storageKeysLookup.LookupCalled).To(BeTrue())
-	})
-
-	It("returns error if lookup fails", func() {
-		storageKeysLookup.LookupErr = fakes.FakeError
-
-		err := t.Execute(utils.PersistedStorageDiff{})
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("creates storage row with decoded data", func() {
-		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
-		storageKeysLookup.Metadata = fakeMetadata
-		rawValue := common.HexToAddress("0x12345")
-		fakeBlockNumber := 123
-		fakeBlockHash := "0x67890"
-		fakeRow := utils.PersistedStorageDiff{
-			ID: rand.Int63(),
-			StorageDiffInput: utils.StorageDiffInput{
-				HashedAddress: common.Hash{},
-				BlockHash:     common.HexToHash(fakeBlockHash),
-				BlockHeight:   fakeBlockNumber,
-				StorageKey:    common.Hash{},
-				StorageValue:  rawValue.Hash(),
-			},
-		}
-
-		err := t.Execute(fakeRow)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(repository.PassedDiffID).To(Equal(fakeRow.ID))
-		Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
-		Expect(repository.PassedValue.(string)).To(Equal(rawValue.Hex()))
-	})
-
-	It("returns error if creating row fails", func() {
-		rawValue := common.HexToAddress("0x12345")
-		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
-		storageKeysLookup.Metadata = fakeMetadata
-		repository.CreateErr = fakes.FakeError
-		diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: rawValue.Hash()}}
-
-		err := t.Execute(diff)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	Describe("when a storage row contains more than one item packed in storage", func() {
-		var (
-			rawValue        = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30")
-			fakeBlockNumber = 123
-			fakeBlockHash   = "0x67890"
-			packedTypes     = make(map[int]utils.ValueType)
-		)
-		packedTypes[0] = utils.Uint48
-		packedTypes[1] = utils.Uint48
-
-		var fakeMetadata = utils.StorageValueMetadata{
-			Name:        "",
-			Keys:        nil,
-			Type:        utils.PackedSlot,
-			PackedTypes: packedTypes,
-		}
-
-		It("passes the decoded data items to the repository", func() {
-			storageKeysLookup.Metadata = fakeMetadata
-			fakeRow := utils.PersistedStorageDiff{
-				ID: rand.Int63(),
-				StorageDiffInput: utils.StorageDiffInput{
-					HashedAddress: common.Hash{},
-					BlockHash:     common.HexToHash(fakeBlockHash),
-					BlockHeight:   fakeBlockNumber,
-					StorageKey:    common.Hash{},
-					StorageValue:  rawValue.Hash(),
-				},
-			}
-
-			err := t.Execute(fakeRow)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(repository.PassedDiffID).To(Equal(fakeRow.ID))
-			Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
-			expectedPassedValue := make(map[int]string)
-			expectedPassedValue[0] = "10800"
-			expectedPassedValue[1] = "172800"
-			Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue))
-		})
-
-		It("returns error if creating a row fails", func() {
-			storageKeysLookup.Metadata = fakeMetadata
-			repository.CreateErr = fakes.FakeError
-			diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: rawValue.Hash()}}
-
-			err := t.Execute(diff)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-	})
-})

libraries/shared/fetcher/csv_tail_storage_fetcher.go

@@ -1,49 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher
-
-import (
-	"strings"
-
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-)
-
-type CsvTailStorageFetcher struct {
-	tailer fs.Tailer
-}
-
-func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher {
-	return CsvTailStorageFetcher{tailer: tailer}
-}
-
-func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffInput, errs chan<- error) {
-	t, tailErr := storageFetcher.tailer.Tail()
-	if tailErr != nil {
-		errs <- tailErr
-	}
-	logrus.Debug("fetching storage diffs...")
-	for line := range t.Lines {
-		diff, parseErr := utils.FromParityCsvRow(strings.Split(line.Text, ","))
-		if parseErr != nil {
-			errs <- parseErr
-		} else {
-			out <- diff
-		}
-	}
-}

libraries/shared/fetcher/csv_tail_storage_fetcher_test.go

@@ -1,98 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher_test
-
-import (
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/hpcloud/tail"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Csv Tail Storage Fetcher", func() {
-	var (
-		errorsChannel  chan error
-		mockTailer     *fakes.MockTailer
-		diffsChannel   chan utils.StorageDiffInput
-		storageFetcher fetcher.CsvTailStorageFetcher
-	)
-
-	BeforeEach(func() {
-		errorsChannel = make(chan error)
-		diffsChannel = make(chan utils.StorageDiffInput)
-		mockTailer = fakes.NewMockTailer()
-		storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer)
-	})
-
-	It("adds error to errors channel if tailing file fails", func(done Done) {
-		mockTailer.TailErr = fakes.FakeError
-
-		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
-
-		Expect(<-errorsChannel).To(MatchError(fakes.FakeError))
-		close(done)
-	})
-
-	It("adds parsed csv row to rows channel for storage diff", func(done Done) {
-		line := getFakeLine()
-
-		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
-		mockTailer.Lines <- line
-
-		expectedRow, err := utils.FromParityCsvRow(strings.Split(line.Text, ","))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(<-diffsChannel).To(Equal(expectedRow))
-		close(done)
-	})
-
-	It("adds error to errors channel if parsing csv fails", func(done Done) {
-		line := &tail.Line{Text: "invalid"}
-
-		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
-		mockTailer.Lines <- line
-
-		Expect(<-errorsChannel).To(HaveOccurred())
-		select {
-		case <-diffsChannel:
-			Fail("value passed to rows channel on error")
-		default:
-			Succeed()
-		}
-		close(done)
-	})
-})
-
-func getFakeLine() *tail.Line {
-	address := common.HexToAddress("0x1234567890abcdef")
-	blockHash := []byte{4, 5, 6}
-	blockHeight := int64(789)
-	storageKey := []byte{9, 8, 7}
-	storageValue := []byte{6, 5, 4}
-	return &tail.Line{
-		Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address.Bytes()), common.Bytes2Hex(blockHash),
-			blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
-		Time: time.Time{},
-		Err:  nil,
-	}
-}

libraries/shared/fetcher/fetcher_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	log "github.com/sirupsen/logrus"
-)
-
-func TestFactories(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Shared Fetcher Suite")
-}
-
-var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
-})

libraries/shared/fetcher/geth_rpc_storage_fetcher.go

@@ -1,86 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-package fetcher
-
-import (
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-)
-
-const (
-	PayloadChanBufferSize = 20000 // the max eth sub buffer size
-)
-
-type GethRPCStorageFetcher struct {
-	StatediffPayloadChan chan statediff.Payload
-	streamer             streamer.Streamer
-}
-
-func NewGethRPCStorageFetcher(streamer streamer.Streamer) GethRPCStorageFetcher {
-	return GethRPCStorageFetcher{
-		StatediffPayloadChan: make(chan statediff.Payload, PayloadChanBufferSize),
-		streamer:             streamer,
-	}
-}
-
-func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffInput, errs chan<- error) {
-	ethStatediffPayloadChan := fetcher.StatediffPayloadChan
-	clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan, statediff.Params{})
-	if clientSubErr != nil {
-		errs <- clientSubErr
-		panic(fmt.Sprintf("Error creating a geth client subscription: %v", clientSubErr))
-	}
-	logrus.Info("Successfully created a geth client subscription: ", clientSubscription)
-
-	for {
-		diff := <-ethStatediffPayloadChan
-		logrus.Trace("received a statediff")
-		stateDiff := new(statediff.StateObject)
-		decodeErr := rlp.DecodeBytes(diff.StateObjectRlp, stateDiff)
-		if decodeErr != nil {
-			logrus.Warn("Error decoding state diff into RLP: ", decodeErr)
-			errs <- decodeErr
-		}
-
-		accounts := utils.GetAccountsFromDiff(*stateDiff)
-		logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber))
-		for _, account := range accounts {
-			logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.StorageNodes), common.BytesToHash(account.LeafKey).Hex()))
-			for _, storage := range account.StorageNodes {
-				diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
-				if formatErr != nil {
-					logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey))
-					errs <- formatErr
-					continue
-				}
-				logrus.Trace("adding storage diff to out channel",
-					"keccak of address: ", diff.HashedAddress.Hex(),
-					"block height: ", diff.BlockHeight,
-					"storage key: ", diff.StorageKey.Hex(),
-					"storage value: ", diff.StorageValue.Hex())
-
-				out <- diff
-			}
-		}
-	}
-}

libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go

@@ -1,174 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-type MockStoragediffStreamer struct {
-	subscribeError    error
-	PassedPayloadChan chan statediff.Payload
-	PassedParams      statediff.Params
-	streamPayloads    []statediff.Payload
-}
-
-func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan statediff.Payload, params statediff.Params) (*rpc.ClientSubscription, error) {
-	clientSubscription := rpc.ClientSubscription{}
-	streamer.PassedPayloadChan = statediffPayloadChan
-	streamer.PassedParams = params
-	go func() {
-		for _, payload := range streamer.streamPayloads {
-			streamer.PassedPayloadChan <- payload
-		}
-	}()
-
-	return &clientSubscription, streamer.subscribeError
-}
-
-func (streamer *MockStoragediffStreamer) SetSubscribeError(err error) {
-	streamer.subscribeError = err
-}
-
-func (streamer *MockStoragediffStreamer) SetPayloads(payloads []statediff.Payload) {
-	streamer.streamPayloads = payloads
-}
-
-var _ = Describe("Geth RPC Storage Fetcher", func() {
-	var streamer MockStoragediffStreamer
-	var statediffFetcher fetcher.GethRPCStorageFetcher
-	var storagediffChan chan utils.StorageDiffInput
-	var errorChan chan error
-
-	BeforeEach(func() {
-		streamer = MockStoragediffStreamer{}
-		statediffFetcher = fetcher.NewGethRPCStorageFetcher(&streamer)
-		storagediffChan = make(chan utils.StorageDiffInput)
-		errorChan = make(chan error)
-	})
-
-	It("adds errors to error channel if the RPC subscription fails and panics", func(done Done) {
-		streamer.SetSubscribeError(fakes.FakeError)
-
-		go func() {
-			failedSub := func() {
-				statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
-			}
-			Expect(failedSub).To(Panic())
-		}()
-
-		Expect(<-errorChan).To(MatchError(fakes.FakeError))
-		close(done)
-	})
-
-	It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) {
-		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
-
-		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
-
-		streamedPayload := <-statediffFetcher.StatediffPayloadChan
-		Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload))
-		Expect(streamer.PassedPayloadChan).To(Equal(statediffFetcher.StatediffPayloadChan))
-		close(done)
-	})
-
-	It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) {
-		badStatediffPayload := statediff.Payload{}
-		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
-
-		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
-
-		Expect(<-errorChan).To(MatchError("EOF"))
-
-		close(done)
-	})
-
-	It("adds parsed statediff payloads to the rows channel", func(done Done) {
-		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
-
-		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
-
-		height := test_data.BlockNumber
-		intHeight := int(height.Int64())
-		createdExpectedStorageDiff := utils.StorageDiffInput{
-			HashedAddress: common.BytesToHash(test_data.ContractLeafKey[:]),
-			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
-			BlockHeight:   intHeight,
-			StorageKey:    common.BytesToHash(test_data.StorageKey),
-			StorageValue:  common.BytesToHash(test_data.SmallStorageValue),
-		}
-		updatedExpectedStorageDiff := utils.StorageDiffInput{
-			HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
-			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
-			BlockHeight:   intHeight,
-			StorageKey:    common.BytesToHash(test_data.StorageKey),
-			StorageValue:  common.BytesToHash(test_data.LargeStorageValue),
-		}
-		deletedExpectedStorageDiff := utils.StorageDiffInput{
-			HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
-			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
-			BlockHeight:   intHeight,
-			StorageKey:    common.BytesToHash(test_data.StorageKey),
-			StorageValue:  common.BytesToHash(test_data.SmallStorageValue),
-		}
-
-		createdStateDiff := <-storagediffChan
-		updatedStateDiff := <-storagediffChan
-		deletedStateDiff := <-storagediffChan
-
-		Expect(createdStateDiff).To(Equal(createdExpectedStorageDiff))
-		Expect(updatedStateDiff).To(Equal(updatedExpectedStorageDiff))
-		Expect(deletedStateDiff).To(Equal(deletedExpectedStorageDiff))
-
-		close(done)
-	})
-
-	It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) {
-		accountDiffs := test_data.CreatedAccountDiffs
-		accountDiffs[0].StorageNodes = []statediff.StorageNode{test_data.StorageWithBadValue}
-
-		stateDiff := statediff.StateObject{
-			BlockNumber: test_data.BlockNumber,
-			BlockHash:   common.HexToHash(test_data.BlockHash),
-			Nodes:       accountDiffs,
-		}
-
-		stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
-		Expect(err).NotTo(HaveOccurred())
-
-		badStatediffPayload := statediff.Payload{
-			StateObjectRlp: stateDiffRlp,
-		}
-		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
-
-		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
-
-		Expect(<-errorChan).To(MatchError("rlp: input contains more than one value"))
-
-		close(done)
-	})
-})

libraries/shared/fetcher/log_fetcher.go

@@ -1,59 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher
-
-import (
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-type ILogFetcher interface {
-	FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
-	// TODO Extend FetchLogs for doing several blocks at a time
-}
-
-type LogFetcher struct {
-	blockChain core.BlockChain
-}
-
-func NewLogFetcher(blockchain core.BlockChain) *LogFetcher {
-	return &LogFetcher{
-		blockChain: blockchain,
-	}
-}
-
-// Checks all topic0s, on all addresses, fetching matching logs for the given header
-func (logFetcher LogFetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
-	blockHash := common.HexToHash(header.Hash)
-	query := ethereum.FilterQuery{
-		BlockHash: &blockHash,
-		Addresses: addresses,
-		// Search for _any_ of the topics in topic0 position; see docs on `FilterQuery`
-		Topics: [][]common.Hash{topic0s},
-	}
-
-	logs, err := logFetcher.blockChain.GetEthLogsWithCustomQuery(query)
-	if err != nil {
-		// TODO review aggregate fetching error handling
-		return []types.Log{}, err
-	}
-
-	return logs, nil
-}

libraries/shared/fetcher/log_fetcher_test.go

@@ -1,70 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher_test
-
-import (
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("LogFetcher", func() {
-	Describe("FetchLogs", func() {
-		It("fetches logs based on the given query", func() {
-			blockChain := fakes.NewMockBlockChain()
-			logFetcher := fetcher.NewLogFetcher(blockChain)
-			header := fakes.FakeHeader
-
-			addresses := []common.Address{
-				common.HexToAddress("0xfakeAddress"),
-				common.HexToAddress("0xanotherFakeAddress"),
-			}
-
-			topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})}
-
-			_, err := logFetcher.FetchLogs(addresses, topicZeros, header)
-
-			address1 := common.HexToAddress("0xfakeAddress")
-			address2 := common.HexToAddress("0xanotherFakeAddress")
-			Expect(err).NotTo(HaveOccurred())
-
-			blockHash := common.HexToHash(header.Hash)
-			expectedQuery := ethereum.FilterQuery{
-				BlockHash: &blockHash,
-				Addresses: []common.Address{address1, address2},
-				Topics:    [][]common.Hash{topicZeros},
-			}
-			blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
-		})
-
-		It("returns an error if fetching the logs fails", func() {
-			blockChain := fakes.NewMockBlockChain()
-			blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
-			logFetcher := fetcher.NewLogFetcher(blockChain)
-
-			_, err := logFetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-	})
-})

libraries/shared/fetcher/state_diff_fetcher.go

@@ -1,79 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher
-
-import (
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/statediff"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-)
-
-// StateDiffFetcher is the state diff fetching interface
-type StateDiffFetcher interface {
-	FetchStateDiffsAt(blockHeights []uint64) ([]statediff.Payload, error)
-}
-
-// BatchClient is an interface to a batch-fetching geth rpc client; created to allow mock insertion
-type BatchClient interface {
-	BatchCall(batch []client.BatchElem) error
-}
-
-// stateDiffFetcher is the state diff fetching struct
-type stateDiffFetcher struct {
-	// stateDiffFetcher is thread-safe as long as the underlying client is thread-safe, since it has/modifies no other state
-	// http.Client is thread-safe
-	client BatchClient
-}
-
-const method = "statediff_stateDiffAt"
-
-// NewStateDiffFetcher returns a IStateDiffFetcher
-func NewStateDiffFetcher(bc BatchClient) StateDiffFetcher {
-	return &stateDiffFetcher{
-		client: bc,
-	}
-}
-
-// FetchStateDiffsAt fetches the statediff payloads at the given block heights
-// Calls StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
-func (fetcher *stateDiffFetcher) FetchStateDiffsAt(blockHeights []uint64) ([]statediff.Payload, error) {
-	batch := make([]client.BatchElem, 0)
-	for _, height := range blockHeights {
-		batch = append(batch, client.BatchElem{
-			Method: method,
-			Args:   []interface{}{height},
-			Result: new(statediff.Payload),
-		})
-	}
-	batchErr := fetcher.client.BatchCall(batch)
-	if batchErr != nil {
-		return nil, fmt.Errorf("stateDiffFetcher err: %s", batchErr.Error())
-	}
-	results := make([]statediff.Payload, 0, len(blockHeights))
-	for _, batchElem := range batch {
-		if batchElem.Error != nil {
-			return nil, fmt.Errorf("stateDiffFetcher err: %s", batchElem.Error.Error())
-		}
-		payload, ok := batchElem.Result.(*statediff.Payload)
-		if ok {
-			results = append(results, *payload)
-		}
-	}
-	return results, nil
-}

libraries/shared/fetcher/state_diff_fetcher_test.go

@@ -1,54 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
-)
-
-var _ = Describe("StateDiffFetcher", func() {
-	Describe("FetchStateDiffsAt", func() {
-		var (
-			mc               *mocks.BackFillerClient
-			stateDiffFetcher fetcher.StateDiffFetcher
-		)
-		BeforeEach(func() {
-			mc = new(mocks.BackFillerClient)
-			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
-			Expect(setDiffAtErr1).ToNot(HaveOccurred())
-			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
-			Expect(setDiffAtErr2).ToNot(HaveOccurred())
-			stateDiffFetcher = fetcher.NewStateDiffFetcher(mc)
-		})
-		It("Batch calls statediff_stateDiffAt", func() {
-			blockHeights := []uint64{
-				test_data.BlockNumber.Uint64(),
-				test_data.BlockNumber2.Uint64(),
-			}
-			stateDiffPayloads, fetchErr := stateDiffFetcher.FetchStateDiffsAt(blockHeights)
-			Expect(fetchErr).ToNot(HaveOccurred())
-			Expect(len(stateDiffPayloads)).To(Equal(2))
-			Expect(stateDiffPayloads[0]).To(Equal(test_data.MockStatediffPayload))
-			Expect(stateDiffPayloads[1]).To(Equal(test_data.MockStatediffPayload2))
-		})
-	})
-})

libraries/shared/fetcher/storage_fetcher_interface.go

@@ -1,21 +0,0 @@
-// Copyright 2018 Vulcanize
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package fetcher
-
-import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-
-type IStorageFetcher interface {
-	FetchStorageDiffs(out chan<- utils.StorageDiffInput, errs chan<- error)
-}

libraries/shared/logs/delegator.go

@@ -1,86 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package logs
-
-import (
-	"errors"
-
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-)
-
-var (
-	ErrNoLogs         = errors.New("no logs available for transforming")
-	ErrNoTransformers = errors.New("no event transformers configured in the log delegator")
-)
-
-type ILogDelegator interface {
-	AddTransformer(t transformer.EventTransformer)
-	DelegateLogs() error
-}
-
-type LogDelegator struct {
-	Chunker       chunker.Chunker
-	LogRepository datastore.HeaderSyncLogRepository
-	Transformers  []transformer.EventTransformer
-}
-
-func (delegator *LogDelegator) AddTransformer(t transformer.EventTransformer) {
-	delegator.Transformers = append(delegator.Transformers, t)
-	delegator.Chunker.AddConfig(t.GetConfig())
-}
-
-func (delegator *LogDelegator) DelegateLogs() error {
-	if len(delegator.Transformers) < 1 {
-		return ErrNoTransformers
-	}
-
-	persistedLogs, fetchErr := delegator.LogRepository.GetUntransformedHeaderSyncLogs()
-	if fetchErr != nil {
-		logrus.Errorf("error loading logs from db: %s", fetchErr.Error())
-		return fetchErr
-	}
-
-	if len(persistedLogs) < 1 {
-		return ErrNoLogs
-	}
-
-	transformErr := delegator.delegateLogs(persistedLogs)
-	if transformErr != nil {
-		logrus.Errorf("error transforming logs: %s", transformErr)
-		return transformErr
-	}
-
-	return nil
-}
-
-func (delegator *LogDelegator) delegateLogs(logs []core.HeaderSyncLog) error {
-	chunkedLogs := delegator.Chunker.ChunkLogs(logs)
-	for _, t := range delegator.Transformers {
-		transformerName := t.GetConfig().TransformerName
-		logChunk := chunkedLogs[transformerName]
-		err := t.Execute(logChunk)
-		if err != nil {
-			logrus.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err)
-			return err
-		}
-	}
-	return nil
-}

libraries/shared/logs/delegator_test.go

@@ -1,166 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package logs_test
-
-import (
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/logs"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Log delegator", func() {
-	Describe("AddTransformer", func() {
-		It("adds transformers to the delegator", func() {
-			fakeTransformer := &mocks.MockEventTransformer{}
-			delegator := logs.LogDelegator{Chunker: chunker.NewLogChunker()}
-
-			delegator.AddTransformer(fakeTransformer)
-
-			Expect(delegator.Transformers).To(Equal([]transformer.EventTransformer{fakeTransformer}))
-		})
-
-		It("passes transformers' configs to the chunker", func() {
-			fakeTransformer := &mocks.MockEventTransformer{}
-			fakeConfig := mocks.FakeTransformerConfig
-			fakeTransformer.SetTransformerConfig(fakeConfig)
-			chunker := chunker.NewLogChunker()
-			delegator := logs.LogDelegator{Chunker: chunker}
-
-			delegator.AddTransformer(fakeTransformer)
-
-			expectedName := fakeConfig.TransformerName
-			expectedTopic := common.HexToHash(fakeConfig.Topic)
-			Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{expectedName: expectedTopic}))
-			expectedAddress := strings.ToLower(fakeConfig.ContractAddresses[0])
-			Expect(chunker.AddressToNames).To(Equal(map[string][]string{expectedAddress: {expectedName}}))
-		})
-	})
-
-	Describe("DelegateLogs", func() {
-		It("returns error if no transformers configured", func() {
-			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
-
-			err := delegator.DelegateLogs()
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(logs.ErrNoTransformers))
-		})
-
-		It("gets untransformed logs", func() {
-			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
-			mockLogRepository.ReturnLogs = []core.HeaderSyncLog{{}}
-			delegator := newDelegator(mockLogRepository)
-			delegator.AddTransformer(&mocks.MockEventTransformer{})
-
-			err := delegator.DelegateLogs()
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(mockLogRepository.GetCalled).To(BeTrue())
-		})
-
-		It("returns error if getting untransformed logs fails", func() {
-			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
-			mockLogRepository.GetError = fakes.FakeError
-			delegator := newDelegator(mockLogRepository)
-			delegator.AddTransformer(&mocks.MockEventTransformer{})
-
-			err := delegator.DelegateLogs()
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-
-		It("returns error that no logs were found if no logs returned", func() {
-			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
-			delegator.AddTransformer(&mocks.MockEventTransformer{})
-
-			err := delegator.DelegateLogs()
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(logs.ErrNoLogs))
-		})
-
-		It("delegates chunked logs to transformers", func() {
-			fakeTransformer := &mocks.MockEventTransformer{}
-			config := mocks.FakeTransformerConfig
-			fakeTransformer.SetTransformerConfig(config)
-			fakeGethLog := types.Log{
-				Address: common.HexToAddress(config.ContractAddresses[0]),
-				Topics:  []common.Hash{common.HexToHash(config.Topic)},
-			}
-			fakeHeaderSyncLogs := []core.HeaderSyncLog{{Log: fakeGethLog}}
-			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
-			mockLogRepository.ReturnLogs = fakeHeaderSyncLogs
-			delegator := newDelegator(mockLogRepository)
-			delegator.AddTransformer(fakeTransformer)
-
-			err := delegator.DelegateLogs()
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
-			Expect(fakeTransformer.PassedLogs).To(Equal(fakeHeaderSyncLogs))
-		})
-
-		It("returns error if transformer returns an error", func() {
-			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
-			mockLogRepository.ReturnLogs = []core.HeaderSyncLog{{}}
-			delegator := newDelegator(mockLogRepository)
-			fakeTransformer := &mocks.MockEventTransformer{ExecuteError: fakes.FakeError}
-			delegator.AddTransformer(fakeTransformer)
-
-			err := delegator.DelegateLogs()
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-
-		It("returns nil for error when logs returned and delegated", func() {
-			fakeTransformer := &mocks.MockEventTransformer{}
-			config := mocks.FakeTransformerConfig
-			fakeTransformer.SetTransformerConfig(config)
-			fakeGethLog := types.Log{
-				Address: common.HexToAddress(config.ContractAddresses[0]),
-				Topics:  []common.Hash{common.HexToHash(config.Topic)},
-			}
-			fakeHeaderSyncLogs := []core.HeaderSyncLog{{Log: fakeGethLog}}
-			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
-			mockLogRepository.ReturnLogs = fakeHeaderSyncLogs
-			delegator := newDelegator(mockLogRepository)
-			delegator.AddTransformer(fakeTransformer)
-
-			err := delegator.DelegateLogs()
-
-			Expect(err).NotTo(HaveOccurred())
-		})
-	})
-})
-
-func newDelegator(headerSyncLogRepository *fakes.MockHeaderSyncLogRepository) *logs.LogDelegator {
-	return &logs.LogDelegator{
-		Chunker:       chunker.NewLogChunker(),
-		LogRepository: headerSyncLogRepository,
-	}
-}

libraries/shared/logs/extractor.go

@@ -1,154 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package logs
-
-import (
-	"errors"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transactions"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-)
-
-var (
-	ErrNoUncheckedHeaders = errors.New("no unchecked headers available for log fetching")
-	ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor")
-)
-
-type ILogExtractor interface {
-	AddTransformerConfig(config transformer.EventTransformerConfig) error
-	ExtractLogs(recheckHeaders constants.TransformerExecution) error
-}
-
-type LogExtractor struct {
-	Addresses                []common.Address
-	CheckedHeadersRepository datastore.CheckedHeadersRepository
-	CheckedLogsRepository    datastore.CheckedLogsRepository
-	Fetcher                  fetcher.ILogFetcher
-	LogRepository            datastore.HeaderSyncLogRepository
-	StartingBlock            *int64
-	Syncer                   transactions.ITransactionsSyncer
-	Topics                   []common.Hash
-}
-
-// Add additional logs to extract
-func (extractor *LogExtractor) AddTransformerConfig(config transformer.EventTransformerConfig) error {
-	checkedHeadersErr := extractor.updateCheckedHeaders(config)
-	if checkedHeadersErr != nil {
-		return checkedHeadersErr
-	}
-
-	if extractor.StartingBlock == nil {
-		extractor.StartingBlock = &config.StartingBlockNumber
-	} else if earlierStartingBlockNumber(config.StartingBlockNumber, *extractor.StartingBlock) {
-		extractor.StartingBlock = &config.StartingBlockNumber
-	}
-
-	addresses := transformer.HexStringsToAddresses(config.ContractAddresses)
-	extractor.Addresses = append(extractor.Addresses, addresses...)
-	extractor.Topics = append(extractor.Topics, common.HexToHash(config.Topic))
-	return nil
-}
-
-// Fetch and persist watched logs
-func (extractor LogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) error {
-	if len(extractor.Addresses) < 1 {
-		logrus.Errorf("error extracting logs: %s", ErrNoWatchedAddresses.Error())
-		return ErrNoWatchedAddresses
-	}
-
-	uncheckedHeaders, uncheckedHeadersErr := extractor.CheckedHeadersRepository.UncheckedHeaders(*extractor.StartingBlock, -1, getCheckCount(recheckHeaders))
-	if uncheckedHeadersErr != nil {
-		logrus.Errorf("error fetching missing headers: %s", uncheckedHeadersErr)
-		return uncheckedHeadersErr
-	}
-
-	if len(uncheckedHeaders) < 1 {
-		return ErrNoUncheckedHeaders
-	}
-
-	for _, header := range uncheckedHeaders {
-		logs, fetchLogsErr := extractor.Fetcher.FetchLogs(extractor.Addresses, extractor.Topics, header)
-		if fetchLogsErr != nil {
-			logError("error fetching logs for header: %s", fetchLogsErr, header)
-			return fetchLogsErr
-		}
-
-		if len(logs) > 0 {
-			transactionsSyncErr := extractor.Syncer.SyncTransactions(header.ID, logs)
-			if transactionsSyncErr != nil {
-				logError("error syncing transactions: %s", transactionsSyncErr, header)
-				return transactionsSyncErr
-			}
-
-			createLogsErr := extractor.LogRepository.CreateHeaderSyncLogs(header.ID, logs)
-			if createLogsErr != nil {
-				logError("error persisting logs: %s", createLogsErr, header)
-				return createLogsErr
-			}
-		}
-
-		markHeaderCheckedErr := extractor.CheckedHeadersRepository.MarkHeaderChecked(header.ID)
-		if markHeaderCheckedErr != nil {
-			logError("error marking header checked: %s", markHeaderCheckedErr, header)
-			return markHeaderCheckedErr
-		}
-	}
-	return nil
-}
-
-func earlierStartingBlockNumber(transformerBlock, watcherBlock int64) bool {
-	return transformerBlock < watcherBlock
-}
-
-func logError(description string, err error, header core.Header) {
-	logrus.WithFields(logrus.Fields{
-		"headerId":    header.ID,
-		"headerHash":  header.Hash,
-		"blockNumber": header.BlockNumber,
-	}).Errorf(description, err.Error())
-}
-
-func getCheckCount(recheckHeaders constants.TransformerExecution) int64 {
-	if recheckHeaders == constants.HeaderUnchecked {
-		return 1
-	}
-	return constants.RecheckHeaderCap
-}
-
-func (extractor *LogExtractor) updateCheckedHeaders(config transformer.EventTransformerConfig) error {
-	alreadyWatchingLog, watchingLogErr := extractor.CheckedLogsRepository.AlreadyWatchingLog(config.ContractAddresses, config.Topic)
-	if watchingLogErr != nil {
-		return watchingLogErr
-	}
-	if !alreadyWatchingLog {
-		uncheckHeadersErr := extractor.CheckedHeadersRepository.MarkHeadersUnchecked(config.StartingBlockNumber)
-		if uncheckHeadersErr != nil {
-			return uncheckHeadersErr
-		}
-		markLogWatchedErr := extractor.CheckedLogsRepository.MarkLogWatched(config.ContractAddresses, config.Topic)
-		if markLogWatchedErr != nil {
-			return markLogWatchedErr
-		}
-	}
-	return nil
-}

libraries/shared/logs/extractor_test.go

@@ -1,415 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package logs_test
-
-import (
-	"math/rand"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/logs"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Log extractor", func() {
-	var (
-		checkedHeadersRepository *fakes.MockCheckedHeadersRepository
-		checkedLogsRepository    *fakes.MockCheckedLogsRepository
-		extractor                *logs.LogExtractor
-	)
-
-	BeforeEach(func() {
-		checkedHeadersRepository = &fakes.MockCheckedHeadersRepository{}
-		checkedLogsRepository = &fakes.MockCheckedLogsRepository{}
-		extractor = &logs.LogExtractor{
-			CheckedHeadersRepository: checkedHeadersRepository,
-			CheckedLogsRepository:    checkedLogsRepository,
-			Fetcher:                  &mocks.MockLogFetcher{},
-			LogRepository:            &fakes.MockHeaderSyncLogRepository{},
-			Syncer:                   &fakes.MockTransactionSyncer{},
-		}
-	})
-
-	Describe("AddTransformerConfig", func() {
-		It("updates extractor's starting block number to earliest available", func() {
-			earlierStartingBlockNumber := rand.Int63()
-			laterStartingBlockNumber := earlierStartingBlockNumber + 1
-
-			errOne := extractor.AddTransformerConfig(getTransformerConfig(laterStartingBlockNumber))
-			Expect(errOne).NotTo(HaveOccurred())
-			errTwo := extractor.AddTransformerConfig(getTransformerConfig(earlierStartingBlockNumber))
-			Expect(errTwo).NotTo(HaveOccurred())
-
-			Expect(*extractor.StartingBlock).To(Equal(earlierStartingBlockNumber))
-		})
-
-		It("adds transformer's addresses to extractor's watched addresses", func() {
-			addresses := []string{"0xA", "0xB"}
-			configWithAddresses := transformer.EventTransformerConfig{
-				ContractAddresses:   addresses,
-				StartingBlockNumber: rand.Int63(),
-			}
-
-			err := extractor.AddTransformerConfig(configWithAddresses)
-
-			Expect(err).NotTo(HaveOccurred())
-			expectedAddresses := transformer.HexStringsToAddresses(addresses)
-			Expect(extractor.Addresses).To(Equal(expectedAddresses))
-		})
-
-		It("adds transformer's topic to extractor's watched topics", func() {
-			topic := "0x1"
-			configWithTopic := transformer.EventTransformerConfig{
-				ContractAddresses:   []string{fakes.FakeAddress.Hex()},
-				Topic:               topic,
-				StartingBlockNumber: rand.Int63(),
-			}
-
-			err := extractor.AddTransformerConfig(configWithTopic)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(extractor.Topics).To(Equal([]common.Hash{common.HexToHash(topic)}))
-		})
-
-		It("returns error if checking whether log has been checked returns error", func() {
-			checkedLogsRepository.AlreadyWatchingLogError = fakes.FakeError
-
-			err := extractor.AddTransformerConfig(getTransformerConfig(rand.Int63()))
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-
-		Describe("when log has previously been checked", func() {
-			It("does not mark any headers unchecked", func() {
-				checkedLogsRepository.AlreadyWatchingLogReturn = true
-
-				err := extractor.AddTransformerConfig(getTransformerConfig(rand.Int63()))
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(checkedHeadersRepository.MarkHeadersUncheckedCalled).To(BeFalse())
-			})
-		})
-
-		Describe("when log has not previously been checked", func() {
-			BeforeEach(func() {
-				checkedLogsRepository.AlreadyWatchingLogReturn = false
-			})
-
-			It("marks headers since transformer's starting block number as unchecked", func() {
-				blockNumber := rand.Int63()
-
-				err := extractor.AddTransformerConfig(getTransformerConfig(blockNumber))
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(checkedHeadersRepository.MarkHeadersUncheckedCalled).To(BeTrue())
-				Expect(checkedHeadersRepository.MarkHeadersUncheckedStartingBlockNumber).To(Equal(blockNumber))
-			})
-
-			It("returns error if marking headers unchecked returns error", func() {
-				checkedHeadersRepository.MarkHeadersUncheckedReturnError = fakes.FakeError
-
-				err := extractor.AddTransformerConfig(getTransformerConfig(rand.Int63()))
-
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(fakes.FakeError))
-			})
-
-			It("persists that tranformer's log has been checked", func() {
-				config := getTransformerConfig(rand.Int63())
-
-				err := extractor.AddTransformerConfig(config)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(checkedLogsRepository.MarkLogWatchedAddresses).To(Equal(config.ContractAddresses))
-				Expect(checkedLogsRepository.MarkLogWatchedTopicZero).To(Equal(config.Topic))
-			})
-
-			It("returns error if marking logs checked returns error", func() {
-				checkedLogsRepository.MarkLogWatchedError = fakes.FakeError
-
-				err := extractor.AddTransformerConfig(getTransformerConfig(rand.Int63()))
-
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(fakes.FakeError))
-			})
-		})
-	})
-
-	Describe("ExtractLogs", func() {
-		It("returns error if no watched addresses configured", func() {
-			err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(logs.ErrNoWatchedAddresses))
-		})
-
-		Describe("when checking unchecked headers", func() {
-			It("gets headers since configured starting block with check_count < 1", func() {
-				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
-				mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
-				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
-				startingBlockNumber := rand.Int63()
-				extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
-
-				err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mockCheckedHeadersRepository.UncheckedHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
-				Expect(mockCheckedHeadersRepository.UncheckedHeadersEndingBlockNumber).To(Equal(int64(-1)))
-				Expect(mockCheckedHeadersRepository.UncheckedHeadersCheckCount).To(Equal(int64(1)))
-			})
-		})
-
-		Describe("when rechecking headers", func() {
-			It("gets headers since configured starting block with check_count < RecheckHeaderCap", func() {
-				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
-				mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
-				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
-				startingBlockNumber := rand.Int63()
-				extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
-
-				err := extractor.ExtractLogs(constants.HeaderRecheck)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mockCheckedHeadersRepository.UncheckedHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
-				Expect(mockCheckedHeadersRepository.UncheckedHeadersEndingBlockNumber).To(Equal(int64(-1)))
-				Expect(mockCheckedHeadersRepository.UncheckedHeadersCheckCount).To(Equal(constants.RecheckHeaderCap))
-			})
-		})
-
-		It("returns error if getting unchecked headers fails", func() {
-			addTransformerConfig(extractor)
-			mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
-			mockCheckedHeadersRepository.UncheckedHeadersReturnError = fakes.FakeError
-			extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
-
-			err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-
-		Describe("when no unchecked headers", func() {
-			It("does not fetch logs", func() {
-				addTransformerConfig(extractor)
-				mockLogFetcher := &mocks.MockLogFetcher{}
-				extractor.Fetcher = mockLogFetcher
-
-				_ = extractor.ExtractLogs(constants.HeaderUnchecked)
-
-				Expect(mockLogFetcher.FetchCalled).To(BeFalse())
-			})
-
-			It("returns error that no unchecked headers were found", func() {
-				addTransformerConfig(extractor)
-				mockLogFetcher := &mocks.MockLogFetcher{}
-				extractor.Fetcher = mockLogFetcher
-
-				err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-				Expect(err).To(MatchError(logs.ErrNoUncheckedHeaders))
-			})
-		})
-
-		Describe("when there are unchecked headers", func() {
-			It("fetches logs for unchecked headers", func() {
-				addUncheckedHeader(extractor)
-				config := transformer.EventTransformerConfig{
-					ContractAddresses:   []string{fakes.FakeAddress.Hex()},
-					Topic:               fakes.FakeHash.Hex(),
-					StartingBlockNumber: rand.Int63(),
-				}
-				addTransformerErr := extractor.AddTransformerConfig(config)
-				Expect(addTransformerErr).NotTo(HaveOccurred())
-				mockLogFetcher := &mocks.MockLogFetcher{}
-				extractor.Fetcher = mockLogFetcher
-
-				err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mockLogFetcher.FetchCalled).To(BeTrue())
-				expectedTopics := []common.Hash{common.HexToHash(config.Topic)}
-				Expect(mockLogFetcher.Topics).To(Equal(expectedTopics))
-				expectedAddresses := transformer.HexStringsToAddresses(config.ContractAddresses)
-				Expect(mockLogFetcher.ContractAddresses).To(Equal(expectedAddresses))
-			})
-
-			It("returns error if fetching logs fails", func() {
-				addUncheckedHeader(extractor)
-				addTransformerConfig(extractor)
-				mockLogFetcher := &mocks.MockLogFetcher{}
-				mockLogFetcher.ReturnError = fakes.FakeError
-				extractor.Fetcher = mockLogFetcher
-
-				err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(fakes.FakeError))
-			})
-
-			Describe("when no fetched logs", func() {
-				It("does not sync transactions", func() {
-					addUncheckedHeader(extractor)
-					addTransformerConfig(extractor)
-					mockTransactionSyncer := &fakes.MockTransactionSyncer{}
-					extractor.Syncer = mockTransactionSyncer
-
-					err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-					Expect(err).NotTo(HaveOccurred())
-					Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeFalse())
-				})
-			})
-
-			Describe("when there are fetched logs", func() {
-				It("syncs transactions", func() {
-					addUncheckedHeader(extractor)
-					addFetchedLog(extractor)
-					addTransformerConfig(extractor)
-					mockTransactionSyncer := &fakes.MockTransactionSyncer{}
-					extractor.Syncer = mockTransactionSyncer
-
-					err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-					Expect(err).NotTo(HaveOccurred())
-					Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue())
-				})
-
-				It("returns error if syncing transactions fails", func() {
-					addUncheckedHeader(extractor)
-					addFetchedLog(extractor)
-					addTransformerConfig(extractor)
-					mockTransactionSyncer := &fakes.MockTransactionSyncer{}
-					mockTransactionSyncer.SyncTransactionsError = fakes.FakeError
-					extractor.Syncer = mockTransactionSyncer
-
-					err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-					Expect(err).To(HaveOccurred())
-					Expect(err).To(MatchError(fakes.FakeError))
-				})
-
-				It("persists fetched logs", func() {
-					addUncheckedHeader(extractor)
-					addTransformerConfig(extractor)
-					fakeLogs := []types.Log{{
-						Address: common.HexToAddress("0xA"),
-						Topics:  []common.Hash{common.HexToHash("0xA")},
-						Data:    []byte{},
-						Index:   0,
-					}}
-					mockLogFetcher := &mocks.MockLogFetcher{ReturnLogs: fakeLogs}
-					extractor.Fetcher = mockLogFetcher
-					mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
-					extractor.LogRepository = mockLogRepository
-
-					err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-					Expect(err).NotTo(HaveOccurred())
-					Expect(mockLogRepository.PassedLogs).To(Equal(fakeLogs))
-				})
-
-				It("returns error if persisting logs fails", func() {
-					addUncheckedHeader(extractor)
-					addFetchedLog(extractor)
-					addTransformerConfig(extractor)
-					mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
-					mockLogRepository.CreateError = fakes.FakeError
-					extractor.LogRepository = mockLogRepository
-
-					err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-					Expect(err).To(HaveOccurred())
-					Expect(err).To(MatchError(fakes.FakeError))
-				})
-			})
-
-			It("marks header checked", func() {
-				addFetchedLog(extractor)
-				addTransformerConfig(extractor)
-				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
-				headerID := rand.Int63()
-				mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{ID: headerID}}
-				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
-
-				err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mockCheckedHeadersRepository.MarkHeaderCheckedHeaderID).To(Equal(headerID))
-			})
-
-			It("returns error if marking header checked fails", func() {
-				addFetchedLog(extractor)
-				addTransformerConfig(extractor)
-				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
-				mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{ID: rand.Int63()}}
-				mockCheckedHeadersRepository.MarkHeaderCheckedReturnError = fakes.FakeError
-				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
-
-				err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(fakes.FakeError))
-			})
-
-			It("returns nil for error if everything succeeds", func() {
-				addUncheckedHeader(extractor)
-				addTransformerConfig(extractor)
-
-				err := extractor.ExtractLogs(constants.HeaderUnchecked)
-
-				Expect(err).NotTo(HaveOccurred())
-			})
-		})
-	})
-})
-
-func addTransformerConfig(extractor *logs.LogExtractor) {
-	fakeConfig := transformer.EventTransformerConfig{
-		ContractAddresses:   []string{fakes.FakeAddress.Hex()},
-		Topic:               fakes.FakeHash.Hex(),
-		StartingBlockNumber: rand.Int63(),
-	}
-	extractor.AddTransformerConfig(fakeConfig)
-}
-
-func addUncheckedHeader(extractor *logs.LogExtractor) {
-	mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
-	mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
-	extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
-}
-
-func addFetchedLog(extractor *logs.LogExtractor) {
-	mockLogFetcher := &mocks.MockLogFetcher{}
-	mockLogFetcher.ReturnLogs = []types.Log{{}}
-	extractor.Fetcher = mockLogFetcher
-}
-
-func getTransformerConfig(startingBlockNumber int64) transformer.EventTransformerConfig {
-	return transformer.EventTransformerConfig{
-		ContractAddresses:   []string{fakes.FakeAddress.Hex()},
-		Topic:               fakes.FakeHash.Hex(),
-		StartingBlockNumber: startingBlockNumber,
-	}
-}