trimming down to ipfs watchers

pkg/btc/cid_retriever.go

@@ -26,9 +26,9 @@ import (
 	"github.com/lib/pq"
 	log "github.com/sirupsen/logrus"
 
-	utils "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/utils"
 )
 
 // CIDRetriever satisfies the CIDRetriever interface for bitcoin

pkg/btc/cleaner.go

@@ -22,8 +22,8 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // Cleaner satisfies the shared.Cleaner interface fo bitcoin

pkg/btc/cleaner_test.go

@@ -23,21 +23,19 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 var (
 	// Block 0
 	// header variables
-	blockHash1      = crypto.Keccak256Hash([]byte{00, 02})
-	blocKNumber1    = big.NewInt(0)
-	headerCid1      = "mockHeader1CID"
-	parentHash      = crypto.Keccak256Hash([]byte{00, 01})
-	totalDifficulty = "50000000000000000000"
-	reward          = "5000000000000000000"
-	headerModel1    = btc.HeaderModel{
+	blockHash1   = crypto.Keccak256Hash([]byte{00, 02})
+	blocKNumber1 = big.NewInt(0)
+	headerCid1   = "mockHeader1CID"
+	parentHash   = crypto.Keccak256Hash([]byte{00, 01})
+	headerModel1 = btc.HeaderModel{
 		BlockHash:   blockHash1.String(),
 		BlockNumber: blocKNumber1.String(),
 		ParentHash:  parentHash.String(),

pkg/btc/converter.go

@@ -23,7 +23,7 @@ import (
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/txscript"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // PayloadConverter satisfies the PayloadConverter interface for bitcoin

pkg/btc/converter_test.go

@@ -21,8 +21,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc/mocks"
 )
 
 var _ = Describe("Converter", func() {

pkg/btc/filterer.go

@@ -23,9 +23,9 @@ import (
 
 	"github.com/multiformats/go-multihash"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/ipld"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // ResponseFilterer satisfies the ResponseFilterer interface for bitcoin

pkg/btc/http_streamer.go

@@ -23,7 +23,7 @@ import (
 	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // HTTPPayloadStreamer satisfies the PayloadStreamer interface for bitcoin over http endpoints (since bitcoin core doesn't support websockets)

pkg/btc/indexer.go

@@ -24,8 +24,8 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 type CIDIndexer struct {

pkg/btc/indexer_test.go

@@ -20,10 +20,10 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 var _ = Describe("Indexer", func() {

pkg/btc/ipld_fetcher.go

@@ -26,8 +26,8 @@ import (
 	"github.com/ipfs/go-cid"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 var (

pkg/btc/ipld_pg_fetcher.go

@@ -22,9 +22,9 @@ import (
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum

pkg/btc/mocks/converter.go

@@ -19,8 +19,8 @@ package mocks
 import (
 	"fmt"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // PayloadConverter is the underlying struct for the Converter interface

pkg/btc/mocks/indexer.go

@@ -19,11 +19,8 @@ package mocks
 import (
 	"fmt"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
-
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
-
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // CIDIndexer is the underlying struct for the Indexer interface
@@ -36,7 +33,7 @@ type CIDIndexer struct {
 func (repo *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
 	cidPayload, ok := cids.(*btc.CIDPayload)
 	if !ok {
-		return fmt.Errorf("index expected cids type %T got %T", &eth.CIDPayload{}, cids)
+		return fmt.Errorf("index expected cids type %T got %T", &btc.CIDPayload{}, cids)
 	}
 	repo.PassedCIDPayload = append(repo.PassedCIDPayload, cidPayload)
 	return repo.ReturnErr

pkg/btc/mocks/publisher.go

@@ -19,9 +19,9 @@ package mocks
 import (
 	"fmt"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // IPLDPublisher is the underlying struct for the Publisher interface

pkg/btc/mocks/test_data.go

@@ -25,7 +25,8 @@ import (
 	"github.com/btcsuite/btcd/txscript"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
+
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
 )
 
 var (

pkg/btc/payload_fetcher.go

@@ -23,7 +23,7 @@ import (
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // PayloadFetcher satisfies the PayloadFetcher interface for bitcoin

pkg/btc/publish_and_indexer.go

@@ -20,9 +20,9 @@ import (
 	"fmt"
 	"strconv"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/ipld"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for bitcoin

pkg/btc/publish_and_indexer_test.go

@@ -25,12 +25,12 @@ import (
 	"github.com/multiformats/go-multihash"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/ipld"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 var _ = Describe("PublishAndIndexer", func() {

pkg/btc/publisher.go

@@ -20,10 +20,10 @@ import (
 	"fmt"
 	"strconv"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/dag_putters"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/dag_putters"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/ipld"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // IPLDPublisher satisfies the IPLDPublisher for ethereum

pkg/btc/publisher_test.go

@@ -23,9 +23,9 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	mocks2 "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/btc/mocks"
+	mocks2 "github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs/mocks"
 )
 
 var (

pkg/btc/streamer.go

@@ -22,7 +22,7 @@ import (
 	"github.com/btcsuite/btcutil"
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 const (

pkg/btc/subscription_config.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // SubscriptionSettings config is used by a subscriber to specify what bitcoin data to stream from the super node

pkg/btc/test_helpers.go

@@ -19,7 +19,7 @@ package btc
 import (
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
 )
 
 // TearDownDB is used to tear down the super node dbs after tests

pkg/btc/types.go

@@ -19,7 +19,7 @@ package btc
 import (
 	"math/big"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
 
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"

pkg/config/config_test.go

@@ -38,7 +38,7 @@ var _ = Describe("Loading the config", func() {
 
 	It("reads the private config using the environment", func() {
 		viper.SetConfigName("config")
-		viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
+		viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/ipfs-chain-watcher/environments/")
 		Expect(viper.Get("client.ipcpath")).To(BeNil())
 
 		testConfig := viper.New()

pkg/config/contract.go

@@ -1,218 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package config
-
-import (
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-)
-
-// Config struct for generic contract transformer
-type ContractConfig struct {
-	// Name for the transformer
-	Name string
-
-	// Ethereum network name; default "" is mainnet
-	Network string
-
-	// List of contract addresses (map to ensure no duplicates)
-	Addresses map[string]bool
-
-	// Map of contract address to abi
-	// If an address has no associated abi the parser will attempt to fetch one from etherscan
-	Abis map[string]string
-
-	// Map of contract address to slice of events
-	// Used to set which addresses to watch
-	// If any events are listed in the slice only those will be watched
-	// Otherwise all events in the contract ABI are watched
-	Events map[string][]string
-
-	// Map of contract address to slice of methods
-	// If any methods are listed in the slice only those will be polled
-	// Otherwise no methods will be polled
-	Methods map[string][]string
-
-	// Map of contract address to slice of event arguments to filter for
-	// If arguments are provided then only events which emit those arguments are watched
-	// Otherwise arguments are not filtered on events
-	EventArgs map[string][]string
-
-	// Map of contract address to slice of method arguments to limit polling to
-	// If arguments are provided then only those arguments are allowed as arguments in method polling
-	// Otherwise any argument of the right type seen emitted from events at that contract will be used in method polling
-	MethodArgs map[string][]string
-
-	// Map of contract address to their starting block
-	StartingBlocks map[string]int64
-
-	// Map of contract address to whether or not to pipe method polling results forward into subsequent method calls
-	Piping map[string]bool
-}
-
-func (contractConfig *ContractConfig) PrepConfig() {
-	addrs := viper.GetStringSlice("contract.addresses")
-	contractConfig.Network = viper.GetString("contract.network")
-	contractConfig.Addresses = make(map[string]bool, len(addrs))
-	contractConfig.Abis = make(map[string]string, len(addrs))
-	contractConfig.Methods = make(map[string][]string, len(addrs))
-	contractConfig.Events = make(map[string][]string, len(addrs))
-	contractConfig.MethodArgs = make(map[string][]string, len(addrs))
-	contractConfig.EventArgs = make(map[string][]string, len(addrs))
-	contractConfig.StartingBlocks = make(map[string]int64, len(addrs))
-	contractConfig.Piping = make(map[string]bool, len(addrs))
-	// De-dupe addresses
-	for _, addr := range addrs {
-		contractConfig.Addresses[strings.ToLower(addr)] = true
-	}
-
-	// Iterate over addresses to pull out config info for each contract
-	for _, addr := range addrs {
-		transformer := viper.GetStringMap("contract." + addr)
-
-		// Get and check abi
-		var abi string
-		abiInterface, abiOK := transformer["abi"]
-		if !abiOK {
-			log.Warnf("contract %s not configured with an ABI, will attempt to fetch it from Etherscan\r\n", addr)
-		} else {
-			abi, abiOK = abiInterface.(string)
-			if !abiOK {
-				log.Fatal(addr, "transformer `abi` not of type []string")
-			}
-		}
-		if abi != "" {
-			if _, abiErr := eth.ParseAbi(abi); abiErr != nil {
-				log.Fatal(addr, "transformer `abi` not valid JSON")
-			}
-		}
-		contractConfig.Abis[strings.ToLower(addr)] = abi
-
-		// Get and check events
-		events := make([]string, 0)
-		eventsInterface, eventsOK := transformer["events"]
-		if !eventsOK {
-			log.Warnf("contract %s not configured with a list of events to watch, will watch all events\r\n", addr)
-			events = []string{}
-		} else {
-			eventsI, eventsOK := eventsInterface.([]interface{})
-			if !eventsOK {
-				log.Fatal(addr, "transformer `events` not of type []string\r\n")
-			}
-			for _, strI := range eventsI {
-				str, strOK := strI.(string)
-				if !strOK {
-					log.Fatal(addr, "transformer `events` not of type []string\r\n")
-				}
-				events = append(events, str)
-			}
-		}
-		contractConfig.Events[strings.ToLower(addr)] = events
-
-		// Get and check methods
-		methods := make([]string, 0)
-		methodsInterface, methodsOK := transformer["methods"]
-		if !methodsOK {
-			log.Warnf("contract %s not configured with a list of methods to poll, will not poll any methods\r\n", addr)
-			methods = []string{}
-		} else {
-			methodsI, methodsOK := methodsInterface.([]interface{})
-			if !methodsOK {
-				log.Fatal(addr, "transformer `methods` not of type []string\r\n")
-			}
-			for _, strI := range methodsI {
-				str, strOK := strI.(string)
-				if !strOK {
-					log.Fatal(addr, "transformer `methods` not of type []string\r\n")
-				}
-				methods = append(methods, str)
-			}
-		}
-		contractConfig.Methods[strings.ToLower(addr)] = methods
-
-		// Get and check eventArgs
-		eventArgs := make([]string, 0)
-		eventArgsInterface, eventArgsOK := transformer["eventArgs"]
-		if !eventArgsOK {
-			log.Warnf("contract %s not configured with a list of event arguments to filter for, will not filter events for specific emitted values\r\n", addr)
-			eventArgs = []string{}
-		} else {
-			eventArgsI, eventArgsOK := eventArgsInterface.([]interface{})
-			if !eventArgsOK {
-				log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n")
-			}
-			for _, strI := range eventArgsI {
-				str, strOK := strI.(string)
-				if !strOK {
-					log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n")
-				}
-				eventArgs = append(eventArgs, str)
-			}
-		}
-		contractConfig.EventArgs[strings.ToLower(addr)] = eventArgs
-
-		// Get and check methodArgs
-		methodArgs := make([]string, 0)
-		methodArgsInterface, methodArgsOK := transformer["methodArgs"]
-		if !methodArgsOK {
-			log.Warnf("contract %s not configured with a list of method argument values to poll with, will poll methods with all available arguments\r\n", addr)
-			methodArgs = []string{}
-		} else {
-			methodArgsI, methodArgsOK := methodArgsInterface.([]interface{})
-			if !methodArgsOK {
-				log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n")
-			}
-			for _, strI := range methodArgsI {
-				str, strOK := strI.(string)
-				if !strOK {
-					log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n")
-				}
-				methodArgs = append(methodArgs, str)
-			}
-		}
-		contractConfig.MethodArgs[strings.ToLower(addr)] = methodArgs
-
-		// Get and check startingBlock
-		startInterface, startOK := transformer["startingblock"]
-		if !startOK {
-			log.Fatal(addr, "transformer config is missing `startingBlock` value\r\n")
-		}
-		start, startOK := startInterface.(int64)
-		if !startOK {
-			log.Fatal(addr, "transformer `startingBlock` not of type int\r\n")
-		}
-		contractConfig.StartingBlocks[strings.ToLower(addr)] = start
-
-		// Get pipping
-		var piping bool
-		_, pipeOK := transformer["piping"]
-		if !pipeOK {
-			log.Warnf("contract %s does not have its `piping` set, by default piping is turned off\r\n", addr)
-			piping = false
-		} else {
-			pipingInterface := transformer["piping"]
-			piping, pipeOK = pipingInterface.(bool)
-			if !pipeOK {
-				log.Fatal(addr, "transformer `piping` not of type bool\r\n")
-			}
-		}
-		contractConfig.Piping[strings.ToLower(addr)] = piping
-	}
-}

pkg/config/plugin.go

@@ -1,167 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package config
-
-import (
-	"errors"
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
-)
-
-type Plugin struct {
-	Transformers map[string]Transformer
-	FilePath     string
-	FileName     string
-	Save         bool
-	Home         string
-}
-
-type Transformer struct {
-	Path           string
-	Type           TransformerType
-	MigrationPath  string
-	MigrationRank  uint64
-	RepositoryPath string
-}
-
-func (pluginConfig *Plugin) GetPluginPaths() (string, string, error) {
-	path, err := helpers.CleanPath(pluginConfig.FilePath)
-	if err != nil {
-		return "", "", err
-	}
-
-	name := strings.Split(pluginConfig.FileName, ".")[0]
-	goFile := filepath.Join(path, name+".go")
-	soFile := filepath.Join(path, name+".so")
-
-	return goFile, soFile, nil
-}
-
-// Removes duplicate migration paths and returns them in ranked order
-func (pluginConfig *Plugin) GetMigrationsPaths() ([]string, error) {
-	paths := make(map[uint64]string)
-	highestRank := -1
-	for name, transformer := range pluginConfig.Transformers {
-		repo := transformer.RepositoryPath
-		mig := transformer.MigrationPath
-		path := filepath.Join("$GOPATH/src", pluginConfig.Home, "vendor", repo, mig)
-		cleanPath, err := helpers.CleanPath(path)
-		if err != nil {
-			return nil, err
-		}
-		// If there is a different path with the same rank then we have a conflict
-		_, ok := paths[transformer.MigrationRank]
-		if ok {
-			conflictingPath := paths[transformer.MigrationRank]
-			if conflictingPath != cleanPath {
-				return nil, fmt.Errorf("transformer %s has the same migration rank (%d) as another transformer", name, transformer.MigrationRank)
-			}
-		}
-		paths[transformer.MigrationRank] = cleanPath
-		if int(transformer.MigrationRank) >= highestRank {
-			highestRank = int(transformer.MigrationRank)
-		}
-	}
-	// Check for gaps and duplicates
-	if len(paths) != (highestRank + 1) {
-		return []string{}, errors.New("number of distinct ranks does not match number of distinct migration paths")
-	}
-	if anyDupes(paths) {
-		return []string{}, errors.New("duplicate paths with different ranks present")
-	}
-
-	sortedPaths := make([]string, len(paths))
-	for rank, path := range paths {
-		sortedPaths[rank] = path
-	}
-
-	return sortedPaths, nil
-}
-
-// Removes duplicate repo paths before returning them
-func (pluginConfig *Plugin) GetRepoPaths() map[string]bool {
-	paths := make(map[string]bool)
-	for _, transformer := range pluginConfig.Transformers {
-		paths[transformer.RepositoryPath] = true
-	}
-
-	return paths
-}
-
-type TransformerType int
-
-const (
-	UnknownTransformerType TransformerType = iota
-	EthEvent
-	EthStorage
-	EthContract
-)
-
-func (transformerType TransformerType) String() string {
-	names := [...]string{
-		"Unknown",
-		"eth_event",
-		"eth_storage",
-		"eth_contract",
-	}
-
-	if transformerType > EthContract || transformerType < EthEvent {
-		return "Unknown"
-	}
-
-	return names[transformerType]
-}
-
-func GetTransformerType(str string) TransformerType {
-	types := [...]TransformerType{
-		EthEvent,
-		EthStorage,
-		EthContract,
-	}
-
-	for _, ty := range types {
-		if ty.String() == str {
-			return ty
-		}
-	}
-
-	return UnknownTransformerType
-}
-
-func anyDupes(list map[uint64]string) bool {
-	seen := make([]string, 0, len(list))
-	for _, str := range list {
-		dupe := inList(str, seen)
-		if dupe {
-			return true
-		}
-		seen = append(seen, str)
-	}
-	return false
-}
-
-func inList(str string, list []string) bool {
-	for _, element := range list {
-		if str == element {
-			return true
-		}
-	}
-	return false
-}

pkg/config/plugin_test.go

@@ -1,242 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package config_test
-
-import (
-	"os"
-	"path/filepath"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-)
-
-var allDifferentPathsConfig = config.Plugin{
-	Transformers: map[string]config.Transformer{
-		"transformer1": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path1",
-			MigrationRank:  0,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer2": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path2",
-			MigrationRank:  2,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer3": {
-			Path:           "test/init/path2",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path3",
-			MigrationRank:  1,
-			RepositoryPath: "test/repo/path",
-		},
-	},
-}
-
-var overlappingPathsConfig = config.Plugin{
-	Transformers: map[string]config.Transformer{
-		"transformer1": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path1",
-			MigrationRank:  0,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer2": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path1",
-			MigrationRank:  0,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer3": {
-			Path:           "test/init/path2",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path3",
-			MigrationRank:  1,
-			RepositoryPath: "test/repo/path",
-		},
-	},
-}
-
-var conflictErrorConfig = config.Plugin{
-	Transformers: map[string]config.Transformer{
-		"transformer1": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path1",
-			MigrationRank:  0,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer2": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path2",
-			MigrationRank:  0,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer3": {
-			Path:           "test/init/path2",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path3",
-			MigrationRank:  1,
-			RepositoryPath: "test/repo/path",
-		},
-	},
-}
-
-var gapErrorConfig = config.Plugin{
-	Transformers: map[string]config.Transformer{
-		"transformer1": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path1",
-			MigrationRank:  0,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer2": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path2",
-			MigrationRank:  3,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer3": {
-			Path:           "test/init/path2",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path3",
-			MigrationRank:  1,
-			RepositoryPath: "test/repo/path",
-		},
-	},
-}
-
-var missingRankErrorConfig = config.Plugin{
-	Transformers: map[string]config.Transformer{
-		"transformer1": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path1",
-			MigrationRank:  0,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer2": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path2",
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer3": {
-			Path:           "test/init/path2",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path3",
-			MigrationRank:  1,
-			RepositoryPath: "test/repo/path",
-		},
-	},
-}
-
-var duplicateErrorConfig = config.Plugin{
-	Transformers: map[string]config.Transformer{
-		"transformer1": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path1",
-			MigrationRank:  0,
-			RepositoryPath: "test/repo/path",
-		},
-		"transformer2": {
-			Path:           "test/init/path",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path1",
-			RepositoryPath: "test/repo/path",
-			MigrationRank:  2,
-		},
-		"transformer3": {
-			Path:           "test/init/path2",
-			Type:           config.EthEvent,
-			MigrationPath:  "test/migration/path3",
-			MigrationRank:  1,
-			RepositoryPath: "test/repo/path",
-		},
-	},
-}
-
-var _ = Describe("GetMigrationsPaths", func() {
-	It("Sorts migration paths by rank", func() {
-		plugin := allDifferentPathsConfig
-		migrationPaths, err := plugin.GetMigrationsPaths()
-		Expect(err).ToNot(HaveOccurred())
-		Expect(len(migrationPaths)).To(Equal(3))
-
-		env := os.Getenv("GOPATH")
-		path1 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path1")
-		path2 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path3")
-		path3 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path2")
-		expectedMigrationPaths := []string{path1, path2, path3}
-		Expect(migrationPaths).To(Equal(expectedMigrationPaths))
-	})
-
-	It("Expects identical migration paths to have the same rank", func() {
-		plugin := overlappingPathsConfig
-		migrationPaths, err := plugin.GetMigrationsPaths()
-		Expect(err).ToNot(HaveOccurred())
-		Expect(len(migrationPaths)).To(Equal(2))
-
-		env := os.Getenv("GOPATH")
-		path1 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path1")
-		path2 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path3")
-		expectedMigrationPaths := []string{path1, path2}
-		Expect(migrationPaths).To(Equal(expectedMigrationPaths))
-	})
-
-	It("Fails if two different migration paths have the same rank", func() {
-		plugin := conflictErrorConfig
-		migrationPaths, err := plugin.GetMigrationsPaths()
-		Expect(err).To(HaveOccurred())
-		Expect(len(migrationPaths)).To(Equal(0))
-		Expect(err.Error()).To(ContainSubstring("has the same migration rank"))
-	})
-
-	It("Fails if there is a gap in the ranks of the migration paths", func() {
-		plugin := gapErrorConfig
-		migrationPaths, err := plugin.GetMigrationsPaths()
-		Expect(err).To(HaveOccurred())
-		Expect(len(migrationPaths)).To(Equal(0))
-		Expect(err.Error()).To(ContainSubstring("number of distinct ranks does not match number of distinct migration paths"))
-	})
-
-	It("Fails if a transformer is missing its rank", func() {
-		plugin := missingRankErrorConfig
-		migrationPaths, err := plugin.GetMigrationsPaths()
-		Expect(err).To(HaveOccurred())
-		Expect(len(migrationPaths)).To(Equal(0))
-		Expect(err.Error()).To(ContainSubstring("has the same migration rank"))
-	})
-
-	It("Fails if the same migration path has more than one rank", func() {
-		plugin := duplicateErrorConfig
-		migrationPaths, err := plugin.GetMigrationsPaths()
-		Expect(err).To(HaveOccurred())
-		Expect(len(migrationPaths)).To(Equal(0))
-		Expect(err.Error()).To(ContainSubstring("duplicate paths with different ranks present"))
-	})
-})

pkg/core/rpc_client.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/rpc"
 
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/client"
 )
 
 type RPCClient interface {

pkg/eth/abi.go

@@ -1,105 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"errors"
-	"strings"
-
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-)
-
-var (
-	ErrInvalidAbiFile   = errors.New("invalid abi")
-	ErrMissingAbiFile   = errors.New("missing abi")
-	ErrAPIRequestFailed = errors.New("etherscan api request failed")
-)
-
-type Response struct {
-	Status  string
-	Message string
-	Result  string
-}
-
-type EtherScanAPI struct {
-	client *http.Client
-	url    string
-}
-
-func NewEtherScanClient(url string) *EtherScanAPI {
-	return &EtherScanAPI{
-		client: &http.Client{Timeout: 10 * time.Second},
-		url:    url,
-	}
-}
-
-func GenURL(network string) string {
-	switch network {
-	case "ropsten":
-		return "https://ropsten.etherscan.io"
-	case "kovan":
-		return "https://kovan.etherscan.io"
-	case "rinkeby":
-		return "https://rinkeby.etherscan.io"
-	default:
-		return "https://api.etherscan.io"
-	}
-}
-
-//https://api.etherscan.io/api?module=contract&action=getabi&address=%s
-func (e *EtherScanAPI) GetAbi(contractHash string) (string, error) {
-	target := new(Response)
-	request := fmt.Sprintf("%s/api?module=contract&action=getabi&address=%s", e.url, contractHash)
-	r, err := e.client.Get(request)
-	if err != nil {
-		return "", ErrAPIRequestFailed
-	}
-	defer r.Body.Close()
-	err = json.NewDecoder(r.Body).Decode(&target)
-	return target.Result, err
-}
-
-func ParseAbiFile(abiFilePath string) (abi.ABI, error) {
-	abiString, err := ReadAbiFile(abiFilePath)
-	if err != nil {
-		return abi.ABI{}, ErrMissingAbiFile
-	}
-	return ParseAbi(abiString)
-}
-
-func ParseAbi(abiString string) (abi.ABI, error) {
-	parsedAbi, err := abi.JSON(strings.NewReader(abiString))
-	if err != nil {
-		return abi.ABI{}, ErrInvalidAbiFile
-	}
-	return parsedAbi, nil
-}
-
-func ReadAbiFile(abiFilePath string) (string, error) {
-	reader := fs.FsReader{}
-	filesBytes, err := reader.Read(abiFilePath)
-	if err != nil {
-		return "", ErrMissingAbiFile
-	}
-	return string(filesBytes), nil
-}

pkg/eth/abi_test.go

@@ -1,135 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"net/http"
-
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/onsi/gomega/ghttp"
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("ABI files", func() {
-
-	Describe("Reading ABI files", func() {
-
-		It("loads a valid ABI file", func() {
-			path := test_config.ABIFilePath + "valid_abi.json"
-
-			contractAbi, err := eth.ParseAbiFile(path)
-
-			Expect(contractAbi).NotTo(BeNil())
-			Expect(err).To(BeNil())
-		})
-
-		It("reads the contents of a valid ABI file", func() {
-			path := test_config.ABIFilePath + "valid_abi.json"
-
-			contractAbi, err := eth.ReadAbiFile(path)
-
-			Expect(contractAbi).To(Equal("[{\"foo\": \"bar\"}]"))
-			Expect(err).To(BeNil())
-		})
-
-		It("returns an error when the file does not exist", func() {
-			path := test_config.ABIFilePath + "missing_abi.json"
-
-			contractAbi, err := eth.ParseAbiFile(path)
-
-			Expect(contractAbi).To(Equal(abi.ABI{}))
-			Expect(err).To(Equal(eth.ErrMissingAbiFile))
-		})
-
-		It("returns an error when the file has invalid contents", func() {
-			path := test_config.ABIFilePath + "invalid_abi.json"
-
-			contractAbi, err := eth.ParseAbiFile(path)
-
-			Expect(contractAbi).To(Equal(abi.ABI{}))
-			Expect(err).To(Equal(eth.ErrInvalidAbiFile))
-		})
-
-		Describe("Request ABI from endpoint", func() {
-
-			var (
-				server    *ghttp.Server
-				client    *eth.EtherScanAPI
-				abiString string
-				err       error
-			)
-
-			BeforeEach(func() {
-				server = ghttp.NewServer()
-				client = eth.NewEtherScanClient(server.URL())
-				path := test_config.ABIFilePath + "sample_abi.json"
-				abiString, err = eth.ReadAbiFile(path)
-
-				Expect(err).NotTo(HaveOccurred())
-				_, err = eth.ParseAbi(abiString)
-				Expect(err).NotTo(HaveOccurred())
-			})
-
-			AfterEach(func() {
-				server.Close()
-			})
-
-			Describe("Fetching ABI from api (etherscan)", func() {
-				BeforeEach(func() {
-
-					response := fmt.Sprintf(`{"status":"1","message":"OK","result":%q}`, abiString)
-					server.AppendHandlers(
-						ghttp.CombineHandlers(
-							ghttp.VerifyRequest("GET", "/api", "module=contract&action=getabi&address=0xd26114cd6EE289AccF82350c8d8487fedB8A0C07"),
-							ghttp.RespondWith(http.StatusOK, response),
-						),
-					)
-				})
-
-				It("should make a GET request with supplied contract hash", func() {
-
-					abi, err := client.GetAbi("0xd26114cd6EE289AccF82350c8d8487fedB8A0C07")
-					Expect(server.ReceivedRequests()).Should(HaveLen(1))
-					Expect(err).ShouldNot(HaveOccurred())
-					Expect(abi).Should(Equal(abiString))
-				})
-			})
-		})
-
-		Describe("Generating etherscan endpoints based on network", func() {
-			It("should return the main endpoint as the default", func() {
-				url := eth.GenURL("")
-				Expect(url).To(Equal("https://api.etherscan.io"))
-			})
-
-			It("generates various test network endpoint if test network is supplied", func() {
-				ropstenUrl := eth.GenURL("ropsten")
-				rinkebyUrl := eth.GenURL("rinkeby")
-				kovanUrl := eth.GenURL("kovan")
-
-				Expect(ropstenUrl).To(Equal("https://ropsten.etherscan.io"))
-				Expect(kovanUrl).To(Equal("https://kovan.etherscan.io"))
-				Expect(rinkebyUrl).To(Equal("https://rinkeby.etherscan.io"))
-			})
-		})
-	})
-})

pkg/eth/api.go

@@ -20,7 +20,7 @@ import (
 	"context"
 	"math/big"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"

pkg/eth/api_test.go

@@ -30,10 +30,10 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 var (

pkg/eth/backend.go

@@ -22,15 +22,15 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
 )
 
 var (

pkg/eth/blockchain.go

@@ -1,279 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"errors"
-
-	"math/big"
-	"strconv"
-
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/net/context"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-var ErrEmptyHeader = errors.New("empty header returned over RPC")
-
-const MAX_BATCH_SIZE = 100
-
-type BlockChain struct {
-	blockConverter       vulcCommon.BlockConverter
-	ethClient            core.EthClient
-	headerConverter      vulcCommon.HeaderConverter
-	node                 core.Node
-	rpcClient            core.RPCClient
-	transactionConverter vulcCommon.TransactionConverter
-}
-
-func NewBlockChain(ethClient core.EthClient, rpcClient core.RPCClient, node core.Node, converter vulcCommon.TransactionConverter) *BlockChain {
-	return &BlockChain{
-		blockConverter:       vulcCommon.NewBlockConverter(converter),
-		ethClient:            ethClient,
-		headerConverter:      vulcCommon.HeaderConverter{},
-		node:                 node,
-		rpcClient:            rpcClient,
-		transactionConverter: converter,
-	}
-}
-
-func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Block, err error) {
-	gethBlock, err := blockChain.ethClient.BlockByNumber(context.Background(), big.NewInt(blockNumber))
-	logrus.Debugf("GetBlockByNumber called with block %d", blockNumber)
-	if err != nil {
-		return block, err
-	}
-	return blockChain.blockConverter.ToCoreBlock(gethBlock)
-}
-
-func (blockChain *BlockChain) GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) {
-	gethLogs, err := blockChain.ethClient.FilterLogs(context.Background(), query)
-	logrus.Debug("GetEthLogsWithCustomQuery called")
-	if err != nil {
-		return []types.Log{}, err
-	}
-	return gethLogs, nil
-}
-
-func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.Header, err error) {
-	logrus.Debugf("GetHeaderByNumber called with block %d", blockNumber)
-	if blockChain.node.NetworkID == string(core.KOVAN_NETWORK_ID) {
-		return blockChain.getPOAHeader(blockNumber)
-	}
-	return blockChain.getPOWHeader(blockNumber)
-}
-
-func (blockChain *BlockChain) GetHeadersByNumbers(blockNumbers []int64) (header []core.Header, err error) {
-	logrus.Debug("GetHeadersByNumbers called")
-	if blockChain.node.NetworkID == string(core.KOVAN_NETWORK_ID) {
-		return blockChain.getPOAHeaders(blockNumbers)
-	}
-	return blockChain.getPOWHeaders(blockNumbers)
-}
-
-func (blockChain *BlockChain) GetFullSyncLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.FullSyncLog, error) {
-	if endingBlockNumber == nil {
-		endingBlockNumber = startingBlockNumber
-	}
-	contractAddress := common.HexToAddress(contract.Hash)
-	fc := ethereum.FilterQuery{
-		FromBlock: startingBlockNumber,
-		ToBlock:   endingBlockNumber,
-		Addresses: []common.Address{contractAddress},
-		Topics:    nil,
-	}
-	gethLogs, err := blockChain.GetEthLogsWithCustomQuery(fc)
-	if err != nil {
-		return []core.FullSyncLog{}, err
-	}
-	logs := vulcCommon.ToFullSyncLogs(gethLogs)
-	return logs, nil
-}
-
-func (blockChain *BlockChain) GetTransactions(transactionHashes []common.Hash) ([]core.TransactionModel, error) {
-	numTransactions := len(transactionHashes)
-	var batch []client.BatchElem
-	transactions := make([]core.RPCTransaction, numTransactions)
-
-	for index, transactionHash := range transactionHashes {
-		batchElem := client.BatchElem{
-			Method: "eth_getTransactionByHash",
-			Result: &transactions[index],
-			Args:   []interface{}{transactionHash},
-		}
-		batch = append(batch, batchElem)
-	}
-
-	rpcErr := blockChain.rpcClient.BatchCall(batch)
-	if rpcErr != nil {
-		return []core.TransactionModel{}, rpcErr
-	}
-
-	return blockChain.transactionConverter.ConvertRPCTransactionsToModels(transactions)
-}
-
-func (blockChain *BlockChain) LastBlock() (*big.Int, error) {
-	block, err := blockChain.ethClient.HeaderByNumber(context.Background(), nil)
-	if err != nil {
-		return big.NewInt(0), err
-	}
-	return block.Number, err
-}
-
-func (blockChain *BlockChain) Node() core.Node {
-	return blockChain.node
-}
-
-func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Header, err error) {
-	var POAHeader core.POAHeader
-	blockNumberArg := hexutil.EncodeBig(big.NewInt(blockNumber))
-	includeTransactions := false
-	err = blockChain.rpcClient.CallContext(context.Background(), &POAHeader, "eth_getBlockByNumber", blockNumberArg, includeTransactions)
-	if err != nil {
-		return header, err
-	}
-	if POAHeader.Number == nil {
-		return header, ErrEmptyHeader
-	}
-	return blockChain.headerConverter.Convert(&types.Header{
-		ParentHash:  POAHeader.ParentHash,
-		UncleHash:   POAHeader.UncleHash,
-		Coinbase:    POAHeader.Coinbase,
-		Root:        POAHeader.Root,
-		TxHash:      POAHeader.TxHash,
-		ReceiptHash: POAHeader.ReceiptHash,
-		Bloom:       POAHeader.Bloom,
-		Difficulty:  POAHeader.Difficulty.ToInt(),
-		Number:      POAHeader.Number.ToInt(),
-		GasLimit:    uint64(POAHeader.GasLimit),
-		GasUsed:     uint64(POAHeader.GasUsed),
-		Time:        uint64(POAHeader.Time),
-		Extra:       POAHeader.Extra,
-	}, POAHeader.Hash.String()), nil
-}
-
-func (blockChain *BlockChain) getPOAHeaders(blockNumbers []int64) (headers []core.Header, err error) {
-
-	var batch []client.BatchElem
-	var POAHeaders [MAX_BATCH_SIZE]core.POAHeader
-	includeTransactions := false
-
-	for index, blockNumber := range blockNumbers {
-
-		if index >= MAX_BATCH_SIZE {
-			break
-		}
-
-		blockNumberArg := hexutil.EncodeBig(big.NewInt(blockNumber))
-
-		batchElem := client.BatchElem{
-			Method: "eth_getBlockByNumber",
-			Result: &POAHeaders[index],
-			Args:   []interface{}{blockNumberArg, includeTransactions},
-		}
-
-		batch = append(batch, batchElem)
-	}
-
-	err = blockChain.rpcClient.BatchCall(batch)
-	if err != nil {
-		return headers, err
-	}
-
-	for _, POAHeader := range POAHeaders {
-		var header core.Header
-		//Header.Number of the newest block will return nil.
-		if _, err := strconv.ParseUint(POAHeader.Number.ToInt().String(), 16, 64); err == nil {
-			header = blockChain.headerConverter.Convert(&types.Header{
-				ParentHash:  POAHeader.ParentHash,
-				UncleHash:   POAHeader.UncleHash,
-				Coinbase:    POAHeader.Coinbase,
-				Root:        POAHeader.Root,
-				TxHash:      POAHeader.TxHash,
-				ReceiptHash: POAHeader.ReceiptHash,
-				Bloom:       POAHeader.Bloom,
-				Difficulty:  POAHeader.Difficulty.ToInt(),
-				Number:      POAHeader.Number.ToInt(),
-				GasLimit:    uint64(POAHeader.GasLimit),
-				GasUsed:     uint64(POAHeader.GasUsed),
-				Time:        uint64(POAHeader.Time),
-				Extra:       POAHeader.Extra,
-			}, POAHeader.Hash.String())
-
-			headers = append(headers, header)
-		}
-	}
-
-	return headers, err
-}
-
-func (blockChain *BlockChain) getPOWHeader(blockNumber int64) (header core.Header, err error) {
-	gethHeader, err := blockChain.ethClient.HeaderByNumber(context.Background(), big.NewInt(blockNumber))
-	if err != nil {
-		return header, err
-	}
-	return blockChain.headerConverter.Convert(gethHeader, gethHeader.Hash().String()), nil
-}
-
-func (blockChain *BlockChain) getPOWHeaders(blockNumbers []int64) (headers []core.Header, err error) {
-	var batch []client.BatchElem
-	var POWHeaders [MAX_BATCH_SIZE]types.Header
-	includeTransactions := false
-
-	for index, blockNumber := range blockNumbers {
-
-		if index >= MAX_BATCH_SIZE {
-			break
-		}
-
-		blockNumberArg := hexutil.EncodeBig(big.NewInt(blockNumber))
-
-		batchElem := client.BatchElem{
-			Method: "eth_getBlockByNumber",
-			Result: &POWHeaders[index],
-			Args:   []interface{}{blockNumberArg, includeTransactions},
-		}
-
-		batch = append(batch, batchElem)
-	}
-
-	err = blockChain.rpcClient.BatchCall(batch)
-	if err != nil {
-		return headers, err
-	}
-
-	for _, POWHeader := range POWHeaders {
-		if POWHeader.Number != nil {
-			header := blockChain.headerConverter.Convert(&POWHeader, POWHeader.Hash().String())
-			headers = append(headers, header)
-		}
-	}
-
-	return headers, err
-}
-
-func (blockChain *BlockChain) GetAccountBalance(address common.Address, blockNumber *big.Int) (*big.Int, error) {
-	logrus.Debug("GetAccountBalance called")
-	return blockChain.ethClient.BalanceAt(context.Background(), address, blockNumber)
-}

pkg/eth/blockchain_test.go

@@ -1,272 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"context"
-	"errors"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	vulcCore "github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Geth blockchain", func() {
-	var (
-		mockClient               *fakes.MockEthClient
-		blockChain               *eth.BlockChain
-		mockRpcClient            *fakes.MockRPCClient
-		mockTransactionConverter *fakes.MockTransactionConverter
-		node                     vulcCore.Node
-	)
-
-	BeforeEach(func() {
-		mockClient = fakes.NewMockEthClient()
-		mockRpcClient = fakes.NewMockRPCClient()
-		mockTransactionConverter = fakes.NewMockTransactionConverter()
-		node = vulcCore.Node{}
-		blockChain = eth.NewBlockChain(mockClient, mockRpcClient, node, mockTransactionConverter)
-	})
-
-	Describe("getting a block", func() {
-		It("fetches block from ethClient", func() {
-			mockClient.SetBlockByNumberReturnBlock(types.NewBlockWithHeader(&types.Header{}))
-			blockNumber := int64(100)
-
-			_, err := blockChain.GetBlockByNumber(blockNumber)
-
-			Expect(err).NotTo(HaveOccurred())
-			mockClient.AssertBlockByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
-		})
-
-		It("returns err if ethClient returns err", func() {
-			mockClient.SetBlockByNumberErr(fakes.FakeError)
-
-			_, err := blockChain.GetBlockByNumber(100)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-	})
-
-	Describe("getting a header", func() {
-		Describe("default/mainnet", func() {
-			It("fetches header from ethClient", func() {
-				blockNumber := int64(100)
-				mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
-
-				_, err := blockChain.GetHeaderByNumber(blockNumber)
-
-				Expect(err).NotTo(HaveOccurred())
-				mockClient.AssertHeaderByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
-			})
-
-			It("returns err if ethClient returns err", func() {
-				mockClient.SetHeaderByNumberErr(fakes.FakeError)
-
-				_, err := blockChain.GetHeaderByNumber(100)
-
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(fakes.FakeError))
-			})
-
-			It("fetches headers with multiple blocks", func() {
-				_, err := blockChain.GetHeadersByNumbers([]int64{100, 99})
-
-				Expect(err).NotTo(HaveOccurred())
-				mockRpcClient.AssertBatchCalledWith("eth_getBlockByNumber", 2)
-			})
-		})
-
-		Describe("POA/Kovan", func() {
-			It("fetches header from rpcClient", func() {
-				node.NetworkID = string(vulcCore.KOVAN_NETWORK_ID)
-				blockNumber := hexutil.Big(*big.NewInt(100))
-				mockRpcClient.SetReturnPOAHeader(vulcCore.POAHeader{Number: &blockNumber})
-				blockChain = eth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter())
-
-				_, err := blockChain.GetHeaderByNumber(100)
-
-				Expect(err).NotTo(HaveOccurred())
-				mockRpcClient.AssertCallContextCalledWith(context.Background(), &vulcCore.POAHeader{}, "eth_getBlockByNumber")
-			})
-
-			It("returns err if rpcClient returns err", func() {
-				node.NetworkID = string(vulcCore.KOVAN_NETWORK_ID)
-				mockRpcClient.SetCallContextErr(fakes.FakeError)
-				blockChain = eth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter())
-
-				_, err := blockChain.GetHeaderByNumber(100)
-
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(fakes.FakeError))
-			})
-
-			It("returns error if returned header is empty", func() {
-				node.NetworkID = string(vulcCore.KOVAN_NETWORK_ID)
-				blockChain = eth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter())
-
-				_, err := blockChain.GetHeaderByNumber(100)
-
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(eth.ErrEmptyHeader))
-			})
-
-			It("returns multiple headers with multiple blocknumbers", func() {
-				node.NetworkID = string(vulcCore.KOVAN_NETWORK_ID)
-				blockNumber := hexutil.Big(*big.NewInt(100))
-				mockRpcClient.SetReturnPOAHeaders([]vulcCore.POAHeader{{Number: &blockNumber}})
-
-				_, err := blockChain.GetHeadersByNumbers([]int64{100, 99})
-
-				Expect(err).NotTo(HaveOccurred())
-				mockRpcClient.AssertBatchCalledWith("eth_getBlockByNumber", 2)
-			})
-		})
-	})
-
-	Describe("getting logs with default FilterQuery", func() {
-		It("fetches logs from ethClient", func() {
-			mockClient.SetFilterLogsReturnLogs([]types.Log{{}})
-			contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
-			startingBlockNumber := big.NewInt(1)
-			endingBlockNumber := big.NewInt(2)
-
-			_, err := blockChain.GetFullSyncLogs(contract, startingBlockNumber, endingBlockNumber)
-
-			Expect(err).NotTo(HaveOccurred())
-			expectedQuery := ethereum.FilterQuery{
-				FromBlock: startingBlockNumber,
-				ToBlock:   endingBlockNumber,
-				Addresses: []common.Address{common.HexToAddress(contract.Hash)},
-			}
-			mockClient.AssertFilterLogsCalledWith(context.Background(), expectedQuery)
-		})
-
-		It("returns err if ethClient returns err", func() {
-			mockClient.SetFilterLogsErr(fakes.FakeError)
-			contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
-			startingBlockNumber := big.NewInt(1)
-			endingBlockNumber := big.NewInt(2)
-
-			_, err := blockChain.GetFullSyncLogs(contract, startingBlockNumber, endingBlockNumber)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-	})
-
-	Describe("getting logs with a custom FilterQuery", func() {
-		It("fetches logs from ethClient", func() {
-			mockClient.SetFilterLogsReturnLogs([]types.Log{{}})
-			address := common.HexToAddress("0x")
-			startingBlockNumber := big.NewInt(1)
-			endingBlockNumber := big.NewInt(2)
-			topic := common.HexToHash("0x")
-			query := ethereum.FilterQuery{
-				FromBlock: startingBlockNumber,
-				ToBlock:   endingBlockNumber,
-				Addresses: []common.Address{address},
-				Topics:    [][]common.Hash{{topic}},
-			}
-
-			_, err := blockChain.GetEthLogsWithCustomQuery(query)
-
-			Expect(err).NotTo(HaveOccurred())
-			mockClient.AssertFilterLogsCalledWith(context.Background(), query)
-		})
-
-		It("returns err if ethClient returns err", func() {
-			mockClient.SetFilterLogsErr(fakes.FakeError)
-			contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
-			startingBlockNumber := big.NewInt(1)
-			endingBlockNumber := big.NewInt(2)
-			query := ethereum.FilterQuery{
-				FromBlock: startingBlockNumber,
-				ToBlock:   endingBlockNumber,
-				Addresses: []common.Address{common.HexToAddress(contract.Hash)},
-				Topics:    nil,
-			}
-
-			_, err := blockChain.GetEthLogsWithCustomQuery(query)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-	})
-
-	Describe("getting transactions", func() {
-		It("fetches transaction for each hash", func() {
-			_, err := blockChain.GetTransactions([]common.Hash{{}, {}})
-
-			Expect(err).NotTo(HaveOccurred())
-			mockRpcClient.AssertBatchCalledWith("eth_getTransactionByHash", 2)
-		})
-
-		It("converts transaction indexes from hex to int", func() {
-			_, err := blockChain.GetTransactions([]common.Hash{{}, {}})
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(mockTransactionConverter.ConvertHeaderTransactionIndexToIntCalled).To(BeTrue())
-		})
-	})
-
-	Describe("getting the most recent block number", func() {
-		It("fetches latest header from ethClient", func() {
-			blockNumber := int64(100)
-			mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
-
-			result, err := blockChain.LastBlock()
-			Expect(err).NotTo(HaveOccurred())
-
-			mockClient.AssertHeaderByNumberCalledWith(context.Background(), nil)
-			Expect(result).To(Equal(big.NewInt(blockNumber)))
-		})
-	})
-
-	Describe("getting an account balance", func() {
-		It("fetches the balance for a given account address at a given block height", func() {
-			balance := big.NewInt(100000)
-			mockClient.SetBalanceAt(balance)
-
-			result, err := blockChain.GetAccountBalance(common.HexToAddress("0x40"), big.NewInt(100))
-			Expect(err).NotTo(HaveOccurred())
-
-			mockClient.AssertBalanceAtCalled(context.Background(), common.HexToAddress("0x40"), big.NewInt(100))
-			Expect(result).To(Equal(balance))
-		})
-
-		It("fails if the client returns an error", func() {
-			balance := big.NewInt(100000)
-			mockClient.SetBalanceAt(balance)
-			setErr := errors.New("testError")
-			mockClient.SetBalanceAtErr(setErr)
-
-			_, err := blockChain.GetAccountBalance(common.HexToAddress("0x40"), big.NewInt(100))
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(Equal(setErr))
-		})
-	})
-})

pkg/eth/cid_retriever.go

@@ -27,9 +27,9 @@ import (
 	"github.com/lib/pq"
 	log "github.com/sirupsen/logrus"
 
-	utils "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/utils"
 )
 
 // CIDRetriever satisfies the CIDRetriever interface for ethereum

pkg/eth/cid_retriever_test.go

@@ -24,11 +24,11 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
+	eth2 "github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 var (

pkg/eth/cleaner.go

@@ -22,8 +22,8 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // Cleaner satisfies the shared.Cleaner interface fo ethereum

pkg/eth/cleaner_test.go

@@ -24,10 +24,9 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/postgres"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 var (
@@ -39,7 +38,7 @@ var (
 	parentHash      = crypto.Keccak256Hash([]byte{00, 01})
 	totalDifficulty = "50000000000000000000"
 	reward          = "5000000000000000000"
-	headerModel     = eth2.HeaderModel{
+	headerModel     = eth.HeaderModel{
 		BlockHash:       blockHash1.String(),
 		BlockNumber:     blocKNumber1.String(),
 		CID:             headerCID1,
@@ -55,7 +54,7 @@ var (
 	tx2Hash   = crypto.Keccak256Hash([]byte{01, 02})
 	txSrc     = common.HexToAddress("0x010a")
 	txDst     = common.HexToAddress("0x020a")
-	txModels1 = []eth2.TxModel{
+	txModels1 = []eth.TxModel{
 		{
 			CID:    tx1CID,
 			TxHash: tx1Hash.String(),
@@ -73,7 +72,7 @@ var (
 	uncleHash       = crypto.Keccak256Hash([]byte{02, 02})
 	uncleParentHash = crypto.Keccak256Hash([]byte{02, 01})
 	uncleReward     = "1000000000000000000"
-	uncleModels1    = []eth2.UncleModel{
+	uncleModels1    = []eth.UncleModel{
 		{
 			CID:        uncleCID,
 			Reward:     uncleReward,
@@ -87,7 +86,7 @@ var (
 	rct2CID        = "mockRct2CID"
 	rct1Contract   = common.Address{}
 	rct2Contract   = common.HexToAddress("0x010c")
-	receiptModels1 = map[common.Hash]eth2.ReceiptModel{
+	receiptModels1 = map[common.Hash]eth.ReceiptModel{
 		tx1Hash: {
 			CID:          rct1CID,
 			ContractHash: crypto.Keccak256Hash(rct1Contract.Bytes()).String(),
@@ -105,7 +104,7 @@ var (
 	state2CID1   = "mockState2CID1"
 	state2Path   = []byte{'\x02'}
 	state2Key    = crypto.Keccak256Hash(txDst.Bytes())
-	stateModels1 = []eth2.StateNodeModel{
+	stateModels1 = []eth.StateNodeModel{
 		{
 			CID:      state1CID1,
 			Path:     state1Path,
@@ -124,7 +123,7 @@ var (
 	storageCID     = "mockStorageCID1"
 	storagePath    = []byte{'\x01'}
 	storageKey     = crypto.Keccak256Hash(common.Hex2Bytes("0x0000000000000000000000000000000000000000000000000000000000000000"))
-	storageModels1 = map[string][]eth2.StorageNodeModel{
+	storageModels1 = map[string][]eth.StorageNodeModel{
 		common.Bytes2Hex(state1Path): {
 			{
 				CID:        storageCID,
@@ -148,7 +147,7 @@ var (
 	blockHash2   = crypto.Keccak256Hash([]byte{00, 03})
 	blocKNumber2 = big.NewInt(1)
 	headerCID2   = "mockHeaderCID2"
-	headerModel2 = eth2.HeaderModel{
+	headerModel2 = eth.HeaderModel{
 		BlockHash:       blockHash2.String(),
 		BlockNumber:     blocKNumber2.String(),
 		CID:             headerCID2,
@@ -159,7 +158,7 @@ var (
 	// tx variables
 	tx3CID    = "mockTx3CID"
 	tx3Hash   = crypto.Keccak256Hash([]byte{01, 03})
-	txModels2 = []eth2.TxModel{
+	txModels2 = []eth.TxModel{
 		{
 			CID:    tx3CID,
 			TxHash: tx3Hash.String(),
@@ -168,7 +167,7 @@ var (
 	}
 	// receipt variables
 	rct3CID        = "mockRct3CID"
-	receiptModels2 = map[common.Hash]eth2.ReceiptModel{
+	receiptModels2 = map[common.Hash]eth.ReceiptModel{
 		tx3Hash: {
 			CID:          rct3CID,
 			ContractHash: crypto.Keccak256Hash(rct1Contract.Bytes()).String(),
@@ -177,7 +176,7 @@ var (
 
 	// state variables
 	state1CID2   = "mockState1CID2"
-	stateModels2 = []eth2.StateNodeModel{
+	stateModels2 = []eth.StateNodeModel{
 		{
 			CID:      state1CID2,
 			Path:     state1Path,
@@ -213,15 +212,15 @@ var (
 var _ = Describe("Cleaner", func() {
 	var (
 		db      *postgres.DB
-		repo    *eth2.CIDIndexer
-		cleaner *eth2.Cleaner
+		repo    *eth.CIDIndexer
+		cleaner *eth.Cleaner
 	)
 	BeforeEach(func() {
 		var err error
 		db, err = shared.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = eth2.NewCIDIndexer(db)
-		cleaner = eth2.NewCleaner(db)
+		repo = eth.NewCIDIndexer(db)
+		cleaner = eth.NewCleaner(db)
 	})
 	Describe("Clean", func() {
 		BeforeEach(func() {

pkg/eth/contract.go

@@ -1,61 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"context"
-	"errors"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-)
-
-var (
-	ErrInvalidStateAttribute = errors.New("invalid state attribute")
-)
-
-func (blockChain *BlockChain) FetchContractData(abiJSON string, address string, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
-	parsed, err := ParseAbi(abiJSON)
-	if err != nil {
-		return err
-	}
-	var input []byte
-	if methodArgs != nil {
-		input, err = parsed.Pack(method, methodArgs...)
-	} else {
-		input, err = parsed.Pack(method)
-	}
-	if err != nil {
-		return err
-	}
-	var bn *big.Int
-	if blockNumber > 0 {
-		bn = big.NewInt(blockNumber)
-	}
-	output, err := blockChain.callContract(address, input, bn)
-	if err != nil {
-		return err
-	}
-	return parsed.Unpack(result, method, output)
-}
-
-func (blockChain *BlockChain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
-	to := common.HexToAddress(contractHash)
-	msg := ethereum.CallMsg{To: &to, Data: input}
-	return blockChain.ethClient.CallContract(context.Background(), msg, blockNumber)
-}

pkg/eth/contract_watcher/header/converter/converter.go

@@ -1,227 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package converter
-
-import (
-	"encoding/json"
-	"fmt"
-	"math/big"
-	"strconv"
-
-	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	gethTypes "github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-)
-
-// ConverterInterface is the interface for converting geth logs to our custom log type
-type ConverterInterface interface {
-	Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error)
-	ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error)
-	Update(info *contract.Contract)
-}
-
-// Converter is the underlying struct for the ConverterInterface
-type Converter struct {
-	ContractInfo *contract.Contract
-}
-
-// Update is used to configure the converter with a specific contract
-func (c *Converter) Update(info *contract.Contract) {
-	c.ContractInfo = info
-}
-
-// Convert the given watched event log into a types.Log for the given event
-func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) {
-	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
-	returnLogs := make([]types.Log, 0, len(logs))
-	for _, log := range logs {
-		values := make(map[string]interface{})
-		for _, field := range event.Fields {
-			var i interface{}
-			values[field.Name] = i
-		}
-
-		err := boundContract.UnpackLogIntoMap(values, event.Name, log)
-		if err != nil {
-			return nil, err
-		}
-
-		strValues := make(map[string]string, len(values))
-		seenAddrs := make([]interface{}, 0, len(values))
-		seenHashes := make([]interface{}, 0, len(values))
-		for fieldName, input := range values {
-			// Postgres cannot handle custom types, resolve everything to strings
-			switch input.(type) {
-			case *big.Int:
-				b := input.(*big.Int)
-				strValues[fieldName] = b.String()
-			case common.Address:
-				a := input.(common.Address)
-				strValues[fieldName] = a.String()
-				seenAddrs = append(seenAddrs, a)
-			case common.Hash:
-				h := input.(common.Hash)
-				strValues[fieldName] = h.String()
-				seenHashes = append(seenHashes, h)
-			case string:
-				strValues[fieldName] = input.(string)
-			case bool:
-				strValues[fieldName] = strconv.FormatBool(input.(bool))
-			case []byte:
-				b := input.([]byte)
-				strValues[fieldName] = hexutil.Encode(b)
-				if len(b) == 32 {
-					seenHashes = append(seenHashes, common.HexToHash(strValues[fieldName]))
-				}
-			case uint8:
-				u := input.(uint8)
-				strValues[fieldName] = strconv.Itoa(int(u))
-			case [32]uint8:
-				raw := input.([32]uint8)
-				converted := convertUintSliceToHash(raw)
-				strValues[fieldName] = converted.String()
-				seenHashes = append(seenHashes, converted)
-			default:
-				return nil, fmt.Errorf("error: unhandled abi type %T", input)
-			}
-		}
-
-		// Only hold onto logs that pass our address filter, if any
-		if c.ContractInfo.PassesEventFilter(strValues) {
-			raw, err := json.Marshal(log)
-			if err != nil {
-				return nil, err
-			}
-
-			returnLogs = append(returnLogs, types.Log{
-				LogIndex:         log.Index,
-				Values:           strValues,
-				Raw:              raw,
-				TransactionIndex: log.TxIndex,
-				ID:               headerID,
-			})
-
-			// Cache emitted values if their caching is turned on
-			if c.ContractInfo.EmittedAddrs != nil {
-				c.ContractInfo.AddEmittedAddr(seenAddrs...)
-			}
-			if c.ContractInfo.EmittedHashes != nil {
-				c.ContractInfo.AddEmittedHash(seenHashes...)
-			}
-		}
-	}
-
-	return returnLogs, nil
-}
-
-// ConvertBatch converts the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
-func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
-	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
-	eventsToLogs := make(map[string][]types.Log)
-	for _, event := range events {
-		eventsToLogs[event.Name] = make([]types.Log, 0, len(logs))
-		// Iterate through all event logs
-		for _, log := range logs {
-			// If the log is of this event type, process it as such
-			if event.Sig() == log.Topics[0] {
-				values := make(map[string]interface{})
-				err := boundContract.UnpackLogIntoMap(values, event.Name, log)
-				if err != nil {
-					return nil, err
-				}
-				// Postgres cannot handle custom types, so we will resolve everything to strings
-				strValues := make(map[string]string, len(values))
-				// Keep track of addresses and hashes emitted from events
-				seenAddrs := make([]interface{}, 0, len(values))
-				seenHashes := make([]interface{}, 0, len(values))
-				for fieldName, input := range values {
-					switch input.(type) {
-					case *big.Int:
-						b := input.(*big.Int)
-						strValues[fieldName] = b.String()
-					case common.Address:
-						a := input.(common.Address)
-						strValues[fieldName] = a.String()
-						seenAddrs = append(seenAddrs, a)
-					case common.Hash:
-						h := input.(common.Hash)
-						strValues[fieldName] = h.String()
-						seenHashes = append(seenHashes, h)
-					case string:
-						strValues[fieldName] = input.(string)
-					case bool:
-						strValues[fieldName] = strconv.FormatBool(input.(bool))
-					case []byte:
-						b := input.([]byte)
-						strValues[fieldName] = hexutil.Encode(b)
-						if len(b) == 32 { // collect byte arrays of size 32 as hashes
-							seenHashes = append(seenHashes, common.BytesToHash(b))
-						}
-					case uint8:
-						u := input.(uint8)
-						strValues[fieldName] = strconv.Itoa(int(u))
-					case [32]uint8:
-						raw := input.([32]uint8)
-						converted := convertUintSliceToHash(raw)
-						strValues[fieldName] = converted.String()
-						seenHashes = append(seenHashes, converted)
-					default:
-						return nil, fmt.Errorf("error: unhandled abi type %T", input)
-					}
-				}
-
-				// Only hold onto logs that pass our argument filter, if any
-				if c.ContractInfo.PassesEventFilter(strValues) {
-					raw, err := json.Marshal(log)
-					if err != nil {
-						return nil, err
-					}
-
-					eventsToLogs[event.Name] = append(eventsToLogs[event.Name], types.Log{
-						LogIndex:         log.Index,
-						Values:           strValues,
-						Raw:              raw,
-						TransactionIndex: log.TxIndex,
-						ID:               headerID,
-					})
-
-					// Cache emitted values that pass the argument filter if their caching is turned on
-					if c.ContractInfo.EmittedAddrs != nil {
-						c.ContractInfo.AddEmittedAddr(seenAddrs...)
-					}
-					if c.ContractInfo.EmittedHashes != nil {
-						c.ContractInfo.AddEmittedHash(seenHashes...)
-					}
-				}
-			}
-		}
-	}
-
-	return eventsToLogs, nil
-}
-
-func convertUintSliceToHash(raw [32]uint8) common.Hash {
-	var asBytes []byte
-	for _, u := range raw {
-		asBytes = append(asBytes, u)
-	}
-	return common.BytesToHash(asBytes)
-}

pkg/eth/contract_watcher/header/converter/converter_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package converter_test
-
-import (
-	"io/ioutil"
-	"log"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestConverter(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Header Sync Converter Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/header/converter/converter_test.go

@@ -1,186 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package converter_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/converter"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-)
-
-var _ = Describe("Converter", func() {
-	var con *contract.Contract
-	var tusdWantedEvents = []string{"Transfer", "Mint"}
-	var ensWantedEvents = []string{"NewOwner"}
-	var marketPlaceWantedEvents = []string{"OrderCreated"}
-	var molochWantedEvents = []string{"SubmitVote"}
-	var err error
-
-	Describe("Update", func() {
-		It("Updates contract info held by the converter", func() {
-			con = test_helpers.SetupTusdContract(tusdWantedEvents, []string{})
-			c := converter.Converter{}
-			c.Update(con)
-			Expect(c.ContractInfo).To(Equal(con))
-
-			info := test_helpers.SetupTusdContract([]string{}, []string{})
-			c.Update(info)
-			Expect(c.ContractInfo).To(Equal(info))
-		})
-	})
-
-	Describe("Convert", func() {
-		It("Converts a watched event log to mapping of event input names to values", func() {
-			con = test_helpers.SetupTusdContract(tusdWantedEvents, []string{})
-			_, ok := con.Events["Approval"]
-			Expect(ok).To(Equal(false))
-
-			event, ok := con.Events["Transfer"]
-			Expect(ok).To(Equal(true))
-
-			c := converter.Converter{}
-			c.Update(con)
-			logs, err := c.Convert([]types.Log{mocks.MockTransferLog1, mocks.MockTransferLog2}, event, 232)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-
-			sender1 := common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")
-			sender2 := common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")
-			value := helpers.BigFromString("1097077688018008265106216665536940668749033598146")
-
-			Expect(logs[0].Values["to"]).To(Equal(sender1.String()))
-			Expect(logs[0].Values["from"]).To(Equal(sender2.String()))
-			Expect(logs[0].Values["value"]).To(Equal(value.String()))
-			Expect(logs[0].ID).To(Equal(int64(232)))
-			Expect(logs[1].Values["to"]).To(Equal(sender2.String()))
-			Expect(logs[1].Values["from"]).To(Equal(sender1.String()))
-			Expect(logs[1].Values["value"]).To(Equal(value.String()))
-			Expect(logs[1].ID).To(Equal(int64(232)))
-		})
-
-		It("Keeps track of addresses it sees if they will be used for method polling", func() {
-			con = test_helpers.SetupTusdContract(tusdWantedEvents, []string{"balanceOf"})
-			event, ok := con.Events["Transfer"]
-			Expect(ok).To(Equal(true))
-
-			c := converter.Converter{}
-			c.Update(con)
-			_, err := c.Convert([]types.Log{mocks.MockTransferLog1, mocks.MockTransferLog2}, event, 232)
-			Expect(err).ToNot(HaveOccurred())
-
-			b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = con.EmittedAddrs[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = con.EmittedAddrs[""]
-			Expect(ok).To(Equal(false))
-
-			_, ok = con.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = con.EmittedHashes[common.HexToHash("0x000000000000000000000000c02aaa39b223helloa0e5c4f27ead9083c752553")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Keeps track of hashes it sees if they will be used for method polling", func() {
-			con = test_helpers.SetupENSContract(ensWantedEvents, []string{"owner"})
-			event, ok := con.Events["NewOwner"]
-			Expect(ok).To(Equal(true))
-
-			c := converter.Converter{}
-			c.Update(con)
-			_, err := c.Convert([]types.Log{mocks.MockNewOwnerLog1, mocks.MockNewOwnerLog2}, event, 232)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(con.EmittedHashes)).To(Equal(3))
-
-			b, ok := con.EmittedHashes[common.HexToHash("0x000000000000000000000000c02aaa39b223helloa0e5c4f27ead9083c752553")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = con.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = con.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba400")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = con.EmittedHashes[common.HexToHash("0x9dd48thiscc444isc242510c0made03upa5975cac061dhashb843bce061ba400")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = con.EmittedHashes[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = con.EmittedHashes[""]
-			Expect(ok).To(Equal(false))
-
-			// Does not keep track of emitted addresses if the methods provided will not use them
-			_, ok = con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("correctly parses bytes32", func() {
-			con = test_helpers.SetupMarketPlaceContract(marketPlaceWantedEvents, []string{})
-			event, ok := con.Events["OrderCreated"]
-			Expect(ok).To(BeTrue())
-
-			c := converter.Converter{}
-			c.Update(con)
-			result, err := c.Convert([]types.Log{mocks.MockOrderCreatedLog}, event, 232)
-			Expect(err).NotTo(HaveOccurred())
-
-			Expect(len(result)).To(Equal(1))
-			Expect(result[0].Values["id"]).To(Equal("0x633f94affdcabe07c000231f85c752c97b9cc43966b432ec4d18641e6d178233"))
-		})
-
-		It("correctly parses uint8", func() {
-			con = test_helpers.SetupMolochContract(molochWantedEvents, []string{})
-			event, ok := con.Events["SubmitVote"]
-			Expect(ok).To(BeTrue())
-
-			c := converter.Converter{}
-			c.Update(con)
-			result, err := c.Convert([]types.Log{mocks.MockSubmitVoteLog}, event, 232)
-			Expect(err).NotTo(HaveOccurred())
-
-			Expect(len(result)).To(Equal(1))
-			Expect(result[0].Values["uintVote"]).To(Equal("1"))
-		})
-
-		It("Fails with an empty contract", func() {
-			event := con.Events["Transfer"]
-			c := converter.Converter{}
-			c.Update(&contract.Contract{})
-			_, err = c.Convert([]types.Log{mocks.MockTransferLog1}, event, 232)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-})

pkg/eth/contract_watcher/header/fetcher/fetcher.go

@@ -1,71 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher
-
-import (
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-// Fetcher is the fetching interface
-type Fetcher interface {
-	FetchLogs(contractAddresses []string, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
-}
-
-type fetcher struct {
-	blockChain core.BlockChain
-}
-
-// NewFetcher returns a new Fetcher
-func NewFetcher(blockchain core.BlockChain) Fetcher {
-	return &fetcher{
-		blockChain: blockchain,
-	}
-}
-
-// FetchLogs checks all topic0s, on all addresses, fetching matching logs for the given header
-func (fetcher *fetcher) FetchLogs(contractAddresses []string, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
-	addresses := hexStringsToAddresses(contractAddresses)
-	blockHash := common.HexToHash(header.Hash)
-	query := ethereum.FilterQuery{
-		BlockHash: &blockHash,
-		Addresses: addresses,
-		// Search for _any_ of the topics in topic0 position; see docs on `FilterQuery`
-		Topics: [][]common.Hash{topic0s},
-	}
-
-	logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
-	if err != nil {
-		// TODO review aggregate fetching error handling
-		return []types.Log{}, err
-	}
-
-	return logs, nil
-}
-
-func hexStringsToAddresses(hexStrings []string) []common.Address {
-	var addresses []common.Address
-	for _, hexString := range hexStrings {
-		address := common.HexToAddress(hexString)
-		addresses = append(addresses, address)
-	}
-
-	return addresses
-}

pkg/eth/contract_watcher/header/fetcher/fetcher_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher_test
-
-import (
-	"io/ioutil"
-	"log"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestFetcher(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Fetcher Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/header/fetcher/fetcher_test.go

@@ -1,66 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher_test
-
-import (
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/fetcher"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Fetcher", func() {
-	Describe("FetchLogs", func() {
-		It("fetches logs based on the given query", func() {
-			blockChain := fakes.NewMockBlockChain()
-			fetcher := fetcher.NewFetcher(blockChain)
-			header := fakes.FakeHeader
-
-			addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"}
-			topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}}
-
-			_, err := fetcher.FetchLogs(addresses, []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})}, header)
-
-			address1 := common.HexToAddress("0xfakeAddress")
-			address2 := common.HexToAddress("0xanotherFakeAddress")
-			Expect(err).NotTo(HaveOccurred())
-
-			blockHash := common.HexToHash(header.Hash)
-			expectedQuery := ethereum.FilterQuery{
-				BlockHash: &blockHash,
-				Addresses: []common.Address{address1, address2},
-				Topics:    topicZeros,
-			}
-			blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery)
-		})
-
-		It("returns an error if fetching the logs fails", func() {
-			blockChain := fakes.NewMockBlockChain()
-			blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
-			fetcher := fetcher.NewFetcher(blockChain)
-
-			_, err := fetcher.FetchLogs([]string{}, []common.Hash{}, core.Header{})
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-	})
-})

pkg/eth/contract_watcher/header/repository/header_repository.go

@@ -1,271 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository
-
-import (
-	"fmt"
-
-	"github.com/hashicorp/golang-lru"
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-const columnCacheSize = 1000
-
-// HeaderRepository interfaces with the header and checked_headers tables
-type HeaderRepository interface {
-	AddCheckColumn(id string) error
-	AddCheckColumns(ids []string) error
-	MarkHeaderChecked(headerID int64, eventID string) error
-	MarkHeaderCheckedForAll(headerID int64, ids []string) error
-	MarkHeadersCheckedForAll(headers []core.Header, ids []string) error
-	MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error)
-	MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error)
-	MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error)
-	CheckCache(key string) (interface{}, bool)
-}
-
-type headerRepository struct {
-	db      *postgres.DB
-	columns *lru.Cache // Cache created columns to minimize db connections
-}
-
-// NewHeaderRepository returns a new HeaderRepository
-func NewHeaderRepository(db *postgres.DB) HeaderRepository {
-	ccs, _ := lru.New(columnCacheSize)
-	return &headerRepository{
-		db:      db,
-		columns: ccs,
-	}
-}
-
-// AddCheckColumn adds a checked_header column for the provided column id
-func (r *headerRepository) AddCheckColumn(id string) error {
-	// Check cache to see if column already exists before querying pg
-	_, ok := r.columns.Get(id)
-	if ok {
-		return nil
-	}
-
-	pgStr := "ALTER TABLE public.checked_headers ADD COLUMN IF NOT EXISTS "
-	pgStr = pgStr + id + " INTEGER NOT NULL DEFAULT 0"
-	_, err := r.db.Exec(pgStr)
-	if err != nil {
-		return err
-	}
-
-	// Add column name to cache
-	r.columns.Add(id, true)
-
-	return nil
-}
-
-// AddCheckColumns adds a checked_header column for all of the provided column ids
-func (r *headerRepository) AddCheckColumns(ids []string) error {
-	var err error
-	baseQuery := "ALTER TABLE public.checked_headers"
-	input := make([]string, 0, len(ids))
-	for _, id := range ids {
-		_, ok := r.columns.Get(id)
-		if !ok {
-			baseQuery += " ADD COLUMN IF NOT EXISTS " + id + " INTEGER NOT NULL DEFAULT 0,"
-			input = append(input, id)
-		}
-	}
-	if len(input) > 0 {
-		_, err = r.db.Exec(baseQuery[:len(baseQuery)-1])
-		if err == nil {
-			for _, id := range input {
-				r.columns.Add(id, true)
-			}
-		}
-	}
-
-	return err
-}
-
-// MarkHeaderChecked marks the header checked for the provided column id
-func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
-	_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`)
-		VALUES ($1, $2) 
-		ON CONFLICT (header_id) DO
-			UPDATE SET `+id+` = checked_headers.`+id+` + 1`, headerID, 1)
-	return err
-}
-
-// MarkHeaderCheckedForAll marks the header checked for all of the provided column ids
-func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
-	pgStr := "INSERT INTO public.checked_headers (header_id, "
-	for _, id := range ids {
-		pgStr += id + ", "
-	}
-	pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, "
-	for i := 0; i < len(ids); i++ {
-		pgStr += "1, "
-	}
-	pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET "
-	for _, id := range ids {
-		pgStr += id + `= checked_headers.` + id + ` + 1, `
-	}
-	pgStr = pgStr[:len(pgStr)-2]
-	_, err := r.db.Exec(pgStr, headerID)
-	return err
-}
-
-// MarkHeadersCheckedForAll marks all of the provided headers checked for each of the provided column ids
-func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
-	tx, err := r.db.Beginx()
-	if err != nil {
-		return err
-	}
-	for _, header := range headers {
-		pgStr := "INSERT INTO public.checked_headers (header_id, "
-		for _, id := range ids {
-			pgStr += id + ", "
-		}
-		pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, "
-		for i := 0; i < len(ids); i++ {
-			pgStr += "1, "
-		}
-		pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET "
-		for _, id := range ids {
-			pgStr += fmt.Sprintf("%s = checked_headers.%s + 1, ", id, id)
-		}
-		pgStr = pgStr[:len(pgStr)-2]
-		_, err = tx.Exec(pgStr, header.ID)
-		if err != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				logrus.Warnf("error rolling back transaction: %s", rollbackErr.Error())
-			}
-			return err
-		}
-	}
-	err = tx.Commit()
-	return err
-}
-
-// MissingHeaders returns missing headers for the provided checked_headers column id
-func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) {
-	var result []core.Header
-	var query string
-	var err error
-	if endingBlockNumber == -1 {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE (header_id ISNULL OR checked_headers.` + id + `=0)
-				AND headers.block_number >= $1
-				AND headers.eth_node_fingerprint = $2
-				ORDER BY headers.block_number`
-		err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID)
-	} else {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE (header_id ISNULL OR checked_headers.` + id + `=0)
-				AND headers.block_number >= $1
-				AND headers.block_number <= $2
-				AND headers.eth_node_fingerprint = $3
-				ORDER BY headers.block_number`
-		err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
-	}
-	return continuousHeaders(result), err
-}
-
-// MissingHeadersForAll returns missing headers for all of the provided checked_headers column ids
-func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
-	var result []core.Header
-	var query string
-	var err error
-	baseQuery := `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				  LEFT JOIN checked_headers on headers.id = header_id
-				  WHERE (header_id ISNULL`
-	for _, id := range ids {
-		baseQuery += ` OR checked_headers.` + id + `= 0`
-	}
-	if endingBlockNumber == -1 {
-		endStr := `) AND headers.block_number >= $1
-				  AND headers.eth_node_fingerprint = $2
-				  ORDER BY headers.block_number`
-		query = baseQuery + endStr
-		err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID)
-	} else {
-		endStr := `) AND headers.block_number >= $1
-				  AND headers.block_number <= $2
-				  AND headers.eth_node_fingerprint = $3
-				  ORDER BY headers.block_number`
-		query = baseQuery + endStr
-		err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
-	}
-	return continuousHeaders(result), err
-}
-
-// MissingMethodsCheckedEventsIntersection returns headers that have been checked for all of the provided event ids but not for the provided method ids
-func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
-	var result []core.Header
-	var query string
-	var err error
-	baseQuery := `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				  LEFT JOIN checked_headers on headers.id = header_id
-				  WHERE (header_id IS NOT NULL`
-	for _, id := range eventIds {
-		baseQuery += ` AND ` + id + `!=0`
-	}
-	baseQuery += `) AND (`
-	for _, id := range methodIds {
-		baseQuery += id + ` =0 AND `
-	}
-	baseQuery = baseQuery[:len(baseQuery)-5] + `) `
-	if endingBlockNumber == -1 {
-		endStr := `AND headers.block_number >= $1
-				  AND headers.eth_node_fingerprint = $2
-				  ORDER BY headers.block_number`
-		query = baseQuery + endStr
-		err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID)
-	} else {
-		endStr := `AND headers.block_number >= $1
-				  AND headers.block_number <= $2
-				  AND headers.eth_node_fingerprint = $3
-				  ORDER BY headers.block_number`
-		query = baseQuery + endStr
-		err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
-	}
-	return continuousHeaders(result), err
-}
-
-// Returns a continuous set of headers
-func continuousHeaders(headers []core.Header) []core.Header {
-	if len(headers) < 1 {
-		logrus.Trace("no headers to arrange continuously")
-		return headers
-	}
-	previousHeader := headers[0].BlockNumber
-	for i := 1; i < len(headers); i++ {
-		previousHeader++
-		if headers[i].BlockNumber != previousHeader {
-			return headers[:i]
-		}
-	}
-
-	return headers
-}
-
-// CheckCache checks the repositories column id cache for a value
-func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
-	return r.columns.Get(key)
-}

pkg/eth/contract_watcher/header/repository/header_repository_test.go

@@ -1,371 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository_test
-
-import (
-	"fmt"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-var _ = Describe("Repository", func() {
-	var db *postgres.DB
-	var contractHeaderRepo repository.HeaderRepository // contract_watcher headerSync header repository
-	var coreHeaderRepo repositories.HeaderRepository   // pkg/datastore header repository
-	var eventIDs = []string{
-		"eventName_contractAddr",
-		"eventName_contractAddr2",
-		"eventName_contractAddr3",
-	}
-	var methodIDs = []string{
-		"methodName_contractAddr",
-		"methodName_contractAddr2",
-		"methodName_contractAddr3",
-	}
-
-	BeforeEach(func() {
-		db, _ = test_helpers.SetupDBandBC()
-		contractHeaderRepo = repository.NewHeaderRepository(db)
-		coreHeaderRepo = repositories.NewHeaderRepository(db)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
-	Describe("AddCheckColumn", func() {
-		It("Creates a column for the given eventID to mark if the header has been checked for that event", func() {
-			query := fmt.Sprintf("SELECT %s FROM checked_headers", eventIDs[0])
-			_, err := db.Exec(query)
-			Expect(err).To(HaveOccurred())
-
-			err = contractHeaderRepo.AddCheckColumn(eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = db.Exec(query)
-			Expect(err).ToNot(HaveOccurred())
-		})
-
-		It("Caches column it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-			_, ok := contractHeaderRepo.CheckCache(eventIDs[0])
-			Expect(ok).To(Equal(false))
-
-			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			v, ok := contractHeaderRepo.CheckCache(eventIDs[0])
-			Expect(ok).To(Equal(true))
-			Expect(v).To(Equal(true))
-		})
-	})
-
-	Describe("AddCheckColumns", func() {
-		It("Creates a column for the given eventIDs to mark if the header has been checked for those events", func() {
-			for _, id := range eventIDs {
-				_, err := db.Exec(fmt.Sprintf("SELECT %s FROM checked_headers", id))
-				Expect(err).To(HaveOccurred())
-			}
-
-			err := contractHeaderRepo.AddCheckColumns(eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-
-			for _, id := range eventIDs {
-				_, err := db.Exec(fmt.Sprintf("SELECT %s FROM checked_headers", id))
-				Expect(err).ToNot(HaveOccurred())
-			}
-		})
-
-		It("Caches columns it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-			for _, id := range eventIDs {
-				_, ok := contractHeaderRepo.CheckCache(id)
-				Expect(ok).To(Equal(false))
-			}
-
-			err := contractHeaderRepo.AddCheckColumns(eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-
-			for _, id := range eventIDs {
-				v, ok := contractHeaderRepo.CheckCache(id)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			}
-		})
-	})
-
-	Describe("MissingHeaders", func() {
-		It("Returns all unchecked headers for the given eventID", func() {
-			addHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-		})
-
-		It("Returns unchecked headers in ascending order", func() {
-			addHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-
-			h1 := missingHeaders[0]
-			h2 := missingHeaders[1]
-			h3 := missingHeaders[2]
-			Expect(h1.BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
-			Expect(h2.BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
-			Expect(h3.BlockNumber).To(Equal(mocks.MockHeader3.BlockNumber))
-		})
-
-		It("Returns only contiguous chunks of headers", func() {
-			addDiscontinuousHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumns(eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
-		})
-
-		It("Fails if eventID does not yet exist in check_headers table", func() {
-			addHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			_, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, "notEventId")
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("MissingHeadersForAll", func() { // HERE
-		It("Returns all headers that have not been checked for all of the ids provided", func() {
-			addHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumns(eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-
-			err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].ID, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-
-			err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].ID, eventIDs[1])
-			Expect(err).ToNot(HaveOccurred())
-			err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].ID, eventIDs[2])
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(2))
-		})
-
-		It("Returns only contiguous chunks of headers", func() {
-			addDiscontinuousHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumns(eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
-		})
-
-		It("returns headers after starting header if starting header not missing", func() {
-			addLaterHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumns(eventIDs)
-			Expect(err).NotTo(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, -1, eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader3.BlockNumber))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader4.BlockNumber))
-		})
-
-		It("Fails if one of the eventIDs does not yet exist in check_headers table", func() {
-			addHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumns(eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-			badEventIDs := append(eventIDs, "notEventId")
-
-			_, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, badEventIDs)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("MarkHeaderChecked", func() {
-		It("Marks the header checked for the given eventID", func() {
-			addHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-
-			headerID := missingHeaders[0].ID
-			err = contractHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(2))
-		})
-
-		It("Fails if eventID does not yet exist in check_headers table", func() {
-			addHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-
-			headerID := missingHeaders[0].ID
-			err = contractHeaderRepo.MarkHeaderChecked(headerID, "notEventId")
-			Expect(err).To(HaveOccurred())
-
-			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-		})
-	})
-
-	Describe("MarkHeaderCheckedForAll", func() {
-		It("Marks the header checked for all provided column ids", func() {
-			addHeaders(coreHeaderRepo)
-			err := contractHeaderRepo.AddCheckColumns(eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-
-			headerID := missingHeaders[0].ID
-			err = contractHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-
-			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(2))
-		})
-	})
-
-	Describe("MarkHeadersCheckedForAll", func() {
-		It("Marks the headers checked for all provided column ids", func() {
-			addHeaders(coreHeaderRepo)
-			methodIDs := []string{
-				"methodName_contractAddr",
-				"methodName_contractAddr2",
-				"methodName_contractAddr3",
-			}
-
-			var missingHeaders []core.Header
-			for _, id := range methodIDs {
-				err := contractHeaderRepo.AddCheckColumn(id)
-				Expect(err).ToNot(HaveOccurred())
-				missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, id)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(len(missingHeaders)).To(Equal(3))
-			}
-
-			err := contractHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs)
-			Expect(err).ToNot(HaveOccurred())
-			for _, id := range methodIDs {
-				missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, id)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(len(missingHeaders)).To(Equal(0))
-			}
-		})
-	})
-
-	Describe("MissingMethodsCheckedEventsIntersection", func() {
-		It("Returns headers that have been checked for all the provided events but have not been checked for all the provided methods", func() {
-			addHeaders(coreHeaderRepo)
-			for i, id := range eventIDs {
-				err := contractHeaderRepo.AddCheckColumn(id)
-				Expect(err).ToNot(HaveOccurred())
-				err = contractHeaderRepo.AddCheckColumn(methodIDs[i])
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(missingHeaders)).To(Equal(3))
-
-			headerID := missingHeaders[0].ID
-			headerID2 := missingHeaders[1].ID
-			for i, id := range eventIDs {
-				err = contractHeaderRepo.MarkHeaderChecked(headerID, id)
-				Expect(err).ToNot(HaveOccurred())
-				err = contractHeaderRepo.MarkHeaderChecked(headerID2, id)
-				Expect(err).ToNot(HaveOccurred())
-				err = contractHeaderRepo.MarkHeaderChecked(headerID, methodIDs[i])
-				Expect(err).ToNot(HaveOccurred())
-			}
-
-			intersectionHeaders, err := contractHeaderRepo.MissingMethodsCheckedEventsIntersection(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, methodIDs, eventIDs)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(intersectionHeaders)).To(Equal(1))
-			Expect(intersectionHeaders[0].ID).To(Equal(headerID2))
-		})
-	})
-})
-
-func addHeaders(coreHeaderRepo repositories.HeaderRepository) {
-	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
-	Expect(err).NotTo(HaveOccurred())
-}
-
-func addDiscontinuousHeaders(coreHeaderRepo repositories.HeaderRepository) {
-	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4)
-	Expect(err).NotTo(HaveOccurred())
-}
-
-func addLaterHeaders(coreHeaderRepo repositories.HeaderRepository) {
-	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4)
-	Expect(err).NotTo(HaveOccurred())
-}

pkg/eth/contract_watcher/header/repository/repository_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
-)
-
-func TestRepository(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Header Sync Repository Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/header/retriever/block_retriever.go

@@ -1,61 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever
-
-import (
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-// BlockRetriever is used to retrieve the first block for a given contract and the most recent block
-// It requires a vDB synced database with blocks, transactions, receipts, and logs
-type BlockRetriever interface {
-	RetrieveFirstBlock() (int64, error)
-	RetrieveMostRecentBlock() (int64, error)
-}
-
-type blockRetriever struct {
-	db *postgres.DB
-}
-
-// NewBlockRetriever returns a new BlockRetriever
-func NewBlockRetriever(db *postgres.DB) BlockRetriever {
-	return &blockRetriever{
-		db: db,
-	}
-}
-
-// RetrieveFirstBlock retrieves block number of earliest header in repo
-func (r *blockRetriever) RetrieveFirstBlock() (int64, error) {
-	var firstBlock int
-	err := r.db.Get(
-		&firstBlock,
-		"SELECT block_number FROM headers ORDER BY block_number LIMIT 1",
-	)
-
-	return int64(firstBlock), err
-}
-
-// RetrieveMostRecentBlock retrieves block number of latest header in repo
-func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
-	var lastBlock int
-	err := r.db.Get(
-		&lastBlock,
-		"SELECT block_number FROM headers ORDER BY block_number DESC LIMIT 1",
-	)
-
-	return int64(lastBlock), err
-}

pkg/eth/contract_watcher/header/retriever/block_retriever_test.go

@@ -1,85 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-var _ = Describe("Block Retriever", func() {
-	var db *postgres.DB
-	var r retriever.BlockRetriever
-	var headerRepository repositories.HeaderRepository
-
-	BeforeEach(func() {
-		db, _ = test_helpers.SetupDBandBC()
-		headerRepository = repositories.NewHeaderRepository(db)
-		r = retriever.NewBlockRetriever(db)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
-	Describe("RetrieveFirstBlock", func() {
-		It("Retrieves block number of earliest header in the database", func() {
-			_, err := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-			Expect(err).ToNot(HaveOccurred())
-			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
-			Expect(err).ToNot(HaveOccurred())
-			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
-			Expect(err).ToNot(HaveOccurred())
-
-			i, err := r.RetrieveFirstBlock()
-			Expect(err).NotTo(HaveOccurred())
-			Expect(i).To(Equal(int64(6194632)))
-		})
-
-		It("Fails if no headers can be found in the database", func() {
-			_, err := r.RetrieveFirstBlock()
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("RetrieveMostRecentBlock", func() {
-		It("Retrieves the latest header's block number", func() {
-			_, err := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-			Expect(err).ToNot(HaveOccurred())
-			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
-			Expect(err).ToNot(HaveOccurred())
-			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
-			Expect(err).ToNot(HaveOccurred())
-
-			i, err := r.RetrieveMostRecentBlock()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(i).To(Equal(int64(6194634)))
-		})
-
-		It("Fails if no headers can be found in the database", func() {
-			i, err := r.RetrieveMostRecentBlock()
-			Expect(err).To(HaveOccurred())
-			Expect(i).To(Equal(int64(0)))
-		})
-	})
-})

pkg/eth/contract_watcher/header/retriever/retriever_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
-)
-
-func TestRetriever(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Header Sync Block Number Retriever Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/header/transformer/transformer.go

@@ -1,337 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package transformer
-
-import (
-	"database/sql"
-	"errors"
-	"fmt"
-	"math"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	gethTypes "github.com/ethereum/go-ethereum/core/types"
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/converter"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/fetcher"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/poller"
-	srep "github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-// Transformer is the top level struct for transforming watched contract data
-// Requires a header synced vDB (headers) and a running eth node (or infura)
-type Transformer struct {
-	// Database interfaces
-	EventRepository  srep.EventRepository        // Holds transformed watched event log data
-	HeaderRepository repository.HeaderRepository // Interface for interaction with header repositories
-
-	// Pre-processing interfaces
-	Parser    parser.Parser            // Parses events and methods out of contract abi fetched using contract address
-	Retriever retriever.BlockRetriever // Retrieves first block for contract
-
-	// Processing interfaces
-	Fetcher   fetcher.Fetcher              // Fetches event logs, using header hashes
-	Converter converter.ConverterInterface // Converts watched event logs into custom log
-	Poller    poller.Poller                // Polls methods using arguments collected from events and persists them using a method datastore
-
-	// Store contract configuration information
-	Config config.ContractConfig
-
-	// Store contract info as mapping to contract address
-	Contracts map[string]*contract.Contract
-
-	// Internally configured transformer variables
-	contractAddresses []string            // Holds all contract addresses, for batch fetching of logs
-	sortedEventIds    map[string][]string // Map to sort event column ids by contract, for post fetch processing and persisting of logs
-	sortedMethodIds   map[string][]string // Map to sort method column ids by contract, for post fetch method polling
-	eventIds          []string            // Holds event column ids across all contract, for batch fetching of headers
-	eventFilters      []common.Hash       // Holds topic0 hashes across all contracts, for batch fetching of logs
-	Start             int64               // Hold the lowest starting block and the highest ending block
-}
-
-// Order-of-operations:
-// 1. Create new transformer
-// 2. Load contract addresses and their parameters
-// 3. Init
-// 4. Execute
-
-// NewTransformer takes in a contract config, blockchain, and database, and returns a new Transformer
-func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer {
-
-	return &Transformer{
-		Poller:           poller.NewPoller(bc, db, types.HeaderSync),
-		Fetcher:          fetcher.NewFetcher(bc),
-		Parser:           parser.NewParser(con.Network),
-		HeaderRepository: repository.NewHeaderRepository(db),
-		Retriever:        retriever.NewBlockRetriever(db),
-		Converter:        &converter.Converter{},
-		Contracts:        map[string]*contract.Contract{},
-		EventRepository:  srep.NewEventRepository(db, types.HeaderSync),
-		Config:           con,
-	}
-}
-
-// Init initialized the Transformer
-// Use after creating and setting transformer
-// Loops over all of the addr => filter sets
-// Uses parser to pull event info from abi
-// Use this info to generate event filters
-func (tr *Transformer) Init() error {
-	// Initialize internally configured transformer settings
-	tr.contractAddresses = make([]string, 0)       // Holds all contract addresses, for batch fetching of logs
-	tr.sortedEventIds = make(map[string][]string)  // Map to sort event column ids by contract, for post fetch processing and persisting of logs
-	tr.sortedMethodIds = make(map[string][]string) // Map to sort method column ids by contract, for post fetch method polling
-	tr.eventIds = make([]string, 0)                // Holds event column ids across all contract, for batch fetching of headers
-	tr.eventFilters = make([]common.Hash, 0)       // Holds topic0 hashes across all contracts, for batch fetching of logs
-	tr.Start = math.MaxInt64
-
-	// Iterate through all internal contract addresses
-	for contractAddr := range tr.Config.Addresses {
-		// Configure Abi
-		if tr.Config.Abis[contractAddr] == "" {
-			// If no abi is given in the config, this method will try fetching from internal look-up table and etherscan
-			parseErr := tr.Parser.Parse(contractAddr)
-			if parseErr != nil {
-				return fmt.Errorf("error parsing contract by address: %s", parseErr.Error())
-			}
-		} else {
-			// If we have an abi from the config, load that into the parser
-			parseErr := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
-			if parseErr != nil {
-				return fmt.Errorf("error parsing contract abi: %s", parseErr.Error())
-			}
-		}
-
-		// Get first block and most recent block number in the header repo
-		firstBlock, retrieveErr := tr.Retriever.RetrieveFirstBlock()
-		if retrieveErr != nil {
-			if retrieveErr == sql.ErrNoRows {
-				logrus.Error(fmt.Errorf("error retrieving first block: %s", retrieveErr.Error()))
-				firstBlock = 0
-			} else {
-				return fmt.Errorf("error retrieving first block: %s", retrieveErr.Error())
-			}
-		}
-
-		// Set to specified range if it falls within the bounds
-		if firstBlock < tr.Config.StartingBlocks[contractAddr] {
-			firstBlock = tr.Config.StartingBlocks[contractAddr]
-		}
-
-		// Get contract name if it has one
-		var name = new(string)
-		pollingErr := tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, -1)
-		if pollingErr != nil {
-			// can't return this error because "name" might not exist on the contract
-			logrus.Warnf("error fetching contract data: %s", pollingErr.Error())
-		}
-
-		// Remove any potential accidental duplicate inputs
-		eventArgs := map[string]bool{}
-		for _, arg := range tr.Config.EventArgs[contractAddr] {
-			eventArgs[arg] = true
-		}
-		methodArgs := map[string]bool{}
-		for _, arg := range tr.Config.MethodArgs[contractAddr] {
-			methodArgs[arg] = true
-		}
-
-		// Aggregate info into contract object and store for execution
-		con := contract.Contract{
-			Name:          *name,
-			Network:       tr.Config.Network,
-			Address:       contractAddr,
-			Abi:           tr.Parser.Abi(),
-			ParsedAbi:     tr.Parser.ParsedAbi(),
-			StartingBlock: firstBlock,
-			Events:        tr.Parser.GetEvents(tr.Config.Events[contractAddr]),
-			Methods:       tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]),
-			FilterArgs:    eventArgs,
-			MethodArgs:    methodArgs,
-			Piping:        tr.Config.Piping[contractAddr],
-		}.Init()
-		tr.Contracts[contractAddr] = con
-		tr.contractAddresses = append(tr.contractAddresses, con.Address)
-
-		// Create checked_headers columns for each event id and append to list of all event ids
-		tr.sortedEventIds[con.Address] = make([]string, 0, len(con.Events))
-		for _, event := range con.Events {
-			eventID := strings.ToLower(event.Name + "_" + con.Address)
-			addColumnErr := tr.HeaderRepository.AddCheckColumn(eventID)
-			if addColumnErr != nil {
-				return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
-			}
-			// Keep track of this event id; sorted and unsorted
-			tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventID)
-			tr.eventIds = append(tr.eventIds, eventID)
-			// Append this event sig to the filters
-			tr.eventFilters = append(tr.eventFilters, event.Sig())
-		}
-
-		// Create checked_headers columns for each method id and append list of all method ids
-		tr.sortedMethodIds[con.Address] = make([]string, 0, len(con.Methods))
-		for _, m := range con.Methods {
-			methodID := strings.ToLower(m.Name + "_" + con.Address)
-			addColumnErr := tr.HeaderRepository.AddCheckColumn(methodID)
-			if addColumnErr != nil {
-				return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
-			}
-			tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodID)
-		}
-
-		// Update start to the lowest block
-		if con.StartingBlock < tr.Start {
-			tr.Start = con.StartingBlock
-		}
-	}
-
-	return nil
-}
-
-// Execute runs the transformation processes
-func (tr *Transformer) Execute() error {
-	if len(tr.Contracts) == 0 {
-		return errors.New("error: transformer has no initialized contracts")
-	}
-
-	// Find unchecked headers for all events across all contracts; these are returned in asc order
-	missingHeaders, missingHeadersErr := tr.HeaderRepository.MissingHeadersForAll(tr.Start, -1, tr.eventIds)
-	if missingHeadersErr != nil {
-		return fmt.Errorf("error getting missing headers: %s", missingHeadersErr.Error())
-	}
-
-	// Iterate over headers
-	for _, header := range missingHeaders {
-		// Set `start` to this header
-		// This way if we throw an error but don't bring the execution cycle down (how it is currently handled)
-		// we restart the cycle at this header
-		tr.Start = header.BlockNumber
-		// Map to sort batch fetched logs by which contract they belong to, for post fetch processing
-		sortedLogs := make(map[string][]gethTypes.Log)
-		// And fetch all event logs across contracts at this header
-		allLogs, fetchErr := tr.Fetcher.FetchLogs(tr.contractAddresses, tr.eventFilters, header)
-		if fetchErr != nil {
-			return fmt.Errorf("error fetching logs: %s", fetchErr.Error())
-		}
-
-		// If no logs are found mark the header checked for all of these eventIDs
-		// and continue to method polling and onto the next iteration
-		if len(allLogs) < 1 {
-			markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.ID, tr.eventIds)
-			if markCheckedErr != nil {
-				return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
-			}
-			pollingErr := tr.methodPolling(header, tr.sortedMethodIds)
-			if pollingErr != nil {
-				return fmt.Errorf("error polling methods: %s", pollingErr.Error())
-			}
-			tr.Start = header.BlockNumber + 1 // Empty header; setup to start at the next header
-			logrus.Tracef("no logs found for block %d, continuing", header.BlockNumber)
-			continue
-		}
-
-		for _, log := range allLogs {
-			addr := strings.ToLower(log.Address.Hex())
-			sortedLogs[addr] = append(sortedLogs[addr], log)
-		}
-
-		// Process logs for each contract
-		for conAddr, logs := range sortedLogs {
-			if logs == nil {
-				logrus.Tracef("no logs found for contract %s at block %d, continuing", conAddr, header.BlockNumber)
-				continue
-			}
-			// Configure converter with this contract
-			con := tr.Contracts[conAddr]
-			tr.Converter.Update(con)
-
-			// Convert logs into batches of log mappings (eventName => []types.Logs
-			convertedLogs, convertErr := tr.Converter.ConvertBatch(logs, con.Events, header.ID)
-			if convertErr != nil {
-				return fmt.Errorf("error converting logs: %s", convertErr.Error())
-			}
-			// Cycle through each type of event log and persist them
-			for eventName, logs := range convertedLogs {
-				// If logs for this event are empty, mark them checked at this header and continue
-				if len(logs) < 1 {
-					logrus.Tracef("no logs found for event %s on contract %s at block %d, continuing", eventName, conAddr, header.BlockNumber)
-					continue
-				}
-				// If logs aren't empty, persist them
-				persistErr := tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name)
-				if persistErr != nil {
-					return fmt.Errorf("error persisting logs: %s", persistErr.Error())
-				}
-			}
-		}
-
-		markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.ID, tr.eventIds)
-		if markCheckedErr != nil {
-			return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
-		}
-
-		// Poll contracts at this block height
-		pollingErr := tr.methodPolling(header, tr.sortedMethodIds)
-		if pollingErr != nil {
-			return fmt.Errorf("error polling methods: %s", pollingErr.Error())
-		}
-		// Success; setup to start at the next header
-		tr.Start = header.BlockNumber + 1
-	}
-
-	return nil
-}
-
-// Used to poll contract methods at a given header
-func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
-	for _, con := range tr.Contracts {
-		// Skip method polling processes if no methods are specified
-		// Also don't try to poll methods below this contract's specified starting block
-		if len(con.Methods) == 0 || header.BlockNumber < con.StartingBlock {
-			logrus.Tracef("not polling contract: %s", con.Address)
-			continue
-		}
-
-		// Poll all methods for this contract at this header
-		pollingErr := tr.Poller.PollContractAt(*con, header.BlockNumber)
-		if pollingErr != nil {
-			return fmt.Errorf("error polling contract %s: %s", con.Address, pollingErr.Error())
-		}
-
-		// Mark this header checked for the methods
-		markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.ID, sortedMethodIds[con.Address])
-		if markCheckedErr != nil {
-			return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
-		}
-	}
-
-	return nil
-}
-
-// GetConfig returns the transformers config; satisfies the transformer interface
-func (tr *Transformer) GetConfig() config.ContractConfig {
-	return tr.Config
-}

pkg/eth/contract_watcher/header/transformer/transformer_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package transformer_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
-)
-
-func TestTransformer(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Header Sync Transformer Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/header/transformer/transformer_test.go

@@ -1,138 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package transformer_test
-
-import (
-	"database/sql"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/poller"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Transformer", func() {
-	var fakeAddress = "0x1234567890abcdef"
-	Describe("Init", func() {
-		It("Initializes transformer's contract objects", func() {
-			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
-			firstBlock := int64(1)
-			blockRetriever.FirstBlock = firstBlock
-
-			parsr := &fakes.MockParser{}
-			fakeAbi := "fake_abi"
-			parsr.AbiToReturn = fakeAbi
-
-			pollr := &fakes.MockPoller{}
-			fakeContractName := "fake_contract_name"
-			pollr.ContractName = fakeContractName
-
-			t := getFakeTransformer(blockRetriever, parsr, pollr)
-
-			err := t.Init()
-
-			Expect(err).ToNot(HaveOccurred())
-
-			c, ok := t.Contracts[fakeAddress]
-			Expect(ok).To(Equal(true))
-
-			Expect(c.StartingBlock).To(Equal(firstBlock))
-			Expect(c.Abi).To(Equal(fakeAbi))
-			Expect(c.Name).To(Equal(fakeContractName))
-			Expect(c.Address).To(Equal(fakeAddress))
-		})
-
-		It("Fails to initialize if first block cannot be fetched from vDB headers table", func() {
-			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
-			blockRetriever.FirstBlockErr = fakes.FakeError
-			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
-
-			err := t.Init()
-
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
-		})
-	})
-
-	Describe("Execute", func() {
-		It("Executes contract transformations", func() {
-			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
-			firstBlock := int64(1)
-			blockRetriever.FirstBlock = firstBlock
-
-			parsr := &fakes.MockParser{}
-			fakeAbi := "fake_abi"
-			parsr.AbiToReturn = fakeAbi
-
-			pollr := &fakes.MockPoller{}
-			fakeContractName := "fake_contract_name"
-			pollr.ContractName = fakeContractName
-
-			t := getFakeTransformer(blockRetriever, parsr, pollr)
-
-			err := t.Init()
-
-			Expect(err).ToNot(HaveOccurred())
-
-			c, ok := t.Contracts[fakeAddress]
-			Expect(ok).To(Equal(true))
-
-			Expect(c.StartingBlock).To(Equal(firstBlock))
-			Expect(c.Abi).To(Equal(fakeAbi))
-			Expect(c.Name).To(Equal(fakeContractName))
-			Expect(c.Address).To(Equal(fakeAddress))
-		})
-
-		It("uses first block from config if vDB headers table has no rows", func() {
-			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
-			blockRetriever.FirstBlockErr = sql.ErrNoRows
-			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
-
-			err := t.Init()
-
-			Expect(err).ToNot(HaveOccurred())
-		})
-
-		It("returns error if fetching first block fails for other reason", func() {
-			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
-			blockRetriever.FirstBlockErr = fakes.FakeError
-			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
-
-			err := t.Init()
-
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
-		})
-	})
-})
-
-func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
-	return transformer.Transformer{
-		Parser:           parsr,
-		Retriever:        blockRetriever,
-		Poller:           pollr,
-		HeaderRepository: &fakes.MockHeaderSyncHeaderRepository{},
-		Contracts:        map[string]*contract.Contract{},
-		Config:           mocks.MockConfig,
-	}
-}

pkg/eth/contract_watcher/shared/constants/interface.go

@@ -1,129 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package constants
-
-import (
-	"github.com/ethereum/go-ethereum/common/hexutil"
-)
-
-// SupportsInterfaceABI is the basic abi needed to check which interfaces are adhered to
-var SupportsInterfaceABI = `[{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]`
-
-// Individual event interfaces for constructing ABI from
-var AddrChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"a","type":"address"}],"name":"AddrChanged","type":"event"}`
-var ContentChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes32"}],"name":"ContentChanged","type":"event"}`
-var NameChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"name","type":"string"}],"name":"NameChanged","type":"event"}`
-var AbiChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":true,"name":"contentType","type":"uint256"}],"name":"ABIChanged","type":"event"}`
-var PubkeyChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"x","type":"bytes32"},{"indexed":false,"name":"y","type":"bytes32"}],"name":"PubkeyChanged","type":"event"}`
-var TextChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"indexedKey","type":"string"},{"indexed":false,"name":"key","type":"string"}],"name":"TextChanged","type":"event"}`
-var MultihashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"MultihashChanged","type":"event"}`
-var ContenthashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"ContenthashChanged","type":"event"}`
-
-// Resolver interface signatures
-type Interface int
-
-// Interface enums
-const (
-	MetaSig Interface = iota
-	AddrChangeSig
-	ContentChangeSig
-	NameChangeSig
-	AbiChangeSig
-	PubkeyChangeSig
-	TextChangeSig
-	MultihashChangeSig
-	ContentHashChangeSig
-)
-
-// Hex returns the hex signature for an interface
-func (e Interface) Hex() string {
-	strings := [...]string{
-		"0x01ffc9a7",
-		"0x3b3b57de",
-		"0xd8389dc5",
-		"0x691f3431",
-		"0x2203ab56",
-		"0xc8690233",
-		"0x59d1d43c",
-		"0xe89401a1",
-		"0xbc1c58d1",
-	}
-
-	if e < MetaSig || e > ContentHashChangeSig {
-		return "Unknown"
-	}
-
-	return strings[e]
-}
-
-// Bytes returns the bytes signature for an interface
-func (e Interface) Bytes() [4]uint8 {
-	if e < MetaSig || e > ContentHashChangeSig {
-		return [4]byte{}
-	}
-
-	str := e.Hex()
-	by, _ := hexutil.Decode(str)
-	var byArray [4]uint8
-	for i := 0; i < 4; i++ {
-		byArray[i] = by[i]
-	}
-
-	return byArray
-}
-
-// EventSig returns the event signature for an interface
-func (e Interface) EventSig() string {
-	strings := [...]string{
-		"",
-		"AddrChanged(bytes32,address)",
-		"ContentChanged(bytes32,bytes32)",
-		"NameChanged(bytes32,string)",
-		"ABIChanged(bytes32,uint256)",
-		"PubkeyChanged(bytes32,bytes32,bytes32)",
-		"TextChanged(bytes32,string,string)",
-		"MultihashChanged(bytes32,bytes)",
-		"ContenthashChanged(bytes32,bytes)",
-	}
-
-	if e < MetaSig || e > ContentHashChangeSig {
-		return "Unknown"
-	}
-
-	return strings[e]
-}
-
-// MethodSig returns the method signature for an interface
-func (e Interface) MethodSig() string {
-	strings := [...]string{
-		"supportsInterface(bytes4)",
-		"addr(bytes32)",
-		"content(bytes32)",
-		"name(bytes32)",
-		"ABI(bytes32,uint256)",
-		"pubkey(bytes32)",
-		"text(bytes32,string)",
-		"multihash(bytes32)",
-		"setContenthash(bytes32,bytes)",
-	}
-
-	if e < MetaSig || e > ContentHashChangeSig {
-		return "Unknown"
-	}
-
-	return strings[e]
-}

pkg/eth/contract_watcher/shared/contract/contract.go

@@ -1,172 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package contract
-
-import (
-	"errors"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/filters"
-)
-
-// Contract object to hold our contract data
-type Contract struct {
-	Name           string                       // Name of the contract
-	Address        string                       // Address of the contract
-	Network        string                       // Network on which the contract is deployed; default empty "" is Ethereum mainnet
-	StartingBlock  int64                        // Starting block of the contract
-	Abi            string                       // Abi string
-	ParsedAbi      abi.ABI                      // Parsed abi
-	Events         map[string]types.Event       // List of events to watch
-	Methods        []types.Method               // List of methods to poll
-	Filters        map[string]filters.LogFilter // Map of event filters to their event names; used only for full sync watcher
-	FilterArgs     map[string]bool              // User-input list of values to filter event logs for
-	MethodArgs     map[string]bool              // User-input list of values to limit method polling to
-	EmittedAddrs   map[interface{}]bool         // List of all unique addresses collected from converted event logs
-	EmittedHashes  map[interface{}]bool         // List of all unique hashes collected from converted event logs
-	CreateAddrList bool                         // Whether or not to persist address list to postgres
-	CreateHashList bool                         // Whether or not to persist hash list to postgres
-	Piping         bool                         // Whether or not to pipe method results forward as arguments to subsequent methods
-}
-
-// Init initializes a contract object
-// If we will be calling methods that use addr, hash, or byte arrays
-// as arguments then we initialize maps to hold these types of values
-func (c Contract) Init() *Contract {
-	for _, method := range c.Methods {
-		for _, arg := range method.Args {
-			switch arg.Type.T {
-			case abi.AddressTy:
-				c.EmittedAddrs = map[interface{}]bool{}
-			case abi.HashTy, abi.BytesTy, abi.FixedBytesTy:
-				c.EmittedHashes = map[interface{}]bool{}
-			default:
-			}
-		}
-	}
-
-	return &c
-}
-
-// GenerateFilters uses contract info to generate event filters - full sync contract watcher only
-func (c *Contract) GenerateFilters() error {
-	c.Filters = map[string]filters.LogFilter{}
-
-	for name, event := range c.Events {
-		c.Filters[name] = filters.LogFilter{
-			Name:      c.Address + "_" + event.Name,
-			FromBlock: c.StartingBlock,
-			ToBlock:   -1,
-			Address:   common.HexToAddress(c.Address).Hex(),
-			Topics:    core.Topics{event.Sig().Hex()},
-		}
-	}
-	// If no filters were generated, throw an error (no point in continuing with this contract)
-	if len(c.Filters) == 0 {
-		return errors.New("error: no filters created")
-	}
-
-	return nil
-}
-
-// WantedEventArg returns true if address is in list of arguments to
-// filter events for or if no filtering is specified
-func (c *Contract) WantedEventArg(arg string) bool {
-	if c.FilterArgs == nil {
-		return false
-	} else if len(c.FilterArgs) == 0 {
-		return true
-	} else if a, ok := c.FilterArgs[arg]; ok {
-		return a
-	}
-
-	return false
-}
-
-// WantedMethodArg returns true if address is in list of arguments to
-// poll methods with or if no filtering is specified
-func (c *Contract) WantedMethodArg(arg interface{}) bool {
-	if c.MethodArgs == nil {
-		return false
-	} else if len(c.MethodArgs) == 0 {
-		return true
-	}
-
-	// resolve interface to one of the three types we handle as arguments
-	str := StringifyArg(arg)
-
-	// See if it's hex string has been filtered for
-	if a, ok := c.MethodArgs[str]; ok {
-		return a
-	}
-
-	return false
-}
-
-// PassesEventFilter returns true if any mapping value matches filtered for address or if no filter exists
-// Used to check if an event log name-value mapping should be filtered or not
-func (c *Contract) PassesEventFilter(args map[string]string) bool {
-	for _, arg := range args {
-		if c.WantedEventArg(arg) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// AddEmittedAddr adds event emitted addresses to our list if it passes filter and method polling is on
-func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
-	for _, addr := range addresses {
-		if c.WantedMethodArg(addr) && c.Methods != nil {
-			c.EmittedAddrs[addr] = true
-		}
-	}
-}
-
-// AddEmittedHash adds event emitted hashes to our list if it passes filter and method polling is on
-func (c *Contract) AddEmittedHash(hashes ...interface{}) {
-	for _, hash := range hashes {
-		if c.WantedMethodArg(hash) && c.Methods != nil {
-			c.EmittedHashes[hash] = true
-		}
-	}
-}
-
-// StringifyArg resolves a method argument type to string type
-func StringifyArg(arg interface{}) (str string) {
-	switch arg.(type) {
-	case string:
-		str = arg.(string)
-	case common.Address:
-		a := arg.(common.Address)
-		str = a.String()
-	case common.Hash:
-		a := arg.(common.Hash)
-		str = a.String()
-	case []byte:
-		a := arg.([]byte)
-		str = hexutil.Encode(a)
-	}
-
-	return
-}

pkg/eth/contract_watcher/shared/contract/contract_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package contract_test
-
-import (
-	"io/ioutil"
-	"log"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestContract(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Contract Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/shared/contract/contract_test.go

@@ -1,236 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package contract_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-)
-
-var _ = Describe("Contract", func() {
-	var err error
-	var info *contract.Contract
-	var wantedEvents = []string{"Transfer", "Approval"}
-
-	Describe("GenerateFilters", func() {
-
-		It("Generates filters from contract data", func() {
-			info = test_helpers.SetupTusdContract(wantedEvents, nil)
-			err = info.GenerateFilters()
-			Expect(err).ToNot(HaveOccurred())
-
-			val, ok := info.Filters["Transfer"]
-			Expect(ok).To(Equal(true))
-			Expect(val).To(Equal(mocks.ExpectedTransferFilter))
-
-			val, ok = info.Filters["Approval"]
-			Expect(ok).To(Equal(true))
-			Expect(val).To(Equal(mocks.ExpectedApprovalFilter))
-
-			val, ok = info.Filters["Mint"]
-			Expect(ok).To(Equal(false))
-
-		})
-
-		It("Fails with an empty contract", func() {
-			info = &contract.Contract{}
-			err = info.GenerateFilters()
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("IsEventAddr", func() {
-
-		BeforeEach(func() {
-			info = &contract.Contract{}
-			info.MethodArgs = map[string]bool{}
-			info.FilterArgs = map[string]bool{}
-		})
-
-		It("Returns true if address is in event address filter list", func() {
-			info.FilterArgs["testAddress1"] = true
-			info.FilterArgs["testAddress2"] = true
-
-			is := info.WantedEventArg("testAddress1")
-			Expect(is).To(Equal(true))
-			is = info.WantedEventArg("testAddress2")
-			Expect(is).To(Equal(true))
-
-			info.MethodArgs["testAddress3"] = true
-			is = info.WantedEventArg("testAddress3")
-			Expect(is).To(Equal(false))
-		})
-
-		It("Returns true if event address filter is empty (no filter)", func() {
-			is := info.WantedEventArg("testAddress1")
-			Expect(is).To(Equal(true))
-			is = info.WantedEventArg("testAddress2")
-			Expect(is).To(Equal(true))
-		})
-
-		It("Returns false if address is not in event address filter list", func() {
-			info.FilterArgs["testAddress1"] = true
-			info.FilterArgs["testAddress2"] = true
-
-			is := info.WantedEventArg("testAddress3")
-			Expect(is).To(Equal(false))
-		})
-
-		It("Returns false if event address filter is nil (block all)", func() {
-			info.FilterArgs = nil
-
-			is := info.WantedEventArg("testAddress1")
-			Expect(is).To(Equal(false))
-			is = info.WantedEventArg("testAddress2")
-			Expect(is).To(Equal(false))
-		})
-	})
-
-	Describe("IsMethodAddr", func() {
-		BeforeEach(func() {
-			info = &contract.Contract{}
-			info.MethodArgs = map[string]bool{}
-			info.FilterArgs = map[string]bool{}
-		})
-
-		It("Returns true if address is in method address filter list", func() {
-			info.MethodArgs["testAddress1"] = true
-			info.MethodArgs["testAddress2"] = true
-
-			is := info.WantedMethodArg("testAddress1")
-			Expect(is).To(Equal(true))
-			is = info.WantedMethodArg("testAddress2")
-			Expect(is).To(Equal(true))
-
-			info.FilterArgs["testAddress3"] = true
-			is = info.WantedMethodArg("testAddress3")
-			Expect(is).To(Equal(false))
-		})
-
-		It("Returns true if method address filter list is empty (no filter)", func() {
-			is := info.WantedMethodArg("testAddress1")
-			Expect(is).To(Equal(true))
-			is = info.WantedMethodArg("testAddress2")
-			Expect(is).To(Equal(true))
-		})
-
-		It("Returns false if address is not in method address filter list", func() {
-			info.MethodArgs["testAddress1"] = true
-			info.MethodArgs["testAddress2"] = true
-
-			is := info.WantedMethodArg("testAddress3")
-			Expect(is).To(Equal(false))
-		})
-
-		It("Returns false if method address filter list is nil (block all)", func() {
-			info.MethodArgs = nil
-
-			is := info.WantedMethodArg("testAddress1")
-			Expect(is).To(Equal(false))
-			is = info.WantedMethodArg("testAddress2")
-			Expect(is).To(Equal(false))
-		})
-	})
-
-	Describe("PassesEventFilter", func() {
-		var mapping map[string]string
-		BeforeEach(func() {
-			info = &contract.Contract{}
-			info.FilterArgs = map[string]bool{}
-			mapping = map[string]string{}
-
-		})
-
-		It("Return true if event log name-value mapping has filtered for address as a value", func() {
-			info.FilterArgs["testAddress1"] = true
-			info.FilterArgs["testAddress2"] = true
-
-			mapping["testInputName1"] = "testAddress1"
-			mapping["testInputName2"] = "testAddress2"
-			mapping["testInputName3"] = "testAddress3"
-
-			pass := info.PassesEventFilter(mapping)
-			Expect(pass).To(Equal(true))
-		})
-
-		It("Return true if event address filter list is empty (no filter)", func() {
-			mapping["testInputName1"] = "testAddress1"
-			mapping["testInputName2"] = "testAddress2"
-			mapping["testInputName3"] = "testAddress3"
-
-			pass := info.PassesEventFilter(mapping)
-			Expect(pass).To(Equal(true))
-		})
-
-		It("Return false if event log name-value mapping does not have filtered for address as a value", func() {
-			info.FilterArgs["testAddress1"] = true
-			info.FilterArgs["testAddress2"] = true
-
-			mapping["testInputName3"] = "testAddress3"
-
-			pass := info.PassesEventFilter(mapping)
-			Expect(pass).To(Equal(false))
-		})
-
-		It("Return false if event address filter list is nil (block all)", func() {
-			info.FilterArgs = nil
-
-			mapping["testInputName1"] = "testAddress1"
-			mapping["testInputName2"] = "testAddress2"
-			mapping["testInputName3"] = "testAddress3"
-
-			pass := info.PassesEventFilter(mapping)
-			Expect(pass).To(Equal(false))
-		})
-	})
-
-	Describe("AddEmittedAddr", func() {
-		BeforeEach(func() {
-			info = &contract.Contract{}
-			info.FilterArgs = map[string]bool{}
-			info.MethodArgs = map[string]bool{}
-			info.Methods = []types.Method{}
-			info.EmittedAddrs = map[interface{}]bool{}
-		})
-
-		It("Adds address to list if it is on the method filter address list", func() {
-			info.MethodArgs["testAddress2"] = true
-			info.AddEmittedAddr("testAddress2")
-			b := info.EmittedAddrs["testAddress2"]
-			Expect(b).To(Equal(true))
-		})
-
-		It("Adds address to list if method filter is empty", func() {
-			info.AddEmittedAddr("testAddress2")
-			b := info.EmittedAddrs["testAddress2"]
-			Expect(b).To(Equal(true))
-		})
-
-		It("Does not add address to list if both filters are closed (nil)", func() {
-			info.FilterArgs = nil // close both
-			info.MethodArgs = nil
-			info.AddEmittedAddr("testAddress1")
-			b := info.EmittedAddrs["testAddress1"]
-			Expect(b).To(Equal(false))
-		})
-	})
-})

pkg/eth/contract_watcher/shared/fetcher/fetcher.go

@@ -1,124 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package fetcher
-
-import (
-	"fmt"
-	"log"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-// Fetcher serves as the lower level data fetcher that calls the underlying
-// blockchain's FetchConctractData method for a given return type
-
-// FetcherInterface is the interface definition for a fetcher
-type FetcherInterface interface {
-	FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error)
-	FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error)
-	FetchAddress(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Address, error)
-	FetchString(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (string, error)
-	FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error)
-}
-
-// Used to create a new Fetcher error for a given error and fetch method
-func newFetcherError(err error, fetchMethod string) *fetcherError {
-	e := fetcherError{err.Error(), fetchMethod}
-	log.Println(e.Error())
-	return &e
-}
-
-// Fetcher struct
-type Fetcher struct {
-	BlockChain core.BlockChain // Underlying Blockchain
-}
-
-// Fetcher error
-type fetcherError struct {
-	err         string
-	fetchMethod string
-}
-
-// Error method
-func (fe *fetcherError) Error() string {
-	return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err)
-}
-
-//  Generic Fetcher methods used by Getters to call contract methods
-
-// FetchBigInt is the method used to fetch big.Int value from contract
-func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) {
-	var result = new(big.Int)
-	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
-
-	if err != nil {
-		return *result, newFetcherError(err, method)
-	}
-
-	return *result, nil
-}
-
-// FetchBool is the method used to fetch bool value from contract
-func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) {
-	var result = new(bool)
-	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
-
-	if err != nil {
-		return *result, newFetcherError(err, method)
-	}
-
-	return *result, nil
-}
-
-// FetchAddress is the method used to fetch address value from contract
-func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Address, error) {
-	var result = new(common.Address)
-	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
-
-	if err != nil {
-		return *result, newFetcherError(err, method)
-	}
-
-	return *result, nil
-}
-
-// FetchString is the method used to fetch string value from contract
-func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (string, error) {
-	var result = new(string)
-	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
-
-	if err != nil {
-		return *result, newFetcherError(err, method)
-	}
-
-	return *result, nil
-}
-
-// FetchHash is the method used to fetch hash value from contract
-func (f Fetcher) FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error) {
-	var result = new(common.Hash)
-	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
-
-	if err != nil {
-		return *result, newFetcherError(err, method)
-	}
-
-	return *result, nil
-}

pkg/eth/contract_watcher/shared/getter/interface_getter.go

@@ -1,113 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package getter
-
-import (
-	"fmt"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-// InterfaceGetter is used to derive the interface of a contract
-type InterfaceGetter interface {
-	GetABI(resolverAddr string, blockNumber int64) (string, error)
-	GetBlockChain() core.BlockChain
-}
-
-type interfaceGetter struct {
-	fetcher.Fetcher
-}
-
-// NewInterfaceGetter returns a new InterfaceGetter
-func NewInterfaceGetter(blockChain core.BlockChain) InterfaceGetter {
-	return &interfaceGetter{
-		Fetcher: fetcher.Fetcher{
-			BlockChain: blockChain,
-		},
-	}
-}
-
-// GetABI is used to construct a custom ABI based on the results from calling supportsInterface
-func (g *interfaceGetter) GetABI(resolverAddr string, blockNumber int64) (string, error) {
-	a := constants.SupportsInterfaceABI
-	args := make([]interface{}, 1)
-	args[0] = constants.MetaSig.Bytes()
-	supports, err := g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err != nil {
-		return "", fmt.Errorf("call to getSupportsInterface failed: %v", err)
-	}
-	if !supports {
-		return "", fmt.Errorf("contract does not support interface")
-	}
-
-	abiStr := `[`
-	args[0] = constants.AddrChangeSig.Bytes()
-	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err == nil && supports {
-		abiStr += constants.AddrChangeInterface + ","
-	}
-	args[0] = constants.NameChangeSig.Bytes()
-	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err == nil && supports {
-		abiStr += constants.NameChangeInterface + ","
-	}
-	args[0] = constants.ContentChangeSig.Bytes()
-	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err == nil && supports {
-		abiStr += constants.ContentChangeInterface + ","
-	}
-	args[0] = constants.AbiChangeSig.Bytes()
-	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err == nil && supports {
-		abiStr += constants.AbiChangeInterface + ","
-	}
-	args[0] = constants.PubkeyChangeSig.Bytes()
-	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err == nil && supports {
-		abiStr += constants.PubkeyChangeInterface + ","
-	}
-	args[0] = constants.ContentHashChangeSig.Bytes()
-	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err == nil && supports {
-		abiStr += constants.ContenthashChangeInterface + ","
-	}
-	args[0] = constants.MultihashChangeSig.Bytes()
-	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err == nil && supports {
-		abiStr += constants.MultihashChangeInterface + ","
-	}
-	args[0] = constants.TextChangeSig.Bytes()
-	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err == nil && supports {
-		abiStr += constants.TextChangeInterface + ","
-	}
-	abiStr = abiStr[:len(abiStr)-1] + `]`
-
-	return abiStr, nil
-}
-
-// Use this method to check whether or not a contract supports a given method/event interface
-func (g *interfaceGetter) getSupportsInterface(contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) {
-	return g.Fetcher.FetchBool("supportsInterface", contractAbi, contractAddress, blockNumber, methodArgs)
-}
-
-// GetBlockChain is a method to retrieve the Getter's blockchain
-func (g *interfaceGetter) GetBlockChain() core.BlockChain {
-	return g.Fetcher.BlockChain
-}

pkg/eth/contract_watcher/shared/helpers/helpers.go

@@ -1,72 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package helpers
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-// ConvertToLog converts a watched event to a log
-func ConvertToLog(watchedEvent core.WatchedEvent) types.Log {
-	allTopics := []string{watchedEvent.Topic0, watchedEvent.Topic1, watchedEvent.Topic2, watchedEvent.Topic3}
-	var nonNilTopics []string
-	for _, topic := range allTopics {
-		if topic != "" {
-			nonNilTopics = append(nonNilTopics, topic)
-		}
-	}
-	return types.Log{
-		Address:     common.HexToAddress(watchedEvent.Address),
-		Topics:      createTopics(nonNilTopics...),
-		Data:        hexutil.MustDecode(watchedEvent.Data),
-		BlockNumber: uint64(watchedEvent.BlockNumber),
-		TxHash:      common.HexToHash(watchedEvent.TxHash),
-		TxIndex:     0,
-		BlockHash:   common.HexToHash("0x0"),
-		Index:       uint(watchedEvent.Index),
-		Removed:     false,
-	}
-}
-
-func createTopics(topics ...string) []common.Hash {
-	var topicsArray []common.Hash
-	for _, topic := range topics {
-		topicsArray = append(topicsArray, common.HexToHash(topic))
-	}
-	return topicsArray
-}
-
-// BigFromString creates a big.Int from a string
-func BigFromString(n string) *big.Int {
-	b := new(big.Int)
-	b.SetString(n, 10)
-	return b
-}
-
-// GenerateSignature returns the keccak256 hash hex of a string
-func GenerateSignature(s string) string {
-	eventSignature := []byte(s)
-	hash := crypto.Keccak256Hash(eventSignature)
-	return hash.Hex()
-}

pkg/eth/contract_watcher/shared/helpers/test_helpers/database.go

@@ -1,243 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package test_helpers
-
-import (
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-type TransferLog struct {
-	ID             int64  `db:"id"`
-	VulcanizeLogID int64  `db:"vulcanize_log_id"`
-	TokenName      string `db:"token_name"`
-	Block          int64  `db:"block"`
-	Tx             string `db:"tx"`
-	From           string `db:"from_"`
-	To             string `db:"to_"`
-	Value          string `db:"value_"`
-}
-
-type NewOwnerLog struct {
-	ID             int64  `db:"id"`
-	VulcanizeLogID int64  `db:"vulcanize_log_id"`
-	TokenName      string `db:"token_name"`
-	Block          int64  `db:"block"`
-	Tx             string `db:"tx"`
-	Node           string `db:"node_"`
-	Label          string `db:"label_"`
-	Owner          string `db:"owner_"`
-}
-
-type HeaderSyncTransferLog struct {
-	ID        int64  `db:"id"`
-	HeaderID  int64  `db:"header_id"`
-	TokenName string `db:"token_name"`
-	LogIndex  int64  `db:"log_idx"`
-	TxIndex   int64  `db:"tx_idx"`
-	From      string `db:"from_"`
-	To        string `db:"to_"`
-	Value     string `db:"value_"`
-	RawLog    []byte `db:"raw_log"`
-}
-
-type HeaderSyncNewOwnerLog struct {
-	ID        int64  `db:"id"`
-	HeaderID  int64  `db:"header_id"`
-	TokenName string `db:"token_name"`
-	LogIndex  int64  `db:"log_idx"`
-	TxIndex   int64  `db:"tx_idx"`
-	Node      string `db:"node_"`
-	Label     string `db:"label_"`
-	Owner     string `db:"owner_"`
-	RawLog    []byte `db:"raw_log"`
-}
-
-type BalanceOf struct {
-	ID        int64  `db:"id"`
-	TokenName string `db:"token_name"`
-	Block     int64  `db:"block"`
-	Address   string `db:"who_"`
-	Balance   string `db:"returned"`
-}
-
-type Resolver struct {
-	ID        int64  `db:"id"`
-	TokenName string `db:"token_name"`
-	Block     int64  `db:"block"`
-	Node      string `db:"node_"`
-	Address   string `db:"returned"`
-}
-
-type Owner struct {
-	ID        int64  `db:"id"`
-	TokenName string `db:"token_name"`
-	Block     int64  `db:"block"`
-	Node      string `db:"node_"`
-	Address   string `db:"returned"`
-}
-
-func SetupDBandBC() (*postgres.DB, core.BlockChain) {
-	con := test_config.TestClient
-	testIPC := con.IPCPath
-	rawRPCClient, err := rpc.Dial(testIPC)
-	Expect(err).NotTo(HaveOccurred())
-	rpcClient := client.NewRPCClient(rawRPCClient, testIPC)
-	ethClient := ethclient.NewClient(rawRPCClient)
-	blockChainClient := client.NewEthClient(ethClient)
-	madeNode := node.MakeNode(rpcClient)
-	transactionConverter := rpc2.NewRPCTransactionConverter(ethClient)
-	blockChain := eth.NewBlockChain(blockChainClient, rpcClient, madeNode, transactionConverter)
-
-	db, err := postgres.NewDB(config.Database{
-		Hostname: "localhost",
-		Name:     "vulcanize_testing",
-		Port:     5432,
-	}, blockChain.Node())
-	Expect(err).NotTo(HaveOccurred())
-
-	return db, blockChain
-}
-
-func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract {
-	p := mocks.NewParser(constants.TusdAbiString)
-	err := p.Parse(constants.TusdContractAddress)
-	Expect(err).ToNot(HaveOccurred())
-
-	return contract.Contract{
-		Name:          "TrueUSD",
-		Address:       constants.TusdContractAddress,
-		Abi:           p.Abi(),
-		ParsedAbi:     p.ParsedAbi(),
-		StartingBlock: 6194634,
-		Events:        p.GetEvents(wantedEvents),
-		Methods:       p.GetSelectMethods(wantedMethods),
-		MethodArgs:    map[string]bool{},
-		FilterArgs:    map[string]bool{},
-	}.Init()
-}
-
-func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract {
-	p := mocks.NewParser(constants.ENSAbiString)
-	err := p.Parse(constants.EnsContractAddress)
-	Expect(err).ToNot(HaveOccurred())
-
-	return contract.Contract{
-		Name:          "ENS-Registry",
-		Address:       constants.EnsContractAddress,
-		Abi:           p.Abi(),
-		ParsedAbi:     p.ParsedAbi(),
-		StartingBlock: 6194634,
-		Events:        p.GetEvents(wantedEvents),
-		Methods:       p.GetSelectMethods(wantedMethods),
-		MethodArgs:    map[string]bool{},
-		FilterArgs:    map[string]bool{},
-	}.Init()
-}
-
-func SetupMarketPlaceContract(wantedEvents, wantedMethods []string) *contract.Contract {
-	p := mocks.NewParser(constants.MarketPlaceAbiString)
-	err := p.Parse(constants.MarketPlaceContractAddress)
-	Expect(err).NotTo(HaveOccurred())
-
-	return contract.Contract{
-		Name:          "Marketplace",
-		Address:       constants.MarketPlaceContractAddress,
-		StartingBlock: 6496012,
-		Abi:           p.Abi(),
-		ParsedAbi:     p.ParsedAbi(),
-		Events:        p.GetEvents(wantedEvents),
-		Methods:       p.GetSelectMethods(wantedMethods),
-		FilterArgs:    map[string]bool{},
-		MethodArgs:    map[string]bool{},
-	}.Init()
-}
-
-func SetupMolochContract(wantedEvents, wantedMethods []string) *contract.Contract {
-	p := mocks.NewParser(constants.MolochAbiString)
-	err := p.Parse(constants.MolochContractAddress)
-	Expect(err).NotTo(HaveOccurred())
-
-	return contract.Contract{
-		Name:          "Moloch",
-		Address:       constants.MolochContractAddress,
-		StartingBlock: 7218566,
-		Abi:           p.Abi(),
-		ParsedAbi:     p.ParsedAbi(),
-		Events:        p.GetEvents(wantedEvents),
-		Methods:       p.GetSelectMethods(wantedMethods),
-		FilterArgs:    map[string]bool{},
-		MethodArgs:    map[string]bool{},
-	}.Init()
-}
-
-// TODO: tear down/setup DB from migrations so this doesn't alter the schema between tests
-func TearDown(db *postgres.DB) {
-	tx, err := db.Beginx()
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DELETE FROM addresses`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DELETE FROM headers`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec("DELETE FROM header_sync_transactions")
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DELETE FROM header_sync_receipts`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DROP TABLE checked_headers`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`CREATE TABLE checked_headers (
-    	id SERIAL PRIMARY KEY,
-    	header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DROP SCHEMA IF EXISTS header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DROP SCHEMA IF EXISTS header_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = tx.Commit()
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = db.Exec(`VACUUM checked_headers`)
-	Expect(err).NotTo(HaveOccurred())
-}

pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks/entities.go

@@ -1,360 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package mocks
-
-import (
-	"encoding/json"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/filters"
-)
-
-var TransferBlock1 = core.Block{
-	Hash:   "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
-	Number: 6194633,
-	Transactions: []core.TransactionModel{{
-		GasLimit: 0,
-		GasPrice: 0,
-		Hash:     "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa",
-		Nonce:    0,
-		Receipt: core.Receipt{
-			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa",
-			ContractAddress: "",
-			Logs: []core.FullSyncLog{{
-				BlockNumber: 6194633,
-				TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa",
-				Address:     constants.TusdContractAddress,
-				Topics: core.Topics{
-					constants.TransferEvent.Signature(),
-					"0x000000000000000000000000000000000000000000000000000000000000af21",
-					"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
-					"",
-				},
-				Index: 1,
-				Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
-			}},
-		},
-		TxIndex: 0,
-		Value:   "0",
-	}},
-}
-
-var TransferBlock2 = core.Block{
-	Hash:   "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ooo",
-	Number: 6194634,
-	Transactions: []core.TransactionModel{{
-		GasLimit: 0,
-		GasPrice: 0,
-		Hash:     "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee",
-		Nonce:    0,
-		Receipt: core.Receipt{
-			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee",
-			ContractAddress: "",
-			Logs: []core.FullSyncLog{{
-				BlockNumber: 6194634,
-				TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee",
-				Address:     constants.TusdContractAddress,
-				Topics: core.Topics{
-					constants.TransferEvent.Signature(),
-					"0x000000000000000000000000000000000000000000000000000000000000af21",
-					"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
-					"",
-				},
-				Index: 1,
-				Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
-			}},
-		},
-		TxIndex: 0,
-		Value:   "0",
-	}},
-}
-
-var NewOwnerBlock1 = core.Block{
-	Hash:   "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ppp",
-	Number: 6194635,
-	Transactions: []core.TransactionModel{{
-		GasLimit: 0,
-		GasPrice: 0,
-		Hash:     "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb",
-		Nonce:    0,
-		Receipt: core.Receipt{
-			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb",
-			ContractAddress: "",
-			Logs: []core.FullSyncLog{{
-				BlockNumber: 6194635,
-				TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb",
-				Address:     constants.EnsContractAddress,
-				Topics: core.Topics{
-					constants.NewOwnerEvent.Signature(),
-					"0x0000000000000000000000000000000000000000000000000000c02aaa39b223",
-					"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
-					"",
-				},
-				Index: 1,
-				Data:  "0x000000000000000000000000000000000000000000000000000000000000af21",
-			}},
-		},
-		TxIndex: 0,
-		Value:   "0",
-	}},
-}
-
-var NewOwnerBlock2 = core.Block{
-	Hash:   "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ggg",
-	Number: 6194636,
-	Transactions: []core.TransactionModel{{
-		GasLimit: 0,
-		GasPrice: 0,
-		Hash:     "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll",
-		Nonce:    0,
-		Receipt: core.Receipt{
-			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll",
-			ContractAddress: "",
-			Logs: []core.FullSyncLog{{
-				BlockNumber: 6194636,
-				TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll",
-				Address:     constants.EnsContractAddress,
-				Topics: core.Topics{
-					constants.NewOwnerEvent.Signature(),
-					"0x0000000000000000000000000000000000000000000000000000c02aaa39b223",
-					"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba400",
-					"",
-				},
-				Index: 1,
-				Data:  "0x000000000000000000000000000000000000000000000000000000000000af21",
-			}},
-		},
-		TxIndex: 0,
-		Value:   "0",
-	}},
-}
-
-var ExpectedTransferFilter = filters.LogFilter{
-	Name:      constants.TusdContractAddress + "_" + "Transfer",
-	Address:   constants.TusdContractAddress,
-	ToBlock:   -1,
-	FromBlock: 6194634,
-	Topics:    core.Topics{constants.TransferEvent.Signature()},
-}
-
-var ExpectedApprovalFilter = filters.LogFilter{
-	Name:      constants.TusdContractAddress + "_" + "Approval",
-	Address:   constants.TusdContractAddress,
-	ToBlock:   -1,
-	FromBlock: 6194634,
-	Topics:    core.Topics{constants.ApprovalEvent.Signature()},
-}
-
-var MockTranferEvent = core.WatchedEvent{
-	LogID:       1,
-	Name:        constants.TransferEvent.String(),
-	BlockNumber: 5488076,
-	Address:     constants.TusdContractAddress,
-	TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-	Index:       110,
-	Topic0:      constants.TransferEvent.Signature(),
-	Topic1:      "0x000000000000000000000000000000000000000000000000000000000000af21",
-	Topic2:      "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
-	Topic3:      "",
-	Data:        "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
-}
-
-var rawFakeHeader, _ = json.Marshal(core.Header{})
-
-var MockHeader1 = core.Header{
-	Hash:        "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
-	BlockNumber: 6194632,
-	Raw:         rawFakeHeader,
-	Timestamp:   "50000000",
-}
-
-var MockHeader2 = core.Header{
-	Hash:        "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
-	BlockNumber: 6194633,
-	Raw:         rawFakeHeader,
-	Timestamp:   "50000015",
-}
-
-var MockHeader3 = core.Header{
-	Hash:        "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
-	BlockNumber: 6194634,
-	Raw:         rawFakeHeader,
-	Timestamp:   "50000030",
-}
-
-var MockHeader4 = core.Header{
-	Hash:        "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
-	BlockNumber: 6194635,
-	Raw:         rawFakeHeader,
-	Timestamp:   "50000030",
-}
-
-var MockTransferLog1 = types.Log{
-	Index:       1,
-	Address:     common.HexToAddress(constants.TusdContractAddress),
-	BlockNumber: 5488076,
-	TxIndex:     110,
-	TxHash:      common.HexToHash("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae"),
-	Topics: []common.Hash{
-		common.HexToHash(constants.TransferEvent.Signature()),
-		common.HexToHash("0x000000000000000000000000000000000000000000000000000000000000af21"),
-		common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"),
-	},
-	Data: hexutil.MustDecode("0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe"),
-}
-
-var MockTransferLog2 = types.Log{
-	Index:       3,
-	Address:     common.HexToAddress(constants.TusdContractAddress),
-	BlockNumber: 5488077,
-	TxIndex:     2,
-	TxHash:      common.HexToHash("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546df"),
-	Topics: []common.Hash{
-		common.HexToHash(constants.TransferEvent.Signature()),
-		common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"),
-		common.HexToHash("0x000000000000000000000000000000000000000000000000000000000000af21"),
-	},
-	Data: hexutil.MustDecode("0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe"),
-}
-
-var MockMintLog = types.Log{
-	Index:       10,
-	Address:     common.HexToAddress(constants.TusdContractAddress),
-	BlockNumber: 5488080,
-	TxIndex:     50,
-	TxHash:      common.HexToHash("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6minty"),
-	Topics: []common.Hash{
-		common.HexToHash(constants.MintEvent.Signature()),
-		common.HexToHash("0x000000000000000000000000000000000000000000000000000000000000af21"),
-	},
-	Data: hexutil.MustDecode("0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe"),
-}
-
-var MockNewOwnerLog1 = types.Log{
-	Index:       1,
-	Address:     common.HexToAddress(constants.EnsContractAddress),
-	BlockNumber: 5488076,
-	TxIndex:     110,
-	TxHash:      common.HexToHash("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae"),
-	Topics: []common.Hash{
-		common.HexToHash(constants.NewOwnerEvent.Signature()),
-		common.HexToHash("0x000000000000000000000000c02aaa39b223helloa0e5c4f27ead9083c752553"),
-		common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"),
-	},
-	Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000af21"),
-}
-
-var MockNewOwnerLog2 = types.Log{
-	Index:       3,
-	Address:     common.HexToAddress(constants.EnsContractAddress),
-	BlockNumber: 5488077,
-	TxIndex:     2,
-	TxHash:      common.HexToHash("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546df"),
-	Topics: []common.Hash{
-		common.HexToHash(constants.NewOwnerEvent.Signature()),
-		common.HexToHash("0x000000000000000000000000c02aaa39b223helloa0e5c4f27ead9083c752553"),
-		common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba400"),
-	},
-	Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000af21"),
-}
-
-var MockOrderCreatedLog = types.Log{
-	Address: common.HexToAddress(constants.MarketPlaceContractAddress),
-	Topics: []common.Hash{
-		common.HexToHash("0x84c66c3f7ba4b390e20e8e8233e2a516f3ce34a72749e4f12bd010dfba238039"),
-		common.HexToHash("0xffffffffffffffffffffffffffffff72ffffffffffffffffffffffffffffffd0"),
-		common.HexToHash("0x00000000000000000000000083b7b6f360a9895d163ea797d9b939b9173b292a"),
-	},
-	Data:        hexutil.MustDecode("0x633f94affdcabe07c000231f85c752c97b9cc43966b432ec4d18641e6d178233000000000000000000000000f87e31492faf9a91b02ee0deaad50d51d56d5d4d0000000000000000000000000000000000000000000003da9fbcf4446d6000000000000000000000000000000000000000000000000000000000016db2524880"),
-	BlockNumber: 8587618,
-	TxHash:      common.HexToHash("0x7ad9e2f88416738f3c7ad0a6d260f71794532206a0e838299f5014b4fe81e66e"),
-	TxIndex:     93,
-	BlockHash:   common.HexToHash("0x06a1762b7f2e070793fc24cd785de0fa485e728832c4f3469790153ae51a56a2"),
-	Index:       59,
-	Removed:     false,
-}
-
-var MockSubmitVoteLog = types.Log{
-	Address: common.HexToAddress(constants.MolochContractAddress),
-	Topics: []common.Hash{
-		common.HexToHash("0x29bf0061f2faa9daa482f061b116195432d435536d8af4ae6b3c5dd78223679b"),
-		common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000061"),
-		common.HexToHash("0x0000000000000000000000006ddf1b8e6d71b5b33912607098be123ffe62ae53"),
-		common.HexToHash("0x00000000000000000000000037385081870ef47e055410fefd582e2a95d2960b"),
-	},
-	Data:        hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000001"),
-	BlockNumber: 8517621,
-	TxHash:      common.HexToHash("0xcc7390a2099812d0dfc9baef201afbc7a44bfae145050c9dc700b77cbd3cd752"),
-	TxIndex:     103,
-	BlockHash:   common.HexToHash("0x3e82681d8036b1225fcaa8bcd4cdbe757b39f13468286b303cde22146385525e"),
-	Index:       132,
-	Removed:     false,
-}
-
-var ens = strings.ToLower(constants.EnsContractAddress)
-var tusd = strings.ToLower(constants.TusdContractAddress)
-
-var MockConfig = config.ContractConfig{
-	Network: "",
-	Addresses: map[string]bool{
-		"0x1234567890abcdef": true,
-	},
-	Abis: map[string]string{
-		"0x1234567890abcdef": "fake_abi",
-	},
-	Events: map[string][]string{
-		"0x1234567890abcdef": {"Transfer"},
-	},
-	Methods: map[string][]string{
-		"0x1234567890abcdef": nil,
-	},
-	MethodArgs: map[string][]string{
-		"0x1234567890abcdef": nil,
-	},
-	EventArgs: map[string][]string{
-		"0x1234567890abcdef": nil,
-	},
-}
-
-var MockEmptyConfig = config.ContractConfig{
-	Network: "",
-	Addresses: map[string]bool{
-		"0x1234567890abcdef": true,
-	},
-	Abis: map[string]string{
-		"0x1234567890abcdef": "fake_abi",
-	},
-	Events: map[string][]string{
-		"0x1234567890abcdef": nil,
-	},
-	Methods: map[string][]string{
-		"0x1234567890abcdef": nil,
-	},
-	MethodArgs: map[string][]string{
-		"0x1234567890abcdef": nil,
-	},
-	EventArgs: map[string][]string{
-		"0x1234567890abcdef": nil,
-	},
-}

pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks/parser.go

@@ -1,165 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package mocks
-
-import (
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-)
-
-// Mock parser
-// Is given ABI string instead of address
-// Performs all other functions of the real parser
-type mockParser struct {
-	abi       string
-	parsedAbi abi.ABI
-}
-
-func NewParser(abi string) parser.Parser {
-	return &mockParser{
-		abi: abi,
-	}
-}
-
-func (p *mockParser) Abi() string {
-	return p.abi
-}
-
-func (p *mockParser) ParsedAbi() abi.ABI {
-	return p.parsedAbi
-}
-
-func (p *mockParser) ParseAbiStr(abiStr string) error {
-	panic("implement me")
-}
-
-// Retrieves and parses the abi string
-// for the given contract address
-func (p *mockParser) Parse(contractAddr string) error {
-	var err error
-	p.parsedAbi, err = eth.ParseAbi(p.abi)
-
-	return err
-}
-
-// Returns only specified methods, if they meet the criteria
-// Returns as array with methods in same order they were specified
-// Nil wanted array => no events are returned
-func (p *mockParser) GetSelectMethods(wanted []string) []types.Method {
-	wLen := len(wanted)
-	if wLen == 0 {
-		return nil
-	}
-	methods := make([]types.Method, wLen)
-	for _, m := range p.parsedAbi.Methods {
-		for i, name := range wanted {
-			if name == m.Name && okTypes(m, wanted) {
-				methods[i] = types.NewMethod(m)
-			}
-		}
-	}
-
-	return methods
-}
-
-// Returns wanted methods
-// Empty wanted array => all methods are returned
-// Nil wanted array => no methods are returned
-func (p *mockParser) GetMethods(wanted []string) []types.Method {
-	if wanted == nil {
-		return nil
-	}
-	methods := make([]types.Method, 0)
-	length := len(wanted)
-	for _, m := range p.parsedAbi.Methods {
-		if length == 0 || stringInSlice(wanted, m.Name) {
-			methods = append(methods, types.NewMethod(m))
-		}
-	}
-
-	return methods
-}
-
-// Returns wanted events as map of types.Events
-// If no events are specified, all events are returned
-func (p *mockParser) GetEvents(wanted []string) map[string]types.Event {
-	events := map[string]types.Event{}
-
-	for _, e := range p.parsedAbi.Events {
-		if len(wanted) == 0 || stringInSlice(wanted, e.Name) {
-			event := types.NewEvent(e)
-			events[e.Name] = event
-		}
-	}
-
-	return events
-}
-
-func stringInSlice(list []string, s string) bool {
-	for _, b := range list {
-		if b == s {
-			return true
-		}
-	}
-
-	return false
-}
-
-func okTypes(m abi.Method, wanted []string) bool {
-	// Only return method if it has less than 3 arguments, a single output value, and it is a method we want or we want all methods (empty 'wanted' slice)
-	if len(m.Inputs) < 3 && len(m.Outputs) == 1 && (len(wanted) == 0 || stringInSlice(wanted, m.Name)) {
-		// Only return methods if inputs are all of accepted types and output is of the accepted types
-		if !okReturnType(m.Outputs[0]) {
-			return false
-		}
-		for _, input := range m.Inputs {
-			switch input.Type.T {
-			case abi.AddressTy, abi.HashTy, abi.BytesTy, abi.FixedBytesTy:
-			default:
-				return false
-			}
-		}
-
-		return true
-	}
-
-	return false
-}
-
-func okReturnType(arg abi.Argument) bool {
-	wantedTypes := []byte{
-		abi.UintTy,
-		abi.IntTy,
-		abi.BoolTy,
-		abi.StringTy,
-		abi.AddressTy,
-		abi.HashTy,
-		abi.BytesTy,
-		abi.FixedBytesTy,
-		abi.FixedPointTy,
-	}
-
-	for _, ty := range wantedTypes {
-		if arg.Type.T == ty {
-			return true
-		}
-	}
-
-	return false
-}

pkg/eth/contract_watcher/shared/helpers/test_helpers/test_data.go

@@ -1,109 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package test_helpers
-
-import (
-	"strings"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-)
-
-var ens = strings.ToLower(constants.EnsContractAddress)
-var tusd = strings.ToLower(constants.TusdContractAddress)
-
-var TusdConfig = config.ContractConfig{
-	Network: "",
-	Addresses: map[string]bool{
-		tusd: true,
-	},
-	Abis: map[string]string{
-		tusd: "",
-	},
-	Events: map[string][]string{
-		tusd: {"Transfer"},
-	},
-	Methods: map[string][]string{
-		tusd: nil,
-	},
-	MethodArgs: map[string][]string{
-		tusd: nil,
-	},
-	EventArgs: map[string][]string{
-		tusd: nil,
-	},
-	StartingBlocks: map[string]int64{
-		tusd: 5197514,
-	},
-}
-
-var ENSConfig = config.ContractConfig{
-	Network: "",
-	Addresses: map[string]bool{
-		ens: true,
-	},
-	Abis: map[string]string{
-		ens: "",
-	},
-	Events: map[string][]string{
-		ens: {"NewOwner"},
-	},
-	Methods: map[string][]string{
-		ens: nil,
-	},
-	MethodArgs: map[string][]string{
-		ens: nil,
-	},
-	EventArgs: map[string][]string{
-		ens: nil,
-	},
-	StartingBlocks: map[string]int64{
-		ens: 3327417,
-	},
-}
-
-var ENSandTusdConfig = config.ContractConfig{
-	Network: "",
-	Addresses: map[string]bool{
-		ens:  true,
-		tusd: true,
-	},
-	Abis: map[string]string{
-		ens:  "",
-		tusd: "",
-	},
-	Events: map[string][]string{
-		ens:  {"NewOwner"},
-		tusd: {"Transfer"},
-	},
-	Methods: map[string][]string{
-		ens:  nil,
-		tusd: nil,
-	},
-	MethodArgs: map[string][]string{
-		ens:  nil,
-		tusd: nil,
-	},
-	EventArgs: map[string][]string{
-		ens:  nil,
-		tusd: nil,
-	},
-	StartingBlocks: map[string]int64{
-		ens:  3327417,
-		tusd: 5197514,
-	},
-}

pkg/eth/contract_watcher/shared/parser/parser.go

@@ -1,221 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package parser
-
-import (
-	"errors"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/common"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-)
-
-// Parser is used to fetch and parse contract ABIs
-// It is dependent on etherscan's api
-type Parser interface {
-	Parse(contractAddr string) error
-	ParseAbiStr(abiStr string) error
-	Abi() string
-	ParsedAbi() abi.ABI
-	GetMethods(wanted []string) []types.Method
-	GetSelectMethods(wanted []string) []types.Method
-	GetEvents(wanted []string) map[string]types.Event
-}
-
-type parser struct {
-	client    *eth.EtherScanAPI
-	abi       string
-	parsedAbi abi.ABI
-}
-
-// NewParser returns a new Parser
-func NewParser(network string) Parser {
-	url := eth.GenURL(network)
-
-	return &parser{
-		client: eth.NewEtherScanClient(url),
-	}
-}
-
-// Abi returns the parser's configured abi string
-func (p *parser) Abi() string {
-	return p.abi
-}
-
-// ParsedAbi returns the parser's parsed abi
-func (p *parser) ParsedAbi() abi.ABI {
-	return p.parsedAbi
-}
-
-// Parse retrieves and parses the abi string
-// for the given contract address
-func (p *parser) Parse(contractAddr string) error {
-	// If the abi is one our locally stored abis, fetch
-	// TODO: Allow users to pass abis through config
-	knownAbi, err := p.lookUp(contractAddr)
-	if err == nil {
-		p.abi = knownAbi
-		p.parsedAbi, err = eth.ParseAbi(knownAbi)
-		return err
-	}
-	// Try getting abi from etherscan
-	abiStr, err := p.client.GetAbi(contractAddr)
-	if err != nil {
-		return err
-	}
-	//TODO: Implement other ways to fetch abi
-	p.abi = abiStr
-	p.parsedAbi, err = eth.ParseAbi(abiStr)
-
-	return err
-}
-
-// ParseAbiStr loads and parses an abi from a given abi string
-func (p *parser) ParseAbiStr(abiStr string) error {
-	var err error
-	p.abi = abiStr
-	p.parsedAbi, err = eth.ParseAbi(abiStr)
-
-	return err
-}
-
-func (p *parser) lookUp(contractAddr string) (string, error) {
-	if v, ok := constants.ABIs[common.HexToAddress(contractAddr)]; ok {
-		return v, nil
-	}
-
-	return "", errors.New("ABI not present in lookup table")
-}
-
-// GetSelectMethods returns only specified methods, if they meet the criteria
-// Returns as array with methods in same order they were specified
-// Nil or empty wanted array => no events are returned
-func (p *parser) GetSelectMethods(wanted []string) []types.Method {
-	wLen := len(wanted)
-	if wLen == 0 {
-		return nil
-	}
-	methods := make([]types.Method, wLen)
-	for _, m := range p.parsedAbi.Methods {
-		for i, name := range wanted {
-			if name == m.Name && okTypes(m, wanted) {
-				methods[i] = types.NewMethod(m)
-			}
-		}
-	}
-
-	return methods
-}
-
-// GetMethods returns wanted methods
-// Empty wanted array => all methods are returned
-// Nil wanted array => no methods are returned
-func (p *parser) GetMethods(wanted []string) []types.Method {
-	if wanted == nil {
-		return nil
-	}
-	methods := make([]types.Method, 0)
-	length := len(wanted)
-	for _, m := range p.parsedAbi.Methods {
-		if length == 0 || stringInSlice(wanted, m.Name) {
-			methods = append(methods, types.NewMethod(m))
-		}
-	}
-
-	return methods
-}
-
-// GetEvents returns wanted events as map of types.Events
-// Empty wanted array => all events are returned
-// Nil wanted array => no events are returned
-func (p *parser) GetEvents(wanted []string) map[string]types.Event {
-	events := map[string]types.Event{}
-	if wanted == nil {
-		return events
-	}
-
-	length := len(wanted)
-	for _, e := range p.parsedAbi.Events {
-		if length == 0 || stringInSlice(wanted, e.Name) {
-			events[e.Name] = types.NewEvent(e)
-		}
-	}
-
-	return events
-}
-
-func okReturnType(arg abi.Argument) bool {
-	wantedTypes := []byte{
-		abi.UintTy,
-		abi.IntTy,
-		abi.BoolTy,
-		abi.StringTy,
-		abi.AddressTy,
-		abi.HashTy,
-		abi.BytesTy,
-		abi.FixedBytesTy,
-		abi.FixedPointTy,
-	}
-
-	for _, ty := range wantedTypes {
-		if arg.Type.T == ty {
-			return true
-		}
-	}
-
-	return false
-}
-
-func okTypes(m abi.Method, wanted []string) bool {
-	// Only return method if it has less than 3 arguments, a single output value, and it is a method we want or we want all methods (empty 'wanted' slice)
-	if len(m.Inputs) < 3 && len(m.Outputs) == 1 && (len(wanted) == 0 || stringInSlice(wanted, m.Name)) {
-		// Only return methods if inputs are all of accepted types and output is of the accepted types
-		if !okReturnType(m.Outputs[0]) {
-			return false
-		}
-		for _, input := range m.Inputs {
-			switch input.Type.T {
-			// Addresses are properly labeled and caught
-			// But hashes tend to not be explicitly labeled and caught
-			// Instead bytes32 are assumed to be hashes
-			case abi.AddressTy, abi.HashTy:
-			case abi.FixedBytesTy:
-				if input.Type.Size != 32 {
-					return false
-				}
-			default:
-				return false
-			}
-		}
-		return true
-	}
-
-	return false
-}
-
-func stringInSlice(list []string, s string) bool {
-	for _, b := range list {
-		if b == s {
-			return true
-		}
-	}
-
-	return false
-}

pkg/eth/contract_watcher/shared/parser/parser_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package parser_test
-
-import (
-	"io/ioutil"
-	"log"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestParser(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Parser Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/shared/parser/parser_test.go

@@ -1,226 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package parser_test
-
-import (
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-)
-
-var _ = Describe("Parser", func() {
-
-	var p parser.Parser
-	var err error
-
-	BeforeEach(func() {
-		p = parser.NewParser("")
-	})
-
-	Describe("Mock Parse", func() {
-		It("Uses parses given abi string", func() {
-			mp := mocks.NewParser(constants.DaiAbiString)
-			err = mp.Parse(constants.DaiContractAddress)
-			Expect(err).ToNot(HaveOccurred())
-
-			parsedAbi := mp.ParsedAbi()
-			expectedAbi, err := eth.ParseAbi(constants.DaiAbiString)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(parsedAbi).To(Equal(expectedAbi))
-
-			methods := mp.GetSelectMethods([]string{"balanceOf"})
-			Expect(len(methods)).To(Equal(1))
-			balOf := methods[0]
-			Expect(balOf.Name).To(Equal("balanceOf"))
-			Expect(len(balOf.Args)).To(Equal(1))
-			Expect(len(balOf.Return)).To(Equal(1))
-
-			events := mp.GetEvents([]string{"Transfer"})
-			_, ok := events["Mint"]
-			Expect(ok).To(Equal(false))
-			e, ok := events["Transfer"]
-			Expect(ok).To(Equal(true))
-			Expect(len(e.Fields)).To(Equal(3))
-		})
-	})
-
-	Describe("Parse", func() {
-		It("Fetches and parses abi from etherscan using contract address", func() {
-			contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359" // dai contract address
-			err = p.Parse(contractAddr)
-			Expect(err).ToNot(HaveOccurred())
-
-			expectedAbi := constants.DaiAbiString
-			Expect(p.Abi()).To(Equal(expectedAbi))
-
-			expectedParsedAbi, err := eth.ParseAbi(expectedAbi)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(p.ParsedAbi()).To(Equal(expectedParsedAbi))
-		})
-
-		It("Fails with a normal, non-contract, account address", func() {
-			addr := "0xAb2A8F7cB56D9EC65573BA1bE0f92Fa2Ff7dd165"
-			err = p.Parse(addr)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("GetEvents", func() {
-		It("Returns parsed events", func() {
-			contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"
-			err = p.Parse(contractAddr)
-			Expect(err).ToNot(HaveOccurred())
-
-			events := p.GetEvents([]string{"Transfer"})
-
-			e, ok := events["Transfer"]
-			Expect(ok).To(Equal(true))
-
-			abiTy := e.Fields[0].Type.T
-			Expect(abiTy).To(Equal(abi.AddressTy))
-
-			pgTy := e.Fields[0].PgType
-			Expect(pgTy).To(Equal("CHARACTER VARYING(66)"))
-
-			abiTy = e.Fields[1].Type.T
-			Expect(abiTy).To(Equal(abi.AddressTy))
-
-			pgTy = e.Fields[1].PgType
-			Expect(pgTy).To(Equal("CHARACTER VARYING(66)"))
-
-			abiTy = e.Fields[2].Type.T
-			Expect(abiTy).To(Equal(abi.UintTy))
-
-			pgTy = e.Fields[2].PgType
-			Expect(pgTy).To(Equal("NUMERIC"))
-
-			_, ok = events["Approval"]
-			Expect(ok).To(Equal(false))
-		})
-	})
-
-	Describe("GetSelectMethods", func() {
-		It("Parses and returns only methods specified in passed array", func() {
-			contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"
-			err = p.Parse(contractAddr)
-			Expect(err).ToNot(HaveOccurred())
-
-			methods := p.GetSelectMethods([]string{"balanceOf"})
-			Expect(len(methods)).To(Equal(1))
-
-			balOf := methods[0]
-			Expect(balOf.Name).To(Equal("balanceOf"))
-			Expect(len(balOf.Args)).To(Equal(1))
-			Expect(len(balOf.Return)).To(Equal(1))
-
-			abiTy := balOf.Args[0].Type.T
-			Expect(abiTy).To(Equal(abi.AddressTy))
-
-			pgTy := balOf.Args[0].PgType
-			Expect(pgTy).To(Equal("CHARACTER VARYING(66)"))
-
-			abiTy = balOf.Return[0].Type.T
-			Expect(abiTy).To(Equal(abi.UintTy))
-
-			pgTy = balOf.Return[0].PgType
-			Expect(pgTy).To(Equal("NUMERIC"))
-
-		})
-
-		It("Parses and returns methods in the order they were specified", func() {
-			contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"
-			err = p.Parse(contractAddr)
-			Expect(err).ToNot(HaveOccurred())
-
-			selectMethods := p.GetSelectMethods([]string{"balanceOf", "allowance"})
-			Expect(len(selectMethods)).To(Equal(2))
-
-			balOf := selectMethods[0]
-			allow := selectMethods[1]
-
-			Expect(balOf.Name).To(Equal("balanceOf"))
-			Expect(allow.Name).To(Equal("allowance"))
-		})
-
-		It("Returns nil if given a nil or empty array", func() {
-			contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"
-			err = p.Parse(contractAddr)
-			Expect(err).ToNot(HaveOccurred())
-
-			var nilArr []types.Method
-			selectMethods := p.GetSelectMethods([]string{})
-			Expect(selectMethods).To(Equal(nilArr))
-			selectMethods = p.GetMethods(nil)
-			Expect(selectMethods).To(Equal(nilArr))
-		})
-
-	})
-
-	Describe("GetMethods", func() {
-		It("Parses and returns only methods specified in passed array", func() {
-			contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"
-			err = p.Parse(contractAddr)
-			Expect(err).ToNot(HaveOccurred())
-
-			methods := p.GetMethods([]string{"balanceOf"})
-			Expect(len(methods)).To(Equal(1))
-
-			balOf := methods[0]
-			Expect(balOf.Name).To(Equal("balanceOf"))
-			Expect(len(balOf.Args)).To(Equal(1))
-			Expect(len(balOf.Return)).To(Equal(1))
-
-			abiTy := balOf.Args[0].Type.T
-			Expect(abiTy).To(Equal(abi.AddressTy))
-
-			pgTy := balOf.Args[0].PgType
-			Expect(pgTy).To(Equal("CHARACTER VARYING(66)"))
-
-			abiTy = balOf.Return[0].Type.T
-			Expect(abiTy).To(Equal(abi.UintTy))
-
-			pgTy = balOf.Return[0].PgType
-			Expect(pgTy).To(Equal("NUMERIC"))
-
-		})
-
-		It("Returns nil if given a nil array", func() {
-			contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"
-			err = p.Parse(contractAddr)
-			Expect(err).ToNot(HaveOccurred())
-
-			var nilArr []types.Method
-			selectMethods := p.GetMethods(nil)
-			Expect(selectMethods).To(Equal(nilArr))
-		})
-
-		It("Returns every method if given an empty array", func() {
-			contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"
-			err = p.Parse(contractAddr)
-			Expect(err).ToNot(HaveOccurred())
-
-			selectMethods := p.GetMethods([]string{})
-			Expect(len(selectMethods)).To(Equal(25))
-		})
-	})
-})

pkg/eth/contract_watcher/shared/poller/poller.go

@@ -1,291 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package poller
-
-import (
-	"errors"
-	"fmt"
-	"math/big"
-	"strconv"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-// Poller is the interface for polling public contract methods
-type Poller interface {
-	PollContract(con contract.Contract, lastBlock int64) error
-	PollContractAt(con contract.Contract, blockNumber int64) error
-	FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error
-}
-
-type poller struct {
-	repository.MethodRepository
-	bc       core.BlockChain
-	contract contract.Contract
-}
-
-// NewPoller returns a new Poller
-func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) Poller {
-	return &poller{
-		MethodRepository: repository.NewMethodRepository(db, mode),
-		bc:               blockChain,
-	}
-}
-
-// PollContract polls a contract's public methods from the contracts starting block to specified last block
-func (p *poller) PollContract(con contract.Contract, lastBlock int64) error {
-	for i := con.StartingBlock; i <= lastBlock; i++ {
-		if err := p.PollContractAt(con, i); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// PollContractAt polls a contract's public getter methods at the specified block height
-func (p *poller) PollContractAt(con contract.Contract, blockNumber int64) error {
-	p.contract = con
-	for _, m := range con.Methods {
-		switch len(m.Args) {
-		case 0:
-			if err := p.pollNoArgAt(m, blockNumber); err != nil {
-				return err
-			}
-		case 1:
-			if err := p.pollSingleArgAt(m, blockNumber); err != nil {
-				return err
-			}
-		case 2:
-			if err := p.pollDoubleArgAt(m, blockNumber); err != nil {
-				return err
-			}
-		default:
-			return errors.New("poller error: too many arguments to handle")
-
-		}
-	}
-
-	return nil
-}
-
-func (p *poller) pollNoArgAt(m types.Method, bn int64) error {
-	result := types.Result{
-		Block:  bn,
-		Method: m,
-		Inputs: nil,
-		PgType: m.Return[0].PgType,
-	}
-
-	var out interface{}
-	err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, bn)
-	if err != nil {
-		return fmt.Errorf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
-	}
-	strOut, err := stringify(out)
-	if err != nil {
-		return err
-	}
-
-	// Cache returned value if piping is turned on
-	p.cache(out)
-	result.Output = strOut
-
-	// Persist result immediately
-	err = p.PersistResults([]types.Result{result}, m, p.contract.Address, p.contract.Name)
-	if err != nil {
-		return fmt.Errorf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
-	}
-
-	return nil
-}
-
-// Use token holder address to poll methods that take 1 address argument (e.g. balanceOf)
-func (p *poller) pollSingleArgAt(m types.Method, bn int64) error {
-	result := types.Result{
-		Block:  bn,
-		Method: m,
-		Inputs: make([]interface{}, 1),
-		PgType: m.Return[0].PgType,
-	}
-
-	// Depending on the type of the arg choose
-	// the correct argument set to iterate over
-	var args map[interface{}]bool
-	switch m.Args[0].Type.T {
-	case abi.HashTy, abi.FixedBytesTy:
-		args = p.contract.EmittedHashes
-	case abi.AddressTy:
-		args = p.contract.EmittedAddrs
-	}
-	if len(args) == 0 { // If we haven't collected any args by now we can't call the method
-		return nil
-	}
-
-	results := make([]types.Result, 0, len(args))
-	for arg := range args {
-		in := []interface{}{arg}
-		strIn := []interface{}{contract.StringifyArg(arg)}
-
-		var out interface{}
-		err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
-		if err != nil {
-			return fmt.Errorf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
-		}
-		strOut, err := stringify(out)
-		if err != nil {
-			return err
-		}
-		p.cache(out)
-
-		// Write inputs and outputs to result and append result to growing set
-		result.Inputs = strIn
-		result.Output = strOut
-		results = append(results, result)
-	}
-	// Persist result set as batch
-	err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
-	if err != nil {
-		return fmt.Errorf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
-	}
-
-	return nil
-}
-
-// Use token holder address to poll methods that take 2 address arguments (e.g. allowance)
-func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error {
-	result := types.Result{
-		Block:  bn,
-		Method: m,
-		Inputs: make([]interface{}, 2),
-		PgType: m.Return[0].PgType,
-	}
-
-	// Depending on the type of the args choose
-	// the correct argument sets to iterate over
-	var firstArgs map[interface{}]bool
-	switch m.Args[0].Type.T {
-	case abi.HashTy, abi.FixedBytesTy:
-		firstArgs = p.contract.EmittedHashes
-	case abi.AddressTy:
-		firstArgs = p.contract.EmittedAddrs
-	}
-	if len(firstArgs) == 0 {
-		return nil
-	}
-
-	var secondArgs map[interface{}]bool
-	switch m.Args[1].Type.T {
-	case abi.HashTy, abi.FixedBytesTy:
-		secondArgs = p.contract.EmittedHashes
-	case abi.AddressTy:
-		secondArgs = p.contract.EmittedAddrs
-	}
-	if len(secondArgs) == 0 {
-		return nil
-	}
-
-	results := make([]types.Result, 0, len(firstArgs)*len(secondArgs))
-	for arg1 := range firstArgs {
-		for arg2 := range secondArgs {
-			in := []interface{}{arg1, arg2}
-			strIn := []interface{}{contract.StringifyArg(arg1), contract.StringifyArg(arg2)}
-
-			var out interface{}
-			err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
-			if err != nil {
-				return fmt.Errorf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
-			}
-			strOut, err := stringify(out)
-			if err != nil {
-				return err
-			}
-			p.cache(out)
-
-			result.Output = strOut
-			result.Inputs = strIn
-			results = append(results, result)
-		}
-	}
-
-	err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
-	if err != nil {
-		return fmt.Errorf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
-	}
-
-	return nil
-}
-
-// FetchContractData is just a wrapper around the poller blockchain's FetchContractData method
-func (p *poller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
-	return p.bc.FetchContractData(contractAbi, contractAddress, method, methodArgs, result, blockNumber)
-}
-
-// This is used to cache a method return value if method piping is turned on
-func (p *poller) cache(out interface{}) {
-	if p.contract.Piping {
-		switch out.(type) {
-		case common.Hash:
-			if p.contract.EmittedHashes != nil {
-				p.contract.AddEmittedHash(out.(common.Hash))
-			}
-		case []byte:
-			if p.contract.EmittedHashes != nil && len(out.([]byte)) == 32 {
-				p.contract.AddEmittedHash(common.BytesToHash(out.([]byte)))
-			}
-		case common.Address:
-			if p.contract.EmittedAddrs != nil {
-				p.contract.AddEmittedAddr(out.(common.Address))
-			}
-		default:
-		}
-	}
-}
-
-func stringify(input interface{}) (string, error) {
-	switch input.(type) {
-	case *big.Int:
-		b := input.(*big.Int)
-		return b.String(), nil
-	case common.Address:
-		a := input.(common.Address)
-		return a.String(), nil
-	case common.Hash:
-		h := input.(common.Hash)
-		return h.String(), nil
-	case string:
-		return input.(string), nil
-	case []byte:
-		b := hexutil.Encode(input.([]byte))
-		return b, nil
-	case byte:
-		b := input.(byte)
-		return string(b), nil
-	case bool:
-		return strconv.FormatBool(input.(bool)), nil
-	default:
-		return "", errors.New("error: unhandled return type")
-	}
-}

pkg/eth/contract_watcher/shared/repository/event_repository.go

@@ -1,316 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-
-	"github.com/hashicorp/golang-lru"
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-const (
-	// Number of contract address and method ids to keep in cache
-	contractCacheSize = 100
-	eventCacheSize    = 1000
-)
-
-// EventRepository is used to persist event data into custom tables
-type EventRepository interface {
-	PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error
-	CreateEventTable(contractAddr string, event types.Event) (bool, error)
-	CreateContractSchema(contractName string) (bool, error)
-	CheckSchemaCache(key string) (interface{}, bool)
-	CheckTableCache(key string) (interface{}, bool)
-}
-
-type eventRepository struct {
-	db      *postgres.DB
-	mode    types.Mode
-	schemas *lru.Cache // Cache names of recently used schemas to minimize db connections
-	tables  *lru.Cache // Cache names of recently used tables to minimize db connections
-}
-
-// NewEventRepository returns a new EventRepository
-func NewEventRepository(db *postgres.DB, mode types.Mode) EventRepository {
-	ccs, _ := lru.New(contractCacheSize)
-	ecs, _ := lru.New(eventCacheSize)
-	return &eventRepository{
-		db:      db,
-		mode:    mode,
-		schemas: ccs,
-		tables:  ecs,
-	}
-}
-
-// PersistLogs creates a schema for the contract if needed
-// Creates table for the watched contract event if needed
-// Persists converted event log data into this custom table
-func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	if len(logs) == 0 {
-		return errors.New("event repository error: passed empty logs slice")
-	}
-	_, schemaErr := r.CreateContractSchema(contractAddr)
-	if schemaErr != nil {
-		return fmt.Errorf("error creating schema for contract %s: %s", contractAddr, schemaErr.Error())
-	}
-
-	_, tableErr := r.CreateEventTable(contractAddr, eventInfo)
-	if tableErr != nil {
-		return fmt.Errorf("error creating table for event %s on contract %s: %s", eventInfo.Name, contractAddr, tableErr.Error())
-	}
-
-	return r.persistLogs(logs, eventInfo, contractAddr, contractName)
-}
-
-func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	var err error
-	switch r.mode {
-	case types.HeaderSync:
-		err = r.persistHeaderSyncLogs(logs, eventInfo, contractAddr, contractName)
-	case types.FullSync:
-		err = r.persistFullSyncLogs(logs, eventInfo, contractAddr, contractName)
-	default:
-		return errors.New("event repository error: unhandled mode")
-	}
-
-	return err
-}
-
-// Creates a custom postgres command to persist logs for the given event (compatible with header synced vDB)
-func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, txErr := r.db.Beginx()
-	if txErr != nil {
-		return fmt.Errorf("error beginning db transaction: %s", txErr.Error())
-	}
-
-	for _, event := range logs {
-		// Begin pg query string
-		pgStr := fmt.Sprintf("INSERT INTO %s_%s.%s_event ", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(eventInfo.Name))
-		pgStr = pgStr + "(header_id, token_name, raw_log, log_idx, tx_idx"
-		el := len(event.Values)
-
-		// Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string
-		data := make([]interface{}, 0, 5+el)
-		data = append(data,
-			event.ID,
-			contractName,
-			event.Raw,
-			event.LogIndex,
-			event.TransactionIndex)
-
-		// Iterate over inputs and append name to query string and value to input data
-		for inputName, input := range event.Values {
-			pgStr = pgStr + fmt.Sprintf(", %s_", strings.ToLower(inputName)) // Add underscore after to avoid any collisions with reserved pg words
-			data = append(data, input)
-		}
-
-		// For each input entry we created we add its postgres command variable to the string
-		pgStr = pgStr + ") VALUES ($1, $2, $3, $4, $5"
-		for i := 0; i < el; i++ {
-			pgStr = pgStr + fmt.Sprintf(", $%d", i+6)
-		}
-		pgStr = pgStr + ") ON CONFLICT DO NOTHING"
-
-		logrus.Tracef("query for inserting log: %s", pgStr)
-		// Add this query to the transaction
-		_, execErr := tx.Exec(pgStr, data...)
-		if execErr != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				logrus.Warnf("error rolling back transactions while persisting logs: %s", rollbackErr.Error())
-			}
-			return fmt.Errorf("error executing query: %s", execErr.Error())
-		}
-	}
-
-	return tx.Commit()
-}
-
-// Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB)
-func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, txErr := r.db.Beginx()
-	if txErr != nil {
-		return fmt.Errorf("error beginning db transaction: %s", txErr.Error())
-	}
-
-	for _, event := range logs {
-		pgStr := fmt.Sprintf("INSERT INTO %s_%s.%s_event ", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(eventInfo.Name))
-		pgStr = pgStr + "(vulcanize_log_id, token_name, block, tx"
-		el := len(event.Values)
-
-		data := make([]interface{}, 0, 4+el)
-		data = append(data,
-			event.ID,
-			contractName,
-			event.Block,
-			event.Tx)
-
-		for inputName, input := range event.Values {
-			pgStr = pgStr + fmt.Sprintf(", %s_", strings.ToLower(inputName))
-			data = append(data, input)
-		}
-
-		pgStr = pgStr + ") VALUES ($1, $2, $3, $4"
-		for i := 0; i < el; i++ {
-			pgStr = pgStr + fmt.Sprintf(", $%d", i+5)
-		}
-		pgStr = pgStr + ") ON CONFLICT (vulcanize_log_id) DO NOTHING"
-
-		logrus.Tracef("query for inserting log: %s", pgStr)
-		_, execErr := tx.Exec(pgStr, data...)
-		if execErr != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				logrus.Warnf("error rolling back transactions while persisting logs: %s", rollbackErr.Error())
-			}
-			return fmt.Errorf("error executing query: %s", execErr.Error())
-		}
-	}
-
-	return tx.Commit()
-}
-
-// CreateEventTable checks for event table and creates it if it does not already exist
-// Returns true if it created a new table; returns false if table already existed
-func (r *eventRepository) CreateEventTable(contractAddr string, event types.Event) (bool, error) {
-	tableID := fmt.Sprintf("%s_%s.%s_event", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(event.Name))
-	// Check cache before querying pq to see if table exists
-	_, ok := r.tables.Get(tableID)
-	if ok {
-		return false, nil
-	}
-	tableExists, checkTableErr := r.checkForTable(contractAddr, event.Name)
-	if checkTableErr != nil {
-		return false, fmt.Errorf("error checking for table: %s", checkTableErr)
-	}
-
-	if !tableExists {
-		createTableErr := r.newEventTable(tableID, event)
-		if createTableErr != nil {
-			return false, fmt.Errorf("error creating table: %s", createTableErr.Error())
-		}
-	}
-
-	// Add table id to cache
-	r.tables.Add(tableID, true)
-
-	return !tableExists, nil
-}
-
-// Creates a table for the given contract and event
-func (r *eventRepository) newEventTable(tableID string, event types.Event) error {
-	// Begin pg string
-	var pgStr = fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s ", tableID)
-	var err error
-
-	// Handle different modes
-	switch r.mode {
-	case types.FullSync:
-		pgStr = pgStr + "(id SERIAL, vulcanize_log_id INTEGER NOT NULL UNIQUE, token_name CHARACTER VARYING(66) NOT NULL, block INTEGER NOT NULL, tx CHARACTER VARYING(66) NOT NULL,"
-
-		// Iterate over event fields, using their name and pgType to grow the string
-		for _, field := range event.Fields {
-			pgStr = pgStr + fmt.Sprintf(" %s_ %s NOT NULL,", strings.ToLower(field.Name), field.PgType)
-		}
-		pgStr = pgStr + " CONSTRAINT log_index_fk FOREIGN KEY (vulcanize_log_id) REFERENCES full_sync_logs (id) ON DELETE CASCADE)"
-	case types.HeaderSync:
-		pgStr = pgStr + "(id SERIAL, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, token_name CHARACTER VARYING(66) NOT NULL, raw_log JSONB, log_idx INTEGER NOT NULL, tx_idx INTEGER NOT NULL,"
-
-		for _, field := range event.Fields {
-			pgStr = pgStr + fmt.Sprintf(" %s_ %s NOT NULL,", strings.ToLower(field.Name), field.PgType)
-		}
-		pgStr = pgStr + " UNIQUE (header_id, tx_idx, log_idx))"
-	default:
-		return errors.New("unhandled repository mode")
-	}
-
-	_, err = r.db.Exec(pgStr)
-
-	return err
-}
-
-// Checks if a table already exists for the given contract and event
-func (r *eventRepository) checkForTable(contractAddr string, eventName string) (bool, error) {
-	pgStr := fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = '%s_%s' AND table_name = '%s_event')", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(eventName))
-
-	var exists bool
-	err := r.db.Get(&exists, pgStr)
-
-	return exists, err
-}
-
-// CreateContractSchema checks for contract schema and creates it if it does not already exist
-// Returns true if it created a new schema; returns false if schema already existed
-func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error) {
-	if contractAddr == "" {
-		return false, errors.New("error: no contract address specified")
-	}
-
-	// Check cache before querying pq to see if schema exists
-	_, ok := r.schemas.Get(contractAddr)
-	if ok {
-		return false, nil
-	}
-	schemaExists, checkSchemaErr := r.checkForSchema(contractAddr)
-	if checkSchemaErr != nil {
-		return false, fmt.Errorf("error checking for schema: %s", checkSchemaErr.Error())
-	}
-	if !schemaExists {
-		createSchemaErr := r.newContractSchema(contractAddr)
-		if createSchemaErr != nil {
-			return false, fmt.Errorf("error creating schema: %s", createSchemaErr.Error())
-		}
-	}
-
-	// Add schema name to cache
-	r.schemas.Add(contractAddr, true)
-
-	return !schemaExists, nil
-}
-
-// Creates a schema for the given contract
-func (r *eventRepository) newContractSchema(contractAddr string) error {
-	_, err := r.db.Exec("CREATE SCHEMA IF NOT EXISTS " + r.mode.String() + "_" + strings.ToLower(contractAddr))
-
-	return err
-}
-
-// Checks if a schema already exists for the given contract
-func (r *eventRepository) checkForSchema(contractAddr string) (bool, error) {
-	pgStr := fmt.Sprintf("SELECT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = '%s_%s')", r.mode.String(), strings.ToLower(contractAddr))
-
-	var exists bool
-	err := r.db.QueryRow(pgStr).Scan(&exists)
-
-	return exists, err
-}
-
-// CheckSchemaCache is used to query the schema name cache
-func (r *eventRepository) CheckSchemaCache(key string) (interface{}, bool) {
-	return r.schemas.Get(key)
-}
-
-// CheckTableCache is used to query the table name cache
-func (r *eventRepository) CheckTableCache(key string) (interface{}, bool) {
-	return r.tables.Get(key)
-}

pkg/eth/contract_watcher/shared/repository/method_repository.go

@@ -1,235 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-
-	"github.com/hashicorp/golang-lru"
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-const methodCacheSize = 1000
-
-// MethodRepository is used to persist public getter method data
-type MethodRepository interface {
-	PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error
-	CreateMethodTable(contractAddr string, method types.Method) (bool, error)
-	CreateContractSchema(contractAddr string) (bool, error)
-	CheckSchemaCache(key string) (interface{}, bool)
-	CheckTableCache(key string) (interface{}, bool)
-}
-
-type methodRepository struct {
-	*postgres.DB
-	mode    types.Mode
-	schemas *lru.Cache // Cache names of recently used schemas to minimize db connections
-	tables  *lru.Cache // Cache names of recently used tables to minimize db connections
-}
-
-// NewMethodRepository returns a new MethodRepository
-func NewMethodRepository(db *postgres.DB, mode types.Mode) MethodRepository {
-	ccs, _ := lru.New(contractCacheSize)
-	mcs, _ := lru.New(methodCacheSize)
-	return &methodRepository{
-		DB:      db,
-		mode:    mode,
-		schemas: ccs,
-		tables:  mcs,
-	}
-}
-
-// PersistResults creates a schema for the contract if needed
-// Creates table for the contract method if needed
-// Persists method polling data into this custom table
-func (r *methodRepository) PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
-	if len(results) == 0 {
-		return errors.New("method repository error: passed empty results slice")
-	}
-	_, err := r.CreateContractSchema(contractAddr)
-	if err != nil {
-		return err
-	}
-
-	_, err = r.CreateMethodTable(contractAddr, methodInfo)
-	if err != nil {
-		return err
-	}
-
-	return r.persistResults(results, methodInfo, contractAddr, contractName)
-}
-
-// Creates a custom postgres command to persist logs for the given event
-func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
-	tx, err := r.DB.Beginx()
-	if err != nil {
-		return err
-	}
-
-	for _, result := range results {
-		// Begin postgres string
-		pgStr := fmt.Sprintf("INSERT INTO %s_%s.%s_method ", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(result.Name))
-		pgStr = pgStr + "(token_name, block"
-		ml := len(result.Args)
-
-		// Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string
-		data := make([]interface{}, 0, 3+ml)
-		data = append(data,
-			contractName,
-			result.Block)
-
-		// Iterate over method args and return value, adding names
-		// to the string and pushing values to the slice
-		for i, arg := range result.Args {
-			pgStr = pgStr + fmt.Sprintf(", %s_", strings.ToLower(arg.Name)) // Add underscore after to avoid any collisions with reserved pg words
-			data = append(data, result.Inputs[i])
-		}
-		pgStr = pgStr + ", returned) VALUES ($1, $2"
-		data = append(data, result.Output)
-
-		// For each input entry we created we add its postgres command variable to the string
-		for i := 0; i <= ml; i++ {
-			pgStr = pgStr + fmt.Sprintf(", $%d", i+3)
-		}
-		pgStr = pgStr + ")"
-
-		// Add this query to the transaction
-		_, err = tx.Exec(pgStr, data...)
-		if err != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				logrus.Warnf("error rolling back transaction: %s", rollbackErr.Error())
-			}
-			return err
-		}
-	}
-
-	return tx.Commit()
-}
-
-// CreateMethodTable checks for event table and creates it if it does not already exist
-func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Method) (bool, error) {
-	tableID := fmt.Sprintf("%s_%s.%s_method", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name))
-
-	// Check cache before querying pq to see if table exists
-	_, ok := r.tables.Get(tableID)
-	if ok {
-		return false, nil
-	}
-	tableExists, err := r.checkForTable(contractAddr, method.Name)
-	if err != nil {
-		return false, err
-	}
-	if !tableExists {
-		err = r.newMethodTable(tableID, method)
-		if err != nil {
-			return false, err
-		}
-	}
-
-	// Add schema name to cache
-	r.tables.Add(tableID, true)
-
-	return !tableExists, nil
-}
-
-// Creates a table for the given contract and event
-func (r *methodRepository) newMethodTable(tableID string, method types.Method) error {
-	// Begin pg string
-	pgStr := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s ", tableID)
-	pgStr = pgStr + "(id SERIAL, token_name CHARACTER VARYING(66) NOT NULL, block INTEGER NOT NULL,"
-
-	// Iterate over method inputs and outputs, using their name and pgType to grow the string
-	for _, arg := range method.Args {
-		pgStr = pgStr + fmt.Sprintf(" %s_ %s NOT NULL,", strings.ToLower(arg.Name), arg.PgType)
-	}
-
-	pgStr = pgStr + fmt.Sprintf(" returned %s NOT NULL)", method.Return[0].PgType)
-
-	_, err := r.DB.Exec(pgStr)
-
-	return err
-}
-
-// Checks if a table already exists for the given contract and event
-func (r *methodRepository) checkForTable(contractAddr string, methodName string) (bool, error) {
-	pgStr := fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = '%s_%s' AND table_name = '%s_method')", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(methodName))
-	var exists bool
-	err := r.DB.Get(&exists, pgStr)
-
-	return exists, err
-}
-
-// CreateContractSchema checks for contract schema and creates it if it does not already exist
-func (r *methodRepository) CreateContractSchema(contractAddr string) (bool, error) {
-	if contractAddr == "" {
-		return false, errors.New("error: no contract address specified")
-	}
-
-	// Check cache before querying pq to see if schema exists
-	_, ok := r.schemas.Get(contractAddr)
-	if ok {
-		return false, nil
-	}
-	schemaExists, err := r.checkForSchema(contractAddr)
-	if err != nil {
-		return false, err
-	}
-	if !schemaExists {
-		err = r.newContractSchema(contractAddr)
-		if err != nil {
-			return false, err
-		}
-	}
-
-	// Add schema name to cache
-	r.schemas.Add(contractAddr, true)
-
-	return !schemaExists, nil
-}
-
-// Creates a schema for the given contract
-func (r *methodRepository) newContractSchema(contractAddr string) error {
-	_, err := r.DB.Exec("CREATE SCHEMA IF NOT EXISTS " + r.mode.String() + "_" + strings.ToLower(contractAddr))
-
-	return err
-}
-
-// Checks if a schema already exists for the given contract
-func (r *methodRepository) checkForSchema(contractAddr string) (bool, error) {
-	pgStr := fmt.Sprintf("SELECT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = '%s_%s')", r.mode.String(), strings.ToLower(contractAddr))
-
-	var exists bool
-	err := r.DB.Get(&exists, pgStr)
-
-	return exists, err
-}
-
-// CheckSchemaCache is used to query the schema name cache
-func (r *methodRepository) CheckSchemaCache(key string) (interface{}, bool) {
-	return r.schemas.Get(key)
-}
-
-// CheckTableCache is used to query the table name cache
-func (r *methodRepository) CheckTableCache(key string) (interface{}, bool) {
-	return r.tables.Get(key)
-}

pkg/eth/contract_watcher/shared/repository/method_repository_test.go

@@ -1,240 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository_test
-
-import (
-	"fmt"
-	"strings"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-var _ = Describe("Repository", func() {
-	var db *postgres.DB
-	var dataStore repository.MethodRepository
-	var con *contract.Contract
-	var err error
-	var mockResult types.Result
-	var method types.Method
-
-	BeforeEach(func() {
-		con = test_helpers.SetupTusdContract([]string{}, []string{"balanceOf"})
-		Expect(len(con.Methods)).To(Equal(1))
-		method = con.Methods[0]
-		mockResult = types.Result{
-			Method: method,
-			PgType: method.Return[0].PgType,
-			Inputs: make([]interface{}, 1),
-			Output: new(interface{}),
-			Block:  6707323,
-		}
-		mockResult.Inputs[0] = "0xfE9e8709d3215310075d67E3ed32A380CCf451C8"
-		mockResult.Output = "66386309548896882859581786"
-		db, _ = test_helpers.SetupDBandBC()
-		dataStore = repository.NewMethodRepository(db, types.FullSync)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
-	Describe("Full Sync Mode", func() {
-		BeforeEach(func() {
-			dataStore = repository.NewMethodRepository(db, types.FullSync)
-		})
-
-		Describe("CreateContractSchema", func() {
-			It("Creates schema if it doesn't exist", func() {
-				created, err := dataStore.CreateContractSchema(constants.TusdContractAddress)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateContractSchema(constants.TusdContractAddress)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(false))
-			})
-
-			It("Caches schema it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-				_, ok := dataStore.CheckSchemaCache(con.Address)
-				Expect(ok).To(Equal(false))
-
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				v, ok := dataStore.CheckSchemaCache(con.Address)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			})
-		})
-
-		Describe("CreateMethodTable", func() {
-			It("Creates table if it doesn't exist", func() {
-				created, err := dataStore.CreateContractSchema(constants.TusdContractAddress)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(false))
-			})
-
-			It("Caches table it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				tableID := fmt.Sprintf("%s_%s.%s_method", types.FullSync, strings.ToLower(con.Address), strings.ToLower(method.Name))
-				_, ok := dataStore.CheckTableCache(tableID)
-				Expect(ok).To(Equal(false))
-
-				created, err = dataStore.CreateMethodTable(con.Address, method)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				v, ok := dataStore.CheckTableCache(tableID)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			})
-		})
-
-		Describe("PersistResult", func() {
-			It("Persists result from method polling in custom pg table", func() {
-				err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.BalanceOf{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method", constants.TusdContractAddress)).StructScan(&scanStruct)
-				expectedLog := test_helpers.BalanceOf{
-					ID:        1,
-					TokenName: "TrueUSD",
-					Block:     6707323,
-					Address:   "0xfE9e8709d3215310075d67E3ed32A380CCf451C8",
-					Balance:   "66386309548896882859581786",
-				}
-				Expect(scanStruct).To(Equal(expectedLog))
-			})
-
-			It("Fails with empty result", func() {
-				err = dataStore.PersistResults([]types.Result{}, method, con.Address, con.Name)
-				Expect(err).To(HaveOccurred())
-			})
-		})
-	})
-
-	Describe("Header Sync Mode", func() {
-		BeforeEach(func() {
-			dataStore = repository.NewMethodRepository(db, types.HeaderSync)
-		})
-
-		Describe("CreateContractSchema", func() {
-			It("Creates schema if it doesn't exist", func() {
-				created, err := dataStore.CreateContractSchema(constants.TusdContractAddress)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateContractSchema(constants.TusdContractAddress)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(false))
-			})
-
-			It("Caches schema it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-				_, ok := dataStore.CheckSchemaCache(con.Address)
-				Expect(ok).To(Equal(false))
-
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				v, ok := dataStore.CheckSchemaCache(con.Address)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			})
-		})
-
-		Describe("CreateMethodTable", func() {
-			It("Creates table if it doesn't exist", func() {
-				created, err := dataStore.CreateContractSchema(constants.TusdContractAddress)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(false))
-			})
-
-			It("Caches table it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				tableID := fmt.Sprintf("%s_%s.%s_method", types.HeaderSync, strings.ToLower(con.Address), strings.ToLower(method.Name))
-				_, ok := dataStore.CheckTableCache(tableID)
-				Expect(ok).To(Equal(false))
-
-				created, err = dataStore.CreateMethodTable(con.Address, method)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				v, ok := dataStore.CheckTableCache(tableID)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			})
-		})
-
-		Describe("PersistResult", func() {
-			It("Persists result from method polling in custom pg table for header sync mode vDB", func() {
-				err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanStruct := test_helpers.BalanceOf{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method", constants.TusdContractAddress)).StructScan(&scanStruct)
-				expectedLog := test_helpers.BalanceOf{
-					ID:        1,
-					TokenName: "TrueUSD",
-					Block:     6707323,
-					Address:   "0xfE9e8709d3215310075d67E3ed32A380CCf451C8",
-					Balance:   "66386309548896882859581786",
-				}
-				Expect(scanStruct).To(Equal(expectedLog))
-			})
-
-			It("Fails with empty result", func() {
-				err = dataStore.PersistResults([]types.Result{}, method, con.Address, con.Name)
-				Expect(err).To(HaveOccurred())
-			})
-		})
-	})
-})

pkg/eth/contract_watcher/shared/repository/repository_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
-)
-
-func TestRepository(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Shared Repository Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/shared/retriever/address_retriever.go

@@ -1,121 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/common"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/postgres"
-)
-
-// AddressRetriever is used to retrieve the addresses associated with a contract
-type AddressRetriever interface {
-	RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error)
-}
-
-type addressRetriever struct {
-	db   *postgres.DB
-	mode types.Mode
-}
-
-// NewAddressRetriever returns a new AddressRetriever
-func NewAddressRetriever(db *postgres.DB, mode types.Mode) AddressRetriever {
-	return &addressRetriever{
-		db:   db,
-		mode: mode,
-	}
-}
-
-// RetrieveTokenHolderAddresses is used to retrieve list of token-holding/contract-related addresses by iterating over available events
-// This generic method should work whether or not the argument/input names of the events meet the expected standard
-// This could be generalized to iterate over ALL events and pull out any address arguments
-func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) {
-	addrList := make([]string, 0)
-
-	_, ok := info.Filters["Transfer"]
-	if ok {
-		addrs, err := r.retrieveTransferAddresses(info)
-		if err != nil {
-			return nil, err
-		}
-		addrList = append(addrList, addrs...)
-	}
-
-	_, ok = info.Filters["Mint"]
-	if ok {
-		addrs, err := r.retrieveTokenMintees(info)
-		if err != nil {
-			return nil, err
-		}
-		addrList = append(addrList, addrs...)
-	}
-
-	contractAddresses := make(map[common.Address]bool)
-	for _, addr := range addrList {
-		contractAddresses[common.HexToAddress(addr)] = true
-	}
-
-	return contractAddresses, nil
-}
-
-func (r *addressRetriever) retrieveTransferAddresses(con contract.Contract) ([]string, error) {
-	transferAddrs := make([]string, 0)
-	event := con.Events["Transfer"]
-
-	for _, field := range event.Fields { // Iterate over event fields, finding the ones with address type
-
-		if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses
-			addrs := make([]string, 0)
-			pgStr := fmt.Sprintf("SELECT %s_ FROM %s_%s.%s_event", strings.ToLower(field.Name), r.mode.String(), strings.ToLower(con.Address), strings.ToLower(event.Name))
-			err := r.db.Select(&addrs, pgStr)
-			if err != nil {
-				return []string{}, err
-			}
-
-			transferAddrs = append(transferAddrs, addrs...) // And append them to the growing list
-		}
-	}
-
-	return transferAddrs, nil
-}
-
-func (r *addressRetriever) retrieveTokenMintees(con contract.Contract) ([]string, error) {
-	mintAddrs := make([]string, 0)
-	event := con.Events["Mint"]
-
-	for _, field := range event.Fields { // Iterate over event fields, finding the ones with address type
-
-		if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses
-			addrs := make([]string, 0)
-			pgStr := fmt.Sprintf("SELECT %s_ FROM %s_%s.%s_event", strings.ToLower(field.Name), r.mode.String(), strings.ToLower(con.Address), strings.ToLower(event.Name))
-			err := r.db.Select(&addrs, pgStr)
-			if err != nil {
-				return []string{}, err
-			}
-
-			mintAddrs = append(mintAddrs, addrs...) // And append them to the growing list
-		}
-	}
-
-	return mintAddrs, nil
-}

pkg/eth/contract_watcher/shared/types/event.go

@@ -1,99 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package types
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-// Event is our custom event type
-type Event struct {
-	Name      string
-	Anonymous bool
-	Fields    []Field
-}
-
-// Field is our custom event field type which associates a postgres type with the field
-type Field struct {
-	abi.Argument        // Name, Type, Indexed
-	PgType       string // Holds type used when committing data held in this field to postgres
-}
-
-// Log is used to hold instance of an event log data
-type Log struct {
-	ID     int64             // VulcanizeIdLog for full sync and header ID for header sync contract watcher
-	Values map[string]string // Map of event input names to their values
-
-	// Used for full sync only
-	Block int64
-	Tx    string
-
-	// Used for headerSync only
-	LogIndex         uint
-	TransactionIndex uint
-	Raw              []byte // json.Unmarshalled byte array of geth/core/types.Log{}
-}
-
-// NewEvent unpacks abi.Event into our custom Event struct
-func NewEvent(e abi.Event) Event {
-	fields := make([]Field, len(e.Inputs))
-	for i, input := range e.Inputs {
-		fields[i] = Field{}
-		fields[i].Name = input.Name
-		fields[i].Type = input.Type
-		fields[i].Indexed = input.Indexed
-		// Fill in pg type based on abi type
-		switch fields[i].Type.T {
-		case abi.HashTy, abi.AddressTy:
-			fields[i].PgType = "CHARACTER VARYING(66)"
-		case abi.IntTy, abi.UintTy:
-			fields[i].PgType = "NUMERIC"
-		case abi.BoolTy:
-			fields[i].PgType = "BOOLEAN"
-		case abi.BytesTy, abi.FixedBytesTy:
-			fields[i].PgType = "BYTEA"
-		case abi.ArrayTy:
-			fields[i].PgType = "TEXT[]"
-		case abi.FixedPointTy:
-			fields[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres?
-		default:
-			fields[i].PgType = "TEXT"
-		}
-	}
-
-	return Event{
-		Name:      e.Name,
-		Anonymous: e.Anonymous,
-		Fields:    fields,
-	}
-}
-
-// Sig returns the hash signature for an event
-func (e Event) Sig() common.Hash {
-	types := make([]string, len(e.Fields))
-
-	for i, input := range e.Fields {
-		types[i] = input.Type.String()
-	}
-
-	return crypto.Keccak256Hash([]byte(fmt.Sprintf("%v(%v)", e.Name, strings.Join(types, ","))))
-}

pkg/eth/contract_watcher/shared/types/method.go

@@ -1,113 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package types
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-// Method is our custom method struct
-type Method struct {
-	Name   string
-	Const  bool
-	Args   []Field
-	Return []Field
-}
-
-// Result is used to hold instance of result from method call with given inputs and block
-type Result struct {
-	Method
-	Inputs []interface{} // Will only use addresses
-	Output interface{}
-	PgType string // Holds output pg type
-	Block  int64
-}
-
-// NewMethod unpacks abi.Method into our custom Method struct
-func NewMethod(m abi.Method) Method {
-	inputs := make([]Field, len(m.Inputs))
-	for i, input := range m.Inputs {
-		inputs[i] = Field{}
-		inputs[i].Name = input.Name
-		inputs[i].Type = input.Type
-		inputs[i].Indexed = input.Indexed
-		switch inputs[i].Type.T {
-		case abi.HashTy, abi.AddressTy:
-			inputs[i].PgType = "CHARACTER VARYING(66)"
-		case abi.IntTy, abi.UintTy:
-			inputs[i].PgType = "NUMERIC"
-		case abi.BoolTy:
-			inputs[i].PgType = "BOOLEAN"
-		case abi.BytesTy, abi.FixedBytesTy:
-			inputs[i].PgType = "BYTEA"
-		case abi.ArrayTy:
-			inputs[i].PgType = "TEXT[]"
-		case abi.FixedPointTy:
-			inputs[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres?
-		default:
-			inputs[i].PgType = "TEXT"
-		}
-	}
-
-	outputs := make([]Field, len(m.Outputs))
-	for i, output := range m.Outputs {
-		outputs[i] = Field{}
-		outputs[i].Name = output.Name
-		outputs[i].Type = output.Type
-		outputs[i].Indexed = output.Indexed
-		switch outputs[i].Type.T {
-		case abi.HashTy, abi.AddressTy:
-			outputs[i].PgType = "CHARACTER VARYING(66)"
-		case abi.IntTy, abi.UintTy:
-			outputs[i].PgType = "NUMERIC"
-		case abi.BoolTy:
-			outputs[i].PgType = "BOOLEAN"
-		case abi.BytesTy, abi.FixedBytesTy:
-			outputs[i].PgType = "BYTEA"
-		case abi.ArrayTy:
-			outputs[i].PgType = "TEXT[]"
-		case abi.FixedPointTy:
-			outputs[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres?
-		default:
-			outputs[i].PgType = "TEXT"
-		}
-	}
-
-	return Method{
-		Name:   m.Name,
-		Const:  m.Const,
-		Args:   inputs,
-		Return: outputs,
-	}
-}
-
-// Sig returns the hash signature for the method
-func (m Method) Sig() common.Hash {
-	types := make([]string, len(m.Args))
-	i := 0
-	for _, arg := range m.Args {
-		types[i] = arg.Type.String()
-		i++
-	}
-
-	return crypto.Keccak256Hash([]byte(fmt.Sprintf("%v(%v)", m.Name, strings.Join(types, ","))))
-}

pkg/eth/contract_watcher/shared/types/mode.go

@@ -1,43 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package types
-
-// Mode is used to explicitly represent the operating mode of the transformer
-type Mode int
-
-// Mode enums
-const (
-	HeaderSync Mode = iota
-	FullSync
-)
-
-// IsValid returns true is the Mode is valid
-func (mode Mode) IsValid() bool {
-	return mode >= HeaderSync && mode <= FullSync
-}
-
-// String returns the string representation of the mode
-func (mode Mode) String() string {
-	switch mode {
-	case HeaderSync:
-		return "header"
-	case FullSync:
-		return "full"
-	default:
-		return "unknown"
-	}
-}

pkg/eth/converter.go

@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 )
 
 // PayloadConverter satisfies the PayloadConverter interface for ethereum

pkg/eth/converter_test.go

@@ -22,8 +22,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth/mocks"
 )
 
 var _ = Describe("Converter", func() {

pkg/eth/converters/cold_db/transaction_converter.go

@@ -1,88 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cold_db
-
-import (
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"golang.org/x/sync/errgroup"
-)
-
-type ColdDbTransactionConverter struct{}
-
-func NewColdDbTransactionConverter() *ColdDbTransactionConverter {
-	return &ColdDbTransactionConverter{}
-}
-
-func (cdtc *ColdDbTransactionConverter) ConvertBlockTransactionsToCore(gethBlock *types.Block) ([]core.TransactionModel, error) {
-	var g errgroup.Group
-	coreTransactions := make([]core.TransactionModel, len(gethBlock.Transactions()))
-
-	for gethTransactionIndex, gethTransaction := range gethBlock.Transactions() {
-		transaction := gethTransaction
-		transactionIndex := uint(gethTransactionIndex)
-		g.Go(func() error {
-			signer := getSigner(transaction)
-			sender, err := signer.Sender(transaction)
-			if err != nil {
-				return err
-			}
-			coreTransaction := transToCoreTrans(transaction, &sender)
-			coreTransactions[transactionIndex] = coreTransaction
-			return nil
-		})
-	}
-	if err := g.Wait(); err != nil {
-		return coreTransactions, err
-	}
-	return coreTransactions, nil
-}
-
-func (cdtc *ColdDbTransactionConverter) ConvertRPCTransactionsToModels(transactions []core.RPCTransaction) ([]core.TransactionModel, error) {
-	panic("converting transaction indexes to integer not supported for cold import")
-}
-
-func getSigner(tx *types.Transaction) types.Signer {
-	v, _, _ := tx.RawSignatureValues()
-	if v.Sign() != 0 && tx.Protected() {
-		return types.NewEIP155Signer(tx.ChainId())
-	}
-	return types.HomesteadSigner{}
-}
-
-func transToCoreTrans(transaction *types.Transaction, from *common.Address) core.TransactionModel {
-	return core.TransactionModel{
-		Hash:     transaction.Hash().Hex(),
-		Nonce:    transaction.Nonce(),
-		To:       strings.ToLower(addressToHex(transaction.To())),
-		From:     strings.ToLower(addressToHex(from)),
-		GasLimit: transaction.Gas(),
-		GasPrice: transaction.GasPrice().Int64(),
-		Value:    transaction.Value().String(),
-		Data:     transaction.Data(),
-	}
-}
-
-func addressToHex(to *common.Address) string {
-	if to == nil {
-		return ""
-	}
-	return to.Hex()
-}

pkg/eth/converters/common/block_converter.go

@@ -1,91 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common
-
-import (
-	"encoding/json"
-	"math/big"
-	"strconv"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-type BlockConverter struct {
-	transactionConverter TransactionConverter
-}
-
-func NewBlockConverter(transactionConverter TransactionConverter) BlockConverter {
-	return BlockConverter{transactionConverter: transactionConverter}
-}
-
-func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error) {
-	transactions, err := bc.transactionConverter.ConvertBlockTransactionsToCore(gethBlock)
-	if err != nil {
-		return core.Block{}, err
-	}
-
-	coreBlock := core.Block{
-		Difficulty:   gethBlock.Difficulty().Int64(),
-		ExtraData:    hexutil.Encode(gethBlock.Extra()),
-		GasLimit:     gethBlock.GasLimit(),
-		GasUsed:      gethBlock.GasUsed(),
-		Hash:         gethBlock.Hash().Hex(),
-		Miner:        strings.ToLower(gethBlock.Coinbase().Hex()),
-		Nonce:        hexutil.Encode(gethBlock.Header().Nonce[:]),
-		Number:       gethBlock.Number().Int64(),
-		ParentHash:   gethBlock.ParentHash().Hex(),
-		Size:         gethBlock.Size().String(),
-		Time:         gethBlock.Time(),
-		Transactions: transactions,
-		UncleHash:    gethBlock.UncleHash().Hex(),
-	}
-	coreBlock.Reward = CalcBlockReward(coreBlock, gethBlock.Uncles()).String()
-	totalUncleReward, uncles := bc.ToCoreUncle(coreBlock, gethBlock.Uncles())
-
-	coreBlock.UnclesReward = totalUncleReward.String()
-	coreBlock.Uncles = uncles
-	return coreBlock, nil
-}
-
-// Rewards for the miners of uncles is calculated as (U_n + 8 - B_n) * R / 8
-// Where U_n is the uncle block number, B_n is the parent block number and R is the static block reward at B_n
-// https://github.com/ethereum/go-ethereum/issues/1591
-// https://ethereum.stackexchange.com/questions/27172/different-uncles-reward
-// https://github.com/ethereum/homestead-guide/issues/399
-// Returns the total uncle reward and the individual processed uncles
-func (bc BlockConverter) ToCoreUncle(block core.Block, uncles []*types.Header) (*big.Int, []core.Uncle) {
-	totalUncleRewards := new(big.Int)
-	coreUncles := make([]core.Uncle, 0, len(uncles))
-	for _, uncle := range uncles {
-		thisUncleReward := calcUncleMinerReward(block.Number, uncle.Number.Int64())
-		raw, _ := json.Marshal(uncle)
-		coreUncle := core.Uncle{
-			Miner:     uncle.Coinbase.Hex(),
-			Hash:      uncle.Hash().Hex(),
-			Raw:       raw,
-			Reward:    thisUncleReward.String(),
-			Timestamp: strconv.FormatUint(uncle.Time, 10),
-		}
-		coreUncles = append(coreUncles, coreUncle)
-		totalUncleRewards.Add(totalUncleRewards, thisUncleReward)
-	}
-	return totalUncleRewards, coreUncles
-}

pkg/eth/converters/common/block_converter_test.go

@@ -1,393 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common_test
-
-import (
-	"bytes"
-	"io/ioutil"
-	"log"
-	"math/big"
-	"os"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Conversion of GethBlock to core.Block", func() {
-
-	It("converts basic Block metadata", func() {
-		difficulty := big.NewInt(1)
-		gasLimit := uint64(100000)
-		gasUsed := uint64(100000)
-		miner := common.HexToAddress("0x0000000000000000000000000000000000000123")
-		extraData, _ := hexutil.Decode("0xe4b883e5bda9e7a59ee4bb99e9b1bc")
-		nonce := types.BlockNonce{10}
-		number := int64(1)
-		time := uint64(140000000)
-
-		header := types.Header{
-			Difficulty: difficulty,
-			GasLimit:   uint64(gasLimit),
-			GasUsed:    uint64(gasUsed),
-			Extra:      extraData,
-			Coinbase:   miner,
-			Nonce:      nonce,
-			Number:     big.NewInt(number),
-			ParentHash: common.Hash{64},
-			Time:       time,
-			UncleHash:  common.Hash{128},
-		}
-		block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
-		client := fakes.NewMockEthClient()
-		transactionConverter := rpc.NewRPCTransactionConverter(client)
-		blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-		coreBlock, err := blockConverter.ToCoreBlock(block)
-
-		Expect(err).ToNot(HaveOccurred())
-		Expect(coreBlock.Difficulty).To(Equal(difficulty.Int64()))
-		Expect(coreBlock.GasLimit).To(Equal(gasLimit))
-		Expect(coreBlock.Miner).To(Equal(miner.Hex()))
-		Expect(coreBlock.GasUsed).To(Equal(gasUsed))
-		Expect(coreBlock.Hash).To(Equal(block.Hash().Hex()))
-		Expect(coreBlock.Nonce).To(Equal(hexutil.Encode(header.Nonce[:])))
-		Expect(coreBlock.Number).To(Equal(number))
-		Expect(coreBlock.ParentHash).To(Equal(block.ParentHash().Hex()))
-		Expect(coreBlock.ExtraData).To(Equal(hexutil.Encode(block.Extra())))
-		Expect(coreBlock.Size).To(Equal(block.Size().String()))
-		Expect(coreBlock.Time).To(Equal(time))
-		Expect(coreBlock.UncleHash).To(Equal(block.UncleHash().Hex()))
-		Expect(coreBlock.IsFinal).To(BeFalse())
-	})
-
-	Describe("The block and uncle rewards calculations", func() {
-		It("calculates block rewards for a block", func() {
-			transaction := types.NewTransaction(
-				uint64(226823),
-				common.HexToAddress("0x108fedb097c1dcfed441480170144d8e19bb217f"),
-				big.NewInt(1080900090000000000),
-				uint64(90000),
-				big.NewInt(50000000000),
-				[]byte{},
-			)
-			transactions := []*types.Transaction{transaction}
-
-			txHash := transaction.Hash()
-			receipt := types.Receipt{
-				TxHash:            txHash,
-				GasUsed:           uint64(21000),
-				CumulativeGasUsed: uint64(21000),
-			}
-			receipts := []*types.Receipt{&receipt}
-
-			client := fakes.NewMockEthClient()
-			client.SetTransactionReceipts(receipts)
-
-			number := int64(1071819)
-			header := types.Header{
-				Number: big.NewInt(number),
-			}
-			uncles := []*types.Header{{Number: big.NewInt(1071817)}, {Number: big.NewInt(1071818)}}
-			block := types.NewBlock(&header, transactions, uncles, []*types.Receipt{&receipt})
-			transactionConverter := rpc.NewRPCTransactionConverter(client)
-			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-			coreBlock, err := blockConverter.ToCoreBlock(block)
-			Expect(err).ToNot(HaveOccurred())
-
-			expectedBlockReward := new(big.Int)
-			expectedBlockReward.SetString("5313550000000000000", 10)
-
-			blockReward := vulcCommon.CalcBlockReward(coreBlock, block.Uncles())
-			Expect(blockReward.String()).To(Equal(expectedBlockReward.String()))
-		})
-
-		It("calculates the uncles reward for a block", func() {
-			transaction := types.NewTransaction(
-				uint64(226823),
-				common.HexToAddress("0x108fedb097c1dcfed441480170144d8e19bb217f"),
-				big.NewInt(1080900090000000000),
-				uint64(90000),
-				big.NewInt(50000000000),
-				[]byte{})
-			transactions := []*types.Transaction{transaction}
-
-			receipt := types.Receipt{
-				TxHash:            transaction.Hash(),
-				GasUsed:           uint64(21000),
-				CumulativeGasUsed: uint64(21000),
-			}
-			receipts := []*types.Receipt{&receipt}
-
-			header := types.Header{
-				Number: big.NewInt(int64(1071819)),
-			}
-			uncles := []*types.Header{
-				{Number: big.NewInt(1071816)},
-				{Number: big.NewInt(1071817)},
-			}
-			block := types.NewBlock(&header, transactions, uncles, receipts)
-
-			client := fakes.NewMockEthClient()
-			client.SetTransactionReceipts(receipts)
-			transactionConverter := rpc.NewRPCTransactionConverter(client)
-			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-			coreBlock, err := blockConverter.ToCoreBlock(block)
-			Expect(err).ToNot(HaveOccurred())
-
-			expectedTotalReward := new(big.Int)
-			expectedTotalReward.SetString("6875000000000000000", 10)
-			totalReward, coreUncles := blockConverter.ToCoreUncle(coreBlock, block.Uncles())
-			Expect(totalReward.String()).To(Equal(expectedTotalReward.String()))
-
-			Expect(len(coreUncles)).To(Equal(2))
-			Expect(coreUncles[0].Reward).To(Equal("3125000000000000000"))
-			Expect(coreUncles[0].Miner).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(coreUncles[0].Hash).To(Equal("0xb629de4014b6e30cf9555ee833f1806fa0d8b8516fde194405f9c98c2deb8772"))
-			Expect(coreUncles[1].Reward).To(Equal("3750000000000000000"))
-			Expect(coreUncles[1].Miner).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(coreUncles[1].Hash).To(Equal("0x673f5231e4888a951e0bc8a25b5774b982e6e9e258362c21affaff6e02dd5a2b"))
-		})
-
-		It("decreases the static block reward from 5 to 3 for blocks after block 4,269,999", func() {
-			transactionOne := types.NewTransaction(
-				uint64(8072),
-				common.HexToAddress("0xebd17720aeb7ac5186c5dfa7bafeb0bb14c02551 "),
-				big.NewInt(0),
-				uint64(500000),
-				big.NewInt(42000000000),
-				[]byte{},
-			)
-
-			transactionTwo := types.NewTransaction(uint64(8071),
-				common.HexToAddress("0x3cdab63d764c8c5048ed5e8f0a4e95534ba7e1ea"),
-				big.NewInt(0),
-				uint64(500000),
-				big.NewInt(42000000000),
-				[]byte{})
-
-			transactions := []*types.Transaction{transactionOne, transactionTwo}
-
-			receiptOne := types.Receipt{
-				TxHash:            transactionOne.Hash(),
-				GasUsed:           uint64(297508),
-				CumulativeGasUsed: uint64(0),
-			}
-			receiptTwo := types.Receipt{
-				TxHash:            transactionTwo.Hash(),
-				GasUsed:           uint64(297508),
-				CumulativeGasUsed: uint64(0),
-			}
-			receipts := []*types.Receipt{&receiptOne, &receiptTwo}
-
-			number := int64(4370055)
-			header := types.Header{
-				Number: big.NewInt(number),
-			}
-			var uncles []*types.Header
-			block := types.NewBlock(&header, transactions, uncles, receipts)
-
-			client := fakes.NewMockEthClient()
-			client.SetTransactionReceipts(receipts)
-			transactionConverter := rpc.NewRPCTransactionConverter(client)
-			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-			coreBlock, err := blockConverter.ToCoreBlock(block)
-			Expect(err).ToNot(HaveOccurred())
-
-			expectedRewards := new(big.Int)
-			expectedRewards.SetString("3024990672000000000", 10)
-			rewards := vulcCommon.CalcBlockReward(coreBlock, block.Uncles())
-			Expect(rewards.String()).To(Equal(expectedRewards.String()))
-		})
-	})
-
-	Describe("the converted transactions", func() {
-		It("is empty", func() {
-			header := types.Header{}
-			block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
-			client := fakes.NewMockEthClient()
-			transactionConverter := rpc.NewRPCTransactionConverter(client)
-			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-			coreBlock, err := blockConverter.ToCoreBlock(block)
-
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(coreBlock.Transactions)).To(Equal(0))
-		})
-
-		It("converts a single transaction", func() {
-			gethTransaction := types.NewTransaction(
-				uint64(10000), common.Address{1},
-				big.NewInt(10),
-				uint64(5000),
-				big.NewInt(3),
-				hexutil.MustDecode("0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14"),
-			)
-			var rawTransaction bytes.Buffer
-			encodeErr := gethTransaction.EncodeRLP(&rawTransaction)
-			Expect(encodeErr).NotTo(HaveOccurred())
-
-			gethReceipt := &types.Receipt{
-				Bloom:             types.BytesToBloom(hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")),
-				ContractAddress:   common.HexToAddress("x123"),
-				CumulativeGasUsed: uint64(7996119),
-				GasUsed:           uint64(21000),
-				Logs:              []*types.Log{},
-				Status:            uint64(1),
-				TxHash:            gethTransaction.Hash(),
-			}
-
-			client := fakes.NewMockEthClient()
-			client.SetTransactionReceipts([]*types.Receipt{gethReceipt})
-
-			header := types.Header{}
-			block := types.NewBlock(
-				&header,
-				[]*types.Transaction{gethTransaction},
-				[]*types.Header{},
-				[]*types.Receipt{gethReceipt},
-			)
-			transactionConverter := rpc.NewRPCTransactionConverter(client)
-			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-			coreBlock, err := blockConverter.ToCoreBlock(block)
-
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(coreBlock.Transactions)).To(Equal(1))
-			coreTransaction := coreBlock.Transactions[0]
-			expectedData := common.FromHex("0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14")
-			Expect(coreTransaction.Data).To(Equal(expectedData))
-			Expect(coreTransaction.To).To(Equal(gethTransaction.To().Hex()))
-			Expect(coreTransaction.From).To(Equal("0x0000000000000000000000000000000000000123"))
-			Expect(coreTransaction.GasLimit).To(Equal(gethTransaction.Gas()))
-			Expect(coreTransaction.GasPrice).To(Equal(gethTransaction.GasPrice().Int64()))
-			Expect(coreTransaction.Raw).To(Equal(rawTransaction.Bytes()))
-			Expect(coreTransaction.TxIndex).To(Equal(int64(0)))
-			Expect(coreTransaction.Value).To(Equal(gethTransaction.Value().String()))
-			Expect(coreTransaction.Nonce).To(Equal(gethTransaction.Nonce()))
-
-			coreReceipt := coreTransaction.Receipt
-			expectedReceipt, err := vulcCommon.ToCoreReceipt(gethReceipt)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(coreReceipt).To(Equal(expectedReceipt))
-		})
-
-		It("has an empty 'To' field when transaction creates a new contract", func() {
-			gethTransaction := types.NewContractCreation(
-				uint64(10000),
-				big.NewInt(10),
-				uint64(5000),
-				big.NewInt(3),
-				[]byte("1234"),
-			)
-
-			gethReceipt := &types.Receipt{
-				CumulativeGasUsed: uint64(1),
-				GasUsed:           uint64(1),
-				TxHash:            gethTransaction.Hash(),
-				ContractAddress:   common.HexToAddress("0x1023342345"),
-			}
-
-			client := fakes.NewMockEthClient()
-			client.SetTransactionReceipts([]*types.Receipt{gethReceipt})
-
-			block := types.NewBlock(
-				&types.Header{},
-				[]*types.Transaction{gethTransaction},
-				[]*types.Header{},
-				[]*types.Receipt{gethReceipt},
-			)
-			transactionConverter := rpc.NewRPCTransactionConverter(client)
-			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-			coreBlock, err := blockConverter.ToCoreBlock(block)
-
-			Expect(err).ToNot(HaveOccurred())
-			coreTransaction := coreBlock.Transactions[0]
-			Expect(coreTransaction.To).To(Equal(""))
-
-			coreReceipt := coreTransaction.Receipt
-			expectedReceipt, err := vulcCommon.ToCoreReceipt(gethReceipt)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(coreReceipt).To(Equal(expectedReceipt))
-		})
-	})
-
-	Describe("transaction error handling", func() {
-		var gethTransaction *types.Transaction
-		var gethReceipt *types.Receipt
-		var header *types.Header
-		var block *types.Block
-
-		BeforeEach(func() {
-			log.SetOutput(ioutil.Discard)
-			gethTransaction = types.NewTransaction(
-				uint64(0),
-				common.Address{},
-				big.NewInt(0),
-				uint64(0),
-				big.NewInt(0),
-				[]byte{},
-			)
-			gethReceipt = &types.Receipt{}
-			header = &types.Header{}
-			block = types.NewBlock(
-				header,
-				[]*types.Transaction{gethTransaction},
-				[]*types.Header{},
-				[]*types.Receipt{gethReceipt},
-			)
-
-		})
-
-		AfterEach(func() {
-			defer log.SetOutput(os.Stdout)
-		})
-
-		It("returns an error when transaction sender call fails", func() {
-			client := fakes.NewMockEthClient()
-			client.SetTransactionSenderErr(fakes.FakeError)
-			transactionConverter := rpc.NewRPCTransactionConverter(client)
-			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-			_, err := blockConverter.ToCoreBlock(block)
-
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-
-		It("returns an error when transaction receipt call fails", func() {
-			client := fakes.NewMockEthClient()
-			client.SetTransactionReceiptErr(fakes.FakeError)
-			transactionConverter := rpc.NewRPCTransactionConverter(client)
-			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
-
-			_, err := blockConverter.ToCoreBlock(block)
-
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-	})
-
-})

pkg/eth/converters/common/common_suite_test.go

@@ -1,29 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common_test
-
-import (
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestCommon(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Common Suite")
-}

pkg/eth/converters/common/full_sync_log_converter.go

@@ -1,57 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common
-
-import (
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-func ToFullSyncLogs(gethLogs []types.Log) []core.FullSyncLog {
-	var logs []core.FullSyncLog
-	for _, log := range gethLogs {
-		log := ToCoreLog(log)
-		logs = append(logs, log)
-	}
-	return logs
-}
-
-func makeTopics(topics []common.Hash) core.Topics {
-	var hexTopics core.Topics
-	for i, topic := range topics {
-		hexTopics[i] = topic.Hex()
-	}
-	return hexTopics
-}
-
-func ToCoreLog(gethLog types.Log) core.FullSyncLog {
-	topics := gethLog.Topics
-	hexTopics := makeTopics(topics)
-	return core.FullSyncLog{
-		Address:     strings.ToLower(gethLog.Address.Hex()),
-		BlockNumber: int64(gethLog.BlockNumber),
-		Topics:      hexTopics,
-		TxHash:      gethLog.TxHash.Hex(),
-		Index:       int64(gethLog.Index),
-		Data:        hexutil.Encode(gethLog.Data),
-	}
-}

pkg/eth/converters/common/full_sync_log_converter_test.go

@@ -1,110 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common_test
-
-import (
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-var _ = Describe("Conversion of GethLog to core.FullSyncLog", func() {
-
-	It("converts geth log to internal log format", func() {
-		gethLog := types.Log{
-			Address:     common.HexToAddress("0x448a5065aeBB8E423F0896E6c5D525C040f59af3"),
-			BlockHash:   common.HexToHash("0x656c34545f90a730a19008c0e7a7cd4fb3895064b48d6d69761bd5abad681056"),
-			BlockNumber: 2019236,
-			Data:        hexutil.MustDecode("0x000000000000000000000000000000000000000000000001a055690d9db80000"),
-			Index:       2,
-			TxIndex:     3,
-			TxHash:      common.HexToHash("0x3b198bfd5d2907285af009e9ae84a0ecd63677110d89d7e030251acb87f6487e"),
-			Topics: []common.Hash{
-				common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"),
-				common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615"),
-			},
-		}
-
-		expected := core.FullSyncLog{
-			Address:     strings.ToLower(gethLog.Address.Hex()),
-			BlockNumber: int64(gethLog.BlockNumber),
-			Data:        hexutil.Encode(gethLog.Data),
-			TxHash:      gethLog.TxHash.Hex(),
-			Index:       2,
-			Topics: core.Topics{
-				gethLog.Topics[0].Hex(),
-				gethLog.Topics[1].Hex(),
-			},
-		}
-
-		coreLog := vulcCommon.ToCoreLog(gethLog)
-
-		Expect(coreLog.Address).To(Equal(expected.Address))
-		Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber))
-		Expect(coreLog.Data).To(Equal(expected.Data))
-		Expect(coreLog.Index).To(Equal(expected.Index))
-		Expect(coreLog.Topics[0]).To(Equal(expected.Topics[0]))
-		Expect(coreLog.Topics[1]).To(Equal(expected.Topics[1]))
-		Expect(coreLog.TxHash).To(Equal(expected.TxHash))
-	})
-
-	It("converts geth log array to array of internal logs", func() {
-		gethLogOne := types.Log{
-			Address:     common.HexToAddress("0xecf8f87f810ecf450940c9f60066b4a7a501d6a7"),
-			BlockHash:   common.HexToHash("0x656c34545f90a730a19008c0e7a7cd4fb3895064b48d6d69761bd5abad681056"),
-			BlockNumber: 2019236,
-			Data:        hexutil.MustDecode("0x000000000000000000000000000000000000000000000001a055690d9db80000"),
-			Index:       2,
-			TxIndex:     3,
-			TxHash:      common.HexToHash("0x3b198bfd5d2907285af009e9ae84a0ecd63677110d89d7e030251acb87f6487e"),
-			Topics: []common.Hash{
-				common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"),
-				common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615"),
-			},
-		}
-
-		gethLogTwo := types.Log{
-			Address:     common.HexToAddress("0x123"),
-			BlockHash:   common.HexToHash("0x576"),
-			BlockNumber: 2019236,
-			Data:        hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000001"),
-			Index:       3,
-			TxIndex:     4,
-			TxHash:      common.HexToHash("0x134"),
-			Topics: []common.Hash{
-				common.HexToHash("0xaaa"),
-				common.HexToHash("0xbbb"),
-			},
-		}
-
-		expectedOne := vulcCommon.ToCoreLog(gethLogOne)
-		expectedTwo := vulcCommon.ToCoreLog(gethLogTwo)
-
-		coreLogs := vulcCommon.ToFullSyncLogs([]types.Log{gethLogOne, gethLogTwo})
-
-		Expect(len(coreLogs)).To(Equal(2))
-		Expect(coreLogs[0]).To(Equal(expectedOne))
-		Expect(coreLogs[1]).To(Equal(expectedTwo))
-	})
-})

pkg/eth/converters/common/header_converter.go

@@ -1,42 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common
-
-import (
-	"encoding/json"
-	"strconv"
-
-	"github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-type HeaderConverter struct{}
-
-func (converter HeaderConverter) Convert(gethHeader *types.Header, blockHash string) core.Header {
-	rawHeader, err := json.Marshal(gethHeader)
-	if err != nil {
-		panic(err)
-	}
-	coreHeader := core.Header{
-		Hash:        blockHash,
-		BlockNumber: gethHeader.Number.Int64(),
-		Raw:         rawHeader,
-		Timestamp:   strconv.FormatUint(gethHeader.Time, 10),
-	}
-	return coreHeader
-}

pkg/eth/converters/common/header_converter_test.go

@@ -1,65 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common_test
-
-import (
-	"encoding/json"
-	"math/big"
-	"strconv"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Block header converter", func() {
-	It("converts geth header to core header", func() {
-		gethHeader := &types.Header{
-			Difficulty:  big.NewInt(1),
-			Number:      big.NewInt(2),
-			ParentHash:  common.HexToHash("0xParent"),
-			ReceiptHash: common.HexToHash("0xReceipt"),
-			Root:        common.HexToHash("0xRoot"),
-			Time:        uint64(123456789),
-			TxHash:      common.HexToHash("0xTransaction"),
-			UncleHash:   common.HexToHash("0xUncle"),
-		}
-		converter := common2.HeaderConverter{}
-		hash := fakes.FakeHash.String()
-
-		coreHeader := converter.Convert(gethHeader, hash)
-
-		Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64()))
-		Expect(coreHeader.Hash).To(Equal(hash))
-		Expect(coreHeader.Timestamp).To(Equal(strconv.FormatUint(gethHeader.Time, 10)))
-	})
-
-	It("includes raw bytes for header as JSON", func() {
-		gethHeader := types.Header{Number: big.NewInt(123)}
-		converter := common2.HeaderConverter{}
-
-		coreHeader := converter.Convert(&gethHeader, fakes.FakeHash.String())
-
-		expectedJSON, err := json.Marshal(gethHeader)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(coreHeader.Raw).To(Equal(expectedJSON))
-	})
-})

pkg/eth/converters/common/receipt_converter.go

@@ -1,89 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common
-
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-func ToCoreReceipts(gethReceipts types.Receipts) ([]core.Receipt, error) {
-	var coreReceipts []core.Receipt
-	for _, receipt := range gethReceipts {
-		coreReceipt, err := ToCoreReceipt(receipt)
-		if err != nil {
-			return nil, err
-		}
-		coreReceipts = append(coreReceipts, coreReceipt)
-	}
-	return coreReceipts, nil
-}
-
-func ToCoreReceipt(gethReceipt *types.Receipt) (core.Receipt, error) {
-	bloom := hexutil.Encode(gethReceipt.Bloom.Bytes())
-	var postState string
-	var status int
-	postState, status = postStateOrStatus(gethReceipt)
-	logs := dereferenceLogs(gethReceipt)
-	contractAddress := setContractAddress(gethReceipt)
-
-	rlpBuff := new(bytes.Buffer)
-	receiptForStorage := types.ReceiptForStorage(*gethReceipt)
-	err := receiptForStorage.EncodeRLP(rlpBuff)
-	if err != nil {
-		return core.Receipt{}, err
-	}
-	return core.Receipt{
-		Bloom:             bloom,
-		ContractAddress:   contractAddress,
-		CumulativeGasUsed: gethReceipt.CumulativeGasUsed,
-		GasUsed:           gethReceipt.GasUsed,
-		Logs:              logs,
-		StateRoot:         postState,
-		TxHash:            gethReceipt.TxHash.Hex(),
-		Status:            status,
-		Rlp:               rlpBuff.Bytes(),
-	}, nil
-}
-
-func setContractAddress(gethReceipt *types.Receipt) string {
-	emptyAddress := common.Address{}.Bytes()
-	if bytes.Equal(gethReceipt.ContractAddress.Bytes(), emptyAddress) {
-		return ""
-	}
-	return gethReceipt.ContractAddress.Hex()
-}
-
-func dereferenceLogs(gethReceipt *types.Receipt) []core.FullSyncLog {
-	logs := []core.FullSyncLog{}
-	for _, log := range gethReceipt.Logs {
-		logs = append(logs, ToCoreLog(*log))
-	}
-	return logs
-}
-
-func postStateOrStatus(gethReceipts *types.Receipt) (string, int) {
-	if len(gethReceipts.PostState) != 0 {
-		return hexutil.Encode(gethReceipts.PostState), -99
-	}
-	return "", int(gethReceipts.Status)
-}

pkg/eth/converters/common/receipt_converter_test.go

@@ -1,115 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common_test
-
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
-
-	It(`converts geth receipt to internal receipt format (pre Byzantium has post-transaction stateroot)`, func() {
-		receipt := types.Receipt{
-			Bloom:             types.BytesToBloom(hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")),
-			ContractAddress:   common.Address{},
-			CumulativeGasUsed: uint64(25000),
-			GasUsed:           uint64(21000),
-			Logs:              []*types.Log{},
-			PostState:         hexutil.MustDecode("0x88abf7e73128227370aa7baa3dd4e18d0af70e92ef1f9ef426942fbe2dddb733"),
-			TxHash:            common.HexToHash("0x97d99bc7729211111a21b12c933c949d4f31684f1d6954ff477d0477538ff017"),
-		}
-
-		rlpBuff := new(bytes.Buffer)
-		receiptForStorage := types.ReceiptForStorage(receipt)
-		err := receiptForStorage.EncodeRLP(rlpBuff)
-		Expect(err).ToNot(HaveOccurred())
-
-		expected := core.Receipt{
-			Bloom:             "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
-			ContractAddress:   "",
-			CumulativeGasUsed: 25000,
-			GasUsed:           21000,
-			Logs:              []core.FullSyncLog{},
-			StateRoot:         "0x88abf7e73128227370aa7baa3dd4e18d0af70e92ef1f9ef426942fbe2dddb733",
-			Status:            -99,
-			TxHash:            receipt.TxHash.Hex(),
-			Rlp:               rlpBuff.Bytes(),
-		}
-
-		coreReceipt, err := vulcCommon.ToCoreReceipt(&receipt)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(coreReceipt.Bloom).To(Equal(expected.Bloom))
-		Expect(coreReceipt.ContractAddress).To(Equal(expected.ContractAddress))
-		Expect(coreReceipt.CumulativeGasUsed).To(Equal(expected.CumulativeGasUsed))
-		Expect(coreReceipt.GasUsed).To(Equal(expected.GasUsed))
-		Expect(coreReceipt.Logs).To(Equal(expected.Logs))
-		Expect(coreReceipt.StateRoot).To(Equal(expected.StateRoot))
-		Expect(coreReceipt.Status).To(Equal(expected.Status))
-		Expect(coreReceipt.TxHash).To(Equal(expected.TxHash))
-		Expect(bytes.Compare(coreReceipt.Rlp, expected.Rlp)).To(Equal(0))
-	})
-
-	It("converts geth receipt to internal receipt format (post Byzantium has status", func() {
-		receipt := types.Receipt{
-			Bloom:             types.BytesToBloom(hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")),
-			ContractAddress:   common.HexToAddress("x0123"),
-			CumulativeGasUsed: uint64(7996119),
-			GasUsed:           uint64(21000),
-			Logs:              []*types.Log{},
-			Status:            uint64(1),
-			TxHash:            common.HexToHash("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223"),
-		}
-
-		rlpBuff := new(bytes.Buffer)
-		receiptForStorage := types.ReceiptForStorage(receipt)
-		err := receiptForStorage.EncodeRLP(rlpBuff)
-		Expect(err).ToNot(HaveOccurred())
-
-		expected := core.Receipt{
-			Bloom:             "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
-			ContractAddress:   receipt.ContractAddress.Hex(),
-			CumulativeGasUsed: 7996119,
-			GasUsed:           21000,
-			Logs:              []core.FullSyncLog{},
-			StateRoot:         "",
-			Status:            1,
-			TxHash:            receipt.TxHash.Hex(),
-			Rlp:               rlpBuff.Bytes(),
-		}
-
-		coreReceipt, err := vulcCommon.ToCoreReceipt(&receipt)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(coreReceipt.Bloom).To(Equal(expected.Bloom))
-		Expect(coreReceipt.ContractAddress).To(Equal(""))
-		Expect(coreReceipt.CumulativeGasUsed).To(Equal(expected.CumulativeGasUsed))
-		Expect(coreReceipt.GasUsed).To(Equal(expected.GasUsed))
-		Expect(coreReceipt.Logs).To(Equal(expected.Logs))
-		Expect(coreReceipt.StateRoot).To(Equal(expected.StateRoot))
-		Expect(coreReceipt.Status).To(Equal(expected.Status))
-		Expect(coreReceipt.TxHash).To(Equal(expected.TxHash))
-		Expect(bytes.Compare(coreReceipt.Rlp, expected.Rlp)).To(Equal(0))
-	})
-})

pkg/eth/converters/common/transaction_converter.go

@@ -1,27 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package common
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-type TransactionConverter interface {
-	ConvertBlockTransactionsToCore(gethBlock *types.Block) ([]core.TransactionModel, error)
-	ConvertRPCTransactionsToModels(transactions []core.RPCTransaction) ([]core.TransactionModel, error)
-}

pkg/eth/converters/rpc/rpc_suite_test.go

@@ -1,13 +0,0 @@
-package rpc_test
-
-import (
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestRpc(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Rpc Suite")
-}