tests and fixes for fetcher, parser, retriever, converter, and repository; update cmd and watcher

cmd/omniWatcher.go

@@ -20,11 +20,13 @@ import (
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/spf13/cobra"
 
+	"github.com/vulcanize/vulcanizedb/libraries/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
 	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
@@ -58,16 +60,24 @@ Requires a .toml config file:
 }
 
 func omniWatcher() {
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
 
-	if contractAddress == "" {
+	if contractAddress == "" && len(contractAddresses) == 0 {
 		log.Fatal("Contract address required")
 	}
 
-	if contractEvents == nil {
+	if len(contractEvents) == 0 || len(contractMethods) == 0 {
 		var str string
 		for str != "y" {
 			reader := bufio.NewReader(os.Stdin)
-			fmt.Print("Warning: no events specified, proceeding to watch every event at address" + contractAddress + "? (Y/n)\n> ")
+			if len(contractEvents) == 0 && len(contractMethods) == 0 {
+				fmt.Print("Warning: no events or methods specified, proceed to watch every event and method? (Y/n)\n> ")
+			} else if len(contractEvents) == 0 {
+				fmt.Print("Warning: no events specified, proceed to watch every event? (Y/n)\n> ")
+			} else {
+				fmt.Print("Warning: no methods specified, proceed to watch every method? (Y/n)\n> ")
+			}
 			resp, err := reader.ReadBytes('\n')
 			if err != nil {
 				log.Fatal(err)
@@ -82,8 +92,9 @@ func omniWatcher() {
 
 	rawRpcClient, err := rpc.Dial(ipc)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal(fmt.Sprintf("Failed to initialize rpc client\r\nerr: %v\r\n", err))
 	}
+
 	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	client := client.NewEthClient(ethClient)
@@ -102,21 +113,31 @@ func omniWatcher() {
 	}
 
 	t := transformer.NewTransformer(&con)
-	t.Set(contractAddress, contractEvents)
+
+	contractAddresses = append(contractAddresses, contractAddress)
+	for _, addr := range contractAddresses {
+		t.Set(addr, contractEvents)
+	}
 
 	err = t.Init()
 	if err != nil {
-		log.Fatal(fmt.Sprintf("Failed to initialized generator\r\nerr: %v\r\n", err))
+		log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err))
 	}
 
-	log.Fatal(t.Execute())
+	w := shared.Watcher{}
+	w.AddTransformer(t)
+
+	for range ticker.C {
+		w.Execute()
+	}
 }
 
 func init() {
 	rootCmd.AddCommand(omniWatcherCmd)
 
 	omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
-	omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch- use only with single address")
-	omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "Addresses of the contracts to generate watchers for")
+	omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "List of addresses to generate watchers for")
+	omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched")
+	omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-methods", "m", []string{}, "Subset of methods to watch; by default all methods are watched")
 	omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)
 }

cmd/root.go

@@ -36,6 +36,7 @@ var (
 	contractAddress     string
 	contractAddresses   []string
 	contractEvents      []string
+	contractMethods     []string
 )
 
 var rootCmd = &cobra.Command{

libraries/shared/watcher.go

@@ -23,6 +23,10 @@ func (watcher *Watcher) AddTransformers(us []TransformerInitializer, con Contrac
 	return nil
 }
 
+func (watcher *Watcher) AddTransformer(t Transformer) {
+	watcher.Transformers = append(watcher.Transformers, t)
+}
+
 func (watcher *Watcher) Execute() error {
 	var err error
 	for _, transformer := range watcher.Transformers {

pkg/omni/converter/converter.go

@@ -24,6 +24,8 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
+// Converter is used to convert watched event logs to
+// custom logs containing event input name => value maps
 type Converter interface {
 	Convert(watchedEvent core.WatchedEvent, event *types.Event) error
 	Update(info types.ContractInfo)
@@ -40,11 +42,12 @@ func NewConverter(info types.ContractInfo) *converter {
 	}
 }
 
-func (c converter) Update(info types.ContractInfo) {
+func (c *converter) Update(info types.ContractInfo) {
 	c.contractInfo = info
 }
 
-func (c converter) Convert(watchedEvent core.WatchedEvent, event *types.Event) error {
+// Convert the given watched event log into a types.Log for the given event
+func (c *converter) Convert(watchedEvent core.WatchedEvent, event *types.Event) error {
 	contract := bind.NewBoundContract(common.HexToAddress(c.contractInfo.Address), c.contractInfo.ParsedAbi, nil, nil, nil)
 	values := make(map[string]interface{})
 
@@ -53,27 +56,24 @@ func (c converter) Convert(watchedEvent core.WatchedEvent, event *types.Event) e
 		values[field.Name] = i
 
 		switch field.Type.T {
-		case abi.StringTy:
-			field.PgType = "CHARACTER VARYING(66) NOT NULL"
+		case abi.StringTy, abi.HashTy, abi.AddressTy:
+			field.PgType = "CHARACTER VARYING(66)"
 		case abi.IntTy, abi.UintTy:
-			field.PgType = "DECIMAL NOT NULL"
+			field.PgType = "DECIMAL"
 		case abi.BoolTy:
-			field.PgType = "BOOLEAN NOT NULL"
+			field.PgType = "BOOLEAN"
 		case abi.BytesTy, abi.FixedBytesTy:
-			field.PgType = "BYTEA NOT NULL"
-		case abi.AddressTy:
-			field.PgType = "CHARACTER VARYING(66) NOT NULL"
-		case abi.HashTy:
-			field.PgType = "CHARACTER VARYING(66) NOT NULL"
+			field.PgType = "BYTEA"
 		case abi.ArrayTy:
-			field.PgType = "TEXT[] NOT NULL"
+			field.PgType = "TEXT[]"
 		case abi.FixedPointTy:
-			field.PgType = "MONEY NOT NULL" // use shopspring/decimal for fixed point numbers in go and money type in postgres?
+			field.PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres?
 		case abi.FunctionTy:
-			field.PgType = "TEXT NOT NULL"
+			field.PgType = "TEXT"
 		default:
-			field.PgType = "TEXT NOT NULL"
+			field.PgType = "TEXT"
 		}
+
 	}
 
 	log := helpers.ConvertToLog(watchedEvent)

pkg/omni/converter/converter_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package converter_test
+
+import (
+	"io/ioutil"
+	"log"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestConverter(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Converter Suite Test")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/omni/converter/converter_test.go

@@ -0,0 +1,80 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package converter_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/examples/constants"
+	"github.com/vulcanize/vulcanizedb/examples/generic/helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
+)
+
+var mockEvent = core.WatchedEvent{
+	LogID:       1,
+	Name:        constants.TransferEvent.String(),
+	BlockNumber: 5488076,
+	Address:     constants.TusdContractAddress,
+	TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+	Index:       110,
+	Topic0:      constants.TransferEvent.Signature(),
+	Topic1:      "0x000000000000000000000000000000000000000000000000000000000000af21",
+	Topic2:      "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
+	Topic3:      "",
+	Data:        "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
+}
+
+var _ = Describe("Converter Test", func() {
+
+	It("Converts watched event log to mapping of event input names to values", func() {
+		p := parser.NewParser("")
+		err := p.Parse(constants.TusdContractAddress)
+		Expect(err).ToNot(HaveOccurred())
+
+		info := types.ContractInfo{
+			Name:          "TrueUSD",
+			Address:       constants.TusdContractAddress,
+			Abi:           p.Abi(),
+			ParsedAbi:     p.ParsedAbi(),
+			StartingBlock: 5197514,
+			Events:        p.GetEvents(),
+			Methods:       p.GetMethods(),
+		}
+
+		event := info.Events["Transfer"]
+
+		info.GenerateFilters([]string{"Transfer"})
+		c := converter.NewConverter(info)
+		err = c.Convert(mockEvent, event)
+		Expect(err).ToNot(HaveOccurred())
+
+		from := common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")
+		to := common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")
+		value := helpers.BigFromString("1097077688018008265106216665536940668749033598146")
+
+		v := event.Logs[1].Values["value"].(*big.Int)
+
+		Expect(event.Logs[1].Values["to"].(common.Address)).To(Equal(to))
+		Expect(event.Logs[1].Values["from"].(common.Address)).To(Equal(from))
+		Expect(v.String()).To(Equal(value.String()))
+	})
+})

pkg/omni/fetcher/fetcher.go

@@ -26,8 +26,7 @@ import (
 
 // Fetcher serves as the lower level data fetcher that calls the underlying
 // blockchain's FetchConctractData method for a given return type
-
-// Interface definition for a Fetcher
+// Used to collect data from contract public methods
 type Fetcher interface {
 	FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error)
 	FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error)
@@ -36,12 +35,10 @@ type Fetcher interface {
 	FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error)
 }
 
-// Fetcher struct
 type fetcher struct {
 	BlockChain core.BlockChain // Underyling Blockchain
 }
 
-// Fetcher error
 type fetcherError struct {
 	err         string
 	fetchMethod string
@@ -53,7 +50,6 @@ func NewFetcher(blockChain core.BlockChain) *fetcher {
 	}
 }
 
-// Fetcher error method
 func (fe *fetcherError) Error() string {
 	return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err)
 }

pkg/omni/fetcher/fetcher_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher_test
+
+import (
+	"io/ioutil"
+	"log"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestFetcher(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Fetcher Suite Test")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/omni/fetcher/fetcher_test.go

@@ -0,0 +1,145 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/examples/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+)
+
+var _ = Describe("Fetcher Test", func() {
+	blockNumber := int64(6194634)
+	var realFetcher fetcher.Fetcher
+
+	BeforeEach(func() {
+		infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
+		rawRpcClient, err := rpc.Dial(infuraIPC)
+		Expect(err).NotTo(HaveOccurred())
+		rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
+		ethClient := ethclient.NewClient(rawRpcClient)
+		blockChainClient := client.NewEthClient(ethClient)
+		node := node.MakeNode(rpcClient)
+		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
+		blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+		realFetcher = fetcher.NewFetcher(blockChain)
+	})
+
+	Describe("Fetch big.Int test", func() {
+
+		It("fetch totalSupply big.Int", func() {
+			bigInt, err := realFetcher.FetchBigInt("totalSupply", constants.DaiAbiString, constants.DaiContractAddress, blockNumber, nil)
+			Expect(err).NotTo(HaveOccurred())
+			expectedBigInt := big.Int{}
+			expectedBigInt.SetString("47327413946297204537985606", 10)
+			Expect(bigInt.String()).To(Equal(expectedBigInt.String()))
+		})
+
+		It("returns an error if the call to the blockchain fails", func() {
+			result, err := realFetcher.FetchBigInt("totalSupply", "", "", 0, nil)
+
+			Expect(result).To(Equal(big.Int{}))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("totalSupply"))
+		})
+	})
+
+	Describe("Fetch bool test", func() {
+
+		It("fetch stopped boolean", func() {
+			boo, err := realFetcher.FetchBool("stopped", constants.DaiAbiString, constants.DaiContractAddress, blockNumber, nil)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(boo).To(Equal(false))
+		})
+
+		It("returns an error if the call to the blockchain fails", func() {
+			boo, err := realFetcher.FetchBool("stopped", "", "", 0, nil)
+
+			Expect(boo).To(Equal(false))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("stopped"))
+		})
+	})
+
+	Describe("Fetch address test", func() {
+
+		It("fetch owner address", func() {
+			expectedAddr := common.HexToAddress("0x0000000000000000000000000000000000000000")
+			addr, err := realFetcher.FetchAddress("owner", constants.DaiAbiString, constants.DaiContractAddress, blockNumber, nil)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(addr).To(Equal(expectedAddr))
+		})
+
+		It("returns an error if the call to the blockchain fails", func() {
+			addr, err := realFetcher.FetchAddress("owner", "", "", 0, nil)
+
+			Expect(addr).To(Equal(common.Address{}))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("owner"))
+		})
+	})
+
+	Describe("Fetch string test", func() {
+
+		It("fetch nae string", func() {
+			expectedStr := "TrueUSD"
+			str, err := realFetcher.FetchString("name", constants.TusdAbiString, constants.TusdContractAddress, blockNumber, nil)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(str).To(Equal(expectedStr))
+		})
+
+		It("returns an error if the call to the blockchain fails", func() {
+			str, err := realFetcher.FetchString("name", "", "", 0, nil)
+
+			Expect(str).To(Equal(""))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("name"))
+		})
+	})
+
+	Describe("Fetch hash test", func() {
+
+		It("fetch name hash", func() {
+			expectedHash := common.HexToHash("0x44616920537461626c65636f696e2076312e3000000000000000000000000000")
+			hash, err := realFetcher.FetchHash("name", constants.DaiAbiString, constants.DaiContractAddress, blockNumber, nil)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(hash).To(Equal(expectedHash))
+		})
+
+		It("returns an error if the call to the blockchain fails", func() {
+			hash, err := realFetcher.FetchHash("name", "", "", 0, nil)
+
+			Expect(hash).To(Equal(common.Hash{}))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("name"))
+		})
+	})
+
+})

pkg/omni/parser/parser.go

@@ -20,10 +20,12 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
+// Parser is used to fetch and parse contract ABIs
+// It is dependent on etherscan's api
 type Parser interface {
 	Parse(contractAddr string) error
-	GetAbi() string
-	GetParsedAbi() abi.ABI
+	Abi() string
+	ParsedAbi() abi.ABI
 	GetMethods() map[string]*types.Method
 	GetEvents() map[string]*types.Event
 }
@@ -42,14 +44,16 @@ func NewParser(network string) *parser {
 	}
 }
 
-func (p *parser) GetAbi() string {
+func (p *parser) Abi() string {
 	return p.abi
 }
 
-func (p *parser) GetParsedAbi() abi.ABI {
+func (p *parser) ParsedAbi() abi.ABI {
 	return p.parsedAbi
 }
 
+// Retrieves and parses the abi string
+// the given contract address
 func (p *parser) Parse(contractAddr string) error {
 	abiStr, err := p.client.GetAbi(contractAddr)
 	if err != nil {
@@ -62,6 +66,7 @@ func (p *parser) Parse(contractAddr string) error {
 	return err
 }
 
+// Parses methods into our custom method type and returns
 func (p *parser) GetMethods() map[string]*types.Method {
 	methods := map[string]*types.Method{}
 
@@ -73,6 +78,7 @@ func (p *parser) GetMethods() map[string]*types.Method {
 	return methods
 }
 
+// Parses events into our custom event type and returns
 func (p *parser) GetEvents() map[string]*types.Event {
 	events := map[string]*types.Event{}
 

pkg/omni/parser/parser_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser_test
+
+import (
+	"io/ioutil"
+	"log"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestParser(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Parser Suite Test")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/omni/parser/parser_test.go

@@ -0,0 +1,62 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/examples/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
+)
+
+var _ = Describe("Parser Test", func() {
+
+	var p parser.Parser
+	var err error
+
+	BeforeEach(func() {
+		p = parser.NewParser("")
+	})
+
+	It("Fetches and parses abi using contract address", func() {
+		contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359" // dai contract address
+		err = p.Parse(contractAddr)
+		Expect(err).ToNot(HaveOccurred())
+
+		expectedAbi := constants.DaiAbiString
+		Expect(p.Abi()).To(Equal(expectedAbi))
+
+		expectedParsedAbi, err := geth.ParseAbi(expectedAbi)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(p.ParsedAbi()).To(Equal(expectedParsedAbi))
+	})
+
+	It("Returns parsed methods and events", func() {
+		contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"
+		err = p.Parse(contractAddr)
+		Expect(err).ToNot(HaveOccurred())
+
+		methods := p.GetMethods()
+		events := p.GetEvents()
+
+		_, ok := methods["totalSupply"]
+		Expect(ok).To(Equal(true))
+
+		_, ok = events["Transfer"]
+		Expect(ok).To(Equal(true))
+	})
+
+})

pkg/omni/repository/repository.go

@@ -16,12 +16,15 @@ package repository
 
 import (
 	"fmt"
+	"github.com/ethereum/go-ethereum/common"
+	"math/big"
 	"strings"
 
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
+// Repository is used to
 type DataStore interface {
 	PersistEvents(info types.ContractInfo) error
 }
@@ -36,6 +39,9 @@ func NewDataStore(db *postgres.DB) *dataStore {
 	}
 }
 
+// Creates a schema for the contract
+// Creates tables for the watched contract events
+// Persists converted event log data into these custom tables
 func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
 
 	schemaExists, err := d.CheckForSchema(contract.Name)
@@ -50,7 +56,9 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
 		}
 	}
 
-	for eventName, event := range contract.Events {
+	for eventName := range contract.Filters {
+
+		event := contract.Events[eventName]
 
 		tableExists, err := d.CheckForTable(contract.Name, eventName)
 		if err != nil {
@@ -79,12 +87,31 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
 
 			counter := 0
 			for inputName, input := range log.Values {
+				// postgres cannot handle custom types, resolve to strings
+				switch input.(type) {
+				case big.Int:
+					var b big.Int
+					b = input.(big.Int)
+					input = b.String()
+				case *big.Int:
+					var b *big.Int
+					b = input.(*big.Int)
+					input = b.String()
+				case common.Address:
+					var a common.Address
+					a = input.(common.Address)
+					input = a.String()
+				case common.Hash:
+					var h common.Hash
+					h = input.(common.Hash)
+					input = h.String()
+				}
+
 				counter += 1
-				pgStr = pgStr + fmt.Sprintf(", %s", strings.ToLower(inputName))
+				pgStr = pgStr + fmt.Sprintf(", _%s", strings.ToLower(inputName))
 				data = append(data, input)
 			}
 
-			pgStr = pgStr + ") "
 			appendStr := "VALUES ($1, $2, $3, $4, $5, $6"
 
 			for i := 0; i < counter; i++ {
@@ -100,51 +127,47 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
 				return err
 			}
 		}
-
 	}
 
 	return nil
 }
 
+// Creates a table for the given contract and event
 func (d *dataStore) CreateEventTable(contractName string, event *types.Event) error {
 	// Create postgres command to create table for any given event
 	pgStr := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s.%s ", strings.ToLower(contractName), strings.ToLower(event.Name))
-	pgStr = pgStr + `(id SERIAL, 
-					vulcanize_log_id INTEGER NOT NULL UNIQUE, 
-					token_name CHARACTER VARYING(66) NOT NULL,
-					token_address CHARACTER VARYING(66) NOT NULL,
-					event_name CHARACTER VARYING(66) NOT NULL,
-					block INTEGER NOT NULL,
-					tx CHARACTER VARYING(66) NOT NULL, `
+	pgStr = pgStr + "(id SERIAL, vulcanize_log_id INTEGER NOT NULL UNIQUE, token_name CHARACTER VARYING(66) NOT NULL, token_address CHARACTER VARYING(66) NOT NULL, event_name CHARACTER VARYING(66) NOT NULL, block INTEGER NOT NULL, tx CHARACTER VARYING(66) NOT NULL,"
 	for _, field := range event.Fields {
-		pgStr = pgStr + fmt.Sprintf("%s %s NOT NULL, ", field.Name, field.PgType)
+		pgStr = pgStr + fmt.Sprintf(" _%s %s NOT NULL,", field.Name, field.PgType)
 	}
 	pgStr = pgStr + " CONSTRAINT log_index_fk FOREIGN KEY (vulcanize_log_id) REFERENCES logs (id) ON DELETE CASCADE)"
-
 	_, err := d.DB.Exec(pgStr)
 
 	return err
 }
 
+// Checks if a table already exists for the given contract and event
 func (d *dataStore) CheckForTable(contractName string, eventName string) (bool, error) {
 	pgStr := fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = '%s' AND table_name = '%s')", contractName, eventName)
 	var exists bool
-	err := d.DB.Get(exists, pgStr)
+	err := d.DB.Get(&exists, pgStr)
 
 	return exists, err
 }
 
+// Creates a schema for the given contract
 func (d *dataStore) CreateContractSchema(contractName string) error {
 	_, err := d.DB.Exec("CREATE SCHEMA IF NOT EXISTS " + contractName)
 
 	return err
 }
 
+// Checks if a schema already exists for the given contract
 func (d *dataStore) CheckForSchema(contractName string) (bool, error) {
 	pgStr := fmt.Sprintf("SELECT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = '%s')", contractName)
 
 	var exists bool
-	err := d.DB.Get(exists, pgStr)
+	err := d.DB.Get(&exists, pgStr)
 
 	return exists, err
 }

pkg/omni/repository/repository_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repository_test
+
+import (
+	"io/ioutil"
+	"log"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestRepository(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Repository Suite Test")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/omni/repository/repository_test.go

@@ -0,0 +1,126 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repository_test
+
+import (
+	"math/rand"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/examples/constants"
+	"github.com/vulcanize/vulcanizedb/examples/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
+)
+
+var mockEvent = core.WatchedEvent{
+	Name:        constants.TransferEvent.String(),
+	BlockNumber: 5488076,
+	Address:     constants.TusdContractAddress,
+	TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+	Index:       110,
+	Topic0:      constants.TransferEvent.Signature(),
+	Topic1:      "0x000000000000000000000000000000000000000000000000000000000000af21",
+	Topic2:      "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
+	Topic3:      "",
+	Data:        "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
+}
+
+var _ = Describe("Converter Test", func() {
+
+	var db *postgres.DB
+	var logRepository repositories.LogRepository
+	var blockRepository repositories.BlockRepository
+	var receiptRepository repositories.ReceiptRepository
+	var dataStore repository.DataStore
+	var err error
+	var info types.ContractInfo
+	var blockNumber int64
+	var blockId int64
+	var vulcanizeLogId int64
+	rand.Seed(time.Now().UnixNano())
+
+	BeforeEach(func() {
+		db, err = postgres.NewDB(config.Database{
+			Hostname: "localhost",
+			Name:     "vulcanize_private",
+			Port:     5432,
+		}, core.Node{})
+		Expect(err).NotTo(HaveOccurred())
+
+		receiptRepository = repositories.ReceiptRepository{DB: db}
+		logRepository = repositories.LogRepository{DB: db}
+		blockRepository = *repositories.NewBlockRepository(db)
+
+		blockNumber = rand.Int63()
+		blockId = test_helpers.CreateBlock(blockNumber, blockRepository)
+
+		log := core.Log{}
+		logs := []core.Log{log}
+		receipt := core.Receipt{
+			Logs: logs,
+		}
+		receipts := []core.Receipt{receipt}
+
+		err = receiptRepository.CreateReceiptsAndLogs(blockId, receipts)
+		Expect(err).ToNot(HaveOccurred())
+
+		err = logRepository.Get(&vulcanizeLogId, `SELECT id FROM logs`)
+		Expect(err).ToNot(HaveOccurred())
+
+		p := parser.NewParser("")
+		err = p.Parse(constants.TusdContractAddress)
+		Expect(err).ToNot(HaveOccurred())
+
+		info = types.ContractInfo{
+			Name:          "TrueUSD",
+			Address:       constants.TusdContractAddress,
+			Abi:           p.Abi(),
+			ParsedAbi:     p.ParsedAbi(),
+			StartingBlock: 5197514,
+			Events:        p.GetEvents(),
+			Methods:       p.GetMethods(),
+		}
+
+		event := info.Events["Transfer"]
+		info.GenerateFilters([]string{"Transfer"})
+		c := converter.NewConverter(info)
+		mockEvent.LogID = vulcanizeLogId
+		err = c.Convert(mockEvent, event)
+		Expect(err).ToNot(HaveOccurred())
+
+		dataStore = repository.NewDataStore(db)
+	})
+
+	AfterEach(func() {
+		db.Query(`DELETE FROM blocks`)
+		db.Query(`DELETE FROM logs`)
+		db.Query(`DELETE FROM receipts`)
+		db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
+	})
+
+	It("Convert watched event test", func() {
+		err = dataStore.PersistEvents(info)
+		Expect(err).ToNot(HaveOccurred())
+	})
+})

pkg/omni/retriever/retriever.go

@@ -18,6 +18,8 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
+// Retriever is used to retrieve the first block for a given contract
+// It requires a vDB synced database with blocks, transactions, receipts, and logs
 type Retriever interface {
 	RetrieveFirstBlock(contractAddr string) (int64, error)
 	RetrieveFirstBlockFromLogs(contractAddr string) (int64, error)
@@ -35,7 +37,7 @@ func NewRetriever(db *postgres.DB) (r *retriever) {
 	}
 }
 
-// For some contracts the creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
+// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
 func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
 	var firstBlock int
 	err := r.DB.Get(
@@ -51,7 +53,7 @@ func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64,
 	return int64(firstBlock), err
 }
 
-// This servers as a heuristic to find the first block by finding the first contract event log
+// In which case this servers as a heuristic to find the first block by finding the first contract event log
 func (r *retriever) RetrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
 	var firstBlock int
 	err := r.DB.Get(

pkg/omni/retriever/retriever_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package retriever_test
+
+import (
+	"io/ioutil"
+	"log"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestRetriever(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Retriever Suite Test")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/omni/retriever/retriever_test.go

@@ -0,0 +1,181 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package retriever_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/examples/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
+)
+
+var _ = Describe("Fetcher Test", func() {
+	var db *postgres.DB
+	var r retriever.Retriever
+	var blockRepository repositories.BlockRepository
+
+	BeforeEach(func() {
+		var err error
+		db, err = postgres.NewDB(config.Database{
+			Hostname: "localhost",
+			Name:     "vulcanize_private",
+			Port:     5432,
+		}, core.Node{})
+		Expect(err).NotTo(HaveOccurred())
+
+		blockRepository = *repositories.NewBlockRepository(db)
+
+		r = retriever.NewRetriever(db)
+	})
+
+	AfterEach(func() {
+		db.Query(`DELETE FROM blocks`)
+		db.Query(`DELETE FROM logs`)
+		db.Query(`DELETE FROM receipts`)
+	})
+
+	It("Retrieve first block number of a contract from receipt if possible", func() {
+		log := core.Log{
+			BlockNumber: 2,
+			TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
+			Address:     constants.TusdContractAddress,
+			Topics: core.Topics{
+				constants.TransferEvent.Signature(),
+				"0x000000000000000000000000000000000000000000000000000000000000af21",
+				"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
+				"",
+			},
+			Index: 1,
+			Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
+		}
+
+		receipt1 := core.Receipt{
+			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+			ContractAddress: constants.TusdContractAddress,
+			Logs:            []core.Log{},
+		}
+
+		receipt2 := core.Receipt{
+			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
+			ContractAddress: constants.TusdContractAddress,
+			Logs:            []core.Log{log},
+		}
+
+		transaction1 := core.Transaction{
+			Hash:    "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+			Receipt: receipt1,
+		}
+
+		transaction2 := core.Transaction{
+			Hash:    "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
+			Receipt: receipt2,
+		}
+
+		block1 := core.Block{
+			Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
+			Number:       1,
+			Transactions: []core.Transaction{transaction1},
+		}
+
+		block2 := core.Block{
+			Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
+			Number:       2,
+			Transactions: []core.Transaction{transaction2},
+		}
+
+		blockRepository.CreateOrUpdateBlock(block1)
+		blockRepository.CreateOrUpdateBlock(block2)
+
+		i, err := r.RetrieveFirstBlock(constants.TusdContractAddress)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(i).To(Equal(int64(1)))
+	})
+
+	It("Retrieves first block number of a contract from event logs", func() {
+		log1 := core.Log{
+			BlockNumber: 1,
+			TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+			Address:     constants.DaiContractAddress,
+			Topics: core.Topics{
+				constants.TransferEvent.Signature(),
+				"0x000000000000000000000000000000000000000000000000000000000000af21",
+				"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
+				"",
+			},
+			Index: 1,
+			Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
+		}
+
+		log2 := core.Log{
+			BlockNumber: 2,
+			TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
+			Address:     constants.DaiContractAddress,
+			Topics: core.Topics{
+				constants.TransferEvent.Signature(),
+				"0x000000000000000000000000000000000000000000000000000000000000af21",
+				"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
+				"",
+			},
+			Index: 1,
+			Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
+		}
+
+		receipt1 := core.Receipt{
+			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+			ContractAddress: "",
+			Logs:            []core.Log{log1},
+		}
+
+		receipt2 := core.Receipt{
+			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
+			ContractAddress: "",
+			Logs:            []core.Log{log2},
+		}
+
+		transaction1 := core.Transaction{
+			Hash:    "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+			Receipt: receipt1,
+		}
+
+		transaction2 := core.Transaction{
+			Hash:    "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
+			Receipt: receipt2,
+		}
+
+		block1 := core.Block{
+			Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
+			Number:       1,
+			Transactions: []core.Transaction{transaction1},
+		}
+
+		block2 := core.Block{
+			Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
+			Number:       2,
+			Transactions: []core.Transaction{transaction2},
+		}
+
+		blockRepository.CreateOrUpdateBlock(block1)
+		blockRepository.CreateOrUpdateBlock(block2)
+
+		i, err := r.RetrieveFirstBlock(constants.DaiContractAddress)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(i).To(Equal(int64(1)))
+	})
+})

pkg/omni/transformer/event_transformer.go

@@ -29,10 +29,11 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
-// Top-level object similar to generator
-// but attempts to solve problem without
-// automated code generation
-
+// Omni event transformer
+// Used to extract all or a subset of event data for
+// any contract and persist it to postgres in a manner
+// that requires no prior knowledge of the contract
+// other than its address and which network it is on
 type EventTransformer interface {
 	Init(contractAddr string) error
 }
@@ -49,13 +50,14 @@ type eventTransformer struct {
 	// Underlying interfaces
 	parser.Parser       // Parses events out of contract abi fetched with addr
 	retriever.Retriever // Retrieves first block with contract addr referenced
-	fetcher.Fetcher     // Fetches data from contract methods
+	fetcher.Fetcher     // Fetches data from public contract methods
+	converter.Converter // Converts watched event logs into custom log
 
 	// Store contract info as mapping to contract address
 	ContractInfo map[string]types.ContractInfo
 
 	// Subset of events of interest, stored as map of contract address to events
-	// By default this
+	// Default/empty list means all events are considered for that address
 	sets map[string][]string
 }
 
@@ -64,6 +66,7 @@ func NewTransformer(c *types.Config) (t *eventTransformer) {
 	t.Parser = parser.NewParser(c.Network)
 	t.Retriever = retriever.NewRetriever(c.DB)
 	t.Fetcher = fetcher.NewFetcher(c.BC)
+	t.Converter = converter.NewConverter(types.ContractInfo{})
 	t.ContractInfo = map[string]types.ContractInfo{}
 	t.WatchedEventRepository = repositories.WatchedEventRepository{DB: c.DB}
 	t.FilterRepository = repositories.FilterRepository{DB: c.DB}
@@ -91,11 +94,11 @@ func (t *eventTransformer) Init() error {
 		}
 
 		var ctrName string
-		strName, err1 := t.Fetcher.FetchString("name", t.Parser.GetAbi(), contractAddr, -1, nil)
-		if err1 != nil || strName == "" {
-			hashName, err2 := t.Fetcher.FetchHash("name", t.Parser.GetAbi(), contractAddr, -1, nil)
-			if err2 != nil || hashName.String() == "" {
-				return errors.New(fmt.Sprintf("fetching string: %s and hash: %s names failed\r\nerr1: %v\r\nerr2: %v\r\n ", strName, hashName, err1, err2))
+		strName, err := t.Fetcher.FetchString("name", t.Parser.Abi(), contractAddr, -1, nil)
+		if err != nil || strName == "" {
+			hashName, err := t.Fetcher.FetchHash("name", t.Parser.Abi(), contractAddr, -1, nil)
+			if err != nil || hashName.String() == "" {
+				return errors.New("unable to fetch contract name") // provide CLI prompt here for user to input a contract name?
 			}
 			ctrName = hashName.String()
 		} else {
@@ -110,8 +113,8 @@ func (t *eventTransformer) Init() error {
 		info := types.ContractInfo{
 			Name:          ctrName,
 			Address:       contractAddr,
-			Abi:           t.Parser.GetAbi(),
-			ParsedAbi:     t.Parser.GetParsedAbi(),
+			Abi:           t.Parser.Abi(),
+			ParsedAbi:     t.Parser.ParsedAbi(),
 			StartingBlock: firstBlock,
 			Events:        t.Parser.GetEvents(),
 			Methods:       t.Parser.GetMethods(),
@@ -129,10 +132,14 @@ func (t *eventTransformer) Init() error {
 	return nil
 }
 
+// Iterate through contracts, creating a new
+// converter for each one and using it to
+// convert watched event logs and persist
+// them into the postgres db
 func (tr eventTransformer) Execute() error {
 	for _, contract := range tr.ContractInfo {
 
-		c := converter.NewConverter(contract)
+		tr.Converter.Update(contract)
 
 		for eventName, filter := range contract.Filters {
 			watchedEvents, err := tr.GetWatchedEvents(eventName)
@@ -142,7 +149,7 @@ func (tr eventTransformer) Execute() error {
 			}
 
 			for _, we := range watchedEvents {
-				err = c.Convert(*we, contract.Events[eventName])
+				err = tr.Converter.Convert(*we, contract.Events[eventName])
 				if err != nil {
 					return err
 				}

pkg/omni/transformer/event_transformer_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformer_test
+
+import (
+	"io/ioutil"
+	"log"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestTransformer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Transformer Suite Test")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/omni/transformer/event_transformer_test.go

@@ -0,0 +1,15 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformer_test

pkg/omni/transformer/integration_test.go

@@ -0,0 +1,15 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformer_test

pkg/omni/types/entities.go

@@ -24,20 +24,19 @@ import (
 type Event struct {
 	Name      string
 	Anonymous bool
-	Fields    []Field
+	Fields    []*Field
 	Logs      map[int64]Log // Map of VulcanizeIdLog to parsed event log
 }
 
 type Method struct {
 	Name    string
 	Const   bool
-	Inputs  []Field
-	Outputs []Field
+	Inputs  []*Field
+	Outputs []*Field
 }
 
 type Field struct {
 	abi.Argument
-	Value  interface{}
 	PgType string
 }
 
@@ -48,8 +47,9 @@ type Log struct {
 }
 
 func NewEvent(e abi.Event) *Event {
-	fields := make([]Field, len(e.Inputs))
+	fields := make([]*Field, len(e.Inputs))
 	for i, input := range e.Inputs {
+		fields[i] = &Field{}
 		fields[i].Name = input.Name
 		fields[i].Type = input.Type
 		fields[i].Indexed = input.Indexed
@@ -64,15 +64,17 @@ func NewEvent(e abi.Event) *Event {
 }
 
 func NewMethod(m abi.Method) *Method {
-	inputs := make([]Field, len(m.Inputs))
+	inputs := make([]*Field, len(m.Inputs))
 	for i, input := range m.Inputs {
+		inputs[i] = &Field{}
 		inputs[i].Name = input.Name
 		inputs[i].Type = input.Type
 		inputs[i].Indexed = input.Indexed
 	}
 
-	outputs := make([]Field, len(m.Outputs))
+	outputs := make([]*Field, len(m.Outputs))
 	for i, output := range m.Outputs {
+		outputs[i] = &Field{}
 		outputs[i].Name = output.Name
 		outputs[i].Type = output.Type
 		outputs[i].Indexed = output.Indexed

vendor/github.com/ethereum/go-ethereum/accounts/abi/argument.go

@@ -192,8 +192,7 @@ func (arguments Arguments) unpackIntoMap(v map[string]interface{}, marshalledVal
 	}
 
 	for i, arg := range arguments.NonIndexed() {
-		reflectValue := reflect.ValueOf(marshalledValues[i])
-		v[arg.Name] = reflectValue
+		v[arg.Name] = marshalledValues[i]
 	}
 
 	return nil