beginning work on method polling; first need to generate list of token holder address in a completely generic/contratc agnostic fashion. Created address retriever that can iterate through any given contract's watched events, finding the inputs/arguments with address type, and generate a list from those values. Edit: Contract objects now cache every event emitted address as its event logs are transformed into the repo to grow a list of contract associated addresses as we go

cmd/omniWatcher.go

@@ -60,13 +60,14 @@ Requires a .toml config file:
 }
 
 func omniWatcher() {
-	ticker := time.NewTicker(5 * time.Second)
-	defer ticker.Stop()
-
 	if contractAddress == "" && len(contractAddresses) == 0 {
 		log.Fatal("Contract address required")
 	}
 
+	if !methodsOn && !eventsOn {
+		log.Fatal("Method polling and event watching turned off- nothing to do!")
+	}
+
 	if len(contractEvents) == 0 || len(contractMethods) == 0 {
 		var str string
 		for str != "y" {
@@ -90,6 +91,9 @@ func omniWatcher() {
 		}
 	}
 
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+
 	rawRpcClient, err := rpc.Dial(ipc)
 	if err != nil {
 		log.Fatal(fmt.Sprintf("Failed to initialize rpc client\r\nerr: %v\r\n", err))
@@ -116,7 +120,8 @@ func omniWatcher() {
 
 	contractAddresses = append(contractAddresses, contractAddress)
 	for _, addr := range contractAddresses {
-		t.Set(addr, contractEvents)
+		t.SetEvents(addr, contractEvents)
+		t.SetMethods(addr, contractMethods)
 	}
 
 	err = t.Init()
@@ -137,6 +142,9 @@ func init() {
 
 	omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
 	omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "List of addresses to generate watchers for")
+	omniWatcherCmd.Flags().BoolVarP(&eventsOn, "events-on", "o", true, "Set to false to turn off watching of any event")
+	omniWatcherCmd.Flags().BoolVarP(&methodsOn, "methods-on", "p", true, "Set to false to turn off polling of any method")
+	omniWatcherCmd.Flags().StringVarP(&contractAddress, "methods-off", "a", "", "Single address to generate watchers for")
 	omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched")
 	omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-methods", "m", []string{}, "Subset of methods to watch; by default all methods are watched")
 	omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)

cmd/root.go

@@ -35,6 +35,8 @@ var (
 	network             string
 	contractAddress     string
 	contractAddresses   []string
+	eventsOn            bool
+	methodsOn           bool
 	contractEvents      []string
 	contractMethods     []string
 )

examples/constants/constants.go

@@ -47,10 +47,10 @@ func (e Event) String() string {
 
 func (e Event) Signature() string {
 	strings := [...]string{
-		helpers.GenerateSignature("Transfer(address,address,uint)"),
+		helpers.GenerateSignature("Transfer(address,address,uint256)"),
-		helpers.GenerateSignature("Approval(address,address,uint)"),
+		helpers.GenerateSignature("Approval(address,address,uint256)"),
-		helpers.GenerateSignature("Burn(address,uint)"),
+		helpers.GenerateSignature("Burn(address,uint256)"),
-		helpers.GenerateSignature("Mint(address,uint)"),
+		helpers.GenerateSignature("Mint(address,uint256)"),
 	}
 
 	if e < TransferEvent || e > MintEvent {

pkg/omni/contract/contract.go

@@ -12,39 +12,54 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package types
+package contract
 
 import (
+	"errors"
+	"github.com/vulcanize/vulcanizedb/examples/generic/helpers"
+
 	"github.com/ethereum/go-ethereum/accounts/abi"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/filters"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
-type ContractInfo struct {
+type Contract struct {
 	Name          string
 	Address       string
 	StartingBlock int64
 	Abi           string
 	ParsedAbi     abi.ABI
-	Events        map[string]*Event            // Map of events to their names
+	Events        map[string]*types.Event      // Map of events to their names
-	Methods       map[string]*Method           // Map of methods to their names
+	Methods       map[string]*types.Method     // Map of methods to their names
 	Filters       map[string]filters.LogFilter // Map of event filters to their names
+	Addresses     map[string]bool              // Map of all contract-associated addresses, populated as events are transformed
 }
 
-func (i *ContractInfo) GenerateFilters(subset []string) {
+func (c *Contract) GenerateFilters(subset []string) error {
-	i.Filters = map[string]filters.LogFilter{}
+	c.Filters = map[string]filters.LogFilter{}
-	for name, event := range i.Events {
+	for name, event := range c.Events {
 		if len(subset) == 0 || stringInSlice(subset, name) {
-			i.Filters[name] = filters.LogFilter{
+			c.Filters[name] = filters.LogFilter{
 				Name:      name,
-				FromBlock: i.StartingBlock,
+				FromBlock: c.StartingBlock,
 				ToBlock:   -1,
-				Address:   i.Address,
+				Address:   c.Address,
-				Topics:    core.Topics{event.Sig()},
+				Topics:    core.Topics{helpers.GenerateSignature(event.Sig())},
 			}
 		}
 	}
+
+	if len(c.Filters) == 0 {
+		return errors.New("error: no filters created")
+	}
+
+	return nil
+}
+
+func (c *Contract) AddAddress(addr string) {
+	c.Addresses[addr] = true
 }
 
 func stringInSlice(list []string, s string) bool {

pkg/omni/contract/contract_suite_test.go

@@ -12,15 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package types
+package contract_test
 
 import (
-	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"io/ioutil"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"log"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
-type Config struct {
+func TestContract(t *testing.T) {
-	Network string
+	RegisterFailHandler(Fail)
-	BC      core.BlockChain
+	RunSpecs(t, "Contract Suite Test")
-	DB      *postgres.DB
 }
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/omni/contract/contract_test.go

@@ -0,0 +1,70 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package contract_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/filters"
+
+	"github.com/vulcanize/vulcanizedb/examples/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
+)
+
+var expectedLogFilter = filters.LogFilter{
+	Name:      "Transfer",
+	Address:   constants.TusdContractAddress,
+	ToBlock:   -1,
+	FromBlock: 5197514,
+	Topics:    core.Topics{constants.TransferEvent.Signature()},
+}
+
+var _ = Describe("Contract test", func() {
+	var p parser.Parser
+	var err error
+
+	BeforeEach(func() {
+		p = parser.NewParser("")
+		err = p.Parse(constants.TusdContractAddress)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	It("Creates filters from stored data", func() {
+		info := contract.Contract{
+			Name:          "TrueUSD",
+			Address:       constants.TusdContractAddress,
+			Abi:           p.Abi(),
+			ParsedAbi:     p.ParsedAbi(),
+			StartingBlock: 5197514,
+			Events:        p.GetEvents(),
+			Methods:       p.GetMethods(),
+			Addresses:     map[string]bool{},
+		}
+
+		err = info.GenerateFilters([]string{"Transfer"})
+		Expect(err).ToNot(HaveOccurred())
+		val, ok := info.Filters["Transfer"]
+		Expect(ok).To(Equal(true))
+		Expect(val).To(Equal(expectedLogFilter))
+	})
+
+	It("Fails with an empty contract", func() {
+		info := contract.Contract{}
+		err = info.GenerateFilters([]string{"Transfer"})
+		Expect(err).To(HaveOccurred())
+	})
+})

pkg/omni/converter/converter.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/vulcanize/vulcanizedb/examples/generic/helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
@@ -28,21 +29,21 @@ import (
 // custom logs containing event input name => value maps
 type Converter interface {
 	Convert(watchedEvent core.WatchedEvent, event *types.Event) error
-	Update(info types.ContractInfo)
+	Update(info contract.Contract)
 }
 
 type converter struct {
-	contractInfo types.ContractInfo
+	contractInfo contract.Contract
 }
 
-func NewConverter(info types.ContractInfo) *converter {
+func NewConverter(info contract.Contract) *converter {
 
 	return &converter{
 		contractInfo: info,
 	}
 }
 
-func (c *converter) Update(info types.ContractInfo) {
+func (c *converter) Update(info contract.Contract) {
 	c.contractInfo = info
 }
 

pkg/omni/converter/converter_test.go

@@ -24,9 +24,9 @@ import (
 	"github.com/vulcanize/vulcanizedb/examples/constants"
 	"github.com/vulcanize/vulcanizedb/examples/generic/helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
 var mockEvent = core.WatchedEvent{
@@ -44,13 +44,17 @@ var mockEvent = core.WatchedEvent{
 }
 
 var _ = Describe("Converter Test", func() {
+	var p parser.Parser
+	var err error
+
+	BeforeEach(func() {
+		p = parser.NewParser("")
+		err = p.Parse(constants.TusdContractAddress)
+		Expect(err).ToNot(HaveOccurred())
+	})
 
 	It("Converts watched event log to mapping of event input names to values", func() {
-		p := parser.NewParser("")
+		info := contract.Contract{
-		err := p.Parse(constants.TusdContractAddress)
-		Expect(err).ToNot(HaveOccurred())
-
-		info := types.ContractInfo{
 			Name:          "TrueUSD",
 			Address:       constants.TusdContractAddress,
 			Abi:           p.Abi(),
@@ -58,11 +62,12 @@ var _ = Describe("Converter Test", func() {
 			StartingBlock: 5197514,
 			Events:        p.GetEvents(),
 			Methods:       p.GetMethods(),
+			Addresses:     map[string]bool{},
 		}
-
 		event := info.Events["Transfer"]
 
-		info.GenerateFilters([]string{"Transfer"})
+		err = info.GenerateFilters([]string{"Transfer"})
+		Expect(err).ToNot(HaveOccurred())
 		c := converter.NewConverter(info)
 		err = c.Convert(mockEvent, event)
 		Expect(err).ToNot(HaveOccurred())
@@ -77,4 +82,11 @@ var _ = Describe("Converter Test", func() {
 		Expect(event.Logs[1].Values["from"].(common.Address)).To(Equal(from))
 		Expect(v.String()).To(Equal(value.String()))
 	})
+
+	It("Fails with an empty contract", func() {
+		event := p.GetEvents()["Transfer"]
+		c := converter.NewConverter(contract.Contract{})
+		err = c.Convert(mockEvent, event)
+		Expect(err).To(HaveOccurred())
+	})
 })

pkg/omni/fetcher/fetcher_test.go

@@ -15,10 +15,9 @@
 package fetcher_test
 
 import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
 	. "github.com/onsi/ginkgo"
@@ -29,6 +28,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
 	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
 	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
 )
 
 var _ = Describe("Fetcher Test", func() {
@@ -106,7 +106,7 @@ var _ = Describe("Fetcher Test", func() {
 
 	Describe("Fetch string test", func() {
 
-		It("fetch nae string", func() {
+		It("fetch name string", func() {
 			expectedStr := "TrueUSD"
 			str, err := realFetcher.FetchString("name", constants.TusdAbiString, constants.TusdContractAddress, blockNumber, nil)
 

pkg/omni/parser/parser.go

@@ -53,7 +53,7 @@ func (p *parser) ParsedAbi() abi.ABI {
 }
 
 // Retrieves and parses the abi string
-// the given contract address
+// for the given contract address
 func (p *parser) Parse(contractAddr string) error {
 	abiStr, err := p.client.GetAbi(contractAddr)
 	if err != nil {

pkg/omni/parser/parser_test.go

@@ -17,6 +17,7 @@ package parser_test
 import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/examples/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
@@ -59,4 +60,10 @@ var _ = Describe("Parser Test", func() {
 		Expect(ok).To(Equal(true))
 	})
 
+	It("Fails with a normal, non-contract, account address", func() {
+		addr := "0xAb2A8F7cB56D9EC65573BA1bE0f92Fa2Ff7dd165"
+		err = p.Parse(addr)
+		Expect(err).To(HaveOccurred())
+	})
+
 })

pkg/omni/repository/repository.go

@@ -16,17 +16,19 @@ package repository
 
 import (
 	"fmt"
-	"github.com/ethereum/go-ethereum/common"
 	"math/big"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/common"
+
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
 // Repository is used to
 type DataStore interface {
-	PersistEvents(info types.ContractInfo) error
+	PersistEvents(info *contract.Contract) error
 }
 
 type dataStore struct {
@@ -34,6 +36,7 @@ type dataStore struct {
 }
 
 func NewDataStore(db *postgres.DB) *dataStore {
+
 	return &dataStore{
 		DB: db,
 	}
@@ -42,31 +45,33 @@ func NewDataStore(db *postgres.DB) *dataStore {
 // Creates a schema for the contract
 // Creates tables for the watched contract events
 // Persists converted event log data into these custom tables
-func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
+func (d *dataStore) PersistEvents(con *contract.Contract) error {
 
-	schemaExists, err := d.CheckForSchema(contract.Name)
+	schemaExists, err := d.CheckForSchema(con.Name)
 	if err != nil {
 		return err
 	}
 
 	if !schemaExists {
-		err = d.CreateContractSchema(contract.Name)
+		err = d.CreateContractSchema(con.Name)
 		if err != nil {
 			return err
 		}
 	}
 
-	for eventName := range contract.Filters {
+	for eventName := range con.Filters {
+		event := con.Events[eventName]
+		if len(event.Logs) == 0 {
+			break
+		}
 
-		event := contract.Events[eventName]
+		tableExists, err := d.CheckForTable(con.Name, eventName)
-
-		tableExists, err := d.CheckForTable(contract.Name, eventName)
 		if err != nil {
 			return err
 		}
 
 		if !tableExists {
-			err = d.CreateEventTable(contract.Name, event)
+			err = d.CreateEventTable(con.Name, event)
 			if err != nil {
 				return err
 			}
@@ -74,13 +79,13 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
 
 		for id, log := range event.Logs {
 			// Create postgres command to persist any given event
-			pgStr := fmt.Sprintf("INSERT INTO %s.%s ", strings.ToLower(contract.Name), strings.ToLower(eventName))
+			pgStr := fmt.Sprintf("INSERT INTO %s.%s ", strings.ToLower(con.Name), strings.ToLower(eventName))
 			pgStr = pgStr + "(vulcanize_log_id, token_name, token_address, event_name, block, tx"
 			var data []interface{}
 			data = append(data,
 				id,
-				strings.ToLower(contract.Name),
+				strings.ToLower(con.Name),
-				strings.ToLower(contract.Address),
+				strings.ToLower(con.Address),
 				strings.ToLower(eventName),
 				log.Block,
 				log.Tx)
@@ -100,7 +105,8 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
 				case common.Address:
 					var a common.Address
 					a = input.(common.Address)
-					input = a.String()
+					input = a.String()         // this also gives us a chance to add any event emitted address
+					con.AddAddress(a.String()) // to a list of token-related addresses, growing it as we go
 				case common.Hash:
 					var h common.Hash
 					h = input.(common.Hash)

pkg/omni/repository/repository_test.go

@@ -27,10 +27,10 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
 )
 
 var mockEvent = core.WatchedEvent{
@@ -46,15 +46,14 @@ var mockEvent = core.WatchedEvent{
 	Data:        "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
 }
 
-var _ = Describe("Converter Test", func() {
+var _ = Describe("Repository Test", func() {
-
 	var db *postgres.DB
 	var logRepository repositories.LogRepository
 	var blockRepository repositories.BlockRepository
 	var receiptRepository repositories.ReceiptRepository
 	var dataStore repository.DataStore
 	var err error
-	var info types.ContractInfo
+	var info *contract.Contract
 	var blockNumber int64
 	var blockId int64
 	var vulcanizeLogId int64
@@ -92,7 +91,7 @@ var _ = Describe("Converter Test", func() {
 		err = p.Parse(constants.TusdContractAddress)
 		Expect(err).ToNot(HaveOccurred())
 
-		info = types.ContractInfo{
+		info = &contract.Contract{
 			Name:          "TrueUSD",
 			Address:       constants.TusdContractAddress,
 			Abi:           p.Abi(),
@@ -100,11 +99,13 @@ var _ = Describe("Converter Test", func() {
 			StartingBlock: 5197514,
 			Events:        p.GetEvents(),
 			Methods:       p.GetMethods(),
+			Addresses:     map[string]bool{},
 		}
 
 		event := info.Events["Transfer"]
-		info.GenerateFilters([]string{"Transfer"})
+		err = info.GenerateFilters([]string{"Transfer"})
-		c := converter.NewConverter(info)
+		Expect(err).ToNot(HaveOccurred())
+		c := converter.NewConverter(*info)
 		mockEvent.LogID = vulcanizeLogId
 		err = c.Convert(mockEvent, event)
 		Expect(err).ToNot(HaveOccurred())
@@ -115,12 +116,26 @@ var _ = Describe("Converter Test", func() {
 	AfterEach(func() {
 		db.Query(`DELETE FROM blocks`)
 		db.Query(`DELETE FROM logs`)
+		db.Query(`DELETE FROM transactions`)
 		db.Query(`DELETE FROM receipts`)
 		db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
 	})
 
-	It("Convert watched event test", func() {
+	It("Persist contract info in custom tables", func() {
 		err = dataStore.PersistEvents(info)
 		Expect(err).ToNot(HaveOccurred())
+
+		b, ok := info.Addresses["0x000000000000000000000000000000000000Af21"]
+		Expect(ok).To(Equal(true))
+		Expect(b).To(Equal(true))
+
+		b, ok = info.Addresses["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
+		Expect(ok).To(Equal(true))
+		Expect(b).To(Equal(true))
+	})
+
+	It("Fails with empty contract", func() {
+		err = dataStore.PersistEvents(&contract.Contract{})
+		Expect(err).To(HaveOccurred())
 	})
 })

pkg/omni/retriever/address_retriever.go

@@ -0,0 +1,117 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package retriever
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
+)
+
+// Address retriever is used to retrieve the addresses associated with a contract
+// It requires a vDB synced database with blocks, transactions, receipts, logs,
+// AND all of the targeted events persisted
+type AddressRetriever interface {
+	RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error)
+}
+
+type addressRetriever struct {
+	*postgres.DB
+}
+
+func NewAddressRetriever(db *postgres.DB) (r *addressRetriever) {
+
+	return &addressRetriever{
+		DB: db,
+	}
+}
+
+// Method to retrieve list of token-holding/contract-related addresses by iterating over available events
+// This generic method should work whether or not the argument/input names of the events meet the expected standard
+// This could be generalized to iterate over ALL events and pull out any address arguments
+func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) {
+	addrList := make([]string, 0)
+
+	_, ok := info.Filters["Transfer"]
+	if ok {
+		addrs, err := r.retrieveTransferAddresses(info)
+		if err != nil {
+			return nil, err
+		}
+		addrList = append(addrList, addrs...)
+	}
+
+	_, ok = info.Filters["Mint"]
+	if ok {
+		addrs, err := r.retrieveTokenMintees(info)
+		if err != nil {
+			return nil, err
+		}
+		addrList = append(addrList, addrs...)
+	}
+
+	contractAddresses := make(map[common.Address]bool)
+	for _, addr := range addrList {
+		contractAddresses[common.HexToAddress(addr)] = true
+	}
+
+	return contractAddresses, nil
+}
+
+func (r *addressRetriever) retrieveTransferAddresses(contract contract.Contract) ([]string, error) {
+	transferAddrs := make([]string, 0)
+	event := contract.Events["Transfer"]
+
+	for _, field := range event.Fields { // Iterate over event fields, finding the ones with address type
+
+		if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses
+			addrs := make([]string, 0)
+			pgStr := fmt.Sprintf("SELECT _%s FROM %s.%s", field.Name, contract.Name, event.Name)
+			err := r.DB.Select(&addrs, pgStr)
+			if err != nil {
+				return []string{}, err
+			}
+
+			transferAddrs = append(transferAddrs, addrs...) // And append them to the growing list
+		}
+	}
+
+	return transferAddrs, nil
+}
+
+func (r *addressRetriever) retrieveTokenMintees(contract contract.Contract) ([]string, error) {
+	mintAddrs := make([]string, 0)
+	event := contract.Events["Mint"]
+
+	for _, field := range event.Fields { // Iterate over event fields, finding the ones with address type
+
+		if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses
+			addrs := make([]string, 0)
+			pgStr := fmt.Sprintf("SELECT _%s FROM %s.%s", field.Name, contract.Name, event.Name)
+			err := r.DB.Select(&addrs, pgStr)
+			if err != nil {
+				return []string{}, err
+			}
+
+			mintAddrs = append(mintAddrs, addrs...) // And append them to the growing list
+		}
+	}
+
+	return mintAddrs, nil
+}

pkg/omni/retriever/address_retriever_test.go

@@ -0,0 +1,152 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package retriever_test
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/examples/constants"
+	"github.com/vulcanize/vulcanizedb/examples/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
+)
+
+var mockEvent = core.WatchedEvent{
+	Name:        constants.TransferEvent.String(),
+	BlockNumber: 5488076,
+	Address:     constants.TusdContractAddress,
+	TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+	Index:       110,
+	Topic0:      constants.TransferEvent.Signature(),
+	Topic1:      "0x000000000000000000000000000000000000000000000000000000000000af21",
+	Topic2:      "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
+	Topic3:      "",
+	Data:        "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
+}
+
+var _ = Describe("Address Retriever Test", func() {
+	var db *postgres.DB
+	var logRepository repositories.LogRepository
+	var blockRepository repositories.BlockRepository
+	var receiptRepository repositories.ReceiptRepository
+	var dataStore repository.DataStore
+	var err error
+	var info *contract.Contract
+	var blockNumber int64
+	var blockId int64
+	var vulcanizeLogId int64
+	var r retriever.AddressRetriever
+	var addresses map[common.Address]bool
+	rand.Seed(time.Now().UnixNano())
+
+	BeforeEach(func() {
+		db, err = postgres.NewDB(config.Database{
+			Hostname: "localhost",
+			Name:     "vulcanize_private",
+			Port:     5432,
+		}, core.Node{})
+		Expect(err).NotTo(HaveOccurred())
+
+		receiptRepository = repositories.ReceiptRepository{DB: db}
+		logRepository = repositories.LogRepository{DB: db}
+		blockRepository = *repositories.NewBlockRepository(db)
+
+		blockNumber = rand.Int63()
+		blockId = test_helpers.CreateBlock(blockNumber, blockRepository)
+
+		log := core.Log{}
+		logs := []core.Log{log}
+		receipt := core.Receipt{
+			Logs: logs,
+		}
+		receipts := []core.Receipt{receipt}
+
+		err = receiptRepository.CreateReceiptsAndLogs(blockId, receipts)
+		Expect(err).ToNot(HaveOccurred())
+
+		err = logRepository.Get(&vulcanizeLogId, `SELECT id FROM logs`)
+		Expect(err).ToNot(HaveOccurred())
+
+		p := parser.NewParser("")
+		err = p.Parse(constants.TusdContractAddress)
+		Expect(err).ToNot(HaveOccurred())
+
+		info = &contract.Contract{
+			Name:          "TrueUSD",
+			Address:       constants.TusdContractAddress,
+			Abi:           p.Abi(),
+			ParsedAbi:     p.ParsedAbi(),
+			StartingBlock: 5197514,
+			Events:        p.GetEvents(),
+			Methods:       p.GetMethods(),
+			Addresses:     map[string]bool{},
+		}
+
+		event := info.Events["Transfer"]
+		err = info.GenerateFilters([]string{"Transfer"})
+		Expect(err).ToNot(HaveOccurred())
+		c := converter.NewConverter(*info)
+		mockEvent.LogID = vulcanizeLogId
+		err = c.Convert(mockEvent, event)
+		Expect(err).ToNot(HaveOccurred())
+
+		dataStore = repository.NewDataStore(db)
+		err = dataStore.PersistEvents(info)
+		Expect(err).ToNot(HaveOccurred())
+
+		r = retriever.NewAddressRetriever(db)
+	})
+
+	AfterEach(func() {
+		db.Query(`DELETE FROM blocks`)
+		db.Query(`DELETE FROM logs`)
+		db.Query(`DELETE FROM transactions`)
+		db.Query(`DELETE FROM receipts`)
+		db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
+	})
+
+	It("Retrieves a list of token holder addresses", func() {
+		addresses, err = r.RetrieveTokenHolderAddresses(*info)
+		Expect(err).ToNot(HaveOccurred())
+
+		_, ok := addresses[common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")]
+		Expect(ok).To(Equal(true))
+
+		_, ok = addresses[common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
+		Expect(ok).To(Equal(true))
+
+		_, ok = addresses[common.HexToAddress("0x")]
+		Expect(ok).To(Equal(false))
+
+	})
+
+	It("Returns empty list when empty contract info is used", func() {
+		addresses, err = r.RetrieveTokenHolderAddresses(contract.Contract{})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(len(addresses)).To(Equal(0))
+	})
+})

pkg/omni/retriever/block_retriever.go

@@ -18,27 +18,36 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-// Retriever is used to retrieve the first block for a given contract
+// Block retriever is used to retrieve the first block for a given contract and the most recent block
 // It requires a vDB synced database with blocks, transactions, receipts, and logs
-type Retriever interface {
+type BlockRetriever interface {
 	RetrieveFirstBlock(contractAddr string) (int64, error)
-	RetrieveFirstBlockFromLogs(contractAddr string) (int64, error)
+	RetrieveMostRecentBlock() (int64, error)
-	RetrieveFirstBlockFromReceipts(contractAddr string) (int64, error)
 }
 
-type retriever struct {
+type blockRetriever struct {
 	*postgres.DB
 }
 
-func NewRetriever(db *postgres.DB) (r *retriever) {
+func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
 
-	return &retriever{
+	return &blockRetriever{
 		DB: db,
 	}
 }
 
+// Try both methods of finding the first block, with the receipt method taking precedence
+func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
+	i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
+	if err != nil {
+		i, err = r.retrieveFirstBlockFromLogs(contractAddr)
+	}
+
+	return i, err
+}
+
 // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
-func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
+func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
 	var firstBlock int
 	err := r.DB.Get(
 		&firstBlock,
@@ -54,7 +63,7 @@ func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64,
 }
 
 // In which case this servers as a heuristic to find the first block by finding the first contract event log
-func (r *retriever) RetrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
+func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
 	var firstBlock int
 	err := r.DB.Get(
 		&firstBlock,
@@ -65,12 +74,13 @@ func (r *retriever) RetrieveFirstBlockFromLogs(contractAddr string) (int64, erro
 	return int64(firstBlock), err
 }
 
-// Try both methods of finding the first block, with the receipt method taking precedence
+// Method to retrieve the most recent block in vDB
-func (r *retriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
+func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
-	i, err := r.RetrieveFirstBlockFromReceipts(contractAddr)
+	var lastBlock int64
-	if err != nil {
+	err := r.DB.Get(
-		i, err = r.RetrieveFirstBlockFromLogs(contractAddr)
+		&lastBlock,
-	}
+		"SELECT number FROM blocks ORDER BY number DESC LIMIT 1",
+	)
 
-	return i, err
+	return lastBlock, err
 }

pkg/omni/retriever/block_retriever_test.go

@@ -26,9 +26,9 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
 )
 
-var _ = Describe("Fetcher Test", func() {
+var _ = Describe("Block Retriever Test", func() {
 	var db *postgres.DB
-	var r retriever.Retriever
+	var r retriever.BlockRetriever
 	var blockRepository repositories.BlockRepository
 
 	BeforeEach(func() {
@@ -42,12 +42,13 @@ var _ = Describe("Fetcher Test", func() {
 
 		blockRepository = *repositories.NewBlockRepository(db)
 
-		r = retriever.NewRetriever(db)
+		r = retriever.NewBlockRetriever(db)
 	})
 
 	AfterEach(func() {
 		db.Query(`DELETE FROM blocks`)
 		db.Query(`DELETE FROM logs`)
+		db.Query(`DELETE FROM transactions`)
 		db.Query(`DELETE FROM receipts`)
 	})
 
@@ -178,4 +179,25 @@ var _ = Describe("Fetcher Test", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(i).To(Equal(int64(1)))
 	})
+
+	It("Fails if a block cannot be found", func() {
+
+		block1 := core.Block{
+			Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
+			Number:       1,
+			Transactions: []core.Transaction{},
+		}
+
+		block2 := core.Block{
+			Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
+			Number:       2,
+			Transactions: []core.Transaction{},
+		}
+
+		blockRepository.CreateOrUpdateBlock(block1)
+		blockRepository.CreateOrUpdateBlock(block2)
+
+		_, err := r.RetrieveFirstBlock(constants.DaiContractAddress)
+		Expect(err).To(HaveOccurred())
+	})
 })

pkg/omni/transformer/event_transformer.go

@@ -1,167 +0,0 @@
-// Copyright 2018 Vulcanize
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package transformer
-
-import (
-	"errors"
-	"fmt"
-	"log"
-
-	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
-)
-
-// Omni event transformer
-// Used to extract all or a subset of event data for
-// any contract and persist it to postgres in a manner
-// that requires no prior knowledge of the contract
-// other than its address and which network it is on
-type EventTransformer interface {
-	Init(contractAddr string) error
-}
-
-type eventTransformer struct {
-	// Network, database, and blockchain config
-	*types.Config
-
-	// Underlying databases
-	datastore.WatchedEventRepository
-	datastore.FilterRepository
-	repository.DataStore
-
-	// Underlying interfaces
-	parser.Parser       // Parses events out of contract abi fetched with addr
-	retriever.Retriever // Retrieves first block with contract addr referenced
-	fetcher.Fetcher     // Fetches data from public contract methods
-	converter.Converter // Converts watched event logs into custom log
-
-	// Store contract info as mapping to contract address
-	ContractInfo map[string]types.ContractInfo
-
-	// Subset of events of interest, stored as map of contract address to events
-	// Default/empty list means all events are considered for that address
-	sets map[string][]string
-}
-
-// Transformer takes in config for blockchain, database, and network id
-func NewTransformer(c *types.Config) (t *eventTransformer) {
-	t.Parser = parser.NewParser(c.Network)
-	t.Retriever = retriever.NewRetriever(c.DB)
-	t.Fetcher = fetcher.NewFetcher(c.BC)
-	t.Converter = converter.NewConverter(types.ContractInfo{})
-	t.ContractInfo = map[string]types.ContractInfo{}
-	t.WatchedEventRepository = repositories.WatchedEventRepository{DB: c.DB}
-	t.FilterRepository = repositories.FilterRepository{DB: c.DB}
-	t.DataStore = repository.NewDataStore(c.DB)
-	t.sets = map[string][]string{}
-
-	return t
-}
-
-// Used to set which contract addresses and which of their events to watch
-func (t *eventTransformer) Set(contractAddr string, filterSet []string) {
-	t.sets[contractAddr] = filterSet
-}
-
-// Use after creating and setting transformer
-// Loops over all of the addr => filter sets
-// Uses parser to pull event info from abi
-// Use this info to generate event filters
-func (t *eventTransformer) Init() error {
-
-	for contractAddr, subset := range t.sets {
-		err := t.Parser.Parse(contractAddr)
-		if err != nil {
-			return err
-		}
-
-		var ctrName string
-		strName, err := t.Fetcher.FetchString("name", t.Parser.Abi(), contractAddr, -1, nil)
-		if err != nil || strName == "" {
-			hashName, err := t.Fetcher.FetchHash("name", t.Parser.Abi(), contractAddr, -1, nil)
-			if err != nil || hashName.String() == "" {
-				return errors.New("unable to fetch contract name") // provide CLI prompt here for user to input a contract name?
-			}
-			ctrName = hashName.String()
-		} else {
-			ctrName = strName
-		}
-
-		firstBlock, err := t.Retriever.RetrieveFirstBlock(contractAddr)
-		if err != nil {
-			return err
-		}
-
-		info := types.ContractInfo{
-			Name:          ctrName,
-			Address:       contractAddr,
-			Abi:           t.Parser.Abi(),
-			ParsedAbi:     t.Parser.ParsedAbi(),
-			StartingBlock: firstBlock,
-			Events:        t.Parser.GetEvents(),
-			Methods:       t.Parser.GetMethods(),
-		}
-
-		info.GenerateFilters(subset)
-
-		for _, filter := range info.Filters {
-			t.CreateFilter(filter)
-		}
-
-		t.ContractInfo[contractAddr] = info
-	}
-
-	return nil
-}
-
-// Iterate through contracts, creating a new
-// converter for each one and using it to
-// convert watched event logs and persist
-// them into the postgres db
-func (tr eventTransformer) Execute() error {
-	for _, contract := range tr.ContractInfo {
-
-		tr.Converter.Update(contract)
-
-		for eventName, filter := range contract.Filters {
-			watchedEvents, err := tr.GetWatchedEvents(eventName)
-			if err != nil {
-				log.Println(fmt.Sprintf("Error fetching events for %s:", filter.Name), err)
-				return err
-			}
-
-			for _, we := range watchedEvents {
-				err = tr.Converter.Convert(*we, contract.Events[eventName])
-				if err != nil {
-					return err
-				}
-			}
-
-		}
-
-		err := tr.PersistEvents(contract)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

pkg/omni/transformer/event_transformer_test.go

@@ -1,15 +0,0 @@
-// Copyright 2018 Vulcanize
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package transformer_test

pkg/omni/transformer/transformer.go

@@ -0,0 +1,199 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformer
+
+import (
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
+)
+
+// Omni transformer
+// Used to extract all or a subset of event and method data
+// for any contract and persist it to postgres in a manner
+// that requires no prior knowledge of the contract
+// other than its address and which network it is on
+type Transformer interface {
+	Init(contractAddr string) error
+}
+
+type transformer struct {
+	// Network, database, and blockchain config
+	*types.Config
+
+	// Underlying databases
+	datastore.WatchedEventRepository
+	datastore.FilterRepository
+	repository.DataStore
+
+	// Underlying interfaces
+	parser.Parser              // Parses events out of contract abi fetched with addr
+	retriever.BlockRetriever   // Retrieves first block with contract addr referenced
+	retriever.AddressRetriever // Retrieves token holder addresses
+	fetcher.Fetcher            // Fetches data from public contract methods
+	converter.Converter        // Converts watched event logs into custom log
+
+	// Store contract info as mapping to contract address
+	Contracts map[string]*contract.Contract
+
+	// Targeted subset of events/methods
+	// Stored as map of contract address to events/method names of interest
+	// Default/empty list means all events/methods are considered for that address
+	targetEvents  map[string][]string
+	targetMethods map[string][]string
+}
+
+// Transformer takes in config for blockchain, database, and network id
+func NewTransformer(c *types.Config) *transformer {
+
+	return &transformer{
+		Parser:                 parser.NewParser(c.Network),
+		BlockRetriever:         retriever.NewBlockRetriever(c.DB),
+		Fetcher:                fetcher.NewFetcher(c.BC),
+		Converter:              converter.NewConverter(contract.Contract{}),
+		Contracts:              map[string]*contract.Contract{},
+		WatchedEventRepository: repositories.WatchedEventRepository{DB: c.DB},
+		FilterRepository:       repositories.FilterRepository{DB: c.DB},
+		DataStore:              repository.NewDataStore(c.DB),
+		targetEvents:           map[string][]string{},
+		targetMethods:          map[string][]string{},
+	}
+}
+
+// Used to set which contract addresses and which of their events to watch
+func (t *transformer) SetEvents(contractAddr string, filterSet []string) {
+	t.targetEvents[contractAddr] = filterSet
+}
+
+// Used to set which contract addresses and which of their methods to call
+func (t *transformer) SetMethods(contractAddr string, filterSet []string) {
+	t.targetMethods[contractAddr] = filterSet
+}
+
+// Use after creating and setting transformer
+// Loops over all of the addr => filter sets
+// Uses parser to pull event info from abi
+// Use this info to generate event filters
+func (t *transformer) Init() error {
+
+	for contractAddr, subset := range t.targetEvents {
+		// Get Abi
+		err := t.Parser.Parse(contractAddr)
+		if err != nil {
+			return err
+		}
+
+		// Get first block for contract
+		firstBlock, err := t.BlockRetriever.RetrieveFirstBlock(contractAddr)
+		if err != nil {
+			return err
+		}
+
+		// Get most recent block
+		lastBlock, err := t.BlockRetriever.RetrieveMostRecentBlock()
+		if err != nil {
+			return err
+		}
+
+		// Get contract name
+		var ctrName string // should change this to check for "name" method and its return type in the abi methods before trying to fetch
+		strName, err := t.Fetcher.FetchString("name", t.Parser.Abi(), contractAddr, lastBlock, nil)
+		if err != nil || strName == "" {
+			hashName, err := t.Fetcher.FetchHash("name", t.Parser.Abi(), contractAddr, lastBlock, nil)
+			if err != nil || hashName.String() == "" {
+				return errors.New(fmt.Sprintf("unable to fetch contract name: %v\r\n", err)) // provide CLI prompt here for user to input a contract name?
+			}
+			ctrName = hashName.String()
+		} else {
+			ctrName = strName
+		}
+
+		// Aggregate info into contract object
+		info := &contract.Contract{
+			Name:          ctrName,
+			Address:       contractAddr,
+			Abi:           t.Parser.Abi(),
+			ParsedAbi:     t.Parser.ParsedAbi(),
+			StartingBlock: firstBlock,
+			Events:        t.Parser.GetEvents(),
+			Methods:       t.Parser.GetMethods(),
+			Addresses:     map[string]bool{},
+		}
+
+		// Use info to create filters
+		err = info.GenerateFilters(subset)
+		if err != nil {
+			return err
+		}
+
+		// Iterate over filters and push them to the repo
+		for _, filter := range info.Filters {
+			t.CreateFilter(filter)
+		}
+
+		t.Contracts[contractAddr] = info
+	}
+
+	return nil
+}
+
+// Iterate through contracts, updating the
+// converter with each one and using it to
+// convert watched event logs.
+// Then persist them into the postgres db
+func (tr transformer) Execute() error {
+	// Iterate through all internal contracts
+	for _, con := range tr.Contracts {
+
+		// Update converter with current contract
+		tr.Converter.Update(*con)
+
+		// Iterate through contract filters and get watched event logs
+		for eventName, filter := range con.Filters {
+			watchedEvents, err := tr.GetWatchedEvents(eventName)
+			if err != nil {
+				log.Println(fmt.Sprintf("Error fetching events for %s:", filter.Name), err)
+				return err
+			}
+
+			// Iterate over watched event logs and convert them
+			for _, we := range watchedEvents {
+				err = tr.Converter.Convert(*we, con.Events[eventName])
+				if err != nil {
+					return err
+				}
+			}
+
+		}
+
+		// After converting all logs for events of interest, persist all of the data
+		err := tr.PersistEvents(con)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/omni/transformer/transformer_test.go

@@ -0,0 +1,154 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformer_test
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/examples/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/types"
+)
+
+var mockEvent = core.WatchedEvent{
+	Name:        constants.TransferEvent.String(),
+	BlockNumber: 5488076,
+	Address:     constants.TusdContractAddress,
+	TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+	Index:       110,
+	Topic0:      constants.TransferEvent.Signature(),
+	Topic1:      "0x000000000000000000000000000000000000000000000000000000000000af21",
+	Topic2:      "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
+	Topic3:      "",
+	Data:        "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
+}
+
+var _ = Describe("Repository Test", func() {
+	var db *postgres.DB
+	var err error
+	var con types.Config
+	var blockRepository repositories.BlockRepository
+	rand.Seed(time.Now().UnixNano())
+
+	BeforeEach(func() {
+		infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
+		rawRpcClient, err := rpc.Dial(infuraIPC)
+		Expect(err).NotTo(HaveOccurred())
+		rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
+		ethClient := ethclient.NewClient(rawRpcClient)
+		blockChainClient := client.NewEthClient(ethClient)
+		node := node.MakeNode(rpcClient)
+		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
+		blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+
+		db, err = postgres.NewDB(config.Database{
+			Hostname: "localhost",
+			Name:     "vulcanize_private",
+			Port:     5432,
+		}, blockChain.Node())
+		Expect(err).NotTo(HaveOccurred())
+
+		con = types.Config{
+			DB:      db,
+			BC:      blockChain,
+			Network: "",
+		}
+
+		blockRepository = *repositories.NewBlockRepository(db)
+	})
+
+	AfterEach(func() {
+		db.Query(`DELETE FROM blocks`)
+		db.Query(`DELETE FROM logs`)
+		db.Query(`DELETE FROM transactions`)
+		db.Query(`DELETE FROM receipts`)
+		db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
+	})
+
+	It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
+		t := transformer.NewTransformer(&con)
+		t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+		err = t.Init()
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("Initializes and executes successfully if first and most recent blocks can be fetched from vDB", func() {
+		log := core.Log{
+			BlockNumber: 6194634,
+			TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
+			Address:     constants.TusdContractAddress,
+			Topics: core.Topics{
+				constants.TransferEvent.Signature(),
+				"0x000000000000000000000000000000000000000000000000000000000000af21",
+				"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
+				"",
+			},
+			Index: 1,
+			Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
+		}
+
+		receipt := core.Receipt{
+			TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+			ContractAddress: constants.TusdContractAddress,
+			Logs:            []core.Log{log},
+		}
+
+		transaction := core.Transaction{
+			Hash:    "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
+			Receipt: receipt,
+		}
+
+		block := core.Block{
+			Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
+			Number:       6194634,
+			Transactions: []core.Transaction{transaction},
+		}
+
+		blockRepository.CreateOrUpdateBlock(block)
+
+		t := transformer.NewTransformer(&con)
+		t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+		err = t.Init()
+		Expect(err).ToNot(HaveOccurred())
+
+		c, ok := t.Contracts[constants.TusdContractAddress]
+		Expect(ok).To(Equal(true))
+
+		err = t.Execute()
+		Expect(err).ToNot(HaveOccurred())
+
+		b, ok := c.Addresses["0x000000000000000000000000000000000000Af21"]
+		Expect(ok).To(Equal(true))
+		Expect(b).To(Equal(true))
+
+		b, ok = c.Addresses["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
+		Expect(ok).To(Equal(true))
+		Expect(b).To(Equal(true))
+	})
+})

pkg/omni/types/entities.go

@@ -19,8 +19,17 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
+type Config struct {
+	Network string
+	BC      core.BlockChain
+	DB      *postgres.DB
+}
+
 type Event struct {
 	Name      string
 	Anonymous bool