beginning work on method polling; first need to generate list of token holder address in a completely generic/contratc agnostic fashion. Created address retriever that can iterate through any given contract's watched events, finding the inputs/arguments with address type, and generate a list from those values. Edit: Contract objects now cache every event emitted address as its event logs are transformed into the repo to grow a list of contract associated addresses as we go

This commit is contained in:
Ian Norden 2018-11-04 15:26:39 -06:00
parent e9dbd771e5
commit b459cf35ed
23 changed files with 889 additions and 265 deletions

View File

@ -60,13 +60,14 @@ Requires a .toml config file:
} }
func omniWatcher() { func omniWatcher() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
if contractAddress == "" && len(contractAddresses) == 0 { if contractAddress == "" && len(contractAddresses) == 0 {
log.Fatal("Contract address required") log.Fatal("Contract address required")
} }
if !methodsOn && !eventsOn {
log.Fatal("Method polling and event watching turned off- nothing to do!")
}
if len(contractEvents) == 0 || len(contractMethods) == 0 { if len(contractEvents) == 0 || len(contractMethods) == 0 {
var str string var str string
for str != "y" { for str != "y" {
@ -90,6 +91,9 @@ func omniWatcher() {
} }
} }
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
rawRpcClient, err := rpc.Dial(ipc) rawRpcClient, err := rpc.Dial(ipc)
if err != nil { if err != nil {
log.Fatal(fmt.Sprintf("Failed to initialize rpc client\r\nerr: %v\r\n", err)) log.Fatal(fmt.Sprintf("Failed to initialize rpc client\r\nerr: %v\r\n", err))
@ -116,7 +120,8 @@ func omniWatcher() {
contractAddresses = append(contractAddresses, contractAddress) contractAddresses = append(contractAddresses, contractAddress)
for _, addr := range contractAddresses { for _, addr := range contractAddresses {
t.Set(addr, contractEvents) t.SetEvents(addr, contractEvents)
t.SetMethods(addr, contractMethods)
} }
err = t.Init() err = t.Init()
@ -137,6 +142,9 @@ func init() {
omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for") omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "List of addresses to generate watchers for") omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "List of addresses to generate watchers for")
omniWatcherCmd.Flags().BoolVarP(&eventsOn, "events-on", "o", true, "Set to false to turn off watching of any event")
omniWatcherCmd.Flags().BoolVarP(&methodsOn, "methods-on", "p", true, "Set to false to turn off polling of any method")
omniWatcherCmd.Flags().StringVarP(&contractAddress, "methods-off", "a", "", "Single address to generate watchers for")
omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched") omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched")
omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-methods", "m", []string{}, "Subset of methods to watch; by default all methods are watched") omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-methods", "m", []string{}, "Subset of methods to watch; by default all methods are watched")
omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`) omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)

View File

@ -35,6 +35,8 @@ var (
network string network string
contractAddress string contractAddress string
contractAddresses []string contractAddresses []string
eventsOn bool
methodsOn bool
contractEvents []string contractEvents []string
contractMethods []string contractMethods []string
) )

View File

@ -47,10 +47,10 @@ func (e Event) String() string {
func (e Event) Signature() string { func (e Event) Signature() string {
strings := [...]string{ strings := [...]string{
helpers.GenerateSignature("Transfer(address,address,uint)"), helpers.GenerateSignature("Transfer(address,address,uint256)"),
helpers.GenerateSignature("Approval(address,address,uint)"), helpers.GenerateSignature("Approval(address,address,uint256)"),
helpers.GenerateSignature("Burn(address,uint)"), helpers.GenerateSignature("Burn(address,uint256)"),
helpers.GenerateSignature("Mint(address,uint)"), helpers.GenerateSignature("Mint(address,uint256)"),
} }
if e < TransferEvent || e > MintEvent { if e < TransferEvent || e > MintEvent {

View File

@ -12,39 +12,54 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package types package contract
import ( import (
"errors"
"github.com/vulcanize/vulcanizedb/examples/generic/helpers"
"github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters" "github.com/vulcanize/vulcanizedb/pkg/filters"
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
) )
type ContractInfo struct { type Contract struct {
Name string Name string
Address string Address string
StartingBlock int64 StartingBlock int64
Abi string Abi string
ParsedAbi abi.ABI ParsedAbi abi.ABI
Events map[string]*Event // Map of events to their names Events map[string]*types.Event // Map of events to their names
Methods map[string]*Method // Map of methods to their names Methods map[string]*types.Method // Map of methods to their names
Filters map[string]filters.LogFilter // Map of event filters to their names Filters map[string]filters.LogFilter // Map of event filters to their names
Addresses map[string]bool // Map of all contract-associated addresses, populated as events are transformed
} }
func (i *ContractInfo) GenerateFilters(subset []string) { func (c *Contract) GenerateFilters(subset []string) error {
i.Filters = map[string]filters.LogFilter{} c.Filters = map[string]filters.LogFilter{}
for name, event := range i.Events { for name, event := range c.Events {
if len(subset) == 0 || stringInSlice(subset, name) { if len(subset) == 0 || stringInSlice(subset, name) {
i.Filters[name] = filters.LogFilter{ c.Filters[name] = filters.LogFilter{
Name: name, Name: name,
FromBlock: i.StartingBlock, FromBlock: c.StartingBlock,
ToBlock: -1, ToBlock: -1,
Address: i.Address, Address: c.Address,
Topics: core.Topics{event.Sig()}, Topics: core.Topics{helpers.GenerateSignature(event.Sig())},
} }
} }
} }
if len(c.Filters) == 0 {
return errors.New("error: no filters created")
}
return nil
}
func (c *Contract) AddAddress(addr string) {
c.Addresses[addr] = true
} }
func stringInSlice(list []string, s string) bool { func stringInSlice(list []string, s string) bool {

View File

@ -12,15 +12,22 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package types package contract_test
import ( import (
"github.com/vulcanize/vulcanizedb/pkg/core" "io/ioutil"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "log"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
) )
type Config struct { func TestContract(t *testing.T) {
Network string RegisterFailHandler(Fail)
BC core.BlockChain RunSpecs(t, "Contract Suite Test")
DB *postgres.DB
} }
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

View File

@ -0,0 +1,70 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package contract_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
)
var expectedLogFilter = filters.LogFilter{
Name: "Transfer",
Address: constants.TusdContractAddress,
ToBlock: -1,
FromBlock: 5197514,
Topics: core.Topics{constants.TransferEvent.Signature()},
}
var _ = Describe("Contract test", func() {
var p parser.Parser
var err error
BeforeEach(func() {
p = parser.NewParser("")
err = p.Parse(constants.TusdContractAddress)
Expect(err).ToNot(HaveOccurred())
})
It("Creates filters from stored data", func() {
info := contract.Contract{
Name: "TrueUSD",
Address: constants.TusdContractAddress,
Abi: p.Abi(),
ParsedAbi: p.ParsedAbi(),
StartingBlock: 5197514,
Events: p.GetEvents(),
Methods: p.GetMethods(),
Addresses: map[string]bool{},
}
err = info.GenerateFilters([]string{"Transfer"})
Expect(err).ToNot(HaveOccurred())
val, ok := info.Filters["Transfer"]
Expect(ok).To(Equal(true))
Expect(val).To(Equal(expectedLogFilter))
})
It("Fails with an empty contract", func() {
info := contract.Contract{}
err = info.GenerateFilters([]string{"Transfer"})
Expect(err).To(HaveOccurred())
})
})

View File

@ -21,6 +21,7 @@ import (
"github.com/vulcanize/vulcanizedb/examples/generic/helpers" "github.com/vulcanize/vulcanizedb/examples/generic/helpers"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/types" "github.com/vulcanize/vulcanizedb/pkg/omni/types"
) )
@ -28,21 +29,21 @@ import (
// custom logs containing event input name => value maps // custom logs containing event input name => value maps
type Converter interface { type Converter interface {
Convert(watchedEvent core.WatchedEvent, event *types.Event) error Convert(watchedEvent core.WatchedEvent, event *types.Event) error
Update(info types.ContractInfo) Update(info contract.Contract)
} }
type converter struct { type converter struct {
contractInfo types.ContractInfo contractInfo contract.Contract
} }
func NewConverter(info types.ContractInfo) *converter { func NewConverter(info contract.Contract) *converter {
return &converter{ return &converter{
contractInfo: info, contractInfo: info,
} }
} }
func (c *converter) Update(info types.ContractInfo) { func (c *converter) Update(info contract.Contract) {
c.contractInfo = info c.contractInfo = info
} }

View File

@ -24,9 +24,9 @@ import (
"github.com/vulcanize/vulcanizedb/examples/constants" "github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/generic/helpers" "github.com/vulcanize/vulcanizedb/examples/generic/helpers"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/converter" "github.com/vulcanize/vulcanizedb/pkg/omni/converter"
"github.com/vulcanize/vulcanizedb/pkg/omni/parser" "github.com/vulcanize/vulcanizedb/pkg/omni/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
) )
var mockEvent = core.WatchedEvent{ var mockEvent = core.WatchedEvent{
@ -44,13 +44,17 @@ var mockEvent = core.WatchedEvent{
} }
var _ = Describe("Converter Test", func() { var _ = Describe("Converter Test", func() {
var p parser.Parser
var err error
BeforeEach(func() {
p = parser.NewParser("")
err = p.Parse(constants.TusdContractAddress)
Expect(err).ToNot(HaveOccurred())
})
It("Converts watched event log to mapping of event input names to values", func() { It("Converts watched event log to mapping of event input names to values", func() {
p := parser.NewParser("") info := contract.Contract{
err := p.Parse(constants.TusdContractAddress)
Expect(err).ToNot(HaveOccurred())
info := types.ContractInfo{
Name: "TrueUSD", Name: "TrueUSD",
Address: constants.TusdContractAddress, Address: constants.TusdContractAddress,
Abi: p.Abi(), Abi: p.Abi(),
@ -58,11 +62,12 @@ var _ = Describe("Converter Test", func() {
StartingBlock: 5197514, StartingBlock: 5197514,
Events: p.GetEvents(), Events: p.GetEvents(),
Methods: p.GetMethods(), Methods: p.GetMethods(),
Addresses: map[string]bool{},
} }
event := info.Events["Transfer"] event := info.Events["Transfer"]
info.GenerateFilters([]string{"Transfer"}) err = info.GenerateFilters([]string{"Transfer"})
Expect(err).ToNot(HaveOccurred())
c := converter.NewConverter(info) c := converter.NewConverter(info)
err = c.Convert(mockEvent, event) err = c.Convert(mockEvent, event)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -77,4 +82,11 @@ var _ = Describe("Converter Test", func() {
Expect(event.Logs[1].Values["from"].(common.Address)).To(Equal(from)) Expect(event.Logs[1].Values["from"].(common.Address)).To(Equal(from))
Expect(v.String()).To(Equal(value.String())) Expect(v.String()).To(Equal(value.String()))
}) })
It("Fails with an empty contract", func() {
event := p.GetEvents()["Transfer"]
c := converter.NewConverter(contract.Contract{})
err = c.Convert(mockEvent, event)
Expect(err).To(HaveOccurred())
})
}) })

View File

@ -15,10 +15,9 @@
package fetcher_test package fetcher_test
import ( import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
"math/big" "math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
@ -29,6 +28,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/geth/client" "github.com/vulcanize/vulcanizedb/pkg/geth/client"
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
) )
var _ = Describe("Fetcher Test", func() { var _ = Describe("Fetcher Test", func() {
@ -106,7 +106,7 @@ var _ = Describe("Fetcher Test", func() {
Describe("Fetch string test", func() { Describe("Fetch string test", func() {
It("fetch nae string", func() { It("fetch name string", func() {
expectedStr := "TrueUSD" expectedStr := "TrueUSD"
str, err := realFetcher.FetchString("name", constants.TusdAbiString, constants.TusdContractAddress, blockNumber, nil) str, err := realFetcher.FetchString("name", constants.TusdAbiString, constants.TusdContractAddress, blockNumber, nil)

View File

@ -53,7 +53,7 @@ func (p *parser) ParsedAbi() abi.ABI {
} }
// Retrieves and parses the abi string // Retrieves and parses the abi string
// the given contract address // for the given contract address
func (p *parser) Parse(contractAddr string) error { func (p *parser) Parse(contractAddr string) error {
abiStr, err := p.client.GetAbi(contractAddr) abiStr, err := p.client.GetAbi(contractAddr)
if err != nil { if err != nil {

View File

@ -17,6 +17,7 @@ package parser_test
import ( import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants" "github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/omni/parser" "github.com/vulcanize/vulcanizedb/pkg/omni/parser"
@ -59,4 +60,10 @@ var _ = Describe("Parser Test", func() {
Expect(ok).To(Equal(true)) Expect(ok).To(Equal(true))
}) })
It("Fails with a normal, non-contract, account address", func() {
addr := "0xAb2A8F7cB56D9EC65573BA1bE0f92Fa2Ff7dd165"
err = p.Parse(addr)
Expect(err).To(HaveOccurred())
})
}) })

View File

@ -16,17 +16,19 @@ package repository
import ( import (
"fmt" "fmt"
"github.com/ethereum/go-ethereum/common"
"math/big" "math/big"
"strings" "strings"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/types" "github.com/vulcanize/vulcanizedb/pkg/omni/types"
) )
// Repository is used to // Repository is used to
type DataStore interface { type DataStore interface {
PersistEvents(info types.ContractInfo) error PersistEvents(info *contract.Contract) error
} }
type dataStore struct { type dataStore struct {
@ -34,6 +36,7 @@ type dataStore struct {
} }
func NewDataStore(db *postgres.DB) *dataStore { func NewDataStore(db *postgres.DB) *dataStore {
return &dataStore{ return &dataStore{
DB: db, DB: db,
} }
@ -42,31 +45,33 @@ func NewDataStore(db *postgres.DB) *dataStore {
// Creates a schema for the contract // Creates a schema for the contract
// Creates tables for the watched contract events // Creates tables for the watched contract events
// Persists converted event log data into these custom tables // Persists converted event log data into these custom tables
func (d *dataStore) PersistEvents(contract types.ContractInfo) error { func (d *dataStore) PersistEvents(con *contract.Contract) error {
schemaExists, err := d.CheckForSchema(contract.Name) schemaExists, err := d.CheckForSchema(con.Name)
if err != nil { if err != nil {
return err return err
} }
if !schemaExists { if !schemaExists {
err = d.CreateContractSchema(contract.Name) err = d.CreateContractSchema(con.Name)
if err != nil { if err != nil {
return err return err
} }
} }
for eventName := range contract.Filters { for eventName := range con.Filters {
event := con.Events[eventName]
if len(event.Logs) == 0 {
break
}
event := contract.Events[eventName] tableExists, err := d.CheckForTable(con.Name, eventName)
tableExists, err := d.CheckForTable(contract.Name, eventName)
if err != nil { if err != nil {
return err return err
} }
if !tableExists { if !tableExists {
err = d.CreateEventTable(contract.Name, event) err = d.CreateEventTable(con.Name, event)
if err != nil { if err != nil {
return err return err
} }
@ -74,13 +79,13 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
for id, log := range event.Logs { for id, log := range event.Logs {
// Create postgres command to persist any given event // Create postgres command to persist any given event
pgStr := fmt.Sprintf("INSERT INTO %s.%s ", strings.ToLower(contract.Name), strings.ToLower(eventName)) pgStr := fmt.Sprintf("INSERT INTO %s.%s ", strings.ToLower(con.Name), strings.ToLower(eventName))
pgStr = pgStr + "(vulcanize_log_id, token_name, token_address, event_name, block, tx" pgStr = pgStr + "(vulcanize_log_id, token_name, token_address, event_name, block, tx"
var data []interface{} var data []interface{}
data = append(data, data = append(data,
id, id,
strings.ToLower(contract.Name), strings.ToLower(con.Name),
strings.ToLower(contract.Address), strings.ToLower(con.Address),
strings.ToLower(eventName), strings.ToLower(eventName),
log.Block, log.Block,
log.Tx) log.Tx)
@ -100,7 +105,8 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
case common.Address: case common.Address:
var a common.Address var a common.Address
a = input.(common.Address) a = input.(common.Address)
input = a.String() input = a.String() // this also gives us a chance to add any event emitted address
con.AddAddress(a.String()) // to a list of token-related addresses, growing it as we go
case common.Hash: case common.Hash:
var h common.Hash var h common.Hash
h = input.(common.Hash) h = input.(common.Hash)

View File

@ -27,10 +27,10 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/converter" "github.com/vulcanize/vulcanizedb/pkg/omni/converter"
"github.com/vulcanize/vulcanizedb/pkg/omni/parser" "github.com/vulcanize/vulcanizedb/pkg/omni/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/repository" "github.com/vulcanize/vulcanizedb/pkg/omni/repository"
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
) )
var mockEvent = core.WatchedEvent{ var mockEvent = core.WatchedEvent{
@ -46,15 +46,14 @@ var mockEvent = core.WatchedEvent{
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
} }
var _ = Describe("Converter Test", func() { var _ = Describe("Repository Test", func() {
var db *postgres.DB var db *postgres.DB
var logRepository repositories.LogRepository var logRepository repositories.LogRepository
var blockRepository repositories.BlockRepository var blockRepository repositories.BlockRepository
var receiptRepository repositories.ReceiptRepository var receiptRepository repositories.ReceiptRepository
var dataStore repository.DataStore var dataStore repository.DataStore
var err error var err error
var info types.ContractInfo var info *contract.Contract
var blockNumber int64 var blockNumber int64
var blockId int64 var blockId int64
var vulcanizeLogId int64 var vulcanizeLogId int64
@ -92,7 +91,7 @@ var _ = Describe("Converter Test", func() {
err = p.Parse(constants.TusdContractAddress) err = p.Parse(constants.TusdContractAddress)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
info = types.ContractInfo{ info = &contract.Contract{
Name: "TrueUSD", Name: "TrueUSD",
Address: constants.TusdContractAddress, Address: constants.TusdContractAddress,
Abi: p.Abi(), Abi: p.Abi(),
@ -100,11 +99,13 @@ var _ = Describe("Converter Test", func() {
StartingBlock: 5197514, StartingBlock: 5197514,
Events: p.GetEvents(), Events: p.GetEvents(),
Methods: p.GetMethods(), Methods: p.GetMethods(),
Addresses: map[string]bool{},
} }
event := info.Events["Transfer"] event := info.Events["Transfer"]
info.GenerateFilters([]string{"Transfer"}) err = info.GenerateFilters([]string{"Transfer"})
c := converter.NewConverter(info) Expect(err).ToNot(HaveOccurred())
c := converter.NewConverter(*info)
mockEvent.LogID = vulcanizeLogId mockEvent.LogID = vulcanizeLogId
err = c.Convert(mockEvent, event) err = c.Convert(mockEvent, event)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -115,12 +116,26 @@ var _ = Describe("Converter Test", func() {
AfterEach(func() { AfterEach(func() {
db.Query(`DELETE FROM blocks`) db.Query(`DELETE FROM blocks`)
db.Query(`DELETE FROM logs`) db.Query(`DELETE FROM logs`)
db.Query(`DELETE FROM transactions`)
db.Query(`DELETE FROM receipts`) db.Query(`DELETE FROM receipts`)
db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`) db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
}) })
It("Convert watched event test", func() { It("Persist contract info in custom tables", func() {
err = dataStore.PersistEvents(info) err = dataStore.PersistEvents(info)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
b, ok := info.Addresses["0x000000000000000000000000000000000000Af21"]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = info.Addresses["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
})
It("Fails with empty contract", func() {
err = dataStore.PersistEvents(&contract.Contract{})
Expect(err).To(HaveOccurred())
}) })
}) })

View File

@ -0,0 +1,117 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package retriever
import (
"fmt"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
)
// Address retriever is used to retrieve the addresses associated with a contract
// It requires a vDB synced database with blocks, transactions, receipts, logs,
// AND all of the targeted events persisted
type AddressRetriever interface {
RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error)
}
type addressRetriever struct {
*postgres.DB
}
func NewAddressRetriever(db *postgres.DB) (r *addressRetriever) {
return &addressRetriever{
DB: db,
}
}
// Method to retrieve list of token-holding/contract-related addresses by iterating over available events
// This generic method should work whether or not the argument/input names of the events meet the expected standard
// This could be generalized to iterate over ALL events and pull out any address arguments
func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) {
addrList := make([]string, 0)
_, ok := info.Filters["Transfer"]
if ok {
addrs, err := r.retrieveTransferAddresses(info)
if err != nil {
return nil, err
}
addrList = append(addrList, addrs...)
}
_, ok = info.Filters["Mint"]
if ok {
addrs, err := r.retrieveTokenMintees(info)
if err != nil {
return nil, err
}
addrList = append(addrList, addrs...)
}
contractAddresses := make(map[common.Address]bool)
for _, addr := range addrList {
contractAddresses[common.HexToAddress(addr)] = true
}
return contractAddresses, nil
}
func (r *addressRetriever) retrieveTransferAddresses(contract contract.Contract) ([]string, error) {
transferAddrs := make([]string, 0)
event := contract.Events["Transfer"]
for _, field := range event.Fields { // Iterate over event fields, finding the ones with address type
if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses
addrs := make([]string, 0)
pgStr := fmt.Sprintf("SELECT _%s FROM %s.%s", field.Name, contract.Name, event.Name)
err := r.DB.Select(&addrs, pgStr)
if err != nil {
return []string{}, err
}
transferAddrs = append(transferAddrs, addrs...) // And append them to the growing list
}
}
return transferAddrs, nil
}
func (r *addressRetriever) retrieveTokenMintees(contract contract.Contract) ([]string, error) {
mintAddrs := make([]string, 0)
event := contract.Events["Mint"]
for _, field := range event.Fields { // Iterate over event fields, finding the ones with address type
if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses
addrs := make([]string, 0)
pgStr := fmt.Sprintf("SELECT _%s FROM %s.%s", field.Name, contract.Name, event.Name)
err := r.DB.Select(&addrs, pgStr)
if err != nil {
return []string{}, err
}
mintAddrs = append(mintAddrs, addrs...) // And append them to the growing list
}
}
return mintAddrs, nil
}

View File

@ -0,0 +1,152 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package retriever_test
import (
"math/rand"
"time"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
)
var mockEvent = core.WatchedEvent{
Name: constants.TransferEvent.String(),
BlockNumber: 5488076,
Address: constants.TusdContractAddress,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
Index: 110,
Topic0: constants.TransferEvent.Signature(),
Topic1: "0x000000000000000000000000000000000000000000000000000000000000af21",
Topic2: "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
Topic3: "",
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
}
var _ = Describe("Address Retriever Test", func() {
var db *postgres.DB
var logRepository repositories.LogRepository
var blockRepository repositories.BlockRepository
var receiptRepository repositories.ReceiptRepository
var dataStore repository.DataStore
var err error
var info *contract.Contract
var blockNumber int64
var blockId int64
var vulcanizeLogId int64
var r retriever.AddressRetriever
var addresses map[common.Address]bool
rand.Seed(time.Now().UnixNano())
BeforeEach(func() {
db, err = postgres.NewDB(config.Database{
Hostname: "localhost",
Name: "vulcanize_private",
Port: 5432,
}, core.Node{})
Expect(err).NotTo(HaveOccurred())
receiptRepository = repositories.ReceiptRepository{DB: db}
logRepository = repositories.LogRepository{DB: db}
blockRepository = *repositories.NewBlockRepository(db)
blockNumber = rand.Int63()
blockId = test_helpers.CreateBlock(blockNumber, blockRepository)
log := core.Log{}
logs := []core.Log{log}
receipt := core.Receipt{
Logs: logs,
}
receipts := []core.Receipt{receipt}
err = receiptRepository.CreateReceiptsAndLogs(blockId, receipts)
Expect(err).ToNot(HaveOccurred())
err = logRepository.Get(&vulcanizeLogId, `SELECT id FROM logs`)
Expect(err).ToNot(HaveOccurred())
p := parser.NewParser("")
err = p.Parse(constants.TusdContractAddress)
Expect(err).ToNot(HaveOccurred())
info = &contract.Contract{
Name: "TrueUSD",
Address: constants.TusdContractAddress,
Abi: p.Abi(),
ParsedAbi: p.ParsedAbi(),
StartingBlock: 5197514,
Events: p.GetEvents(),
Methods: p.GetMethods(),
Addresses: map[string]bool{},
}
event := info.Events["Transfer"]
err = info.GenerateFilters([]string{"Transfer"})
Expect(err).ToNot(HaveOccurred())
c := converter.NewConverter(*info)
mockEvent.LogID = vulcanizeLogId
err = c.Convert(mockEvent, event)
Expect(err).ToNot(HaveOccurred())
dataStore = repository.NewDataStore(db)
err = dataStore.PersistEvents(info)
Expect(err).ToNot(HaveOccurred())
r = retriever.NewAddressRetriever(db)
})
AfterEach(func() {
db.Query(`DELETE FROM blocks`)
db.Query(`DELETE FROM logs`)
db.Query(`DELETE FROM transactions`)
db.Query(`DELETE FROM receipts`)
db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
})
It("Retrieves a list of token holder addresses", func() {
addresses, err = r.RetrieveTokenHolderAddresses(*info)
Expect(err).ToNot(HaveOccurred())
_, ok := addresses[common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")]
Expect(ok).To(Equal(true))
_, ok = addresses[common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
Expect(ok).To(Equal(true))
_, ok = addresses[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
})
It("Returns empty list when empty contract info is used", func() {
addresses, err = r.RetrieveTokenHolderAddresses(contract.Contract{})
Expect(err).ToNot(HaveOccurred())
Expect(len(addresses)).To(Equal(0))
})
})

View File

@ -18,27 +18,36 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
// Retriever is used to retrieve the first block for a given contract // Block retriever is used to retrieve the first block for a given contract and the most recent block
// It requires a vDB synced database with blocks, transactions, receipts, and logs // It requires a vDB synced database with blocks, transactions, receipts, and logs
type Retriever interface { type BlockRetriever interface {
RetrieveFirstBlock(contractAddr string) (int64, error) RetrieveFirstBlock(contractAddr string) (int64, error)
RetrieveFirstBlockFromLogs(contractAddr string) (int64, error) RetrieveMostRecentBlock() (int64, error)
RetrieveFirstBlockFromReceipts(contractAddr string) (int64, error)
} }
type retriever struct { type blockRetriever struct {
*postgres.DB *postgres.DB
} }
func NewRetriever(db *postgres.DB) (r *retriever) { func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
return &retriever{ return &blockRetriever{
DB: db, DB: db,
} }
} }
// Try both methods of finding the first block, with the receipt method taking precedence
func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
if err != nil {
i, err = r.retrieveFirstBlockFromLogs(contractAddr)
}
return i, err
}
// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai) // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64, error) { func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
var firstBlock int var firstBlock int
err := r.DB.Get( err := r.DB.Get(
&firstBlock, &firstBlock,
@ -54,7 +63,7 @@ func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64,
} }
// In which case this servers as a heuristic to find the first block by finding the first contract event log // In which case this servers as a heuristic to find the first block by finding the first contract event log
func (r *retriever) RetrieveFirstBlockFromLogs(contractAddr string) (int64, error) { func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
var firstBlock int var firstBlock int
err := r.DB.Get( err := r.DB.Get(
&firstBlock, &firstBlock,
@ -65,12 +74,13 @@ func (r *retriever) RetrieveFirstBlockFromLogs(contractAddr string) (int64, erro
return int64(firstBlock), err return int64(firstBlock), err
} }
// Try both methods of finding the first block, with the receipt method taking precedence // Method to retrieve the most recent block in vDB
func (r *retriever) RetrieveFirstBlock(contractAddr string) (int64, error) { func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
i, err := r.RetrieveFirstBlockFromReceipts(contractAddr) var lastBlock int64
if err != nil { err := r.DB.Get(
i, err = r.RetrieveFirstBlockFromLogs(contractAddr) &lastBlock,
} "SELECT number FROM blocks ORDER BY number DESC LIMIT 1",
)
return i, err return lastBlock, err
} }

View File

@ -26,9 +26,9 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/omni/retriever" "github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
) )
var _ = Describe("Fetcher Test", func() { var _ = Describe("Block Retriever Test", func() {
var db *postgres.DB var db *postgres.DB
var r retriever.Retriever var r retriever.BlockRetriever
var blockRepository repositories.BlockRepository var blockRepository repositories.BlockRepository
BeforeEach(func() { BeforeEach(func() {
@ -42,12 +42,13 @@ var _ = Describe("Fetcher Test", func() {
blockRepository = *repositories.NewBlockRepository(db) blockRepository = *repositories.NewBlockRepository(db)
r = retriever.NewRetriever(db) r = retriever.NewBlockRetriever(db)
}) })
AfterEach(func() { AfterEach(func() {
db.Query(`DELETE FROM blocks`) db.Query(`DELETE FROM blocks`)
db.Query(`DELETE FROM logs`) db.Query(`DELETE FROM logs`)
db.Query(`DELETE FROM transactions`)
db.Query(`DELETE FROM receipts`) db.Query(`DELETE FROM receipts`)
}) })
@ -178,4 +179,25 @@ var _ = Describe("Fetcher Test", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(i).To(Equal(int64(1))) Expect(i).To(Equal(int64(1)))
}) })
It("Fails if a block cannot be found", func() {
block1 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
Number: 1,
Transactions: []core.Transaction{},
}
block2 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
Number: 2,
Transactions: []core.Transaction{},
}
blockRepository.CreateOrUpdateBlock(block1)
blockRepository.CreateOrUpdateBlock(block2)
_, err := r.RetrieveFirstBlock(constants.DaiContractAddress)
Expect(err).To(HaveOccurred())
})
}) })

View File

@ -1,167 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transformer
import (
"errors"
"fmt"
"log"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
)
// Omni event transformer
// Used to extract all or a subset of event data for
// any contract and persist it to postgres in a manner
// that requires no prior knowledge of the contract
// other than its address and which network it is on
type EventTransformer interface {
Init(contractAddr string) error
}
type eventTransformer struct {
// Network, database, and blockchain config
*types.Config
// Underlying databases
datastore.WatchedEventRepository
datastore.FilterRepository
repository.DataStore
// Underlying interfaces
parser.Parser // Parses events out of contract abi fetched with addr
retriever.Retriever // Retrieves first block with contract addr referenced
fetcher.Fetcher // Fetches data from public contract methods
converter.Converter // Converts watched event logs into custom log
// Store contract info as mapping to contract address
ContractInfo map[string]types.ContractInfo
// Subset of events of interest, stored as map of contract address to events
// Default/empty list means all events are considered for that address
sets map[string][]string
}
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(c *types.Config) (t *eventTransformer) {
t.Parser = parser.NewParser(c.Network)
t.Retriever = retriever.NewRetriever(c.DB)
t.Fetcher = fetcher.NewFetcher(c.BC)
t.Converter = converter.NewConverter(types.ContractInfo{})
t.ContractInfo = map[string]types.ContractInfo{}
t.WatchedEventRepository = repositories.WatchedEventRepository{DB: c.DB}
t.FilterRepository = repositories.FilterRepository{DB: c.DB}
t.DataStore = repository.NewDataStore(c.DB)
t.sets = map[string][]string{}
return t
}
// Used to set which contract addresses and which of their events to watch
func (t *eventTransformer) Set(contractAddr string, filterSet []string) {
t.sets[contractAddr] = filterSet
}
// Use after creating and setting transformer
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (t *eventTransformer) Init() error {
for contractAddr, subset := range t.sets {
err := t.Parser.Parse(contractAddr)
if err != nil {
return err
}
var ctrName string
strName, err := t.Fetcher.FetchString("name", t.Parser.Abi(), contractAddr, -1, nil)
if err != nil || strName == "" {
hashName, err := t.Fetcher.FetchHash("name", t.Parser.Abi(), contractAddr, -1, nil)
if err != nil || hashName.String() == "" {
return errors.New("unable to fetch contract name") // provide CLI prompt here for user to input a contract name?
}
ctrName = hashName.String()
} else {
ctrName = strName
}
firstBlock, err := t.Retriever.RetrieveFirstBlock(contractAddr)
if err != nil {
return err
}
info := types.ContractInfo{
Name: ctrName,
Address: contractAddr,
Abi: t.Parser.Abi(),
ParsedAbi: t.Parser.ParsedAbi(),
StartingBlock: firstBlock,
Events: t.Parser.GetEvents(),
Methods: t.Parser.GetMethods(),
}
info.GenerateFilters(subset)
for _, filter := range info.Filters {
t.CreateFilter(filter)
}
t.ContractInfo[contractAddr] = info
}
return nil
}
// Iterate through contracts, creating a new
// converter for each one and using it to
// convert watched event logs and persist
// them into the postgres db
func (tr eventTransformer) Execute() error {
for _, contract := range tr.ContractInfo {
tr.Converter.Update(contract)
for eventName, filter := range contract.Filters {
watchedEvents, err := tr.GetWatchedEvents(eventName)
if err != nil {
log.Println(fmt.Sprintf("Error fetching events for %s:", filter.Name), err)
return err
}
for _, we := range watchedEvents {
err = tr.Converter.Convert(*we, contract.Events[eventName])
if err != nil {
return err
}
}
}
err := tr.PersistEvents(contract)
if err != nil {
return err
}
}
return nil
}

View File

@ -1,15 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transformer_test

View File

@ -0,0 +1,199 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transformer
import (
"errors"
"fmt"
"log"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
)
// Omni transformer
// Used to extract all or a subset of event and method data
// for any contract and persist it to postgres in a manner
// that requires no prior knowledge of the contract
// other than its address and which network it is on
type Transformer interface {
Init(contractAddr string) error
}
type transformer struct {
// Network, database, and blockchain config
*types.Config
// Underlying databases
datastore.WatchedEventRepository
datastore.FilterRepository
repository.DataStore
// Underlying interfaces
parser.Parser // Parses events out of contract abi fetched with addr
retriever.BlockRetriever // Retrieves first block with contract addr referenced
retriever.AddressRetriever // Retrieves token holder addresses
fetcher.Fetcher // Fetches data from public contract methods
converter.Converter // Converts watched event logs into custom log
// Store contract info as mapping to contract address
Contracts map[string]*contract.Contract
// Targeted subset of events/methods
// Stored as map of contract address to events/method names of interest
// Default/empty list means all events/methods are considered for that address
targetEvents map[string][]string
targetMethods map[string][]string
}
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(c *types.Config) *transformer {
return &transformer{
Parser: parser.NewParser(c.Network),
BlockRetriever: retriever.NewBlockRetriever(c.DB),
Fetcher: fetcher.NewFetcher(c.BC),
Converter: converter.NewConverter(contract.Contract{}),
Contracts: map[string]*contract.Contract{},
WatchedEventRepository: repositories.WatchedEventRepository{DB: c.DB},
FilterRepository: repositories.FilterRepository{DB: c.DB},
DataStore: repository.NewDataStore(c.DB),
targetEvents: map[string][]string{},
targetMethods: map[string][]string{},
}
}
// Used to set which contract addresses and which of their events to watch
func (t *transformer) SetEvents(contractAddr string, filterSet []string) {
t.targetEvents[contractAddr] = filterSet
}
// Used to set which contract addresses and which of their methods to call
func (t *transformer) SetMethods(contractAddr string, filterSet []string) {
t.targetMethods[contractAddr] = filterSet
}
// Use after creating and setting transformer
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (t *transformer) Init() error {
for contractAddr, subset := range t.targetEvents {
// Get Abi
err := t.Parser.Parse(contractAddr)
if err != nil {
return err
}
// Get first block for contract
firstBlock, err := t.BlockRetriever.RetrieveFirstBlock(contractAddr)
if err != nil {
return err
}
// Get most recent block
lastBlock, err := t.BlockRetriever.RetrieveMostRecentBlock()
if err != nil {
return err
}
// Get contract name
var ctrName string // should change this to check for "name" method and its return type in the abi methods before trying to fetch
strName, err := t.Fetcher.FetchString("name", t.Parser.Abi(), contractAddr, lastBlock, nil)
if err != nil || strName == "" {
hashName, err := t.Fetcher.FetchHash("name", t.Parser.Abi(), contractAddr, lastBlock, nil)
if err != nil || hashName.String() == "" {
return errors.New(fmt.Sprintf("unable to fetch contract name: %v\r\n", err)) // provide CLI prompt here for user to input a contract name?
}
ctrName = hashName.String()
} else {
ctrName = strName
}
// Aggregate info into contract object
info := &contract.Contract{
Name: ctrName,
Address: contractAddr,
Abi: t.Parser.Abi(),
ParsedAbi: t.Parser.ParsedAbi(),
StartingBlock: firstBlock,
Events: t.Parser.GetEvents(),
Methods: t.Parser.GetMethods(),
Addresses: map[string]bool{},
}
// Use info to create filters
err = info.GenerateFilters(subset)
if err != nil {
return err
}
// Iterate over filters and push them to the repo
for _, filter := range info.Filters {
t.CreateFilter(filter)
}
t.Contracts[contractAddr] = info
}
return nil
}
// Iterate through contracts, updating the
// converter with each one and using it to
// convert watched event logs.
// Then persist them into the postgres db
func (tr transformer) Execute() error {
// Iterate through all internal contracts
for _, con := range tr.Contracts {
// Update converter with current contract
tr.Converter.Update(*con)
// Iterate through contract filters and get watched event logs
for eventName, filter := range con.Filters {
watchedEvents, err := tr.GetWatchedEvents(eventName)
if err != nil {
log.Println(fmt.Sprintf("Error fetching events for %s:", filter.Name), err)
return err
}
// Iterate over watched event logs and convert them
for _, we := range watchedEvents {
err = tr.Converter.Convert(*we, con.Events[eventName])
if err != nil {
return err
}
}
}
// After converting all logs for events of interest, persist all of the data
err := tr.PersistEvents(con)
if err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,154 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package transformer_test
import (
"math/rand"
"time"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/omni/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
)
var mockEvent = core.WatchedEvent{
Name: constants.TransferEvent.String(),
BlockNumber: 5488076,
Address: constants.TusdContractAddress,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
Index: 110,
Topic0: constants.TransferEvent.Signature(),
Topic1: "0x000000000000000000000000000000000000000000000000000000000000af21",
Topic2: "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
Topic3: "",
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
}
var _ = Describe("Repository Test", func() {
var db *postgres.DB
var err error
var con types.Config
var blockRepository repositories.BlockRepository
rand.Seed(time.Now().UnixNano())
BeforeEach(func() {
infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
rawRpcClient, err := rpc.Dial(infuraIPC)
Expect(err).NotTo(HaveOccurred())
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
db, err = postgres.NewDB(config.Database{
Hostname: "localhost",
Name: "vulcanize_private",
Port: 5432,
}, blockChain.Node())
Expect(err).NotTo(HaveOccurred())
con = types.Config{
DB: db,
BC: blockChain,
Network: "",
}
blockRepository = *repositories.NewBlockRepository(db)
})
AfterEach(func() {
db.Query(`DELETE FROM blocks`)
db.Query(`DELETE FROM logs`)
db.Query(`DELETE FROM transactions`)
db.Query(`DELETE FROM receipts`)
db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
})
It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
t := transformer.NewTransformer(&con)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
Expect(err).To(HaveOccurred())
})
It("Initializes and executes successfully if first and most recent blocks can be fetched from vDB", func() {
log := core.Log{
BlockNumber: 6194634,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
Address: constants.TusdContractAddress,
Topics: core.Topics{
constants.TransferEvent.Signature(),
"0x000000000000000000000000000000000000000000000000000000000000af21",
"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
"",
},
Index: 1,
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
}
receipt := core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
ContractAddress: constants.TusdContractAddress,
Logs: []core.Log{log},
}
transaction := core.Transaction{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
Receipt: receipt,
}
block := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
Number: 6194634,
Transactions: []core.Transaction{transaction},
}
blockRepository.CreateOrUpdateBlock(block)
t := transformer.NewTransformer(&con)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[constants.TusdContractAddress]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.Addresses["0x000000000000000000000000000000000000Af21"]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.Addresses["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
})
})

View File

@ -19,8 +19,17 @@ import (
"strings" "strings"
"github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
type Config struct {
Network string
BC core.BlockChain
DB *postgres.DB
}
type Event struct { type Event struct {
Name string Name string
Anonymous bool Anonymous bool