beginning work on method polling; first need to generate list of token holder address in a completely generic/contratc agnostic fashion. Created address retriever that can iterate through any given contract's watched events, finding the inputs/arguments with address type, and generate a list from those values. Edit: Contract objects now cache every event emitted address as its event logs are transformed into the repo to grow a list of contract associated addresses as we go
This commit is contained in:
Ian Norden
parent
e9dbd771e5
commit
b459cf35ed
@ -60,13 +60,14 @@ Requires a .toml config file:
|
||||
}
|
||||
|
||||
func omniWatcher() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
if contractAddress == "" && len(contractAddresses) == 0 {
|
||||
log.Fatal("Contract address required")
|
||||
}
|
||||
|
||||
if !methodsOn && !eventsOn {
|
||||
log.Fatal("Method polling and event watching turned off- nothing to do!")
|
||||
}
|
||||
|
||||
if len(contractEvents) == 0 || len(contractMethods) == 0 {
|
||||
var str string
|
||||
for str != "y" {
|
||||
@ -90,6 +91,9 @@ func omniWatcher() {
|
||||
}
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
rawRpcClient, err := rpc.Dial(ipc)
|
||||
if err != nil {
|
||||
log.Fatal(fmt.Sprintf("Failed to initialize rpc client\r\nerr: %v\r\n", err))
|
||||
@ -116,7 +120,8 @@ func omniWatcher() {
|
||||
|
||||
contractAddresses = append(contractAddresses, contractAddress)
|
||||
for _, addr := range contractAddresses {
|
||||
t.Set(addr, contractEvents)
|
||||
t.SetEvents(addr, contractEvents)
|
||||
t.SetMethods(addr, contractMethods)
|
||||
}
|
||||
|
||||
err = t.Init()
|
||||
@ -137,6 +142,9 @@ func init() {
|
||||
|
||||
omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
|
||||
omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "List of addresses to generate watchers for")
|
||||
omniWatcherCmd.Flags().BoolVarP(&eventsOn, "events-on", "o", true, "Set to false to turn off watching of any event")
|
||||
omniWatcherCmd.Flags().BoolVarP(&methodsOn, "methods-on", "p", true, "Set to false to turn off polling of any method")
|
||||
omniWatcherCmd.Flags().StringVarP(&contractAddress, "methods-off", "a", "", "Single address to generate watchers for")
|
||||
omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched")
|
||||
omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-methods", "m", []string{}, "Subset of methods to watch; by default all methods are watched")
|
||||
omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)
|
||||
|
||||
@ -35,6 +35,8 @@ var (
|
||||
network string
|
||||
contractAddress string
|
||||
contractAddresses []string
|
||||
eventsOn bool
|
||||
methodsOn bool
|
||||
contractEvents []string
|
||||
contractMethods []string
|
||||
)
|
||||
|
||||
@ -47,10 +47,10 @@ func (e Event) String() string {
|
||||
|
||||
func (e Event) Signature() string {
|
||||
strings := [...]string{
|
||||
helpers.GenerateSignature("Transfer(address,address,uint)"),
|
||||
helpers.GenerateSignature("Approval(address,address,uint)"),
|
||||
helpers.GenerateSignature("Burn(address,uint)"),
|
||||
helpers.GenerateSignature("Mint(address,uint)"),
|
||||
helpers.GenerateSignature("Transfer(address,address,uint256)"),
|
||||
helpers.GenerateSignature("Approval(address,address,uint256)"),
|
||||
helpers.GenerateSignature("Burn(address,uint256)"),
|
||||
helpers.GenerateSignature("Mint(address,uint256)"),
|
||||
}
|
||||
|
||||
if e < TransferEvent || e > MintEvent {
|
||||
|
||||
@ -12,39 +12,54 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package types
|
||||
package contract
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/vulcanize/vulcanizedb/examples/generic/helpers"
|
||||
|
||||
"github.com/ethereum/go-ethereum/accounts/abi"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/filters"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
|
||||
)
|
||||
|
||||
type ContractInfo struct {
|
||||
type Contract struct {
|
||||
Name string
|
||||
Address string
|
||||
StartingBlock int64
|
||||
Abi string
|
||||
ParsedAbi abi.ABI
|
||||
Events map[string]*Event // Map of events to their names
|
||||
Methods map[string]*Method // Map of methods to their names
|
||||
Events map[string]*types.Event // Map of events to their names
|
||||
Methods map[string]*types.Method // Map of methods to their names
|
||||
Filters map[string]filters.LogFilter // Map of event filters to their names
|
||||
Addresses map[string]bool // Map of all contract-associated addresses, populated as events are transformed
|
||||
}
|
||||
|
||||
func (i *ContractInfo) GenerateFilters(subset []string) {
|
||||
i.Filters = map[string]filters.LogFilter{}
|
||||
for name, event := range i.Events {
|
||||
func (c *Contract) GenerateFilters(subset []string) error {
|
||||
c.Filters = map[string]filters.LogFilter{}
|
||||
for name, event := range c.Events {
|
||||
if len(subset) == 0 || stringInSlice(subset, name) {
|
||||
i.Filters[name] = filters.LogFilter{
|
||||
c.Filters[name] = filters.LogFilter{
|
||||
Name: name,
|
||||
FromBlock: i.StartingBlock,
|
||||
FromBlock: c.StartingBlock,
|
||||
ToBlock: -1,
|
||||
Address: i.Address,
|
||||
Topics: core.Topics{event.Sig()},
|
||||
Address: c.Address,
|
||||
Topics: core.Topics{helpers.GenerateSignature(event.Sig())},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(c.Filters) == 0 {
|
||||
return errors.New("error: no filters created")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Contract) AddAddress(addr string) {
|
||||
c.Addresses[addr] = true
|
||||
}
|
||||
|
||||
func stringInSlice(list []string, s string) bool {
|
||||
@ -12,15 +12,22 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package types
|
||||
package contract_test
|
||||
|
||||
import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Network string
|
||||
BC core.BlockChain
|
||||
DB *postgres.DB
|
||||
func TestContract(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Contract Suite Test")
|
||||
}
|
||||
|
||||
var _ = BeforeSuite(func() {
|
||||
log.SetOutput(ioutil.Discard)
|
||||
})
|
||||
70
pkg/omni/contract/contract_test.go
Normal file
70
pkg/omni/contract/contract_test.go
Normal file
@ -0,0 +1,70 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package contract_test
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/filters"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
|
||||
)
|
||||
|
||||
var expectedLogFilter = filters.LogFilter{
|
||||
Name: "Transfer",
|
||||
Address: constants.TusdContractAddress,
|
||||
ToBlock: -1,
|
||||
FromBlock: 5197514,
|
||||
Topics: core.Topics{constants.TransferEvent.Signature()},
|
||||
}
|
||||
|
||||
var _ = Describe("Contract test", func() {
|
||||
var p parser.Parser
|
||||
var err error
|
||||
|
||||
BeforeEach(func() {
|
||||
p = parser.NewParser("")
|
||||
err = p.Parse(constants.TusdContractAddress)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("Creates filters from stored data", func() {
|
||||
info := contract.Contract{
|
||||
Name: "TrueUSD",
|
||||
Address: constants.TusdContractAddress,
|
||||
Abi: p.Abi(),
|
||||
ParsedAbi: p.ParsedAbi(),
|
||||
StartingBlock: 5197514,
|
||||
Events: p.GetEvents(),
|
||||
Methods: p.GetMethods(),
|
||||
Addresses: map[string]bool{},
|
||||
}
|
||||
|
||||
err = info.GenerateFilters([]string{"Transfer"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
val, ok := info.Filters["Transfer"]
|
||||
Expect(ok).To(Equal(true))
|
||||
Expect(val).To(Equal(expectedLogFilter))
|
||||
})
|
||||
|
||||
It("Fails with an empty contract", func() {
|
||||
info := contract.Contract{}
|
||||
err = info.GenerateFilters([]string{"Transfer"})
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/generic/helpers"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
|
||||
)
|
||||
|
||||
@ -28,21 +29,21 @@ import (
|
||||
// custom logs containing event input name => value maps
|
||||
type Converter interface {
|
||||
Convert(watchedEvent core.WatchedEvent, event *types.Event) error
|
||||
Update(info types.ContractInfo)
|
||||
Update(info contract.Contract)
|
||||
}
|
||||
|
||||
type converter struct {
|
||||
contractInfo types.ContractInfo
|
||||
contractInfo contract.Contract
|
||||
}
|
||||
|
||||
func NewConverter(info types.ContractInfo) *converter {
|
||||
func NewConverter(info contract.Contract) *converter {
|
||||
|
||||
return &converter{
|
||||
contractInfo: info,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *converter) Update(info types.ContractInfo) {
|
||||
func (c *converter) Update(info contract.Contract) {
|
||||
c.contractInfo = info
|
||||
}
|
||||
|
||||
|
||||
@ -24,9 +24,9 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/examples/generic/helpers"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
|
||||
)
|
||||
|
||||
var mockEvent = core.WatchedEvent{
|
||||
@ -44,13 +44,17 @@ var mockEvent = core.WatchedEvent{
|
||||
}
|
||||
|
||||
var _ = Describe("Converter Test", func() {
|
||||
var p parser.Parser
|
||||
var err error
|
||||
|
||||
BeforeEach(func() {
|
||||
p = parser.NewParser("")
|
||||
err = p.Parse(constants.TusdContractAddress)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("Converts watched event log to mapping of event input names to values", func() {
|
||||
p := parser.NewParser("")
|
||||
err := p.Parse(constants.TusdContractAddress)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
info := types.ContractInfo{
|
||||
info := contract.Contract{
|
||||
Name: "TrueUSD",
|
||||
Address: constants.TusdContractAddress,
|
||||
Abi: p.Abi(),
|
||||
@ -58,11 +62,12 @@ var _ = Describe("Converter Test", func() {
|
||||
StartingBlock: 5197514,
|
||||
Events: p.GetEvents(),
|
||||
Methods: p.GetMethods(),
|
||||
Addresses: map[string]bool{},
|
||||
}
|
||||
|
||||
event := info.Events["Transfer"]
|
||||
|
||||
info.GenerateFilters([]string{"Transfer"})
|
||||
err = info.GenerateFilters([]string{"Transfer"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
c := converter.NewConverter(info)
|
||||
err = c.Convert(mockEvent, event)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@ -77,4 +82,11 @@ var _ = Describe("Converter Test", func() {
|
||||
Expect(event.Logs[1].Values["from"].(common.Address)).To(Equal(from))
|
||||
Expect(v.String()).To(Equal(value.String()))
|
||||
})
|
||||
|
||||
It("Fails with an empty contract", func() {
|
||||
event := p.GetEvents()["Transfer"]
|
||||
c := converter.NewConverter(contract.Contract{})
|
||||
err = c.Convert(mockEvent, event)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
@ -15,10 +15,9 @@
|
||||
package fetcher_test
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/ethclient"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
. "github.com/onsi/ginkgo"
|
||||
@ -29,6 +28,7 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
|
||||
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
|
||||
)
|
||||
|
||||
var _ = Describe("Fetcher Test", func() {
|
||||
@ -106,7 +106,7 @@ var _ = Describe("Fetcher Test", func() {
|
||||
|
||||
Describe("Fetch string test", func() {
|
||||
|
||||
It("fetch nae string", func() {
|
||||
It("fetch name string", func() {
|
||||
expectedStr := "TrueUSD"
|
||||
str, err := realFetcher.FetchString("name", constants.TusdAbiString, constants.TusdContractAddress, blockNumber, nil)
|
||||
|
||||
|
||||
@ -53,7 +53,7 @@ func (p *parser) ParsedAbi() abi.ABI {
|
||||
}
|
||||
|
||||
// Retrieves and parses the abi string
|
||||
// the given contract address
|
||||
// for the given contract address
|
||||
func (p *parser) Parse(contractAddr string) error {
|
||||
abiStr, err := p.client.GetAbi(contractAddr)
|
||||
if err != nil {
|
||||
|
||||
@ -17,6 +17,7 @@ package parser_test
|
||||
import (
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
|
||||
@ -59,4 +60,10 @@ var _ = Describe("Parser Test", func() {
|
||||
Expect(ok).To(Equal(true))
|
||||
})
|
||||
|
||||
It("Fails with a normal, non-contract, account address", func() {
|
||||
addr := "0xAb2A8F7cB56D9EC65573BA1bE0f92Fa2Ff7dd165"
|
||||
err = p.Parse(addr)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
@ -16,17 +16,19 @@ package repository
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"math/big"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
|
||||
)
|
||||
|
||||
// Repository is used to
|
||||
type DataStore interface {
|
||||
PersistEvents(info types.ContractInfo) error
|
||||
PersistEvents(info *contract.Contract) error
|
||||
}
|
||||
|
||||
type dataStore struct {
|
||||
@ -34,6 +36,7 @@ type dataStore struct {
|
||||
}
|
||||
|
||||
func NewDataStore(db *postgres.DB) *dataStore {
|
||||
|
||||
return &dataStore{
|
||||
DB: db,
|
||||
}
|
||||
@ -42,31 +45,33 @@ func NewDataStore(db *postgres.DB) *dataStore {
|
||||
// Creates a schema for the contract
|
||||
// Creates tables for the watched contract events
|
||||
// Persists converted event log data into these custom tables
|
||||
func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
|
||||
func (d *dataStore) PersistEvents(con *contract.Contract) error {
|
||||
|
||||
schemaExists, err := d.CheckForSchema(contract.Name)
|
||||
schemaExists, err := d.CheckForSchema(con.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !schemaExists {
|
||||
err = d.CreateContractSchema(contract.Name)
|
||||
err = d.CreateContractSchema(con.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for eventName := range contract.Filters {
|
||||
for eventName := range con.Filters {
|
||||
event := con.Events[eventName]
|
||||
if len(event.Logs) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
event := contract.Events[eventName]
|
||||
|
||||
tableExists, err := d.CheckForTable(contract.Name, eventName)
|
||||
tableExists, err := d.CheckForTable(con.Name, eventName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !tableExists {
|
||||
err = d.CreateEventTable(contract.Name, event)
|
||||
err = d.CreateEventTable(con.Name, event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -74,13 +79,13 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
|
||||
|
||||
for id, log := range event.Logs {
|
||||
// Create postgres command to persist any given event
|
||||
pgStr := fmt.Sprintf("INSERT INTO %s.%s ", strings.ToLower(contract.Name), strings.ToLower(eventName))
|
||||
pgStr := fmt.Sprintf("INSERT INTO %s.%s ", strings.ToLower(con.Name), strings.ToLower(eventName))
|
||||
pgStr = pgStr + "(vulcanize_log_id, token_name, token_address, event_name, block, tx"
|
||||
var data []interface{}
|
||||
data = append(data,
|
||||
id,
|
||||
strings.ToLower(contract.Name),
|
||||
strings.ToLower(contract.Address),
|
||||
strings.ToLower(con.Name),
|
||||
strings.ToLower(con.Address),
|
||||
strings.ToLower(eventName),
|
||||
log.Block,
|
||||
log.Tx)
|
||||
@ -100,7 +105,8 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error {
|
||||
case common.Address:
|
||||
var a common.Address
|
||||
a = input.(common.Address)
|
||||
input = a.String()
|
||||
input = a.String() // this also gives us a chance to add any event emitted address
|
||||
con.AddAddress(a.String()) // to a list of token-related addresses, growing it as we go
|
||||
case common.Hash:
|
||||
var h common.Hash
|
||||
h = input.(common.Hash)
|
||||
|
||||
@ -27,10 +27,10 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
|
||||
)
|
||||
|
||||
var mockEvent = core.WatchedEvent{
|
||||
@ -46,15 +46,14 @@ var mockEvent = core.WatchedEvent{
|
||||
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
|
||||
}
|
||||
|
||||
var _ = Describe("Converter Test", func() {
|
||||
|
||||
var _ = Describe("Repository Test", func() {
|
||||
var db *postgres.DB
|
||||
var logRepository repositories.LogRepository
|
||||
var blockRepository repositories.BlockRepository
|
||||
var receiptRepository repositories.ReceiptRepository
|
||||
var dataStore repository.DataStore
|
||||
var err error
|
||||
var info types.ContractInfo
|
||||
var info *contract.Contract
|
||||
var blockNumber int64
|
||||
var blockId int64
|
||||
var vulcanizeLogId int64
|
||||
@ -92,7 +91,7 @@ var _ = Describe("Converter Test", func() {
|
||||
err = p.Parse(constants.TusdContractAddress)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
info = types.ContractInfo{
|
||||
info = &contract.Contract{
|
||||
Name: "TrueUSD",
|
||||
Address: constants.TusdContractAddress,
|
||||
Abi: p.Abi(),
|
||||
@ -100,11 +99,13 @@ var _ = Describe("Converter Test", func() {
|
||||
StartingBlock: 5197514,
|
||||
Events: p.GetEvents(),
|
||||
Methods: p.GetMethods(),
|
||||
Addresses: map[string]bool{},
|
||||
}
|
||||
|
||||
event := info.Events["Transfer"]
|
||||
info.GenerateFilters([]string{"Transfer"})
|
||||
c := converter.NewConverter(info)
|
||||
err = info.GenerateFilters([]string{"Transfer"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
c := converter.NewConverter(*info)
|
||||
mockEvent.LogID = vulcanizeLogId
|
||||
err = c.Convert(mockEvent, event)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@ -115,12 +116,26 @@ var _ = Describe("Converter Test", func() {
|
||||
AfterEach(func() {
|
||||
db.Query(`DELETE FROM blocks`)
|
||||
db.Query(`DELETE FROM logs`)
|
||||
db.Query(`DELETE FROM transactions`)
|
||||
db.Query(`DELETE FROM receipts`)
|
||||
db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
|
||||
})
|
||||
|
||||
It("Convert watched event test", func() {
|
||||
It("Persist contract info in custom tables", func() {
|
||||
err = dataStore.PersistEvents(info)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
b, ok := info.Addresses["0x000000000000000000000000000000000000Af21"]
|
||||
Expect(ok).To(Equal(true))
|
||||
Expect(b).To(Equal(true))
|
||||
|
||||
b, ok = info.Addresses["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
|
||||
Expect(ok).To(Equal(true))
|
||||
Expect(b).To(Equal(true))
|
||||
})
|
||||
|
||||
It("Fails with empty contract", func() {
|
||||
err = dataStore.PersistEvents(&contract.Contract{})
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
117
pkg/omni/retriever/address_retriever.go
Normal file
117
pkg/omni/retriever/address_retriever.go
Normal file
@ -0,0 +1,117 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package retriever
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/accounts/abi"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
|
||||
)
|
||||
|
||||
// Address retriever is used to retrieve the addresses associated with a contract
|
||||
// It requires a vDB synced database with blocks, transactions, receipts, logs,
|
||||
// AND all of the targeted events persisted
|
||||
type AddressRetriever interface {
|
||||
RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error)
|
||||
}
|
||||
|
||||
type addressRetriever struct {
|
||||
*postgres.DB
|
||||
}
|
||||
|
||||
func NewAddressRetriever(db *postgres.DB) (r *addressRetriever) {
|
||||
|
||||
return &addressRetriever{
|
||||
DB: db,
|
||||
}
|
||||
}
|
||||
|
||||
// Method to retrieve list of token-holding/contract-related addresses by iterating over available events
|
||||
// This generic method should work whether or not the argument/input names of the events meet the expected standard
|
||||
// This could be generalized to iterate over ALL events and pull out any address arguments
|
||||
func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) {
|
||||
addrList := make([]string, 0)
|
||||
|
||||
_, ok := info.Filters["Transfer"]
|
||||
if ok {
|
||||
addrs, err := r.retrieveTransferAddresses(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addrList = append(addrList, addrs...)
|
||||
}
|
||||
|
||||
_, ok = info.Filters["Mint"]
|
||||
if ok {
|
||||
addrs, err := r.retrieveTokenMintees(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addrList = append(addrList, addrs...)
|
||||
}
|
||||
|
||||
contractAddresses := make(map[common.Address]bool)
|
||||
for _, addr := range addrList {
|
||||
contractAddresses[common.HexToAddress(addr)] = true
|
||||
}
|
||||
|
||||
return contractAddresses, nil
|
||||
}
|
||||
|
||||
func (r *addressRetriever) retrieveTransferAddresses(contract contract.Contract) ([]string, error) {
|
||||
transferAddrs := make([]string, 0)
|
||||
event := contract.Events["Transfer"]
|
||||
|
||||
for _, field := range event.Fields { // Iterate over event fields, finding the ones with address type
|
||||
|
||||
if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses
|
||||
addrs := make([]string, 0)
|
||||
pgStr := fmt.Sprintf("SELECT _%s FROM %s.%s", field.Name, contract.Name, event.Name)
|
||||
err := r.DB.Select(&addrs, pgStr)
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
|
||||
transferAddrs = append(transferAddrs, addrs...) // And append them to the growing list
|
||||
}
|
||||
}
|
||||
|
||||
return transferAddrs, nil
|
||||
}
|
||||
|
||||
func (r *addressRetriever) retrieveTokenMintees(contract contract.Contract) ([]string, error) {
|
||||
mintAddrs := make([]string, 0)
|
||||
event := contract.Events["Mint"]
|
||||
|
||||
for _, field := range event.Fields { // Iterate over event fields, finding the ones with address type
|
||||
|
||||
if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses
|
||||
addrs := make([]string, 0)
|
||||
pgStr := fmt.Sprintf("SELECT _%s FROM %s.%s", field.Name, contract.Name, event.Name)
|
||||
err := r.DB.Select(&addrs, pgStr)
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
|
||||
mintAddrs = append(mintAddrs, addrs...) // And append them to the growing list
|
||||
}
|
||||
}
|
||||
|
||||
return mintAddrs, nil
|
||||
}
|
||||
152
pkg/omni/retriever/address_retriever_test.go
Normal file
152
pkg/omni/retriever/address_retriever_test.go
Normal file
@ -0,0 +1,152 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package retriever_test
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/examples/test_helpers"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/config"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
|
||||
)
|
||||
|
||||
var mockEvent = core.WatchedEvent{
|
||||
Name: constants.TransferEvent.String(),
|
||||
BlockNumber: 5488076,
|
||||
Address: constants.TusdContractAddress,
|
||||
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
|
||||
Index: 110,
|
||||
Topic0: constants.TransferEvent.Signature(),
|
||||
Topic1: "0x000000000000000000000000000000000000000000000000000000000000af21",
|
||||
Topic2: "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
|
||||
Topic3: "",
|
||||
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
|
||||
}
|
||||
|
||||
var _ = Describe("Address Retriever Test", func() {
|
||||
var db *postgres.DB
|
||||
var logRepository repositories.LogRepository
|
||||
var blockRepository repositories.BlockRepository
|
||||
var receiptRepository repositories.ReceiptRepository
|
||||
var dataStore repository.DataStore
|
||||
var err error
|
||||
var info *contract.Contract
|
||||
var blockNumber int64
|
||||
var blockId int64
|
||||
var vulcanizeLogId int64
|
||||
var r retriever.AddressRetriever
|
||||
var addresses map[common.Address]bool
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
|
||||
BeforeEach(func() {
|
||||
db, err = postgres.NewDB(config.Database{
|
||||
Hostname: "localhost",
|
||||
Name: "vulcanize_private",
|
||||
Port: 5432,
|
||||
}, core.Node{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
receiptRepository = repositories.ReceiptRepository{DB: db}
|
||||
logRepository = repositories.LogRepository{DB: db}
|
||||
blockRepository = *repositories.NewBlockRepository(db)
|
||||
|
||||
blockNumber = rand.Int63()
|
||||
blockId = test_helpers.CreateBlock(blockNumber, blockRepository)
|
||||
|
||||
log := core.Log{}
|
||||
logs := []core.Log{log}
|
||||
receipt := core.Receipt{
|
||||
Logs: logs,
|
||||
}
|
||||
receipts := []core.Receipt{receipt}
|
||||
|
||||
err = receiptRepository.CreateReceiptsAndLogs(blockId, receipts)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = logRepository.Get(&vulcanizeLogId, `SELECT id FROM logs`)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
p := parser.NewParser("")
|
||||
err = p.Parse(constants.TusdContractAddress)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
info = &contract.Contract{
|
||||
Name: "TrueUSD",
|
||||
Address: constants.TusdContractAddress,
|
||||
Abi: p.Abi(),
|
||||
ParsedAbi: p.ParsedAbi(),
|
||||
StartingBlock: 5197514,
|
||||
Events: p.GetEvents(),
|
||||
Methods: p.GetMethods(),
|
||||
Addresses: map[string]bool{},
|
||||
}
|
||||
|
||||
event := info.Events["Transfer"]
|
||||
err = info.GenerateFilters([]string{"Transfer"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
c := converter.NewConverter(*info)
|
||||
mockEvent.LogID = vulcanizeLogId
|
||||
err = c.Convert(mockEvent, event)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
dataStore = repository.NewDataStore(db)
|
||||
err = dataStore.PersistEvents(info)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
r = retriever.NewAddressRetriever(db)
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
db.Query(`DELETE FROM blocks`)
|
||||
db.Query(`DELETE FROM logs`)
|
||||
db.Query(`DELETE FROM transactions`)
|
||||
db.Query(`DELETE FROM receipts`)
|
||||
db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
|
||||
})
|
||||
|
||||
It("Retrieves a list of token holder addresses", func() {
|
||||
addresses, err = r.RetrieveTokenHolderAddresses(*info)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
_, ok := addresses[common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")]
|
||||
Expect(ok).To(Equal(true))
|
||||
|
||||
_, ok = addresses[common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
|
||||
Expect(ok).To(Equal(true))
|
||||
|
||||
_, ok = addresses[common.HexToAddress("0x")]
|
||||
Expect(ok).To(Equal(false))
|
||||
|
||||
})
|
||||
|
||||
It("Returns empty list when empty contract info is used", func() {
|
||||
addresses, err = r.RetrieveTokenHolderAddresses(contract.Contract{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(addresses)).To(Equal(0))
|
||||
})
|
||||
})
|
||||
@ -18,27 +18,36 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
// Retriever is used to retrieve the first block for a given contract
|
||||
// Block retriever is used to retrieve the first block for a given contract and the most recent block
|
||||
// It requires a vDB synced database with blocks, transactions, receipts, and logs
|
||||
type Retriever interface {
|
||||
type BlockRetriever interface {
|
||||
RetrieveFirstBlock(contractAddr string) (int64, error)
|
||||
RetrieveFirstBlockFromLogs(contractAddr string) (int64, error)
|
||||
RetrieveFirstBlockFromReceipts(contractAddr string) (int64, error)
|
||||
RetrieveMostRecentBlock() (int64, error)
|
||||
}
|
||||
|
||||
type retriever struct {
|
||||
type blockRetriever struct {
|
||||
*postgres.DB
|
||||
}
|
||||
|
||||
func NewRetriever(db *postgres.DB) (r *retriever) {
|
||||
func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
|
||||
|
||||
return &retriever{
|
||||
return &blockRetriever{
|
||||
DB: db,
|
||||
}
|
||||
}
|
||||
|
||||
// Try both methods of finding the first block, with the receipt method taking precedence
|
||||
func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
|
||||
i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
|
||||
if err != nil {
|
||||
i, err = r.retrieveFirstBlockFromLogs(contractAddr)
|
||||
}
|
||||
|
||||
return i, err
|
||||
}
|
||||
|
||||
// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
|
||||
func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
|
||||
func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
|
||||
var firstBlock int
|
||||
err := r.DB.Get(
|
||||
&firstBlock,
|
||||
@ -54,7 +63,7 @@ func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64,
|
||||
}
|
||||
|
||||
// In which case this servers as a heuristic to find the first block by finding the first contract event log
|
||||
func (r *retriever) RetrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
|
||||
func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
|
||||
var firstBlock int
|
||||
err := r.DB.Get(
|
||||
&firstBlock,
|
||||
@ -65,12 +74,13 @@ func (r *retriever) RetrieveFirstBlockFromLogs(contractAddr string) (int64, erro
|
||||
return int64(firstBlock), err
|
||||
}
|
||||
|
||||
// Try both methods of finding the first block, with the receipt method taking precedence
|
||||
func (r *retriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
|
||||
i, err := r.RetrieveFirstBlockFromReceipts(contractAddr)
|
||||
if err != nil {
|
||||
i, err = r.RetrieveFirstBlockFromLogs(contractAddr)
|
||||
}
|
||||
// Method to retrieve the most recent block in vDB
|
||||
func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
|
||||
var lastBlock int64
|
||||
err := r.DB.Get(
|
||||
&lastBlock,
|
||||
"SELECT number FROM blocks ORDER BY number DESC LIMIT 1",
|
||||
)
|
||||
|
||||
return i, err
|
||||
return lastBlock, err
|
||||
}
|
||||
@ -26,9 +26,9 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
|
||||
)
|
||||
|
||||
var _ = Describe("Fetcher Test", func() {
|
||||
var _ = Describe("Block Retriever Test", func() {
|
||||
var db *postgres.DB
|
||||
var r retriever.Retriever
|
||||
var r retriever.BlockRetriever
|
||||
var blockRepository repositories.BlockRepository
|
||||
|
||||
BeforeEach(func() {
|
||||
@ -42,12 +42,13 @@ var _ = Describe("Fetcher Test", func() {
|
||||
|
||||
blockRepository = *repositories.NewBlockRepository(db)
|
||||
|
||||
r = retriever.NewRetriever(db)
|
||||
r = retriever.NewBlockRetriever(db)
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
db.Query(`DELETE FROM blocks`)
|
||||
db.Query(`DELETE FROM logs`)
|
||||
db.Query(`DELETE FROM transactions`)
|
||||
db.Query(`DELETE FROM receipts`)
|
||||
})
|
||||
|
||||
@ -178,4 +179,25 @@ var _ = Describe("Fetcher Test", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(i).To(Equal(int64(1)))
|
||||
})
|
||||
|
||||
It("Fails if a block cannot be found", func() {
|
||||
|
||||
block1 := core.Block{
|
||||
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
|
||||
Number: 1,
|
||||
Transactions: []core.Transaction{},
|
||||
}
|
||||
|
||||
block2 := core.Block{
|
||||
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
|
||||
Number: 2,
|
||||
Transactions: []core.Transaction{},
|
||||
}
|
||||
|
||||
blockRepository.CreateOrUpdateBlock(block1)
|
||||
blockRepository.CreateOrUpdateBlock(block2)
|
||||
|
||||
_, err := r.RetrieveFirstBlock(constants.DaiContractAddress)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
@ -1,167 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package transformer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
|
||||
)
|
||||
|
||||
// Omni event transformer
|
||||
// Used to extract all or a subset of event data for
|
||||
// any contract and persist it to postgres in a manner
|
||||
// that requires no prior knowledge of the contract
|
||||
// other than its address and which network it is on
|
||||
type EventTransformer interface {
|
||||
Init(contractAddr string) error
|
||||
}
|
||||
|
||||
type eventTransformer struct {
|
||||
// Network, database, and blockchain config
|
||||
*types.Config
|
||||
|
||||
// Underlying databases
|
||||
datastore.WatchedEventRepository
|
||||
datastore.FilterRepository
|
||||
repository.DataStore
|
||||
|
||||
// Underlying interfaces
|
||||
parser.Parser // Parses events out of contract abi fetched with addr
|
||||
retriever.Retriever // Retrieves first block with contract addr referenced
|
||||
fetcher.Fetcher // Fetches data from public contract methods
|
||||
converter.Converter // Converts watched event logs into custom log
|
||||
|
||||
// Store contract info as mapping to contract address
|
||||
ContractInfo map[string]types.ContractInfo
|
||||
|
||||
// Subset of events of interest, stored as map of contract address to events
|
||||
// Default/empty list means all events are considered for that address
|
||||
sets map[string][]string
|
||||
}
|
||||
|
||||
// Transformer takes in config for blockchain, database, and network id
|
||||
func NewTransformer(c *types.Config) (t *eventTransformer) {
|
||||
t.Parser = parser.NewParser(c.Network)
|
||||
t.Retriever = retriever.NewRetriever(c.DB)
|
||||
t.Fetcher = fetcher.NewFetcher(c.BC)
|
||||
t.Converter = converter.NewConverter(types.ContractInfo{})
|
||||
t.ContractInfo = map[string]types.ContractInfo{}
|
||||
t.WatchedEventRepository = repositories.WatchedEventRepository{DB: c.DB}
|
||||
t.FilterRepository = repositories.FilterRepository{DB: c.DB}
|
||||
t.DataStore = repository.NewDataStore(c.DB)
|
||||
t.sets = map[string][]string{}
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
// Used to set which contract addresses and which of their events to watch
|
||||
func (t *eventTransformer) Set(contractAddr string, filterSet []string) {
|
||||
t.sets[contractAddr] = filterSet
|
||||
}
|
||||
|
||||
// Use after creating and setting transformer
|
||||
// Loops over all of the addr => filter sets
|
||||
// Uses parser to pull event info from abi
|
||||
// Use this info to generate event filters
|
||||
func (t *eventTransformer) Init() error {
|
||||
|
||||
for contractAddr, subset := range t.sets {
|
||||
err := t.Parser.Parse(contractAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var ctrName string
|
||||
strName, err := t.Fetcher.FetchString("name", t.Parser.Abi(), contractAddr, -1, nil)
|
||||
if err != nil || strName == "" {
|
||||
hashName, err := t.Fetcher.FetchHash("name", t.Parser.Abi(), contractAddr, -1, nil)
|
||||
if err != nil || hashName.String() == "" {
|
||||
return errors.New("unable to fetch contract name") // provide CLI prompt here for user to input a contract name?
|
||||
}
|
||||
ctrName = hashName.String()
|
||||
} else {
|
||||
ctrName = strName
|
||||
}
|
||||
|
||||
firstBlock, err := t.Retriever.RetrieveFirstBlock(contractAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info := types.ContractInfo{
|
||||
Name: ctrName,
|
||||
Address: contractAddr,
|
||||
Abi: t.Parser.Abi(),
|
||||
ParsedAbi: t.Parser.ParsedAbi(),
|
||||
StartingBlock: firstBlock,
|
||||
Events: t.Parser.GetEvents(),
|
||||
Methods: t.Parser.GetMethods(),
|
||||
}
|
||||
|
||||
info.GenerateFilters(subset)
|
||||
|
||||
for _, filter := range info.Filters {
|
||||
t.CreateFilter(filter)
|
||||
}
|
||||
|
||||
t.ContractInfo[contractAddr] = info
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Iterate through contracts, creating a new
|
||||
// converter for each one and using it to
|
||||
// convert watched event logs and persist
|
||||
// them into the postgres db
|
||||
func (tr eventTransformer) Execute() error {
|
||||
for _, contract := range tr.ContractInfo {
|
||||
|
||||
tr.Converter.Update(contract)
|
||||
|
||||
for eventName, filter := range contract.Filters {
|
||||
watchedEvents, err := tr.GetWatchedEvents(eventName)
|
||||
if err != nil {
|
||||
log.Println(fmt.Sprintf("Error fetching events for %s:", filter.Name), err)
|
||||
return err
|
||||
}
|
||||
|
||||
for _, we := range watchedEvents {
|
||||
err = tr.Converter.Convert(*we, contract.Events[eventName])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
err := tr.PersistEvents(contract)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package transformer_test
|
||||
199
pkg/omni/transformer/transformer.go
Normal file
199
pkg/omni/transformer/transformer.go
Normal file
@ -0,0 +1,199 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package transformer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/contract"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/converter"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/fetcher"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/parser"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/repository"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/retriever"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
|
||||
)
|
||||
|
||||
// Omni transformer
|
||||
// Used to extract all or a subset of event and method data
|
||||
// for any contract and persist it to postgres in a manner
|
||||
// that requires no prior knowledge of the contract
|
||||
// other than its address and which network it is on
|
||||
type Transformer interface {
|
||||
Init(contractAddr string) error
|
||||
}
|
||||
|
||||
type transformer struct {
|
||||
// Network, database, and blockchain config
|
||||
*types.Config
|
||||
|
||||
// Underlying databases
|
||||
datastore.WatchedEventRepository
|
||||
datastore.FilterRepository
|
||||
repository.DataStore
|
||||
|
||||
// Underlying interfaces
|
||||
parser.Parser // Parses events out of contract abi fetched with addr
|
||||
retriever.BlockRetriever // Retrieves first block with contract addr referenced
|
||||
retriever.AddressRetriever // Retrieves token holder addresses
|
||||
fetcher.Fetcher // Fetches data from public contract methods
|
||||
converter.Converter // Converts watched event logs into custom log
|
||||
|
||||
// Store contract info as mapping to contract address
|
||||
Contracts map[string]*contract.Contract
|
||||
|
||||
// Targeted subset of events/methods
|
||||
// Stored as map of contract address to events/method names of interest
|
||||
// Default/empty list means all events/methods are considered for that address
|
||||
targetEvents map[string][]string
|
||||
targetMethods map[string][]string
|
||||
}
|
||||
|
||||
// Transformer takes in config for blockchain, database, and network id
|
||||
func NewTransformer(c *types.Config) *transformer {
|
||||
|
||||
return &transformer{
|
||||
Parser: parser.NewParser(c.Network),
|
||||
BlockRetriever: retriever.NewBlockRetriever(c.DB),
|
||||
Fetcher: fetcher.NewFetcher(c.BC),
|
||||
Converter: converter.NewConverter(contract.Contract{}),
|
||||
Contracts: map[string]*contract.Contract{},
|
||||
WatchedEventRepository: repositories.WatchedEventRepository{DB: c.DB},
|
||||
FilterRepository: repositories.FilterRepository{DB: c.DB},
|
||||
DataStore: repository.NewDataStore(c.DB),
|
||||
targetEvents: map[string][]string{},
|
||||
targetMethods: map[string][]string{},
|
||||
}
|
||||
}
|
||||
|
||||
// Used to set which contract addresses and which of their events to watch
|
||||
func (t *transformer) SetEvents(contractAddr string, filterSet []string) {
|
||||
t.targetEvents[contractAddr] = filterSet
|
||||
}
|
||||
|
||||
// Used to set which contract addresses and which of their methods to call
|
||||
func (t *transformer) SetMethods(contractAddr string, filterSet []string) {
|
||||
t.targetMethods[contractAddr] = filterSet
|
||||
}
|
||||
|
||||
// Use after creating and setting transformer
|
||||
// Loops over all of the addr => filter sets
|
||||
// Uses parser to pull event info from abi
|
||||
// Use this info to generate event filters
|
||||
func (t *transformer) Init() error {
|
||||
|
||||
for contractAddr, subset := range t.targetEvents {
|
||||
// Get Abi
|
||||
err := t.Parser.Parse(contractAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get first block for contract
|
||||
firstBlock, err := t.BlockRetriever.RetrieveFirstBlock(contractAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get most recent block
|
||||
lastBlock, err := t.BlockRetriever.RetrieveMostRecentBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get contract name
|
||||
var ctrName string // should change this to check for "name" method and its return type in the abi methods before trying to fetch
|
||||
strName, err := t.Fetcher.FetchString("name", t.Parser.Abi(), contractAddr, lastBlock, nil)
|
||||
if err != nil || strName == "" {
|
||||
hashName, err := t.Fetcher.FetchHash("name", t.Parser.Abi(), contractAddr, lastBlock, nil)
|
||||
if err != nil || hashName.String() == "" {
|
||||
return errors.New(fmt.Sprintf("unable to fetch contract name: %v\r\n", err)) // provide CLI prompt here for user to input a contract name?
|
||||
}
|
||||
ctrName = hashName.String()
|
||||
} else {
|
||||
ctrName = strName
|
||||
}
|
||||
|
||||
// Aggregate info into contract object
|
||||
info := &contract.Contract{
|
||||
Name: ctrName,
|
||||
Address: contractAddr,
|
||||
Abi: t.Parser.Abi(),
|
||||
ParsedAbi: t.Parser.ParsedAbi(),
|
||||
StartingBlock: firstBlock,
|
||||
Events: t.Parser.GetEvents(),
|
||||
Methods: t.Parser.GetMethods(),
|
||||
Addresses: map[string]bool{},
|
||||
}
|
||||
|
||||
// Use info to create filters
|
||||
err = info.GenerateFilters(subset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Iterate over filters and push them to the repo
|
||||
for _, filter := range info.Filters {
|
||||
t.CreateFilter(filter)
|
||||
}
|
||||
|
||||
t.Contracts[contractAddr] = info
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Iterate through contracts, updating the
|
||||
// converter with each one and using it to
|
||||
// convert watched event logs.
|
||||
// Then persist them into the postgres db
|
||||
func (tr transformer) Execute() error {
|
||||
// Iterate through all internal contracts
|
||||
for _, con := range tr.Contracts {
|
||||
|
||||
// Update converter with current contract
|
||||
tr.Converter.Update(*con)
|
||||
|
||||
// Iterate through contract filters and get watched event logs
|
||||
for eventName, filter := range con.Filters {
|
||||
watchedEvents, err := tr.GetWatchedEvents(eventName)
|
||||
if err != nil {
|
||||
log.Println(fmt.Sprintf("Error fetching events for %s:", filter.Name), err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Iterate over watched event logs and convert them
|
||||
for _, we := range watchedEvents {
|
||||
err = tr.Converter.Convert(*we, con.Events[eventName])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// After converting all logs for events of interest, persist all of the data
|
||||
err := tr.PersistEvents(con)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
154
pkg/omni/transformer/transformer_test.go
Normal file
154
pkg/omni/transformer/transformer_test.go
Normal file
@ -0,0 +1,154 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package transformer_test
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/ethclient"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/config"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
|
||||
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/transformer"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/omni/types"
|
||||
)
|
||||
|
||||
var mockEvent = core.WatchedEvent{
|
||||
Name: constants.TransferEvent.String(),
|
||||
BlockNumber: 5488076,
|
||||
Address: constants.TusdContractAddress,
|
||||
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
|
||||
Index: 110,
|
||||
Topic0: constants.TransferEvent.Signature(),
|
||||
Topic1: "0x000000000000000000000000000000000000000000000000000000000000af21",
|
||||
Topic2: "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
|
||||
Topic3: "",
|
||||
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
|
||||
}
|
||||
|
||||
var _ = Describe("Repository Test", func() {
|
||||
var db *postgres.DB
|
||||
var err error
|
||||
var con types.Config
|
||||
var blockRepository repositories.BlockRepository
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
|
||||
BeforeEach(func() {
|
||||
infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
|
||||
rawRpcClient, err := rpc.Dial(infuraIPC)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
|
||||
ethClient := ethclient.NewClient(rawRpcClient)
|
||||
blockChainClient := client.NewEthClient(ethClient)
|
||||
node := node.MakeNode(rpcClient)
|
||||
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
|
||||
blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
|
||||
|
||||
db, err = postgres.NewDB(config.Database{
|
||||
Hostname: "localhost",
|
||||
Name: "vulcanize_private",
|
||||
Port: 5432,
|
||||
}, blockChain.Node())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
con = types.Config{
|
||||
DB: db,
|
||||
BC: blockChain,
|
||||
Network: "",
|
||||
}
|
||||
|
||||
blockRepository = *repositories.NewBlockRepository(db)
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
db.Query(`DELETE FROM blocks`)
|
||||
db.Query(`DELETE FROM logs`)
|
||||
db.Query(`DELETE FROM transactions`)
|
||||
db.Query(`DELETE FROM receipts`)
|
||||
db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`)
|
||||
})
|
||||
|
||||
It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
|
||||
t := transformer.NewTransformer(&con)
|
||||
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
|
||||
err = t.Init()
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("Initializes and executes successfully if first and most recent blocks can be fetched from vDB", func() {
|
||||
log := core.Log{
|
||||
BlockNumber: 6194634,
|
||||
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
|
||||
Address: constants.TusdContractAddress,
|
||||
Topics: core.Topics{
|
||||
constants.TransferEvent.Signature(),
|
||||
"0x000000000000000000000000000000000000000000000000000000000000af21",
|
||||
"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
|
||||
"",
|
||||
},
|
||||
Index: 1,
|
||||
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
|
||||
}
|
||||
|
||||
receipt := core.Receipt{
|
||||
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
|
||||
ContractAddress: constants.TusdContractAddress,
|
||||
Logs: []core.Log{log},
|
||||
}
|
||||
|
||||
transaction := core.Transaction{
|
||||
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
|
||||
Receipt: receipt,
|
||||
}
|
||||
|
||||
block := core.Block{
|
||||
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
|
||||
Number: 6194634,
|
||||
Transactions: []core.Transaction{transaction},
|
||||
}
|
||||
|
||||
blockRepository.CreateOrUpdateBlock(block)
|
||||
|
||||
t := transformer.NewTransformer(&con)
|
||||
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
|
||||
err = t.Init()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
c, ok := t.Contracts[constants.TusdContractAddress]
|
||||
Expect(ok).To(Equal(true))
|
||||
|
||||
err = t.Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
b, ok := c.Addresses["0x000000000000000000000000000000000000Af21"]
|
||||
Expect(ok).To(Equal(true))
|
||||
Expect(b).To(Equal(true))
|
||||
|
||||
b, ok = c.Addresses["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
|
||||
Expect(ok).To(Equal(true))
|
||||
Expect(b).To(Equal(true))
|
||||
})
|
||||
})
|
||||
@ -19,8 +19,17 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/accounts/abi"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Network string
|
||||
BC core.BlockChain
|
||||
DB *postgres.DB
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
Name string
|
||||
Anonymous bool
|
||||
|
||||
Loading…
Reference in New Issue
Block a user