cerc-io/ipld-eth-server
17
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

refactor omni code; rename omniWatcher => contractWatcher; use config instead of excessive number of CLI flags

Browse Source
This commit is contained in:
Ian Norden 2019-03-11 11:19:18 -05:00
parent 84b65caedd
commit 8174ce4aee
20 changed files with 1278 additions and 706 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
cmd/contractWatcher.go Normal file
View File

@ -0,0 +1,123 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
ft "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
lt "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
"github.com/vulcanize/vulcanizedb/utils"
)
// contractWatcherCmd represents the contractWatcher command
var contractWatcherCmd = &cobra.Command{
Use: "contractWatcher",
Short: "Watches events at the provided contract address using fully synced vDB",
Long: `Uses input contract address and event filters to watch events
Expects an ethereum node to be running
Expects an archival node synced into vulcanizeDB
Requires a .toml config file:
[database]
name = "vulcanize_public"
hostname = "localhost"
port = 5432
[client]
ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
[contract]
network = ""
addresses = [
"contractAddress1",
"contractAddress2"
]
[contract.contractAddress1]
abi = 'ABI for contract 1'
startingBlock = 982463
[contract.contractAddress2]
abi = 'ABI for contract 2'
events = [
"event1",
"event2"
]
eventArgs = [
"arg1",
"arg2"
]
methods = [
"method1",
"method2"
]
methodArgs = [
"arg1",
"arg2"
]
startingBlock = 4448566
piping = true
`,
Run: func(cmd *cobra.Command, args []string) {
contractWatcher()
},
}
var (
mode string
)
func contractWatcher() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
blockChain := getBlockChain()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
var t st.GenericTransformer
con := config.ContractConfig{}
con.PrepConfig()
switch mode {
case "light":
t = lt.NewTransformer(con, blockChain, &db)
case "full":
t = ft.NewTransformer(con, blockChain, &db)
default:
log.Fatal("Invalid mode")
}
err := t.Init()
if err != nil {
log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err))
}
for range ticker.C {
t.Execute()
}
}
func init() {
rootCmd.AddCommand(contractWatcherCmd)
contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)")
}

121
cmd/omniWatcher.go
View File

@ -1,121 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"log"
"time"
"github.com/spf13/cobra"
ft "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
lt "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
st "github.com/vulcanize/vulcanizedb/pkg/omni/shared/transformer"
"github.com/vulcanize/vulcanizedb/utils"
)
// omniWatcherCmd represents the omniWatcher command
var omniWatcherCmd = &cobra.Command{
Use: "omniWatcher",
Short: "Watches events at the provided contract address using fully synced vDB",
Long: `Uses input contract address and event filters to watch events
Expects an ethereum node to be running
Expects an archival node synced into vulcanizeDB
Requires a .toml config file:
[database]
name = "vulcanize_public"
hostname = "localhost"
port = 5432
[client]
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
omniWatcher()
},
}
var (
network string
contractAddress string
contractAddresses []string
contractEvents []string
contractMethods []string
eventArgs []string
methodArgs []string
methodPiping bool
mode string
)
func omniWatcher() {
if contractAddress == "" && len(contractAddresses) == 0 {
log.Fatal("Contract address required")
}
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
blockChain := getBlockChain()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
var t st.Transformer
switch mode {
case "light":
t = lt.NewTransformer(network, blockChain, &db)
case "full":
t = ft.NewTransformer(network, blockChain, &db)
default:
log.Fatal("Invalid mode")
}
contractAddresses = append(contractAddresses, contractAddress)
for _, addr := range contractAddresses {
t.SetEvents(addr, contractEvents)
t.SetMethods(addr, contractMethods)
t.SetEventArgs(addr, eventArgs)
t.SetMethodArgs(addr, methodArgs)
t.SetPiping(addr, methodPiping)
t.SetStartingBlock(addr, startingBlockNumber)
}
err := t.Init()
if err != nil {
log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err))
}
for range ticker.C {
t.Execute()
}
}
func init() {
rootCmd.AddCommand(omniWatcherCmd)
omniWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)")
omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address")
omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "events", "e", []string{}, "Subset of events to watch; by default all events are watched")
omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "methods", "m", nil, "Subset of methods to poll; by default no methods are polled")
omniWatcherCmd.Flags().StringArrayVarP(&eventArgs, "event-args", "f", []string{}, "Argument values to filter event logs for; will only persist event logs that emit at least one of the value specified")
omniWatcherCmd.Flags().StringArrayVarP(&methodArgs, "method-args", "g", []string{}, "Argument values to limit methods to; will only call methods with emitted values that were specified here")
omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)
omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists")
omniWatcherCmd.Flags().BoolVarP(&methodPiping, "piping", "p", false, "Turn on method output piping: methods listed first will be polled first and their output used as input to subsequent methods")
}

75
integration_test/omni_full_transformer_test.go
View File

@ -2,6 +2,7 @@ package integration
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"math/rand"
"strings"
"time"
@ -41,8 +42,7 @@ var _ = Describe("Omni full transformer", func() {
It("Initializes transformer's contract objects", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -64,9 +64,7 @@ var _ = Describe("Omni full transformer", func() {
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -85,9 +83,12 @@ var _ = Describe("Omni full transformer", func() {
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -119,9 +120,12 @@ var _ = Describe("Omni full transformer", func() {
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -145,9 +149,7 @@ var _ = Describe("Omni full transformer", func() {
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Execute()
Expect(err).To(HaveOccurred())
@ -161,9 +163,7 @@ var _ = Describe("Omni full transformer", func() {
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -183,9 +183,12 @@ var _ = Describe("Omni full transformer", func() {
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -210,9 +213,12 @@ var _ = Describe("Omni full transformer", func() {
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -235,10 +241,12 @@ var _ = Describe("Omni full transformer", func() {
})
It("It does not perist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.EventArgs = map[string][]string{
ensAddr: {"fake_filter_value"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -252,10 +260,15 @@ var _ = Describe("Omni full transformer", func() {
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.MethodArgs = map[string][]string{
ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"},
}
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())

105
integration_test/omni_light_transformer_test.go
View File

@ -2,6 +2,7 @@ package integration
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"strings"
"github.com/ethereum/go-ethereum/common"
@ -39,8 +40,7 @@ var _ = Describe("Omnit light transformer", func() {
It("Initializes transformer's contract objects", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -70,9 +70,7 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
@ -89,9 +87,12 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
@ -131,9 +132,12 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
@ -150,9 +154,7 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
@ -173,9 +175,7 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
@ -192,9 +192,12 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
@ -218,9 +221,12 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Polls given method using list of collected hashes", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
@ -242,10 +248,12 @@ var _ = Describe("Omnit light transformer", func() {
})
It("It does not persist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.EventArgs = map[string][]string{
ensAddr: {"fake_filter_value"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
@ -257,10 +265,15 @@ var _ = Describe("Omnit light transformer", func() {
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.MethodArgs = map[string][]string{
ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"},
}
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
@ -302,11 +315,7 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
t := transformer.NewTransformer(mocks.ENSandTusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
@ -332,11 +341,13 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
var testConf config.ContractConfig
testConf = mocks.ENSandTusdConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
ens, ok := t.Contracts[ensAddr]
@ -376,11 +387,13 @@ var _ = Describe("Omnit light transformer", func() {
})
It("Polls given methods for each contract, using list of collected values", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
var testConf config.ContractConfig
testConf = mocks.ENSandTusdConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()

177
pkg/config/contract.go Normal file
View File

@ -0,0 +1,177 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"strings"
)
// Config struct for generic contract transformer
type ContractConfig struct {
// Name for the transformer
Name string
// Ethereum network name; default "" is mainnet
Network string
// List of contract addresses (map to ensure no duplicates)
Addresses map[string]bool
// Map of contract address to abi
// If an address has no associated abi the parser will attempt to fetch one from etherscan
Abis map[string]string
// Map of contract address to slice of events
// Used to set which addresses to watch
// If any events are listed in the slice only those will be watched
// Otherwise all events in the contract ABI are watched
Events map[string][]string
// Map of contract address to slice of methods
// If any methods are listed in the slice only those will be polled
// Otherwise no methods will be polled
Methods map[string][]string
// Map of contract address to slice of event arguments to filter for
// If arguments are provided then only events which emit those arguments are watched
// Otherwise arguments are not filtered on events
EventArgs map[string][]string
// Map of contract address to slice of method arguments to limit polling to
// If arguments are provided then only those arguments are allowed as arguments in method polling
// Otherwise any argument of the right type seen emitted from events at that contract will be used in method polling
MethodArgs map[string][]string
// Map of contract address to their starting block
StartingBlocks map[string]int64
// Map of contract address to whether or not to pipe method polling results forward into subsequent method calls
Piping map[string]bool
}
func (oc *ContractConfig) PrepConfig() {
addrs := viper.GetStringSlice("contract.addresses")
oc.Network = viper.GetString("contract.network")
oc.Addresses = make(map[string]bool, len(addrs))
oc.Abis = make(map[string]string, len(addrs))
oc.Methods = make(map[string][]string, len(addrs))
oc.EventArgs = make(map[string][]string, len(addrs))
oc.MethodArgs = make(map[string][]string, len(addrs))
oc.EventArgs = make(map[string][]string, len(addrs))
oc.StartingBlocks = make(map[string]int64, len(addrs))
oc.Piping = make(map[string]bool, len(addrs))
// De-dupe addresses
for _, addr := range addrs {
oc.Addresses[strings.ToLower(addr)] = true
}
// Iterate over addresses to pull out config info for each contract
for _, addr := range addrs {
transformer := viper.GetStringMap("contract." + addr)
// Get and check abi
abi, abiOK := transformer["abi"]
if !abiOK || abi == nil {
log.Fatal(addr, "transformer config is missing `abi` value")
}
abiRef, abiOK := abi.(string)
if !abiOK {
log.Fatal(addr, "transformer `events` not of type []string")
}
oc.Abis[strings.ToLower(addr)] = abiRef
// Get and check events
events, eventsOK := transformer["events"]
if !eventsOK || events == nil {
log.Fatal(addr, "transformer config is missing `events` value")
}
eventsRef, eventsOK := events.([]string)
if !eventsOK {
log.Fatal(addr, "transformer `events` not of type []string")
}
if eventsRef == nil {
eventsRef = []string{}
}
oc.Events[strings.ToLower(addr)] = eventsRef
// Get and check methods
methods, methodsOK := transformer["methods"]
if !methodsOK || methods == nil {
log.Fatal(addr, "transformer config is missing `methods` value")
}
methodsRef, methodsOK := methods.([]string)
if !methodsOK {
log.Fatal(addr, "transformer `methods` not of type []string")
}
if methodsRef == nil {
methodsRef = []string{}
}
oc.Methods[strings.ToLower(addr)] = methodsRef
// Get and check eventArgs
eventArgs, eventArgsOK := transformer["eventArgs"]
if !eventArgsOK || eventArgs == nil {
log.Fatal(addr, "transformer config is missing `eventArgs` value")
}
eventArgsRef, eventArgsOK := eventArgs.([]string)
if !eventArgsOK {
log.Fatal(addr, "transformer `eventArgs` not of type []string")
}
if eventArgsRef == nil {
eventArgsRef = []string{}
}
oc.EventArgs[strings.ToLower(addr)] = eventArgsRef
// Get and check methodArgs
methodArgs, methodArgsOK := transformer["methodArgs"]
if !methodArgsOK || methodArgs == nil {
log.Fatal(addr, "transformer config is missing `methodArgs` value")
}
methodArgsRef, methodArgsOK := methodArgs.([]string)
if !methodArgsOK {
log.Fatal(addr, "transformer `methodArgs` not of type []string")
}
if methodArgsRef == nil {
methodArgsRef = []string{}
}
oc.MethodArgs[strings.ToLower(addr)] = methodArgsRef
// Get and check startingBlock
start, startOK := transformer["startingBlock"]
if !startOK || start == nil {
log.Fatal(addr, "transformer config is missing `startingBlock` value")
}
startRef, startOK := start.(int64)
if !startOK {
log.Fatal(addr, "transformer `startingBlock` not of type int")
}
oc.StartingBlocks[strings.ToLower(addr)] = startRef
// Get pipping
pipe, pipeOK := transformer["pipping"]
if !pipeOK || pipe == nil {
log.Fatal(addr, "transformer config is missing `pipping` value")
}
pipeRef, pipeOK := pipe.(bool)
if !pipeOK {
log.Fatal(addr, "transformer `piping` not of type bool")
}
oc.Piping[strings.ToLower(addr)] = pipeRef
}
}

4
pkg/omni/full/converter/converter.go
View File

@ -55,10 +55,10 @@ func (c *converter) Update(info *contract.Contract) {
// Convert the given watched event log into a types.Log for the given event
func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
values := make(map[string]interface{})
log := helpers.ConvertToLog(watchedEvent)
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
err := contract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}

164
pkg/omni/full/transformer/transformer.go
View File

@ -18,8 +18,8 @@ package transformer
import (
"errors"
"strings"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@ -34,7 +34,7 @@ import (
)
// Requires a fully synced vDB and a running eth node (or infura)
type Transformer struct {
type transformer struct {
// Database interfaces
datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters()
datastore.WatchedEventRepository // Watched event log views, created by the log filters
@ -48,52 +48,25 @@ type Transformer struct {
converter.Converter // Converts watched event logs into custom log
poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore
// Ethereum network name; default "" is mainnet
Network string
// Store contract configuration information
Config config.ContractConfig
// Store contract info as mapping to contract address
Contracts map[string]*contract.Contract
// Targeted subset of events/methods
// Stored as map sof contract address to events/method names of interest
WatchedEvents map[string][]string // Default/empty event list means all are watched
WantedMethods map[string][]string // Default/empty method list means none are polled
// Starting block for contracts
ContractStart map[string]int64
// Lists of addresses to filter event or method data
// before persisting; if empty no filter is applied
EventArgs map[string][]string
MethodArgs map[string][]string
// Whether or not to create a list of emitted address or hashes for the contract in postgres
CreateAddrList map[string]bool
CreateHashList map[string]bool
// Method piping on/off for a contract
Piping map[string]bool
}
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transformer {
return &Transformer{
func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *transformer {
return &transformer{
Poller: poller.NewPoller(BC, DB, types.FullSync),
Parser: parser.NewParser(network),
Parser: parser.NewParser(con.Network),
BlockRetriever: retriever.NewBlockRetriever(DB),
Converter: converter.NewConverter(&contract.Contract{}),
Contracts: map[string]*contract.Contract{},
WatchedEventRepository: repositories.WatchedEventRepository{DB: DB},
FilterRepository: repositories.FilterRepository{DB: DB},
EventRepository: repository.NewEventRepository(DB, types.FullSync),
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractStart: map[string]int64{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
CreateHashList: map[string]bool{},
Piping: map[string]bool{},
Config: con,
}
}
@ -101,59 +74,66 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transf
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (transformer *Transformer) Init() error {
for contractAddr, subset := range transformer.WatchedEvents {
// Get Abi
err := transformer.Parser.Parse(contractAddr)
if err != nil {
return err
func (tr *transformer) Init() error {
for contractAddr := range tr.Config.Addresses {
// Configure Abi
if tr.Config.Abis[contractAddr] == "" {
// If no abi is given in the config, this method will try fetching from internal look-up table and etherscan
err := tr.Parser.Parse(contractAddr)
if err != nil {
return err
}
} else {
// If we have an abi from the config, load that into the parser
err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
if err != nil {
return err
}
}
// Get first block and most recent block number in the header repo
firstBlock, err := transformer.BlockRetriever.RetrieveFirstBlock(contractAddr)
firstBlock, err := tr.BlockRetriever.RetrieveFirstBlock(contractAddr)
if err != nil {
return err
}
lastBlock, err := transformer.BlockRetriever.RetrieveMostRecentBlock()
lastBlock, err := tr.BlockRetriever.RetrieveMostRecentBlock()
if err != nil {
return err
}
// Set to specified range if it falls within the bounds
if firstBlock < transformer.ContractStart[contractAddr] {
firstBlock = transformer.ContractStart[contractAddr]
if firstBlock < tr.Config.StartingBlocks[contractAddr] {
firstBlock = tr.Config.StartingBlocks[contractAddr]
}
// Get contract name if it has one
var name = new(string)
transformer.Poller.FetchContractData(transformer.Abi(), contractAddr, "name", nil, name, lastBlock)
tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
for _, arg := range transformer.EventArgs[contractAddr] {
for _, arg := range tr.Config.EventArgs[contractAddr] {
eventArgs[arg] = true
}
methodArgs := map[string]bool{}
for _, arg := range transformer.MethodArgs[contractAddr] {
for _, arg := range tr.Config.MethodArgs[contractAddr] {
methodArgs[arg] = true
}
// Aggregate info into contract object
info := contract.Contract{
Name: *name,
Network: transformer.Network,
Address: contractAddr,
Abi: transformer.Parser.Abi(),
ParsedAbi: transformer.Parser.ParsedAbi(),
StartingBlock: firstBlock,
LastBlock: lastBlock,
Events: transformer.Parser.GetEvents(subset),
Methods: transformer.Parser.GetSelectMethods(transformer.WantedMethods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
CreateAddrList: transformer.CreateAddrList[contractAddr],
CreateHashList: transformer.CreateHashList[contractAddr],
Piping: transformer.Piping[contractAddr],
Name: *name,
Network: tr.Config.Network,
Address: contractAddr,
Abi: tr.Parser.Abi(),
ParsedAbi: tr.Parser.ParsedAbi(),
StartingBlock: firstBlock,
LastBlock: lastBlock,
Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]),
Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
Piping: tr.Config.Piping[contractAddr],
}.Init()
// Use info to create filters
@ -164,14 +144,14 @@ func (transformer *Transformer) Init() error {
// Iterate over filters and push them to the repo using filter repository interface
for _, filter := range info.Filters {
err = transformer.FilterRepository.CreateFilter(filter)
err = tr.CreateFilter(filter)
if err != nil {
return err
}
}
// Store contract info for further processing
transformer.Contracts[contractAddr] = info
tr.Contracts[contractAddr] = info
}
return nil
@ -182,18 +162,18 @@ func (transformer *Transformer) Init() error {
// Uses converter to convert logs into custom log type
// Persists converted logs into custuom postgres tables
// Calls selected methods, using token holder address generated during event log conversion
func (transformer Transformer) Execute() error {
if len(transformer.Contracts) == 0 {
func (tr *transformer) Execute() error {
if len(tr.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts to work with")
}
// Iterate through all internal contracts
for _, con := range transformer.Contracts {
for _, con := range tr.Contracts {
// Update converter with current contract
transformer.Update(con)
tr.Update(con)
// Iterate through contract filters and get watched event logs
for eventSig, filter := range con.Filters {
watchedEvents, err := transformer.GetWatchedEvents(filter.Name)
watchedEvents, err := tr.GetWatchedEvents(filter.Name)
if err != nil {
return err
}
@ -201,7 +181,7 @@ func (transformer Transformer) Execute() error {
// Iterate over watched event logs
for _, we := range watchedEvents {
// Convert them to our custom log type
cstm, err := transformer.Converter.Convert(*we, con.Events[eventSig])
cstm, err := tr.Converter.Convert(*we, con.Events[eventSig])
if err != nil {
return err
}
@ -211,7 +191,7 @@ func (transformer Transformer) Execute() error {
// If log is not empty, immediately persist in repo
// Run this in seperate goroutine?
err = transformer.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
if err != nil {
return err
}
@ -222,7 +202,7 @@ func (transformer Transformer) Execute() error {
// poller polls select contract methods
// and persists the results into custom pg tables
// Run this in seperate goroutine?
if err := transformer.PollContract(*con); err != nil {
if err := tr.PollContract(*con); err != nil {
return err
}
}
@ -230,42 +210,6 @@ func (transformer Transformer) Execute() error {
return nil
}
// Used to set which contract addresses and which of their events to watch
func (transformer *Transformer) SetEvents(contractAddr string, filterSet []string) {
transformer.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to watch events for
func (transformer *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
transformer.EventArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set which contract addresses and which of their methods to call
func (transformer *Transformer) SetMethods(contractAddr string, filterSet []string) {
transformer.WantedMethods[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to poll methods on
func (transformer *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
transformer.MethodArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set the block range to watch for a given address
func (transformer *Transformer) SetStartingBlock(contractAddr string, start int64) {
transformer.ContractStart[strings.ToLower(contractAddr)] = start
}
// Used to set whether or not to persist an account address list
func (transformer *Transformer) SetCreateAddrList(contractAddr string, on bool) {
transformer.CreateAddrList[strings.ToLower(contractAddr)] = on
}
// Used to set whether or not to persist an hash list
func (transformer *Transformer) SetCreateHashList(contractAddr string, on bool) {
transformer.CreateHashList[strings.ToLower(contractAddr)] = on
}
// Used to turn method piping on for a contract
func (transformer *Transformer) SetPiping(contractAddr string, on bool) {
transformer.Piping[strings.ToLower(contractAddr)] = on
func (tr *transformer) GetConfig() config.ContractConfig {
return tr.Config
}

393
pkg/omni/full/transformer/transformer_test.go
View File

@ -17,160 +17,307 @@
package transformer_test
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"math/rand"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
)
var _ = Describe("Transformer", func() {
var fakeAddress = "0x1234567890abcdef"
var db *postgres.DB
var err error
var blockChain core.BlockChain
var blockRepository repositories.BlockRepository
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
rand.Seed(time.Now().UnixNano())
Describe("SetEvents", func() {
It("Sets which events to watch from the given contract address", func() {
watchedEvents := []string{"Transfer", "Mint"}
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, watchedEvents)
Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
})
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
blockRepository = *repositories.NewBlockRepository(db)
})
Describe("SetEventAddrs", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEventArgs(fakeAddress, eventAddrs)
Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
})
})
Describe("SetMethods", func() {
It("Sets which methods to poll at the given contract address", func() {
watchedMethods := []string{"balanceOf", "totalSupply"}
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethods(fakeAddress, watchedMethods)
Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
})
})
Describe("SetMethodAddrs", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethodArgs(fakeAddress, methodAddrs)
Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
})
})
Describe("SetStartingBlock", func() {
It("Sets the block range that the contract should be watched within", func() {
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetStartingBlock(fakeAddress, 11)
Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
})
})
Describe("SetCreateAddrList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateAddrList(fakeAddress, true)
Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
})
})
Describe("SetCreateHashList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateHashList(fakeAddress, true)
Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRetriever := &fakes.MockFullBlockRetriever{}
firstBlock := int64(1)
mostRecentBlock := int64(2)
blockRetriever.FirstBlock = firstBlock
blockRetriever.MostRecentBlock = mostRecentBlock
parsr := &fakes.MockParser{}
fakeAbi := "fake_abi"
eventName := "Transfer"
event := types.Event{}
parsr.AbiToReturn = fakeAbi
parsr.EventName = eventName
parsr.Event = event
pollr := &fakes.MockPoller{}
fakeContractName := "fake_contract_name"
pollr.ContractName = fakeContractName
t := getTransformer(blockRetriever, parsr, pollr)
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[fakeAddress]
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(firstBlock))
Expect(c.LastBlock).To(Equal(mostRecentBlock))
Expect(c.Abi).To(Equal(fakeAbi))
Expect(c.Name).To(Equal(fakeContractName))
Expect(c.Address).To(Equal(fakeAddress))
Expect(c.StartingBlock).To(Equal(int64(6194633)))
Expect(c.LastBlock).To(Equal(int64(6194634)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
})
It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
blockRetriever := &fakes.MockFullBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("Does nothing if watched events are unset", func() {
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Events = nil
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).To(HaveOccurred())
err := t.Init()
Expect(err).ToNot(HaveOccurred())
_, ok := t.Contracts[fakeAddress]
_, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(false))
})
})
})
func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
return transformer.Transformer{
FilterRepository: &fakes.MockFilterRepository{},
Parser: parsr,
BlockRetriever: blockRetriever,
Poller: pollr,
Contracts: map[string]*contract.Contract{},
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractStart: map[string]int64{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
CreateHashList: map[string]bool{},
}
}
Describe("Execute", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.TransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
Expect(log.Block).To(Equal(int64(6194634)))
Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.NewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
Expect(log.Block).To(Equal(int64(6194635)))
Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(3))
b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not perist events if they do not pass the emitted arg filter", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.EventArgs = map[string][]string{
ensAddr: {"fake_filter_value"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
testConf.MethodArgs = map[string][]string{
ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
})

8
pkg/omni/light/converter/converter.go
View File

@ -54,7 +54,7 @@ func (c *converter) Update(info *contract.Contract) {
// Convert the given watched event log into a types.Log for the given event
func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) {
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
returnLogs := make([]types.Log, 0, len(logs))
for _, log := range logs {
values := make(map[string]interface{})
@ -63,7 +63,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
values[field.Name] = i
}
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
err := contract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}
@ -133,7 +133,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
// Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
eventsToLogs := make(map[string][]types.Log)
for _, event := range events {
eventsToLogs[event.Name] = make([]types.Log, 0, len(logs))
@ -142,7 +142,7 @@ func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
// If the log is of this event type, process it as such
if event.Sig() == log.Topics[0] {
values := make(map[string]interface{})
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
err := contract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}

6
pkg/omni/light/repository/header_repository.go
View File

@ -17,8 +17,8 @@
package repository
import (
"database/sql"
"fmt"
"github.com/jmoiron/sqlx"
"github.com/hashicorp/golang-lru"
@ -125,7 +125,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
}
func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
tx, err := r.db.Beginx()
tx, err := r.db.Begin()
if err != nil {
return err
}
@ -250,7 +250,7 @@ func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
return r.columns.Get(key)
}
func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error {
func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO

2
pkg/omni/light/retriever/retriever_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestRetriever(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Light Block Number Retriever Suite Test")
RunSpecs(t, "Light BLock Number Retriever Suite Test")
}
var _ = BeforeSuite(func() {

146
pkg/omni/light/transformer/transformer.go
View File

@ -18,6 +18,7 @@ package transformer
import (
"errors"
"github.com/vulcanize/vulcanizedb/pkg/config"
"strings"
"github.com/ethereum/go-ethereum/common"
@ -37,7 +38,7 @@ import (
)
// Requires a light synced vDB (headers) and a running eth node (or infura)
type Transformer struct {
type transformer struct {
// Database interfaces
srep.EventRepository // Holds transformed watched event log data
repository.HeaderRepository // Interface for interaction with header repositories
@ -51,32 +52,12 @@ type Transformer struct {
converter.Converter // Converts watched event logs into custom log
poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore
// Ethereum network name; default "" is mainnet
Network string
// Store contract configuration information
Config config.ContractConfig
// Store contract info as mapping to contract address
Contracts map[string]*contract.Contract
// Targeted subset of events/methods
// Stored as maps of contract address to events/method names of interest
WatchedEvents map[string][]string // Default/empty event list means all are watched
WantedMethods map[string][]string // Default/empty method list means none are polled
// Starting block number for each contract
ContractStart map[string]int64
// Lists of argument values to filter event or
// method data with; if empty no filter is applied
EventArgs map[string][]string
MethodArgs map[string][]string
// Whether or not to create a list of emitted address or hashes for the contract in postgres
CreateAddrList map[string]bool
CreateHashList map[string]bool
// Method piping on/off for a contract
Piping map[string]bool
// Internally configured transformer variables
contractAddresses []string // Holds all contract addresses, for batch fetching of logs
sortedEventIds map[string][]string // Map to sort event column ids by contract, for post fetch processing and persisting of logs
@ -93,26 +74,18 @@ type Transformer struct {
// 4. Execute
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transformer {
func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *transformer {
return &Transformer{
return &transformer{
Poller: poller.NewPoller(bc, db, types.LightSync),
Fetcher: fetcher.NewFetcher(bc),
Parser: parser.NewParser(network),
Parser: parser.NewParser(con.Network),
HeaderRepository: repository.NewHeaderRepository(db),
BlockRetriever: retriever.NewBlockRetriever(db),
Converter: converter.NewConverter(&contract.Contract{}),
Contracts: map[string]*contract.Contract{},
EventRepository: srep.NewEventRepository(db, types.LightSync),
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractStart: map[string]int64{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
CreateHashList: map[string]bool{},
Piping: map[string]bool{},
Network: network,
Config: con,
}
}
@ -120,7 +93,7 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transf
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (tr *Transformer) Init() error {
func (tr *transformer) Init() error {
// Initialize internally configured transformer settings
tr.contractAddresses = make([]string, 0) // Holds all contract addresses, for batch fetching of logs
tr.sortedEventIds = make(map[string][]string) // Map to sort event column ids by contract, for post fetch processing and persisting of logs
@ -130,11 +103,20 @@ func (tr *Transformer) Init() error {
tr.start = 100000000000 // Hold the lowest starting block and the highest ending block
// Iterate through all internal contract addresses
for contractAddr, subset := range tr.WatchedEvents {
// Get Abi
err := tr.Parser.Parse(contractAddr)
if err != nil {
return err
for contractAddr := range tr.Config.Addresses {
// Configure Abi
if tr.Config.Abis[contractAddr] == "" {
// If no abi is given in the config, this method will try fetching from internal look-up table and etherscan
err := tr.Parser.Parse(contractAddr)
if err != nil {
return err
}
} else {
// If we have an abi from the config, load that into the parser
err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
if err != nil {
return err
}
}
// Get first block and most recent block number in the header repo
@ -148,40 +130,38 @@ func (tr *Transformer) Init() error {
}
// Set to specified range if it falls within the bounds
if firstBlock < tr.ContractStart[contractAddr] {
firstBlock = tr.ContractStart[contractAddr]
if firstBlock < tr.Config.StartingBlocks[contractAddr] {
firstBlock = tr.Config.StartingBlocks[contractAddr]
}
// Get contract name if it has one
var name = new(string)
tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock)
tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
// Remove any potential accidental duplicate inputs in arg filter values
// Remove any potential accidental duplicate inputs
eventArgs := map[string]bool{}
for _, arg := range tr.EventArgs[contractAddr] {
for _, arg := range tr.Config.EventArgs[contractAddr] {
eventArgs[arg] = true
}
methodArgs := map[string]bool{}
for _, arg := range tr.MethodArgs[contractAddr] {
for _, arg := range tr.Config.MethodArgs[contractAddr] {
methodArgs[arg] = true
}
// Aggregate info into contract object and store for execution
con := contract.Contract{
Name: *name,
Network: tr.Network,
Address: contractAddr,
Abi: tr.Parser.Abi(),
ParsedAbi: tr.Parser.ParsedAbi(),
StartingBlock: firstBlock,
LastBlock: -1,
Events: tr.Parser.GetEvents(subset),
Methods: tr.Parser.GetSelectMethods(tr.WantedMethods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
CreateAddrList: tr.CreateAddrList[contractAddr],
CreateHashList: tr.CreateHashList[contractAddr],
Piping: tr.Piping[contractAddr],
Name: *name,
Network: tr.Config.Network,
Address: contractAddr,
Abi: tr.Parser.Abi(),
ParsedAbi: tr.Parser.ParsedAbi(),
StartingBlock: firstBlock,
LastBlock: -1,
Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]),
Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
Piping: tr.Config.Piping[contractAddr],
}.Init()
tr.Contracts[contractAddr] = con
tr.contractAddresses = append(tr.contractAddresses, con.Address)
@ -221,7 +201,7 @@ func (tr *Transformer) Init() error {
return nil
}
func (tr *Transformer) Execute() error {
func (tr *transformer) Execute() error {
if len(tr.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts")
}
@ -311,7 +291,7 @@ func (tr *Transformer) Execute() error {
}
// Used to poll contract methods at a given header
func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
for _, con := range tr.Contracts {
// Skip method polling processes if no methods are specified
// Also don't try to poll methods below this contract's specified starting block
@ -335,42 +315,6 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str
return nil
}
// Used to set which contract addresses and which of their events to watch
func (tr *Transformer) SetEvents(contractAddr string, filterSet []string) {
tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to watch events for
func (tr *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set which contract addresses and which of their methods to call
func (tr *Transformer) SetMethods(contractAddr string, filterSet []string) {
tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to poll methods on
func (tr *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set the block range to watch for a given address
func (tr *Transformer) SetStartingBlock(contractAddr string, start int64) {
tr.ContractStart[strings.ToLower(contractAddr)] = start
}
// Used to set whether or not to persist an account address list
func (tr *Transformer) SetCreateAddrList(contractAddr string, on bool) {
tr.CreateAddrList[strings.ToLower(contractAddr)] = on
}
// Used to set whether or not to persist an hash list
func (tr *Transformer) SetCreateHashList(contractAddr string, on bool) {
tr.CreateHashList[strings.ToLower(contractAddr)] = on
}
// Used to turn method piping on for a contract
func (tr *Transformer) SetPiping(contractAddr string, on bool) {
tr.Piping[strings.ToLower(contractAddr)] = on
func (tr *transformer) GetConfig() config.ContractConfig {
return tr.Config
}

526
pkg/omni/light/transformer/transformer_test.go
View File

@ -17,150 +17,446 @@
package transformer_test
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"strings"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
)
var _ = Describe("Transformer", func() {
var fakeAddress = "0x1234567890abcdef"
var db *postgres.DB
var err error
var blockChain core.BlockChain
var headerRepository repositories.HeaderRepository
var headerID, headerID2 int64
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
Describe("SetEvents", func() {
It("Sets which events to watch from the given contract address", func() {
watchedEvents := []string{"Transfer", "Mint"}
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, watchedEvents)
Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
})
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
headerRepository = repositories.NewHeaderRepository(db)
})
Describe("SetEventAddrs", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEventArgs(fakeAddress, eventAddrs)
Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
})
})
Describe("SetMethods", func() {
It("Sets which methods to poll at the given contract address", func() {
watchedMethods := []string{"balanceOf", "totalSupply"}
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethods(fakeAddress, watchedMethods)
Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
})
})
Describe("SetMethodAddrs", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethodArgs(fakeAddress, methodAddrs)
Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
})
})
Describe("SetStartingBlock", func() {
It("Sets the block range that the contract should be watched within", func() {
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetStartingBlock(fakeAddress, 11)
Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
})
})
Describe("SetCreateAddrList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateAddrList(fakeAddress, true)
Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
})
})
Describe("SetCreateHashList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateHashList(fakeAddress, true)
Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRetriever := &fakes.MockLightBlockRetriever{}
firstBlock := int64(1)
blockRetriever.FirstBlock = firstBlock
parsr := &fakes.MockParser{}
fakeAbi := "fake_abi"
parsr.AbiToReturn = fakeAbi
pollr := &fakes.MockPoller{}
fakeContractName := "fake_contract_name"
pollr.ContractName = fakeContractName
t := getFakeTransformer(blockRetriever, parsr, pollr)
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[fakeAddress]
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(firstBlock))
Expect(c.StartingBlock).To(Equal(int64(6194632)))
Expect(c.LastBlock).To(Equal(int64(-1)))
Expect(c.Abi).To(Equal(fakeAbi))
Expect(c.Name).To(Equal(fakeContractName))
Expect(c.Address).To(Equal(fakeAddress))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
})
It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() {
blockRetriever := &fakes.MockLightBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("Does nothing if watched events are unset", func() {
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
err := t.Init()
It("Does nothing if nothing if no addresses are configured", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Addresses = nil
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
_, ok := t.Contracts[fakeAddress]
_, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(false))
})
})
})
Describe("Execute- against TrueUSD contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
return transformer.Transformer{
Parser: parsr,
BlockRetriever: blockRetriever,
Poller: pollr,
HeaderRepository: &fakes.MockLightHeaderRepository{},
Contracts: map[string]*contract.Contract{},
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractStart: map[string]int64{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
CreateHashList: map[string]bool{},
}
}
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(log.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedAddrs)).To(Equal(4))
Expect(len(c.EmittedHashes)).To(Equal(0))
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("55849938025000000000000"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(2))
Expect(len(c.EmittedAddrs)).To(Equal(0))
b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
Expect(ok).To(Equal(false))
})
It("Polls given method using list of collected hashes", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not persist events if they do not pass the emitted arg filter", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.EventArgs = map[string][]string{
ensAddr: {"fake_filter_value"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
testConf.MethodArgs = map[string][]string{
ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against both ENS and TrueUSD", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
header4, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header5, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header6, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
headerRepository.CreateOrUpdateHeader(header4)
headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header6)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer(mocks.ENSandTusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
newOwnerLog := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
transferLog := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(transferLog.HeaderID).To(Equal(headerID))
Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(transferLog.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig
testConf = mocks.ENSandTusdConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
ens, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
tusd, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(ens.EmittedHashes)).To(Equal(2))
Expect(len(ens.EmittedAddrs)).To(Equal(0))
Expect(len(tusd.EmittedAddrs)).To(Equal(4))
Expect(len(tusd.EmittedHashes)).To(Equal(0))
b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
})
It("Polls given methods for each contract, using list of collected values", func() {
var testConf config.ContractConfig
testConf = mocks.ENSandTusdConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
owner := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).To(HaveOccurred())
bal := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).ToNot(HaveOccurred())
Expect(bal.Balance).To(Equal("55849938025000000000000"))
Expect(bal.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).To(HaveOccurred())
})
})
})

1
pkg/omni/shared/constants/constants.go
View File

@ -72,7 +72,6 @@ var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b"
var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5"
// TODO: Consider whether these should be moved to plugins
// Contract Owner
var DaiContractOwner = "0x0000000000000000000000000000000000000000"
var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f"

6
pkg/omni/shared/contract/contract.go
View File

@ -64,12 +64,6 @@ func (c Contract) Init() *Contract {
}
}
// If we are creating an address list in postgres
// we initialize the map despite what method call, if any
if c.CreateAddrList {
c.EmittedAddrs = map[interface{}]bool{}
}
return &c
}

12
pkg/omni/shared/helpers/test_helpers/database.go
View File

@ -106,7 +106,6 @@ type Owner struct {
Address string `db:"returned"`
}
// TODO: consider whether this should be moved to libraries/shared
func SetupBC() core.BlockChain {
infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
rawRpcClient, err := rpc.Dial(infuraIPC)
@ -114,9 +113,9 @@ func SetupBC() core.BlockChain {
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
blockChainNode := node.MakeNode(rpcClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
return blockChain
}
@ -128,9 +127,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
blockChainNode := node.MakeNode(rpcClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",
@ -189,7 +188,6 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract
}.Init()
}
// TODO: consider whether this can be moved to plugin or libraries/shared
func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) {
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",
@ -238,7 +236,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract {
}
func TearDown(db *postgres.DB) {
tx, err := db.Beginx()
tx, err := db.Begin()
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM blocks`)

77
pkg/omni/shared/helpers/test_helpers/mocks/entities.go
View File

@ -18,6 +18,8 @@ package mocks
import (
"encoding/json"
"github.com/vulcanize/vulcanizedb/pkg/config"
"strings"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -249,3 +251,78 @@ var MockNewOwnerLog2 = types.Log{
},
Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000af21"),
}
var ens = strings.ToLower(constants.EnsContractAddress)
var tusd = strings.ToLower(constants.TusdContractAddress)
var TusdConfig = config.ContractConfig{
Network: "",
Addresses: map[string]bool{
tusd: true,
},
Abis: map[string]string{
tusd: "",
},
Events: map[string][]string{
tusd: []string{"Transfer"},
},
Methods: map[string][]string{
tusd: nil,
},
MethodArgs: map[string][]string{
tusd: nil,
},
EventArgs: map[string][]string{
tusd: nil,
},
}
var ENSConfig = config.ContractConfig{
Network: "",
Addresses: map[string]bool{
ens: true,
},
Abis: map[string]string{
ens: "",
},
Events: map[string][]string{
ens: []string{"NewOwner"},
},
Methods: map[string][]string{
ens: nil,
},
MethodArgs: map[string][]string{
ens: nil,
},
EventArgs: map[string][]string{
ens: nil,
},
}
var ENSandTusdConfig = config.ContractConfig{
Network: "",
Addresses: map[string]bool{
ens: true,
tusd: true,
},
Abis: map[string]string{
ens: "",
tusd: "",
},
Events: map[string][]string{
ens: []string{"NewOwner"},
tusd: []string{"Transfer"},
},
Methods: map[string][]string{
ens: nil,
tusd: nil,
},
MethodArgs: map[string][]string{
ens: nil,
tusd: nil,
},
EventArgs: map[string][]string{
ens: nil,
tusd: nil,
},
}

4
pkg/omni/shared/repository/event_repository.go
View File

@ -97,7 +97,7 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
// Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB)
func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
tx, err := r.db.Beginx()
tx, err := r.db.Begin()
if err != nil {
return err
}
@ -151,7 +151,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types
// Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB)
func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
tx, err := r.db.Beginx()
tx, err := r.db.Begin()
if err != nil {
return err
}

2
pkg/omni/shared/repository/method_repository.go
View File

@ -77,7 +77,7 @@ func (r *methodRepository) PersistResults(results []types.Result, methodInfo typ
// Creates a custom postgres command to persist logs for the given event
func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
tx, err := r.DB.Beginx()
tx, err := r.DB.Begin()
if err != nil {
return err
}

32
pkg/omni/shared/transformer/interface.go
View File

@ -1,32 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer
// Used to extract any/all events and a subset of method (state variable)
// data for any contract and persists it to custom postgres tables in vDB
type Transformer interface {
SetEvents(contractAddr string, filterSet []string)
SetEventArgs(contractAddr string, filterSet []string)
SetMethods(contractAddr string, filterSet []string)
SetMethodArgs(contractAddr string, filterSet []string)
SetStartingBlock(contractAddr string, start int64)
SetCreateAddrList(contractAddr string, on bool)
SetCreateHashList(contractAddr string, on bool)
SetPiping(contractAddr string, on bool)
Init() error
Execute() error
}