deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

mv pkg/omni pkg/contract_watcher

Browse Source
This commit is contained in:
Ian Norden
2019-03-21 18:33:56 -05:00
parent 073378de6d
commit 24879a85aa
60 changed files with 117 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/contract_watcher/full/converter/converter.go
+122
View File
@@ -0,0 +1,122 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package converter
import (
"errors"
"fmt"
"math/big"
"strconv"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// Converter is used to convert watched event logs to
// custom logs containing event input name => value maps
type Converter interface {
Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error)
Update(info *contract.Contract)
}
type converter struct {
ContractInfo *contract.Contract
}
func NewConverter(info *contract.Contract) *converter {
return &converter{
ContractInfo: info,
}
}
func (c *converter) Update(info *contract.Contract) {
c.ContractInfo = info
}
// Convert the given watched event log into a types.Log for the given event
func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
values := make(map[string]interface{})
log := helpers.ConvertToLog(watchedEvent)
err := contract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}
strValues := make(map[string]string, len(values))
seenAddrs := make([]interface{}, 0, len(values))
seenHashes := make([]interface{}, 0, len(values))
for fieldName, input := range values {
// Postgres cannot handle custom types, resolve to strings
switch input.(type) {
case *big.Int:
b := input.(*big.Int)
strValues[fieldName] = b.String()
case common.Address:
a := input.(common.Address)
strValues[fieldName] = a.String()
seenAddrs = append(seenAddrs, a)
case common.Hash:
h := input.(common.Hash)
strValues[fieldName] = h.String()
seenHashes = append(seenHashes, h)
case string:
strValues[fieldName] = input.(string)
case bool:
strValues[fieldName] = strconv.FormatBool(input.(bool))
case []byte:
b := input.([]byte)
strValues[fieldName] = hexutil.Encode(b)
if len(b) == 32 { // collect byte arrays of size 32 as hashes
seenHashes = append(seenHashes, common.HexToHash(strValues[fieldName]))
}
case byte:
b := input.(byte)
strValues[fieldName] = string(b)
default:
return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
}
}
// Only hold onto logs that pass our address filter, if any
if c.ContractInfo.PassesEventFilter(strValues) {
eventLog := &types.Log{
Id: watchedEvent.LogID,
Values: strValues,
Block: watchedEvent.BlockNumber,
Tx: watchedEvent.TxHash,
}
// Cache emitted values if their caching is turned on
if c.ContractInfo.EmittedAddrs != nil {
c.ContractInfo.AddEmittedAddr(seenAddrs...)
}
if c.ContractInfo.EmittedHashes != nil {
c.ContractInfo.AddEmittedHash(seenHashes...)
}
return eventLog, nil
}
return nil, nil
}
pkg/contract_watcher/full/converter/converter_suite_test.go
+35
View File
@@ -0,0 +1,35 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package converter_test
import (
"io/ioutil"
"log"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestConverter(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Full Converter Suite Test")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})
pkg/contract_watcher/full/converter/converter_test.go
+109
View File
@@ -0,0 +1,109 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package converter_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
)
var _ = Describe("Converter", func() {
var con *contract.Contract
var wantedEvents = []string{"Transfer"}
var err error
BeforeEach(func() {
con = test_helpers.SetupTusdContract(wantedEvents, []string{"balanceOf"})
})
Describe("Update", func() {
It("Updates contract con held by the converter", func() {
c := converter.NewConverter(con)
Expect(c.ContractInfo).To(Equal(con))
con := test_helpers.SetupTusdContract([]string{}, []string{})
c.Update(con)
Expect(c.ContractInfo).To(Equal(con))
})
})
Describe("Convert", func() {
It("Converts a watched event log to mapping of event input names to values", func() {
_, ok := con.Events["Approval"]
Expect(ok).To(Equal(false))
event, ok := con.Events["Transfer"]
Expect(ok).To(Equal(true))
err = con.GenerateFilters()
Expect(err).ToNot(HaveOccurred())
c := converter.NewConverter(con)
log, err := c.Convert(mocks.MockTranferEvent, event)
Expect(err).ToNot(HaveOccurred())
from := common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")
to := common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")
value := helpers.BigFromString("1097077688018008265106216665536940668749033598146")
v := log.Values["value"]
Expect(log.Values["to"]).To(Equal(to.String()))
Expect(log.Values["from"]).To(Equal(from.String()))
Expect(v).To(Equal(value.String()))
})
It("Keeps track of addresses it sees to grow a token holder address list for the contract", func() {
event, ok := con.Events["Transfer"]
Expect(ok).To(Equal(true))
c := converter.NewConverter(con)
_, err := c.Convert(mocks.MockTranferEvent, event)
Expect(err).ToNot(HaveOccurred())
b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = con.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = con.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = con.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Fails with an empty contract", func() {
event := con.Events["Transfer"]
c := converter.NewConverter(&contract.Contract{})
_, err = c.Convert(mocks.MockTranferEvent, event)
Expect(err).To(HaveOccurred())
})
})
})
pkg/contract_watcher/full/retriever/block_retriever.go
+87
View File
@@ -0,0 +1,87 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package retriever
import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// Block retriever is used to retrieve the first block for a given contract and the most recent block
// It requires a vDB synced database with blocks, transactions, receipts, and logs
type BlockRetriever interface {
RetrieveFirstBlock(contractAddr string) (int64, error)
RetrieveMostRecentBlock() (int64, error)
}
type blockRetriever struct {
db *postgres.DB
}
func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
return &blockRetriever{
db: db,
}
}
// Try both methods of finding the first block, with the receipt method taking precedence
func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
if err != nil {
i, err = r.retrieveFirstBlockFromLogs(contractAddr)
}
return i, err
}
// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
var firstBlock int
err := r.db.Get(
&firstBlock,
`SELECT number FROM blocks
WHERE id = (SELECT block_id FROM receipts
WHERE lower(contract_address) = $1
ORDER BY block_id ASC
LIMIT 1)`,
contractAddr,
)
return int64(firstBlock), err
}
// In which case this servers as a heuristic to find the first block by finding the first contract event log
func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
var firstBlock int
err := r.db.Get(
&firstBlock,
"SELECT block_number FROM logs WHERE lower(address) = $1 ORDER BY block_number ASC LIMIT 1",
contractAddr,
)
return int64(firstBlock), err
}
// Method to retrieve the most recent block in vDB
func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
var lastBlock int64
err := r.db.Get(
&lastBlock,
"SELECT number FROM blocks ORDER BY number DESC LIMIT 1",
)
return lastBlock, err
}
pkg/contract_watcher/full/retriever/block_retriever_test.go
+218
View File
@@ -0,0 +1,218 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package retriever_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"strings"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
var _ = Describe("Block Retriever", func() {
var db *postgres.DB
var r retriever.BlockRetriever
var blockRepository repositories.BlockRepository
// Contains no contract address
var block1 = core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
Number: 1,
Transactions: []core.Transaction{},
}
BeforeEach(func() {
db, _ = test_helpers.SetupDBandBC()
blockRepository = *repositories.NewBlockRepository(db)
r = retriever.NewBlockRetriever(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("RetrieveFirstBlock", func() {
It("Retrieves block number where contract first appears in receipt, if available", func() {
// Contains the address in the receipt
block2 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
Number: 2,
Transactions: []core.Transaction{{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
ContractAddress: constants.TusdContractAddress,
Logs: []core.Log{},
},
}},
}
// Contains address in logs
block3 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
Number: 3,
Transactions: []core.Transaction{{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
ContractAddress: constants.TusdContractAddress,
Logs: []core.Log{{
BlockNumber: 3,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
Address: constants.TusdContractAddress,
Topics: core.Topics{
constants.TransferEvent.Signature(),
"0x000000000000000000000000000000000000000000000000000000000000af21",
"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
"",
},
Index: 1,
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
}},
},
}},
}
blockRepository.CreateOrUpdateBlock(block1)
blockRepository.CreateOrUpdateBlock(block2)
blockRepository.CreateOrUpdateBlock(block3)
i, err := r.RetrieveFirstBlock(strings.ToLower(constants.TusdContractAddress))
Expect(err).NotTo(HaveOccurred())
Expect(i).To(Equal(int64(2)))
})
It("Retrieves block number where contract first appears in event logs if it cannot find the address in a receipt", func() {
block2 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
Number: 2,
Transactions: []core.Transaction{{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
ContractAddress: "",
Logs: []core.Log{{
BlockNumber: 2,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
Address: constants.DaiContractAddress,
Topics: core.Topics{
constants.TransferEvent.Signature(),
"0x000000000000000000000000000000000000000000000000000000000000af21",
"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
"",
},
Index: 1,
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
}},
},
}},
}
block3 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
Number: 3,
Transactions: []core.Transaction{{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
Receipt: core.Receipt{
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
ContractAddress: "",
Logs: []core.Log{{
BlockNumber: 3,
TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
Address: constants.DaiContractAddress,
Topics: core.Topics{
constants.TransferEvent.Signature(),
"0x000000000000000000000000000000000000000000000000000000000000af21",
"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
"",
},
Index: 1,
Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
}},
},
}},
}
blockRepository.CreateOrUpdateBlock(block1)
blockRepository.CreateOrUpdateBlock(block2)
blockRepository.CreateOrUpdateBlock(block3)
i, err := r.RetrieveFirstBlock(constants.DaiContractAddress)
Expect(err).NotTo(HaveOccurred())
Expect(i).To(Equal(int64(2)))
})
It("Fails if the contract address cannot be found in any blocks", func() {
block2 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
Number: 2,
Transactions: []core.Transaction{},
}
block3 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
Number: 3,
Transactions: []core.Transaction{},
}
blockRepository.CreateOrUpdateBlock(block1)
blockRepository.CreateOrUpdateBlock(block2)
blockRepository.CreateOrUpdateBlock(block3)
_, err := r.RetrieveFirstBlock(constants.DaiContractAddress)
Expect(err).To(HaveOccurred())
})
})
Describe("RetrieveMostRecentBlock", func() {
It("Retrieves the latest block", func() {
block2 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
Number: 2,
Transactions: []core.Transaction{},
}
block3 := core.Block{
Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
Number: 3,
Transactions: []core.Transaction{},
}
blockRepository.CreateOrUpdateBlock(block1)
blockRepository.CreateOrUpdateBlock(block2)
blockRepository.CreateOrUpdateBlock(block3)
i, err := r.RetrieveMostRecentBlock()
Expect(err).ToNot(HaveOccurred())
Expect(i).To(Equal(int64(3)))
})
It("Fails if it cannot retrieve the latest block", func() {
i, err := r.RetrieveMostRecentBlock()
Expect(err).To(HaveOccurred())
Expect(i).To(Equal(int64(0)))
})
})
})
pkg/contract_watcher/full/retriever/retriever_suite_test.go
+35
View File
@@ -0,0 +1,35 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package retriever_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestRetriever(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Full Block Number Retriever Suite Test")
}
var _ = BeforeSuite(func() {
logrus.SetOutput(ioutil.Discard)
})
pkg/contract_watcher/full/transformer/transformer.go
+215
View File
@@ -0,0 +1,215 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer
import (
"errors"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
// Requires a fully synced vDB and a running eth node (or infura)
type transformer struct {
// Database interfaces
datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters()
datastore.WatchedEventRepository // Watched event log views, created by the log filters
repository.EventRepository // Holds transformed watched event log data
// Pre-processing interfaces
parser.Parser // Parses events and methods out of contract abi fetched using contract address
retriever.BlockRetriever // Retrieves first block for contract and current block height
// Processing interfaces
converter.Converter // Converts watched event logs into custom log
poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore
// Store contract configuration information
Config config.ContractConfig
// Store contract info as mapping to contract address
Contracts map[string]*contract.Contract
}
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *transformer {
return &transformer{
Poller: poller.NewPoller(BC, DB, types.FullSync),
Parser: parser.NewParser(con.Network),
BlockRetriever: retriever.NewBlockRetriever(DB),
Converter: converter.NewConverter(&contract.Contract{}),
Contracts: map[string]*contract.Contract{},
WatchedEventRepository: repositories.WatchedEventRepository{DB: DB},
FilterRepository: repositories.FilterRepository{DB: DB},
EventRepository: repository.NewEventRepository(DB, types.FullSync),
Config: con,
}
}
// Use after creating and setting transformer
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (tr *transformer) Init() error {
for contractAddr := range tr.Config.Addresses {
// Configure Abi
if tr.Config.Abis[contractAddr] == "" {
// If no abi is given in the config, this method will try fetching from internal look-up table and etherscan
err := tr.Parser.Parse(contractAddr)
if err != nil {
return err
}
} else {
// If we have an abi from the config, load that into the parser
err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
if err != nil {
return err
}
}
// Get first block and most recent block number in the header repo
firstBlock, err := tr.BlockRetriever.RetrieveFirstBlock(contractAddr)
if err != nil {
return err
}
lastBlock, err := tr.BlockRetriever.RetrieveMostRecentBlock()
if err != nil {
return err
}
// Set to specified range if it falls within the bounds
if firstBlock < tr.Config.StartingBlocks[contractAddr] {
firstBlock = tr.Config.StartingBlocks[contractAddr]
}
// Get contract name if it has one
var name = new(string)
tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
for _, arg := range tr.Config.EventArgs[contractAddr] {
eventArgs[arg] = true
}
methodArgs := map[string]bool{}
for _, arg := range tr.Config.MethodArgs[contractAddr] {
methodArgs[arg] = true
}
// Aggregate info into contract object
info := contract.Contract{
Name: *name,
Network: tr.Config.Network,
Address: contractAddr,
Abi: tr.Parser.Abi(),
ParsedAbi: tr.Parser.ParsedAbi(),
StartingBlock: firstBlock,
LastBlock: lastBlock,
Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]),
Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
Piping: tr.Config.Piping[contractAddr],
}.Init()
// Use info to create filters
err = info.GenerateFilters()
if err != nil {
return err
}
// Iterate over filters and push them to the repo using filter repository interface
for _, filter := range info.Filters {
err = tr.CreateFilter(filter)
if err != nil {
return err
}
}
// Store contract info for further processing
tr.Contracts[contractAddr] = info
}
return nil
}
// Iterates through stored, initialized contract objects
// Iterates through contract's event filters, grabbing watched event logs
// Uses converter to convert logs into custom log type
// Persists converted logs into custuom postgres tables
// Calls selected methods, using token holder address generated during event log conversion
func (tr *transformer) Execute() error {
if len(tr.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts to work with")
}
// Iterate through all internal contracts
for _, con := range tr.Contracts {
// Update converter with current contract
tr.Update(con)
// Iterate through contract filters and get watched event logs
for eventSig, filter := range con.Filters {
watchedEvents, err := tr.GetWatchedEvents(filter.Name)
if err != nil {
return err
}
// Iterate over watched event logs
for _, we := range watchedEvents {
// Convert them to our custom log type
cstm, err := tr.Converter.Convert(*we, con.Events[eventSig])
if err != nil {
return err
}
if cstm == nil {
continue
}
// If log is not empty, immediately persist in repo
// Run this in seperate goroutine?
err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
if err != nil {
return err
}
}
}
// After persisting all watched event logs
// poller polls select contract methods
// and persists the results into custom pg tables
// Run this in seperate goroutine?
if err := tr.PollContract(*con); err != nil {
return err
}
}
return nil
}
func (tr *transformer) GetConfig() config.ContractConfig {
return tr.Config
}
pkg/contract_watcher/full/transformer/transformer_suite_test.go
+35
View File
@@ -0,0 +1,35 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestTransformer(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Full Transformer Suite Test")
}
var _ = BeforeSuite(func() {
logrus.SetOutput(ioutil.Discard)
})
pkg/contract_watcher/full/transformer/transformer_test.go
+323
View File
@@ -0,0 +1,323 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer_test
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"math/rand"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
var _ = Describe("Transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var blockRepository repositories.BlockRepository
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
rand.Seed(time.Now().UnixNano())
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
blockRepository = *repositories.NewBlockRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194633)))
Expect(c.LastBlock).To(Equal(int64(6194634)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
})
It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).To(HaveOccurred())
})
It("Does nothing if watched events are unset", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Events = nil
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).To(HaveOccurred())
_, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(false))
})
})
Describe("Execute", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.TransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
Expect(log.Block).To(Equal(int64(6194634)))
Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
var testConf config.ContractConfig
testConf = mocks.TusdConfig
testConf.Methods = map[string][]string{
tusdAddr: {"balanceOf"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.NewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
Expect(log.Block).To(Equal(int64(6194635)))
Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(3))
b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not perist events if they do not pass the emitted arg filter", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.EventArgs = map[string][]string{
ensAddr: {"fake_filter_value"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
var testConf config.ContractConfig
testConf = mocks.ENSConfig
testConf.Methods = map[string][]string{
ensAddr: {"owner"},
}
testConf.MethodArgs = map[string][]string{
ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"},
}
t := transformer.NewTransformer(testConf, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
})