Merge pull request #158 from vulcanize/missed_marked_checked_headers

Fix for issue #146
This commit is contained in:
Ian Norden 2019-10-28 12:34:47 -05:00 committed by GitHub
commit b7675316b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 214 additions and 290 deletions

View File

@ -18,7 +18,6 @@ package integration
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"math/rand" "math/rand"
"strings" "strings"
"time" "time"
@ -27,6 +26,7 @@ import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"

View File

@ -40,8 +40,8 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
var blockChain core.BlockChain var blockChain core.BlockChain
var headerRepository repositories.HeaderRepository var headerRepository repositories.HeaderRepository
var headerID int64 var headerID int64
var ensAddr = strings.ToLower(constants.EnsContractAddress) var ensAddr = strings.ToLower(constants.EnsContractAddress) // 0x314159265dd8dbb310642f98f50c066173c1259b
var tusdAddr = strings.ToLower(constants.TusdContractAddress) var tusdAddr = strings.ToLower(constants.TusdContractAddress) // 0x8dd5fbce2f6a956c3022ba3663759011dd51e73e
BeforeEach(func() { BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC() db, blockChain = test_helpers.SetupDBandBC()
@ -377,6 +377,42 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
Expect(transferLog.Value).To(Equal("2800000000000000000000")) Expect(transferLog.Value).To(Equal("2800000000000000000000"))
}) })
It("Marks header checked for a contract that has no logs at that header", func() {
t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(t.Start).To(Equal(int64(6885702)))
newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
Expect(err).ToNot(HaveOccurred())
transferLog := test_helpers.HeaderSyncTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
Expect(err).ToNot(HaveOccurred())
Expect(transferLog.HeaderID).ToNot(Equal(newOwnerLog.HeaderID))
type checkedHeader struct {
ID int64 `db:"id"`
HeaderID int64 `db:"header_id"`
NewOwner int64 `db:"newowner_0x314159265dd8dbb310642f98f50c066173c1259b"`
Transfer int64 `db:"transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e"`
}
transferCheckedHeader := new(checkedHeader)
err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", transferLog.HeaderID).StructScan(transferCheckedHeader)
Expect(err).ToNot(HaveOccurred())
Expect(transferCheckedHeader.Transfer).To(Equal(int64(1)))
Expect(transferCheckedHeader.NewOwner).To(Equal(int64(1)))
newOwnerCheckedHeader := new(checkedHeader)
err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", newOwnerLog.HeaderID).StructScan(newOwnerCheckedHeader)
Expect(err).ToNot(HaveOccurred())
Expect(newOwnerCheckedHeader.NewOwner).To(Equal(int64(1)))
Expect(newOwnerCheckedHeader.Transfer).To(Equal(int64(1)))
})
It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig var testConf config.ContractConfig
testConf = test_helpers.ENSandTusdConfig testConf = test_helpers.ENSandTusdConfig

View File

@ -1,27 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repository
import "github.com/jmoiron/sqlx"
func MarkContractWatcherHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
return err
}

View File

@ -1,67 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repository_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("", func() {
Describe("MarkContractWatcherHeaderCheckedInTransaction", func() {
var (
checkedHeadersColumn string
db *postgres.DB
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
It("marks passed header as checked within a passed transaction", func() {
headerRepository := repositories.NewHeaderRepository(db)
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
tx, txErr := db.Beginx()
Expect(txErr).NotTo(HaveOccurred())
err := repository.MarkContractWatcherHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
commitErr := tx.Commit()
Expect(commitErr).NotTo(HaveOccurred())
var checkedCount int
fetchErr := db.Get(&checkedCount, `SELECT COUNT(*) FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(fetchErr).NotTo(HaveOccurred())
Expect(checkedCount).To(Equal(1))
})
})
})

View File

@ -17,7 +17,6 @@
package converter package converter
import ( import (
"errors"
"fmt" "fmt"
"math/big" "math/big"
"strconv" "strconv"
@ -32,22 +31,24 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
) )
// Converter is used to convert watched event logs to // ConverterInterface is used to convert watched event logs to
// custom logs containing event input name => value maps // custom logs containing event input name => value maps
type ConverterInterface interface { type ConverterInterface interface {
Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error)
Update(info *contract.Contract) Update(info *contract.Contract)
} }
// Converter is the underlying struct for the ConverterInterface
type Converter struct { type Converter struct {
ContractInfo *contract.Contract ContractInfo *contract.Contract
} }
// Update configures the converter for a specific contract
func (c *Converter) Update(info *contract.Contract) { func (c *Converter) Update(info *contract.Contract) {
c.ContractInfo = info c.ContractInfo = info
} }
// Convert the given watched event log into a types.Log for the given event // Convert converts the given watched event log into a types.Log for the given event
func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
values := make(map[string]interface{}) values := make(map[string]interface{})
@ -88,14 +89,14 @@ func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
b := input.(byte) b := input.(byte)
strValues[fieldName] = string(b) strValues[fieldName] = string(b)
default: default:
return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input)) return nil, fmt.Errorf("error: unhandled abi type %T", input)
} }
} }
// Only hold onto logs that pass our address filter, if any // Only hold onto logs that pass our address filter, if any
if c.ContractInfo.PassesEventFilter(strValues) { if c.ContractInfo.PassesEventFilter(strValues) {
eventLog := &types.Log{ eventLog := &types.Log{
Id: watchedEvent.LogID, ID: watchedEvent.LogID,
Values: strValues, Values: strValues,
Block: watchedEvent.BlockNumber, Block: watchedEvent.BlockNumber,
Tx: watchedEvent.TxHash, Tx: watchedEvent.TxHash,

View File

@ -18,11 +18,12 @@ package retriever
import ( import (
"database/sql" "database/sql"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
// Block retriever is used to retrieve the first block for a given contract and the most recent block // BlockRetriever is used to retrieve the first block for a given contract and the most recent block
// It requires a vDB synced database with blocks, transactions, receipts, and logs // It requires a vDB synced database with blocks, transactions, receipts, and logs
type BlockRetriever interface { type BlockRetriever interface {
RetrieveFirstBlock(contractAddr string) (int64, error) RetrieveFirstBlock(contractAddr string) (int64, error)
@ -33,13 +34,15 @@ type blockRetriever struct {
db *postgres.DB db *postgres.DB
} }
func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) { // NewBlockRetriever returns a new BlockRetriever
func NewBlockRetriever(db *postgres.DB) BlockRetriever {
return &blockRetriever{ return &blockRetriever{
db: db, db: db,
} }
} }
// Try both methods of finding the first block, with the receipt method taking precedence // RetrieveFirstBlock fetches the block number for the earliest block in the db
// Tries both methods of finding the first block, with the receipt method taking precedence
func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) { func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
i, err := r.retrieveFirstBlockFromReceipts(contractAddr) i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
if err != nil { if err != nil {
@ -55,7 +58,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error)
// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai) // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) { func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
var firstBlock int64 var firstBlock int64
addressId, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr) addressID, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
if getAddressErr != nil { if getAddressErr != nil {
return firstBlock, getAddressErr return firstBlock, getAddressErr
} }
@ -66,7 +69,7 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in
WHERE contract_address_id = $1 WHERE contract_address_id = $1
ORDER BY block_id ASC ORDER BY block_id ASC
LIMIT 1)`, LIMIT 1)`,
addressId, addressID,
) )
return firstBlock, err return firstBlock, err
@ -84,7 +87,7 @@ func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64,
return int64(firstBlock), err return int64(firstBlock), err
} }
// Method to retrieve the most recent block in vDB // RetrieveMostRecentBlock retrieves the most recent block number in vDB
func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) { func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
var lastBlock int64 var lastBlock int64
err := r.db.Get( err := r.db.Get(

View File

@ -17,10 +17,11 @@
package retriever_test package retriever_test
import ( import (
"strings"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"strings"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"

View File

@ -17,12 +17,12 @@
package retriever_test package retriever_test
import ( import (
"github.com/sirupsen/logrus"
"io/ioutil" "io/ioutil"
"testing" "testing"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
) )
func TestRetriever(t *testing.T) { func TestRetriever(t *testing.T) {

View File

@ -35,6 +35,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
) )
// Transformer is the top level struct for transforming watched contract data
// Requires a fully synced vDB and a running eth node (or infura) // Requires a fully synced vDB and a running eth node (or infura)
type Transformer struct { type Transformer struct {
// Database interfaces // Database interfaces
@ -60,7 +61,7 @@ type Transformer struct {
LastBlock int64 LastBlock int64
} }
// Transformer takes in config for blockchain, database, and network id // NewTransformer takes in contract config, blockchain, and database, and returns a new Transformer
func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer { func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer {
return &Transformer{ return &Transformer{
Poller: poller.NewPoller(BC, DB, types.FullSync), Poller: poller.NewPoller(BC, DB, types.FullSync),
@ -75,6 +76,7 @@ func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.
} }
} }
// Init initializes the transformer
// Use after creating and setting transformer // Use after creating and setting transformer
// Loops over all of the addr => filter sets // Loops over all of the addr => filter sets
// Uses parser to pull event info from abi // Uses parser to pull event info from abi
@ -167,6 +169,7 @@ func (tr *Transformer) Init() error {
return nil return nil
} }
// Execute runs the transformation processes
// Iterates through stored, initialized contract objects // Iterates through stored, initialized contract objects
// Iterates through contract's event filters, grabbing watched event logs // Iterates through contract's event filters, grabbing watched event logs
// Uses converter to convert logs into custom log type // Uses converter to convert logs into custom log type
@ -227,6 +230,7 @@ func (tr *Transformer) Execute() error {
return nil return nil
} }
// GetConfig returns the transformers config; satisfies the transformer interface
func (tr *Transformer) GetConfig() config.ContractConfig { func (tr *Transformer) GetConfig() config.ContractConfig {
return tr.Config return tr.Config
} }

View File

@ -17,12 +17,12 @@
package transformer_test package transformer_test
import ( import (
"github.com/sirupsen/logrus"
"io/ioutil" "io/ioutil"
"testing" "testing"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
) )
func TestTransformer(t *testing.T) { func TestTransformer(t *testing.T) {

View File

@ -18,7 +18,6 @@ package converter
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"math/big" "math/big"
"strconv" "strconv"
@ -32,16 +31,19 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
) )
// ConverterInterface is the interface for converting geth logs to our custom log type
type ConverterInterface interface { type ConverterInterface interface {
Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error)
ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error)
Update(info *contract.Contract) Update(info *contract.Contract)
} }
// Converter is the underlying struct for the ConverterInterface
type Converter struct { type Converter struct {
ContractInfo *contract.Contract ContractInfo *contract.Contract
} }
// Update is used to configure the converter with a specific contract
func (c *Converter) Update(info *contract.Contract) { func (c *Converter) Update(info *contract.Contract) {
c.ContractInfo = info c.ContractInfo = info
} }
@ -98,7 +100,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
strValues[fieldName] = converted.String() strValues[fieldName] = converted.String()
seenHashes = append(seenHashes, converted) seenHashes = append(seenHashes, converted)
default: default:
return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input)) return nil, fmt.Errorf("error: unhandled abi type %T", input)
} }
} }
@ -114,7 +116,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
Values: strValues, Values: strValues,
Raw: raw, Raw: raw,
TransactionIndex: log.TxIndex, TransactionIndex: log.TxIndex,
Id: headerID, ID: headerID,
}) })
// Cache emitted values if their caching is turned on // Cache emitted values if their caching is turned on
@ -130,7 +132,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
return returnLogs, nil return returnLogs, nil
} }
// Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs // ConvertBatch converts the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) { func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
eventsToLogs := make(map[string][]types.Log) eventsToLogs := make(map[string][]types.Log)
@ -182,7 +184,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
strValues[fieldName] = converted.String() strValues[fieldName] = converted.String()
seenHashes = append(seenHashes, converted) seenHashes = append(seenHashes, converted)
default: default:
return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input)) return nil, fmt.Errorf("error: unhandled abi type %T", input)
} }
} }
@ -198,7 +200,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
Values: strValues, Values: strValues,
Raw: raw, Raw: raw,
TransactionIndex: log.TxIndex, TransactionIndex: log.TxIndex,
Id: headerID, ID: headerID,
}) })
// Cache emitted values that pass the argument filter if their caching is turned on // Cache emitted values that pass the argument filter if their caching is turned on

View File

@ -72,11 +72,11 @@ var _ = Describe("Converter", func() {
Expect(logs[0].Values["to"]).To(Equal(sender1.String())) Expect(logs[0].Values["to"]).To(Equal(sender1.String()))
Expect(logs[0].Values["from"]).To(Equal(sender2.String())) Expect(logs[0].Values["from"]).To(Equal(sender2.String()))
Expect(logs[0].Values["value"]).To(Equal(value.String())) Expect(logs[0].Values["value"]).To(Equal(value.String()))
Expect(logs[0].Id).To(Equal(int64(232))) Expect(logs[0].ID).To(Equal(int64(232)))
Expect(logs[1].Values["to"]).To(Equal(sender2.String())) Expect(logs[1].Values["to"]).To(Equal(sender2.String()))
Expect(logs[1].Values["from"]).To(Equal(sender1.String())) Expect(logs[1].Values["from"]).To(Equal(sender1.String()))
Expect(logs[1].Values["value"]).To(Equal(value.String())) Expect(logs[1].Values["value"]).To(Equal(value.String()))
Expect(logs[1].Id).To(Equal(int64(232))) Expect(logs[1].ID).To(Equal(int64(232)))
}) })
It("Keeps track of addresses it sees if they will be used for method polling", func() { It("Keeps track of addresses it sees if they will be used for method polling", func() {

View File

@ -24,6 +24,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
) )
// Fetcher is the fetching interface
type Fetcher interface { type Fetcher interface {
FetchLogs(contractAddresses []string, topics []common.Hash, missingHeader core.Header) ([]types.Log, error) FetchLogs(contractAddresses []string, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
} }
@ -32,13 +33,14 @@ type fetcher struct {
blockChain core.BlockChain blockChain core.BlockChain
} }
func NewFetcher(blockchain core.BlockChain) *fetcher { // NewFetcher returns a new Fetcher
func NewFetcher(blockchain core.BlockChain) Fetcher {
return &fetcher{ return &fetcher{
blockChain: blockchain, blockChain: blockchain,
} }
} }
// Checks all topic0s, on all addresses, fetching matching logs for the given header // FetchLogs checks all topic0s, on all addresses, fetching matching logs for the given header
func (fetcher *fetcher) FetchLogs(contractAddresses []string, topic0s []common.Hash, header core.Header) ([]types.Log, error) { func (fetcher *fetcher) FetchLogs(contractAddresses []string, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
addresses := hexStringsToAddresses(contractAddresses) addresses := hexStringsToAddresses(contractAddresses)
blockHash := common.HexToHash(header.Hash) blockHash := common.HexToHash(header.Hash)

View File

@ -20,7 +20,6 @@ import (
"fmt" "fmt"
"github.com/hashicorp/golang-lru" "github.com/hashicorp/golang-lru"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
@ -29,6 +28,7 @@ import (
const columnCacheSize = 1000 const columnCacheSize = 1000
// HeaderRepository interfaces with the header and checked_headers tables
type HeaderRepository interface { type HeaderRepository interface {
AddCheckColumn(id string) error AddCheckColumn(id string) error
AddCheckColumns(ids []string) error AddCheckColumns(ids []string) error
@ -46,7 +46,8 @@ type headerRepository struct {
columns *lru.Cache // Cache created columns to minimize db connections columns *lru.Cache // Cache created columns to minimize db connections
} }
func NewHeaderRepository(db *postgres.DB) *headerRepository { // NewHeaderRepository returns a new HeaderRepository
func NewHeaderRepository(db *postgres.DB) HeaderRepository {
ccs, _ := lru.New(columnCacheSize) ccs, _ := lru.New(columnCacheSize)
return &headerRepository{ return &headerRepository{
db: db, db: db,
@ -54,7 +55,7 @@ func NewHeaderRepository(db *postgres.DB) *headerRepository {
} }
} }
// Adds a checked_header column for the provided column id // AddCheckColumn adds a checked_header column for the provided column id
func (r *headerRepository) AddCheckColumn(id string) error { func (r *headerRepository) AddCheckColumn(id string) error {
// Check cache to see if column already exists before querying pg // Check cache to see if column already exists before querying pg
_, ok := r.columns.Get(id) _, ok := r.columns.Get(id)
@ -75,7 +76,7 @@ func (r *headerRepository) AddCheckColumn(id string) error {
return nil return nil
} }
// Adds a checked_header column for all of the provided column ids // AddCheckColumns adds a checked_header column for all of the provided column ids
func (r *headerRepository) AddCheckColumns(ids []string) error { func (r *headerRepository) AddCheckColumns(ids []string) error {
var err error var err error
baseQuery := "ALTER TABLE public.checked_headers" baseQuery := "ALTER TABLE public.checked_headers"
@ -99,7 +100,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error {
return err return err
} }
// Marks the header checked for the provided column id // MarkHeaderChecked marks the header checked for the provided column id
func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error { func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`) _, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`)
VALUES ($1, $2) VALUES ($1, $2)
@ -108,7 +109,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
return err return err
} }
// Marks the header checked for all of the provided column ids // MarkHeaderCheckedForAll marks the header checked for all of the provided column ids
func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error { func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
pgStr := "INSERT INTO public.checked_headers (header_id, " pgStr := "INSERT INTO public.checked_headers (header_id, "
for _, id := range ids { for _, id := range ids {
@ -127,7 +128,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
return err return err
} }
// Marks all of the provided headers checked for each of the provided column ids // MarkHeadersCheckedForAll marks all of the provided headers checked for each of the provided column ids
func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error { func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
tx, err := r.db.Beginx() tx, err := r.db.Beginx()
if err != nil { if err != nil {
@ -160,7 +161,7 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [
return err return err
} }
// Returns missing headers for the provided checked_headers column id // MissingHeaders returns missing headers for the provided checked_headers column id
func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) { func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) {
var result []core.Header var result []core.Header
var query string var query string
@ -186,7 +187,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
return continuousHeaders(result), err return continuousHeaders(result), err
} }
// Returns missing headers for all of the provided checked_headers column ids // MissingHeadersForAll returns missing headers for all of the provided checked_headers column ids
func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) { func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
var result []core.Header var result []core.Header
var query string var query string
@ -214,7 +215,7 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock
return continuousHeaders(result), err return continuousHeaders(result), err
} }
// Returns headers that have been checked for all of the provided event ids but not for the provided method ids // MissingMethodsCheckedEventsIntersection returns headers that have been checked for all of the provided event ids but not for the provided method ids
func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) { func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
var result []core.Header var result []core.Header
var query string var query string
@ -264,16 +265,7 @@ func continuousHeaders(headers []core.Header) []core.Header {
return headers return headers
} }
// Check the repositories column id cache for a value // CheckCache checks the repositories column id cache for a value
func (r *headerRepository) CheckCache(key string) (interface{}, bool) { func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
return r.columns.Get(key) return r.columns.Get(key)
} }
// Used to mark a header checked as part of some external transaction so as to group into one commit
func (r *headerRepository) MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET `+eventID+` = checked_headers.`+eventID+` + 1`, headerID, 1)
return err
}

View File

@ -18,7 +18,6 @@ package repository_test
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
@ -26,6 +25,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
) )

View File

@ -17,12 +17,12 @@
package repository_test package repository_test
import ( import (
"github.com/sirupsen/logrus"
"io/ioutil" "io/ioutil"
"testing" "testing"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
) )
func TestRepository(t *testing.T) { func TestRepository(t *testing.T) {

View File

@ -20,7 +20,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
// Block retriever is used to retrieve the first block for a given contract and the most recent block // BlockRetriever is used to retrieve the first block for a given contract and the most recent block
// It requires a vDB synced database with blocks, transactions, receipts, and logs // It requires a vDB synced database with blocks, transactions, receipts, and logs
type BlockRetriever interface { type BlockRetriever interface {
RetrieveFirstBlock() (int64, error) RetrieveFirstBlock() (int64, error)
@ -31,13 +31,14 @@ type blockRetriever struct {
db *postgres.DB db *postgres.DB
} }
func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) { // NewBlockRetriever returns a new BlockRetriever
func NewBlockRetriever(db *postgres.DB) BlockRetriever {
return &blockRetriever{ return &blockRetriever{
db: db, db: db,
} }
} }
// Retrieve block number of earliest header in repo // RetrieveFirstBlock retrieves block number of earliest header in repo
func (r *blockRetriever) RetrieveFirstBlock() (int64, error) { func (r *blockRetriever) RetrieveFirstBlock() (int64, error) {
var firstBlock int var firstBlock int
err := r.db.Get( err := r.db.Get(
@ -48,7 +49,7 @@ func (r *blockRetriever) RetrieveFirstBlock() (int64, error) {
return int64(firstBlock), err return int64(firstBlock), err
} }
// Retrieve block number of latest header in repo // RetrieveMostRecentBlock retrieves block number of latest header in repo
func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) { func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
var lastBlock int var lastBlock int
err := r.db.Get( err := r.db.Get(

View File

@ -17,12 +17,12 @@
package retriever_test package retriever_test
import ( import (
"github.com/sirupsen/logrus"
"io/ioutil" "io/ioutil"
"testing" "testing"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
) )
func TestRetriever(t *testing.T) { func TestRetriever(t *testing.T) {

View File

@ -40,6 +40,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
// Transformer is the top level struct for transforming watched contract data
// Requires a header synced vDB (headers) and a running eth node (or infura) // Requires a header synced vDB (headers) and a running eth node (or infura)
type Transformer struct { type Transformer struct {
// Database interfaces // Database interfaces
@ -76,7 +77,7 @@ type Transformer struct {
// 3. Init // 3. Init
// 4. Execute // 4. Execute
// Transformer takes in config for blockchain, database, and network id // NewTransformer takes in a contract config, blockchain, and database, and returns a new Transformer
func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer { func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer {
return &Transformer{ return &Transformer{
@ -92,6 +93,7 @@ func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.
} }
} }
// Init initialized the Transformer
// Use after creating and setting transformer // Use after creating and setting transformer
// Loops over all of the addr => filter sets // Loops over all of the addr => filter sets
// Uses parser to pull event info from abi // Uses parser to pull event info from abi
@ -126,6 +128,7 @@ func (tr *Transformer) Init() error {
firstBlock, retrieveErr := tr.Retriever.RetrieveFirstBlock() firstBlock, retrieveErr := tr.Retriever.RetrieveFirstBlock()
if retrieveErr != nil { if retrieveErr != nil {
if retrieveErr == sql.ErrNoRows { if retrieveErr == sql.ErrNoRows {
logrus.Error(fmt.Errorf("error retrieving first block: %s", retrieveErr.Error()))
firstBlock = 0 firstBlock = 0
} else { } else {
return fmt.Errorf("error retrieving first block: %s", retrieveErr.Error()) return fmt.Errorf("error retrieving first block: %s", retrieveErr.Error())
@ -175,14 +178,14 @@ func (tr *Transformer) Init() error {
// Create checked_headers columns for each event id and append to list of all event ids // Create checked_headers columns for each event id and append to list of all event ids
tr.sortedEventIds[con.Address] = make([]string, 0, len(con.Events)) tr.sortedEventIds[con.Address] = make([]string, 0, len(con.Events))
for _, event := range con.Events { for _, event := range con.Events {
eventId := strings.ToLower(event.Name + "_" + con.Address) eventID := strings.ToLower(event.Name + "_" + con.Address)
addColumnErr := tr.HeaderRepository.AddCheckColumn(eventId) addColumnErr := tr.HeaderRepository.AddCheckColumn(eventID)
if addColumnErr != nil { if addColumnErr != nil {
return fmt.Errorf("error adding check column: %s", addColumnErr.Error()) return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
} }
// Keep track of this event id; sorted and unsorted // Keep track of this event id; sorted and unsorted
tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventId) tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventID)
tr.eventIds = append(tr.eventIds, eventId) tr.eventIds = append(tr.eventIds, eventID)
// Append this event sig to the filters // Append this event sig to the filters
tr.eventFilters = append(tr.eventFilters, event.Sig()) tr.eventFilters = append(tr.eventFilters, event.Sig())
} }
@ -190,12 +193,12 @@ func (tr *Transformer) Init() error {
// Create checked_headers columns for each method id and append list of all method ids // Create checked_headers columns for each method id and append list of all method ids
tr.sortedMethodIds[con.Address] = make([]string, 0, len(con.Methods)) tr.sortedMethodIds[con.Address] = make([]string, 0, len(con.Methods))
for _, m := range con.Methods { for _, m := range con.Methods {
methodId := strings.ToLower(m.Name + "_" + con.Address) methodID := strings.ToLower(m.Name + "_" + con.Address)
addColumnErr := tr.HeaderRepository.AddCheckColumn(methodId) addColumnErr := tr.HeaderRepository.AddCheckColumn(methodID)
if addColumnErr != nil { if addColumnErr != nil {
return fmt.Errorf("error adding check column: %s", addColumnErr.Error()) return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
} }
tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodId) tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodID)
} }
// Update start to the lowest block // Update start to the lowest block
@ -207,6 +210,7 @@ func (tr *Transformer) Init() error {
return nil return nil
} }
// Execute runs the transformation processes
func (tr *Transformer) Execute() error { func (tr *Transformer) Execute() error {
if len(tr.Contracts) == 0 { if len(tr.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts") return errors.New("error: transformer has no initialized contracts")
@ -248,7 +252,6 @@ func (tr *Transformer) Execute() error {
continue continue
} }
// Sort logs by the contract they belong to
for _, log := range allLogs { for _, log := range allLogs {
addr := strings.ToLower(log.Address.Hex()) addr := strings.ToLower(log.Address.Hex())
sortedLogs[addr] = append(sortedLogs[addr], log) sortedLogs[addr] = append(sortedLogs[addr], log)
@ -273,16 +276,10 @@ func (tr *Transformer) Execute() error {
for eventName, logs := range convertedLogs { for eventName, logs := range convertedLogs {
// If logs for this event are empty, mark them checked at this header and continue // If logs for this event are empty, mark them checked at this header and continue
if len(logs) < 1 { if len(logs) < 1 {
eventId := strings.ToLower(eventName + "_" + con.Address)
markCheckedErr := tr.HeaderRepository.MarkHeaderChecked(header.Id, eventId)
if markCheckedErr != nil {
return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
}
logrus.Tracef("no logs found for event %s on contract %s at block %d, continuing", eventName, conAddr, header.BlockNumber) logrus.Tracef("no logs found for event %s on contract %s at block %d, continuing", eventName, conAddr, header.BlockNumber)
continue continue
} }
// If logs aren't empty, persist them // If logs aren't empty, persist them
// Header is marked checked in the transactions
persistErr := tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name) persistErr := tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name)
if persistErr != nil { if persistErr != nil {
return fmt.Errorf("error persisting logs: %s", persistErr.Error()) return fmt.Errorf("error persisting logs: %s", persistErr.Error())
@ -290,6 +287,11 @@ func (tr *Transformer) Execute() error {
} }
} }
markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, tr.eventIds)
if markCheckedErr != nil {
return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
}
// Poll contracts at this block height // Poll contracts at this block height
pollingErr := tr.methodPolling(header, tr.sortedMethodIds) pollingErr := tr.methodPolling(header, tr.sortedMethodIds)
if pollingErr != nil { if pollingErr != nil {
@ -328,6 +330,7 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str
return nil return nil
} }
// GetConfig returns the transformers config; satisfies the transformer interface
func (tr *Transformer) GetConfig() config.ContractConfig { func (tr *Transformer) GetConfig() config.ContractConfig {
return tr.Config return tr.Config
} }

View File

@ -17,12 +17,12 @@
package transformer_test package transformer_test
import ( import (
"github.com/sirupsen/logrus"
"io/ioutil" "io/ioutil"
"testing" "testing"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
) )
func TestTransformer(t *testing.T) { func TestTransformer(t *testing.T) {

View File

@ -18,6 +18,7 @@ package transformer_test
import ( import (
"database/sql" "database/sql"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"

File diff suppressed because one or more lines are too long

View File

@ -20,11 +20,10 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/hexutil"
) )
// Basic abi needed to check which interfaces are adhered to // SupportsInterfaceABI is the basic abi needed to check which interfaces are adhered to
var SupportsInterfaceABI = `[{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]` var SupportsInterfaceABI = `[{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]`
// Individual event interfaces for constructing ABI from // Individual event interfaces for constructing ABI from
var SupportsInterface = `{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}`
var AddrChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"a","type":"address"}],"name":"AddrChanged","type":"event"}` var AddrChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"a","type":"address"}],"name":"AddrChanged","type":"event"}`
var ContentChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes32"}],"name":"ContentChanged","type":"event"}` var ContentChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes32"}],"name":"ContentChanged","type":"event"}`
var NameChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"name","type":"string"}],"name":"NameChanged","type":"event"}` var NameChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"name","type":"string"}],"name":"NameChanged","type":"event"}`
@ -34,11 +33,10 @@ var TextChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"
var MultihashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"MultihashChanged","type":"event"}` var MultihashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"MultihashChanged","type":"event"}`
var ContenthashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"ContenthashChanged","type":"event"}` var ContenthashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"ContenthashChanged","type":"event"}`
var StartingBlock = int64(3648359)
// Resolver interface signatures // Resolver interface signatures
type Interface int type Interface int
// Interface enums
const ( const (
MetaSig Interface = iota MetaSig Interface = iota
AddrChangeSig AddrChangeSig
@ -51,6 +49,7 @@ const (
ContentHashChangeSig ContentHashChangeSig
) )
// Hex returns the hex signature for an interface
func (e Interface) Hex() string { func (e Interface) Hex() string {
strings := [...]string{ strings := [...]string{
"0x01ffc9a7", "0x01ffc9a7",
@ -71,6 +70,7 @@ func (e Interface) Hex() string {
return strings[e] return strings[e]
} }
// Bytes returns the bytes signature for an interface
func (e Interface) Bytes() [4]uint8 { func (e Interface) Bytes() [4]uint8 {
if e < MetaSig || e > ContentHashChangeSig { if e < MetaSig || e > ContentHashChangeSig {
return [4]byte{} return [4]byte{}
@ -86,6 +86,7 @@ func (e Interface) Bytes() [4]uint8 {
return byArray return byArray
} }
// EventSig returns the event signature for an interface
func (e Interface) EventSig() string { func (e Interface) EventSig() string {
strings := [...]string{ strings := [...]string{
"", "",
@ -106,6 +107,7 @@ func (e Interface) EventSig() string {
return strings[e] return strings[e]
} }
// MethodSig returns the method signature for an interface
func (e Interface) MethodSig() string { func (e Interface) MethodSig() string {
strings := [...]string{ strings := [...]string{
"supportsInterface(bytes4)", "supportsInterface(bytes4)",

View File

@ -48,6 +48,7 @@ type Contract struct {
Piping bool // Whether or not to pipe method results forward as arguments to subsequent methods Piping bool // Whether or not to pipe method results forward as arguments to subsequent methods
} }
// Init initializes a contract object
// If we will be calling methods that use addr, hash, or byte arrays // If we will be calling methods that use addr, hash, or byte arrays
// as arguments then we initialize maps to hold these types of values // as arguments then we initialize maps to hold these types of values
func (c Contract) Init() *Contract { func (c Contract) Init() *Contract {
@ -66,7 +67,7 @@ func (c Contract) Init() *Contract {
return &c return &c
} }
// Use contract info to generate event filters - full sync contract watcher only // GenerateFilters uses contract info to generate event filters - full sync contract watcher only
func (c *Contract) GenerateFilters() error { func (c *Contract) GenerateFilters() error {
c.Filters = map[string]filters.LogFilter{} c.Filters = map[string]filters.LogFilter{}
@ -87,7 +88,7 @@ func (c *Contract) GenerateFilters() error {
return nil return nil
} }
// Returns true if address is in list of arguments to // WantedEventArg returns true if address is in list of arguments to
// filter events for or if no filtering is specified // filter events for or if no filtering is specified
func (c *Contract) WantedEventArg(arg string) bool { func (c *Contract) WantedEventArg(arg string) bool {
if c.FilterArgs == nil { if c.FilterArgs == nil {
@ -101,7 +102,7 @@ func (c *Contract) WantedEventArg(arg string) bool {
return false return false
} }
// Returns true if address is in list of arguments to // WantedMethodArg returns true if address is in list of arguments to
// poll methods with or if no filtering is specified // poll methods with or if no filtering is specified
func (c *Contract) WantedMethodArg(arg interface{}) bool { func (c *Contract) WantedMethodArg(arg interface{}) bool {
if c.MethodArgs == nil { if c.MethodArgs == nil {
@ -121,7 +122,7 @@ func (c *Contract) WantedMethodArg(arg interface{}) bool {
return false return false
} }
// Returns true if any mapping value matches filtered for address or if no filter exists // PassesEventFilter returns true if any mapping value matches filtered for address or if no filter exists
// Used to check if an event log name-value mapping should be filtered or not // Used to check if an event log name-value mapping should be filtered or not
func (c *Contract) PassesEventFilter(args map[string]string) bool { func (c *Contract) PassesEventFilter(args map[string]string) bool {
for _, arg := range args { for _, arg := range args {
@ -133,7 +134,7 @@ func (c *Contract) PassesEventFilter(args map[string]string) bool {
return false return false
} }
// Add event emitted address to our list if it passes filter and method polling is on // AddEmittedAddr adds event emitted addresses to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedAddr(addresses ...interface{}) { func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
for _, addr := range addresses { for _, addr := range addresses {
if c.WantedMethodArg(addr) && c.Methods != nil { if c.WantedMethodArg(addr) && c.Methods != nil {
@ -142,7 +143,7 @@ func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
} }
} }
// Add event emitted hash to our list if it passes filter and method polling is on // AddEmittedHash adds event emitted hashes to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedHash(hashes ...interface{}) { func (c *Contract) AddEmittedHash(hashes ...interface{}) {
for _, hash := range hashes { for _, hash := range hashes {
if c.WantedMethodArg(hash) && c.Methods != nil { if c.WantedMethodArg(hash) && c.Methods != nil {
@ -151,6 +152,7 @@ func (c *Contract) AddEmittedHash(hashes ...interface{}) {
} }
} }
// StringifyArg resolves a method argument type to string type
func StringifyArg(arg interface{}) (str string) { func StringifyArg(arg interface{}) (str string) {
switch arg.(type) { switch arg.(type) {
case string: case string:

View File

@ -29,7 +29,7 @@ import (
// Fetcher serves as the lower level data fetcher that calls the underlying // Fetcher serves as the lower level data fetcher that calls the underlying
// blockchain's FetchConctractData method for a given return type // blockchain's FetchConctractData method for a given return type
// Interface definition for a Fetcher // FetcherInterface is the interface definition for a fetcher
type FetcherInterface interface { type FetcherInterface interface {
FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error)
FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error)
@ -56,14 +56,14 @@ type fetcherError struct {
fetchMethod string fetchMethod string
} }
// Fetcher error method // Error method
func (fe *fetcherError) Error() string { func (fe *fetcherError) Error() string {
return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err) return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err)
} }
// Generic Fetcher methods used by Getters to call contract methods // Generic Fetcher methods used by Getters to call contract methods
// Method used to fetch big.Int value from contract // FetchBigInt is the method used to fetch big.Int value from contract
func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) { func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) {
var result = new(big.Int) var result = new(big.Int)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@ -75,7 +75,7 @@ func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockN
return *result, nil return *result, nil
} }
// Method used to fetch bool value from contract // FetchBool is the method used to fetch bool value from contract
func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) { func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) {
var result = new(bool) var result = new(bool)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@ -87,7 +87,7 @@ func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNum
return *result, nil return *result, nil
} }
// Method used to fetch address value from contract // FetchAddress is the method used to fetch address value from contract
func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Address, error) { func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Address, error) {
var result = new(common.Address) var result = new(common.Address)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@ -99,7 +99,7 @@ func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, block
return *result, nil return *result, nil
} }
// Method used to fetch string value from contract // FetchString is the method used to fetch string value from contract
func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (string, error) { func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (string, error) {
var result = new(string) var result = new(string)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@ -111,7 +111,7 @@ func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockN
return *result, nil return *result, nil
} }
// Method used to fetch hash value from contract // FetchHash is the method used to fetch hash value from contract
func (f Fetcher) FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error) { func (f Fetcher) FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error) {
var result = new(common.Hash) var result = new(common.Hash)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)

View File

@ -18,13 +18,15 @@ package getter
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/fetcher" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
) )
// InterfaceGetter is used to derive the interface of a contract
type InterfaceGetter interface { type InterfaceGetter interface {
GetABI(resolverAddr string, blockNumber int64) string GetABI(resolverAddr string, blockNumber int64) (string, error)
GetBlockChain() core.BlockChain GetBlockChain() core.BlockChain
} }
@ -32,7 +34,8 @@ type interfaceGetter struct {
fetcher.Fetcher fetcher.Fetcher
} }
func NewInterfaceGetter(blockChain core.BlockChain) *interfaceGetter { // NewInterfaceGetter returns a new InterfaceGetter
func NewInterfaceGetter(blockChain core.BlockChain) InterfaceGetter {
return &interfaceGetter{ return &interfaceGetter{
Fetcher: fetcher.Fetcher{ Fetcher: fetcher.Fetcher{
BlockChain: blockChain, BlockChain: blockChain,
@ -40,7 +43,7 @@ func NewInterfaceGetter(blockChain core.BlockChain) *interfaceGetter {
} }
} }
// Used to construct a custom ABI based on the results from calling supportsInterface // GetABI is used to construct a custom ABI based on the results from calling supportsInterface
func (g *interfaceGetter) GetABI(resolverAddr string, blockNumber int64) (string, error) { func (g *interfaceGetter) GetABI(resolverAddr string, blockNumber int64) (string, error) {
a := constants.SupportsInterfaceABI a := constants.SupportsInterfaceABI
args := make([]interface{}, 1) args := make([]interface{}, 1)
@ -104,7 +107,7 @@ func (g *interfaceGetter) getSupportsInterface(contractAbi, contractAddress stri
return g.Fetcher.FetchBool("supportsInterface", contractAbi, contractAddress, blockNumber, methodArgs) return g.Fetcher.FetchBool("supportsInterface", contractAbi, contractAddress, blockNumber, methodArgs)
} }
// Method to retrieve the Getter's blockchain // GetBlockChain is a method to retrieve the Getter's blockchain
func (g *interfaceGetter) GetBlockChain() core.BlockChain { func (g *interfaceGetter) GetBlockChain() core.BlockChain {
return g.Fetcher.BlockChain return g.Fetcher.BlockChain
} }

View File

@ -27,6 +27,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
) )
// ConvertToLog converts a watched event to a log
func ConvertToLog(watchedEvent core.WatchedEvent) types.Log { func ConvertToLog(watchedEvent core.WatchedEvent) types.Log {
allTopics := []string{watchedEvent.Topic0, watchedEvent.Topic1, watchedEvent.Topic2, watchedEvent.Topic3} allTopics := []string{watchedEvent.Topic0, watchedEvent.Topic1, watchedEvent.Topic2, watchedEvent.Topic3}
var nonNilTopics []string var nonNilTopics []string
@ -56,12 +57,14 @@ func createTopics(topics ...string) []common.Hash {
return topicsArray return topicsArray
} }
// BigFromString creates a big.Int from a string
func BigFromString(n string) *big.Int { func BigFromString(n string) *big.Int {
b := new(big.Int) b := new(big.Int)
b.SetString(n, 10) b.SetString(n, 10)
return b return b
} }
// GenerateSignature returns the keccak256 hash hex of a string
func GenerateSignature(s string) string { func GenerateSignature(s string) string {
eventSignature := []byte(s) eventSignature := []byte(s)
hash := crypto.Keccak256Hash(eventSignature) hash := crypto.Keccak256Hash(eventSignature)

View File

@ -294,8 +294,7 @@ func TearDown(db *postgres.DB) {
_, err = tx.Exec(`CREATE TABLE checked_headers ( _, err = tx.Exec(`CREATE TABLE checked_headers (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE, header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`)
check_count INTEGER NOT NULL DEFAULT 1);`)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`) _, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)

View File

@ -45,7 +45,8 @@ type parser struct {
parsedAbi abi.ABI parsedAbi abi.ABI
} }
func NewParser(network string) *parser { // NewParser returns a new Parser
func NewParser(network string) Parser {
url := eth.GenURL(network) url := eth.GenURL(network)
return &parser{ return &parser{
@ -53,15 +54,17 @@ func NewParser(network string) *parser {
} }
} }
// Abi returns the parser's configured abi string
func (p *parser) Abi() string { func (p *parser) Abi() string {
return p.abi return p.abi
} }
// ParsedAbi returns the parser's parsed abi
func (p *parser) ParsedAbi() abi.ABI { func (p *parser) ParsedAbi() abi.ABI {
return p.parsedAbi return p.parsedAbi
} }
// Retrieves and parses the abi string // Parse retrieves and parses the abi string
// for the given contract address // for the given contract address
func (p *parser) Parse(contractAddr string) error { func (p *parser) Parse(contractAddr string) error {
// If the abi is one our locally stored abis, fetch // If the abi is one our locally stored abis, fetch
@ -84,7 +87,7 @@ func (p *parser) Parse(contractAddr string) error {
return err return err
} }
// Loads and parses an abi from a given abi string // ParseAbiStr loads and parses an abi from a given abi string
func (p *parser) ParseAbiStr(abiStr string) error { func (p *parser) ParseAbiStr(abiStr string) error {
var err error var err error
p.abi = abiStr p.abi = abiStr
@ -94,14 +97,14 @@ func (p *parser) ParseAbiStr(abiStr string) error {
} }
func (p *parser) lookUp(contractAddr string) (string, error) { func (p *parser) lookUp(contractAddr string) (string, error) {
if v, ok := constants.Abis[common.HexToAddress(contractAddr)]; ok { if v, ok := constants.ABIs[common.HexToAddress(contractAddr)]; ok {
return v, nil return v, nil
} }
return "", errors.New("ABI not present in lookup table") return "", errors.New("ABI not present in lookup table")
} }
// Returns only specified methods, if they meet the criteria // GetSelectMethods returns only specified methods, if they meet the criteria
// Returns as array with methods in same order they were specified // Returns as array with methods in same order they were specified
// Nil or empty wanted array => no events are returned // Nil or empty wanted array => no events are returned
func (p *parser) GetSelectMethods(wanted []string) []types.Method { func (p *parser) GetSelectMethods(wanted []string) []types.Method {
@ -121,7 +124,7 @@ func (p *parser) GetSelectMethods(wanted []string) []types.Method {
return methods return methods
} }
// Returns wanted methods // GetMethods returns wanted methods
// Empty wanted array => all methods are returned // Empty wanted array => all methods are returned
// Nil wanted array => no methods are returned // Nil wanted array => no methods are returned
func (p *parser) GetMethods(wanted []string) []types.Method { func (p *parser) GetMethods(wanted []string) []types.Method {
@ -139,7 +142,7 @@ func (p *parser) GetMethods(wanted []string) []types.Method {
return methods return methods
} }
// Returns wanted events as map of types.Events // GetEvents returns wanted events as map of types.Events
// Empty wanted array => all events are returned // Empty wanted array => all events are returned
// Nil wanted array => no events are returned // Nil wanted array => no events are returned
func (p *parser) GetEvents(wanted []string) map[string]types.Event { func (p *parser) GetEvents(wanted []string) map[string]types.Event {

View File

@ -33,6 +33,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
// Poller is the interface for polling public contract methods
type Poller interface { type Poller interface {
PollContract(con contract.Contract, lastBlock int64) error PollContract(con contract.Contract, lastBlock int64) error
PollContractAt(con contract.Contract, blockNumber int64) error PollContractAt(con contract.Contract, blockNumber int64) error
@ -45,13 +46,15 @@ type poller struct {
contract contract.Contract contract contract.Contract
} }
func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *poller { // NewPoller returns a new Poller
func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) Poller {
return &poller{ return &poller{
MethodRepository: repository.NewMethodRepository(db, mode), MethodRepository: repository.NewMethodRepository(db, mode),
bc: blockChain, bc: blockChain,
} }
} }
// PollContract polls a contract's public methods from the contracts starting block to specified last block
func (p *poller) PollContract(con contract.Contract, lastBlock int64) error { func (p *poller) PollContract(con contract.Contract, lastBlock int64) error {
for i := con.StartingBlock; i <= lastBlock; i++ { for i := con.StartingBlock; i <= lastBlock; i++ {
if err := p.PollContractAt(con, i); err != nil { if err := p.PollContractAt(con, i); err != nil {
@ -62,6 +65,7 @@ func (p *poller) PollContract(con contract.Contract, lastBlock int64) error {
return nil return nil
} }
// PollContractAt polls a contract's public getter methods at the specified block height
func (p *poller) PollContractAt(con contract.Contract, blockNumber int64) error { func (p *poller) PollContractAt(con contract.Contract, blockNumber int64) error {
p.contract = con p.contract = con
for _, m := range con.Methods { for _, m := range con.Methods {
@ -98,7 +102,7 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error {
var out interface{} var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, bn) err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, bn)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) return fmt.Errorf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
} }
strOut, err := stringify(out) strOut, err := stringify(out)
if err != nil { if err != nil {
@ -112,7 +116,7 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error {
// Persist result immediately // Persist result immediately
err = p.PersistResults([]types.Result{result}, m, p.contract.Address, p.contract.Name) err = p.PersistResults([]types.Result{result}, m, p.contract.Address, p.contract.Name)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) return fmt.Errorf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
} }
return nil return nil
@ -148,7 +152,7 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error {
var out interface{} var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn) err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) return fmt.Errorf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
} }
strOut, err := stringify(out) strOut, err := stringify(out)
if err != nil { if err != nil {
@ -164,7 +168,7 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error {
// Persist result set as batch // Persist result set as batch
err := p.PersistResults(results, m, p.contract.Address, p.contract.Name) err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) return fmt.Errorf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
} }
return nil return nil
@ -212,7 +216,7 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error {
var out interface{} var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn) err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) return fmt.Errorf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
} }
strOut, err := stringify(out) strOut, err := stringify(out)
if err != nil { if err != nil {
@ -228,13 +232,13 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error {
err := p.PersistResults(results, m, p.contract.Address, p.contract.Name) err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) return fmt.Errorf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
} }
return nil return nil
} }
// This is just a wrapper around the poller blockchain's FetchContractData method // FetchContractData is just a wrapper around the poller blockchain's FetchContractData method
func (p *poller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error { func (p *poller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
return p.bc.FetchContractData(contractAbi, contractAddress, method, methodArgs, result, blockNumber) return p.bc.FetchContractData(contractAbi, contractAddress, method, methodArgs, result, blockNumber)
} }

View File

@ -24,7 +24,6 @@ import (
"github.com/hashicorp/golang-lru" "github.com/hashicorp/golang-lru"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
@ -35,7 +34,7 @@ const (
eventCacheSize = 1000 eventCacheSize = 1000
) )
// Event repository is used to persist event data into custom tables // EventRepository is used to persist event data into custom tables
type EventRepository interface { type EventRepository interface {
PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error
CreateEventTable(contractAddr string, event types.Event) (bool, error) CreateEventTable(contractAddr string, event types.Event) (bool, error)
@ -51,7 +50,8 @@ type eventRepository struct {
tables *lru.Cache // Cache names of recently used tables to minimize db connections tables *lru.Cache // Cache names of recently used tables to minimize db connections
} }
func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository { // NewEventRepository returns a new EventRepository
func NewEventRepository(db *postgres.DB, mode types.Mode) EventRepository {
ccs, _ := lru.New(contractCacheSize) ccs, _ := lru.New(contractCacheSize)
ecs, _ := lru.New(eventCacheSize) ecs, _ := lru.New(eventCacheSize)
return &eventRepository{ return &eventRepository{
@ -62,7 +62,7 @@ func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository {
} }
} }
// Creates a schema for the contract if needed // PersistLogs creates a schema for the contract if needed
// Creates table for the watched contract event if needed // Creates table for the watched contract event if needed
// Persists converted event log data into this custom table // Persists converted event log data into this custom table
func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
@ -112,7 +112,7 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type
// Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string // Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string
data := make([]interface{}, 0, 5+el) data := make([]interface{}, 0, 5+el)
data = append(data, data = append(data,
event.Id, event.ID,
contractName, contractName,
event.Raw, event.Raw,
event.LogIndex, event.LogIndex,
@ -143,17 +143,6 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type
} }
} }
// Mark header as checked for this eventId
eventId := strings.ToLower(eventInfo.Name + "_" + contractAddr)
markCheckedErr := repository.MarkContractWatcherHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block
if markCheckedErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
logrus.Warnf("error rolling back transaction while marking header checked: %s", rollbackErr.Error())
}
return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
}
return tx.Commit() return tx.Commit()
} }
@ -171,7 +160,7 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.
data := make([]interface{}, 0, 4+el) data := make([]interface{}, 0, 4+el)
data = append(data, data = append(data,
event.Id, event.ID,
contractName, contractName,
event.Block, event.Block,
event.Tx) event.Tx)
@ -201,7 +190,7 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.
return tx.Commit() return tx.Commit()
} }
// Checks for event table and creates it if it does not already exist // CreateEventTable checks for event table and creates it if it does not already exist
// Returns true if it created a new table; returns false if table already existed // Returns true if it created a new table; returns false if table already existed
func (r *eventRepository) CreateEventTable(contractAddr string, event types.Event) (bool, error) { func (r *eventRepository) CreateEventTable(contractAddr string, event types.Event) (bool, error) {
tableID := fmt.Sprintf("%s_%s.%s_event", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(event.Name)) tableID := fmt.Sprintf("%s_%s.%s_event", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(event.Name))
@ -270,7 +259,7 @@ func (r *eventRepository) checkForTable(contractAddr string, eventName string) (
return exists, err return exists, err
} }
// Checks for contract schema and creates it if it does not already exist // CreateContractSchema checks for contract schema and creates it if it does not already exist
// Returns true if it created a new schema; returns false if schema already existed // Returns true if it created a new schema; returns false if schema already existed
func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error) { func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error) {
if contractAddr == "" { if contractAddr == "" {
@ -316,10 +305,12 @@ func (r *eventRepository) checkForSchema(contractAddr string) (bool, error) {
return exists, err return exists, err
} }
// CheckSchemaCache is used to query the schema name cache
func (r *eventRepository) CheckSchemaCache(key string) (interface{}, bool) { func (r *eventRepository) CheckSchemaCache(key string) (interface{}, bool) {
return r.schemas.Get(key) return r.schemas.Get(key)
} }
// CheckTableCache is used to query the table name cache
func (r *eventRepository) CheckTableCache(key string) (interface{}, bool) { func (r *eventRepository) CheckTableCache(key string) (interface{}, bool) {
return r.tables.Get(key) return r.tables.Get(key)
} }

View File

@ -355,11 +355,6 @@ var _ = Describe("Repository", func() {
Expect(count).To(Equal(2)) Expect(count).To(Equal(2))
}) })
It("Fails if the persisted event does not have a corresponding eventID column in the checked_headers table", func() {
err = dataStore.PersistLogs(logs, event, con.Address, con.Name)
Expect(err).To(HaveOccurred())
})
It("Fails with empty log", func() { It("Fails with empty log", func() {
err = dataStore.PersistLogs([]types.Log{}, event, con.Address, con.Name) err = dataStore.PersistLogs([]types.Log{}, event, con.Address, con.Name)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())

View File

@ -30,6 +30,7 @@ import (
const methodCacheSize = 1000 const methodCacheSize = 1000
// MethodRepository is used to persist public getter method data
type MethodRepository interface { type MethodRepository interface {
PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error
CreateMethodTable(contractAddr string, method types.Method) (bool, error) CreateMethodTable(contractAddr string, method types.Method) (bool, error)
@ -45,7 +46,8 @@ type methodRepository struct {
tables *lru.Cache // Cache names of recently used tables to minimize db connections tables *lru.Cache // Cache names of recently used tables to minimize db connections
} }
func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository { // NewMethodRepository returns a new MethodRepository
func NewMethodRepository(db *postgres.DB, mode types.Mode) MethodRepository {
ccs, _ := lru.New(contractCacheSize) ccs, _ := lru.New(contractCacheSize)
mcs, _ := lru.New(methodCacheSize) mcs, _ := lru.New(methodCacheSize)
return &methodRepository{ return &methodRepository{
@ -56,7 +58,7 @@ func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository {
} }
} }
// Creates a schema for the contract if needed // PersistResults creates a schema for the contract if needed
// Creates table for the contract method if needed // Creates table for the contract method if needed
// Persists method polling data into this custom table // Persists method polling data into this custom table
func (r *methodRepository) PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error { func (r *methodRepository) PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
@ -124,7 +126,7 @@ func (r *methodRepository) persistResults(results []types.Result, methodInfo typ
return tx.Commit() return tx.Commit()
} }
// Checks for event table and creates it if it does not already exist // CreateMethodTable checks for event table and creates it if it does not already exist
func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Method) (bool, error) { func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Method) (bool, error) {
tableID := fmt.Sprintf("%s_%s.%s_method", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name)) tableID := fmt.Sprintf("%s_%s.%s_method", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name))
@ -177,7 +179,7 @@ func (r *methodRepository) checkForTable(contractAddr string, methodName string)
return exists, err return exists, err
} }
// Checks for contract schema and creates it if it does not already exist // CreateContractSchema checks for contract schema and creates it if it does not already exist
func (r *methodRepository) CreateContractSchema(contractAddr string) (bool, error) { func (r *methodRepository) CreateContractSchema(contractAddr string) (bool, error) {
if contractAddr == "" { if contractAddr == "" {
return false, errors.New("error: no contract address specified") return false, errors.New("error: no contract address specified")
@ -222,10 +224,12 @@ func (r *methodRepository) checkForSchema(contractAddr string) (bool, error) {
return exists, err return exists, err
} }
// CheckSchemaCache is used to query the schema name cache
func (r *methodRepository) CheckSchemaCache(key string) (interface{}, bool) { func (r *methodRepository) CheckSchemaCache(key string) (interface{}, bool) {
return r.schemas.Get(key) return r.schemas.Get(key)
} }
// CheckTableCache is used to query the table name cache
func (r *methodRepository) CheckTableCache(key string) (interface{}, bool) { func (r *methodRepository) CheckTableCache(key string) (interface{}, bool) {
return r.tables.Get(key) return r.tables.Get(key)
} }

View File

@ -17,12 +17,12 @@
package repository_test package repository_test
import ( import (
"github.com/sirupsen/logrus"
"io/ioutil" "io/ioutil"
"testing" "testing"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
) )
func TestRepository(t *testing.T) { func TestRepository(t *testing.T) {

View File

@ -18,17 +18,17 @@ package retriever
import ( import (
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
"strings" "strings"
"github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
// Address retriever is used to retrieve the addresses associated with a contract // AddressRetriever is used to retrieve the addresses associated with a contract
type AddressRetriever interface { type AddressRetriever interface {
RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error)
} }
@ -38,14 +38,15 @@ type addressRetriever struct {
mode types.Mode mode types.Mode
} }
func NewAddressRetriever(db *postgres.DB, mode types.Mode) (r *addressRetriever) { // NewAddressRetriever returns a new AddressRetriever
func NewAddressRetriever(db *postgres.DB, mode types.Mode) AddressRetriever {
return &addressRetriever{ return &addressRetriever{
db: db, db: db,
mode: mode, mode: mode,
} }
} }
// Method to retrieve list of token-holding/contract-related addresses by iterating over available events // RetrieveTokenHolderAddresses is used to retrieve list of token-holding/contract-related addresses by iterating over available events
// This generic method should work whether or not the argument/input names of the events meet the expected standard // This generic method should work whether or not the argument/input names of the events meet the expected standard
// This could be generalized to iterate over ALL events and pull out any address arguments // This could be generalized to iterate over ALL events and pull out any address arguments
func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) { func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) {

View File

@ -17,12 +17,12 @@
package retriever_test package retriever_test
import ( import (
"github.com/sirupsen/logrus"
"io/ioutil" "io/ioutil"
"testing" "testing"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
) )
func TestRetriever(t *testing.T) { func TestRetriever(t *testing.T) {

View File

@ -25,20 +25,22 @@ import (
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
) )
// Event is our custom event type
type Event struct { type Event struct {
Name string Name string
Anonymous bool Anonymous bool
Fields []Field Fields []Field
} }
// Field is our custom event field type which associates a postgres type with the field
type Field struct { type Field struct {
abi.Argument // Name, Type, Indexed abi.Argument // Name, Type, Indexed
PgType string // Holds type used when committing data held in this field to postgres PgType string // Holds type used when committing data held in this field to postgres
} }
// Struct to hold instance of an event log data // Log is used to hold instance of an event log data
type Log struct { type Log struct {
Id int64 // VulcanizeIdLog for full sync and header ID for header sync contract watcher ID int64 // VulcanizeIdLog for full sync and header ID for header sync contract watcher
Values map[string]string // Map of event input names to their values Values map[string]string // Map of event input names to their values
// Used for full sync only // Used for full sync only
@ -51,7 +53,7 @@ type Log struct {
Raw []byte // json.Unmarshalled byte array of geth/core/types.Log{} Raw []byte // json.Unmarshalled byte array of geth/core/types.Log{}
} }
// Unpack abi.Event into our custom Event struct // NewEvent unpacks abi.Event into our custom Event struct
func NewEvent(e abi.Event) Event { func NewEvent(e abi.Event) Event {
fields := make([]Field, len(e.Inputs)) fields := make([]Field, len(e.Inputs))
for i, input := range e.Inputs { for i, input := range e.Inputs {
@ -85,6 +87,7 @@ func NewEvent(e abi.Event) Event {
} }
} }
// Sig returns the hash signature for an event
func (e Event) Sig() common.Hash { func (e Event) Sig() common.Hash {
types := make([]string, len(e.Fields)) types := make([]string, len(e.Fields))

View File

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
) )
// Method is our custom method struct
type Method struct { type Method struct {
Name string Name string
Const bool Const bool
@ -32,7 +33,7 @@ type Method struct {
Return []Field Return []Field
} }
// Struct to hold instance of result from method call with given inputs and block // Result is used to hold instance of result from method call with given inputs and block
type Result struct { type Result struct {
Method Method
Inputs []interface{} // Will only use addresses Inputs []interface{} // Will only use addresses
@ -41,7 +42,7 @@ type Result struct {
Block int64 Block int64
} }
// Unpack abi.Method into our custom Method struct // NewMethod unpacks abi.Method into our custom Method struct
func NewMethod(m abi.Method) Method { func NewMethod(m abi.Method) Method {
inputs := make([]Field, len(m.Inputs)) inputs := make([]Field, len(m.Inputs))
for i, input := range m.Inputs { for i, input := range m.Inputs {
@ -99,6 +100,7 @@ func NewMethod(m abi.Method) Method {
} }
} }
// Sig returns the hash signature for the method
func (m Method) Sig() common.Hash { func (m Method) Sig() common.Hash {
types := make([]string, len(m.Args)) types := make([]string, len(m.Args))
i := 0 i := 0

View File

@ -16,19 +16,21 @@
package types package types
import "fmt" // Mode is used to explicitly represent the operating mode of the transformer
type Mode int type Mode int
// Mode enums
const ( const (
HeaderSync Mode = iota HeaderSync Mode = iota
FullSync FullSync
) )
// IsValid returns true is the Mode is valid
func (mode Mode) IsValid() bool { func (mode Mode) IsValid() bool {
return mode >= HeaderSync && mode <= FullSync return mode >= HeaderSync && mode <= FullSync
} }
// String returns the string representation of the mode
func (mode Mode) String() string { func (mode Mode) String() string {
switch mode { switch mode {
case HeaderSync: case HeaderSync:
@ -39,26 +41,3 @@ func (mode Mode) String() string {
return "unknown" return "unknown"
} }
} }
func (mode Mode) MarshalText() ([]byte, error) {
switch mode {
case HeaderSync:
return []byte("header"), nil
case FullSync:
return []byte("full"), nil
default:
return nil, fmt.Errorf("contract watcher: unknown mode %d, want HeaderSync or FullSync", mode)
}
}
func (mode *Mode) UnmarshalText(text []byte) error {
switch string(text) {
case "header":
*mode = HeaderSync
case "full":
*mode = FullSync
default:
return fmt.Errorf(`contract watcher: unknown mode %q, want "header" or "full"`, text)
}
return nil
}