cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Merge pull request #158 from vulcanize/missed_marked_checked_headers

Browse Source 
Fix for issue #146
This commit is contained in:
Ian Norden 2019-10-28 12:34:47 -05:00 committed by GitHub
commit b7675316b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 214 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
integration_test/contract_watcher_full_transformer_test.go
View File

@ -18,7 +18,6 @@ package integration
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"math/rand"
"strings"
"time"
@ -27,6 +26,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"

40
integration_test/contract_watcher_header_sync_transformer_test.go
View File

@ -40,8 +40,8 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
var blockChain core.BlockChain
var headerRepository repositories.HeaderRepository
var headerID int64
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
var ensAddr = strings.ToLower(constants.EnsContractAddress) // 0x314159265dd8dbb310642f98f50c066173c1259b
var tusdAddr = strings.ToLower(constants.TusdContractAddress) // 0x8dd5fbce2f6a956c3022ba3663759011dd51e73e
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
@ -377,6 +377,42 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
Expect(transferLog.Value).To(Equal("2800000000000000000000"))
})
It("Marks header checked for a contract that has no logs at that header", func() {
t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(t.Start).To(Equal(int64(6885702)))
newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
Expect(err).ToNot(HaveOccurred())
transferLog := test_helpers.HeaderSyncTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
Expect(err).ToNot(HaveOccurred())
Expect(transferLog.HeaderID).ToNot(Equal(newOwnerLog.HeaderID))
type checkedHeader struct {
ID int64 `db:"id"`
HeaderID int64 `db:"header_id"`
NewOwner int64 `db:"newowner_0x314159265dd8dbb310642f98f50c066173c1259b"`
Transfer int64 `db:"transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e"`
}
transferCheckedHeader := new(checkedHeader)
err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", transferLog.HeaderID).StructScan(transferCheckedHeader)
Expect(err).ToNot(HaveOccurred())
Expect(transferCheckedHeader.Transfer).To(Equal(int64(1)))
Expect(transferCheckedHeader.NewOwner).To(Equal(int64(1)))
newOwnerCheckedHeader := new(checkedHeader)
err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", newOwnerLog.HeaderID).StructScan(newOwnerCheckedHeader)
Expect(err).ToNot(HaveOccurred())
Expect(newOwnerCheckedHeader.NewOwner).To(Equal(int64(1)))
Expect(newOwnerCheckedHeader.Transfer).To(Equal(int64(1)))
})
It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
var testConf config.ContractConfig
testConf = test_helpers.ENSandTusdConfig

27
libraries/shared/repository/repository.go
View File

@ -1,27 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repository
import "github.com/jmoiron/sqlx"
func MarkContractWatcherHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
return err
}

67
libraries/shared/repository/repository_test.go
View File

@ -1,67 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repository_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("", func() {
Describe("MarkContractWatcherHeaderCheckedInTransaction", func() {
var (
checkedHeadersColumn string
db *postgres.DB
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
It("marks passed header as checked within a passed transaction", func() {
headerRepository := repositories.NewHeaderRepository(db)
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
tx, txErr := db.Beginx()
Expect(txErr).NotTo(HaveOccurred())
err := repository.MarkContractWatcherHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
commitErr := tx.Commit()
Expect(commitErr).NotTo(HaveOccurred())
var checkedCount int
fetchErr := db.Get(&checkedCount, `SELECT COUNT(*) FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(fetchErr).NotTo(HaveOccurred())
Expect(checkedCount).To(Equal(1))
})
})
})

11
pkg/contract_watcher/full/converter/converter.go
View File

@ -17,7 +17,6 @@
package converter
import (
"errors"
"fmt"
"math/big"
"strconv"
@ -32,22 +31,24 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// Converter is used to convert watched event logs to
// ConverterInterface is used to convert watched event logs to
// custom logs containing event input name => value maps
type ConverterInterface interface {
Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error)
Update(info *contract.Contract)
}
// Converter is the underlying struct for the ConverterInterface
type Converter struct {
ContractInfo *contract.Contract
}
// Update configures the converter for a specific contract
func (c *Converter) Update(info *contract.Contract) {
c.ContractInfo = info
}
// Convert the given watched event log into a types.Log for the given event
// Convert converts the given watched event log into a types.Log for the given event
func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
values := make(map[string]interface{})
@ -88,14 +89,14 @@ func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
b := input.(byte)
strValues[fieldName] = string(b)
default:
return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
return nil, fmt.Errorf("error: unhandled abi type %T", input)
}
}
// Only hold onto logs that pass our address filter, if any
if c.ContractInfo.PassesEventFilter(strValues) {
eventLog := &types.Log{
Id: watchedEvent.LogID,
ID: watchedEvent.LogID,
Values: strValues,
Block: watchedEvent.BlockNumber,
Tx: watchedEvent.TxHash,

15
pkg/contract_watcher/full/retriever/block_retriever.go
View File

@ -18,11 +18,12 @@ package retriever
import (
"database/sql"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// Block retriever is used to retrieve the first block for a given contract and the most recent block
// BlockRetriever is used to retrieve the first block for a given contract and the most recent block
// It requires a vDB synced database with blocks, transactions, receipts, and logs
type BlockRetriever interface {
RetrieveFirstBlock(contractAddr string) (int64, error)
@ -33,13 +34,15 @@ type blockRetriever struct {
db *postgres.DB
}
func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
// NewBlockRetriever returns a new BlockRetriever
func NewBlockRetriever(db *postgres.DB) BlockRetriever {
return &blockRetriever{
db: db,
}
}
// Try both methods of finding the first block, with the receipt method taking precedence
// RetrieveFirstBlock fetches the block number for the earliest block in the db
// Tries both methods of finding the first block, with the receipt method taking precedence
func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
if err != nil {
@ -55,7 +58,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error)
// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
var firstBlock int64
addressId, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
addressID, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
if getAddressErr != nil {
return firstBlock, getAddressErr
}
@ -66,7 +69,7 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in
WHERE contract_address_id = $1
ORDER BY block_id ASC
LIMIT 1)`,
addressId,
addressID,
)
return firstBlock, err
@ -84,7 +87,7 @@ func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64,
return int64(firstBlock), err
}
// Method to retrieve the most recent block in vDB
// RetrieveMostRecentBlock retrieves the most recent block number in vDB
func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
var lastBlock int64
err := r.db.Get(

3
pkg/contract_watcher/full/retriever/block_retriever_test.go
View File

@ -17,10 +17,11 @@
package retriever_test
import (
"strings"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"strings"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"

2
pkg/contract_watcher/full/retriever/retriever_suite_test.go
View File

@ -17,12 +17,12 @@
package retriever_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestRetriever(t *testing.T) {

6
pkg/contract_watcher/full/transformer/transformer.go
View File

@ -35,6 +35,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
// Transformer is the top level struct for transforming watched contract data
// Requires a fully synced vDB and a running eth node (or infura)
type Transformer struct {
// Database interfaces
@ -60,7 +61,7 @@ type Transformer struct {
LastBlock int64
}
// Transformer takes in config for blockchain, database, and network id
// NewTransformer takes in contract config, blockchain, and database, and returns a new Transformer
func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer {
return &Transformer{
Poller: poller.NewPoller(BC, DB, types.FullSync),
@ -75,6 +76,7 @@ func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.
}
}
// Init initializes the transformer
// Use after creating and setting transformer
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
@ -167,6 +169,7 @@ func (tr *Transformer) Init() error {
return nil
}
// Execute runs the transformation processes
// Iterates through stored, initialized contract objects
// Iterates through contract's event filters, grabbing watched event logs
// Uses converter to convert logs into custom log type
@ -227,6 +230,7 @@ func (tr *Transformer) Execute() error {
return nil
}
// GetConfig returns the transformers config; satisfies the transformer interface
func (tr *Transformer) GetConfig() config.ContractConfig {
return tr.Config
}

2
pkg/contract_watcher/full/transformer/transformer_suite_test.go
View File

@ -17,12 +17,12 @@
package transformer_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestTransformer(t *testing.T) {

14
pkg/contract_watcher/header/converter/converter.go
View File

@ -18,7 +18,6 @@ package converter
import (
"encoding/json"
"errors"
"fmt"
"math/big"
"strconv"
@ -32,16 +31,19 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
)
// ConverterInterface is the interface for converting geth logs to our custom log type
type ConverterInterface interface {
Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error)
ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error)
Update(info *contract.Contract)
}
// Converter is the underlying struct for the ConverterInterface
type Converter struct {
ContractInfo *contract.Contract
}
// Update is used to configure the converter with a specific contract
func (c *Converter) Update(info *contract.Contract) {
c.ContractInfo = info
}
@ -98,7 +100,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
strValues[fieldName] = converted.String()
seenHashes = append(seenHashes, converted)
default:
return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
return nil, fmt.Errorf("error: unhandled abi type %T", input)
}
}
@ -114,7 +116,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
Values: strValues,
Raw: raw,
TransactionIndex: log.TxIndex,
Id: headerID,
ID: headerID,
})
// Cache emitted values if their caching is turned on
@ -130,7 +132,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
return returnLogs, nil
}
// Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
// ConvertBatch converts the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
eventsToLogs := make(map[string][]types.Log)
@ -182,7 +184,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
strValues[fieldName] = converted.String()
seenHashes = append(seenHashes, converted)
default:
return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
return nil, fmt.Errorf("error: unhandled abi type %T", input)
}
}
@ -198,7 +200,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
Values: strValues,
Raw: raw,
TransactionIndex: log.TxIndex,
Id: headerID,
ID: headerID,
})
// Cache emitted values that pass the argument filter if their caching is turned on

4
pkg/contract_watcher/header/converter/converter_test.go
View File

@ -72,11 +72,11 @@ var _ = Describe("Converter", func() {
Expect(logs[0].Values["to"]).To(Equal(sender1.String()))
Expect(logs[0].Values["from"]).To(Equal(sender2.String()))
Expect(logs[0].Values["value"]).To(Equal(value.String()))
Expect(logs[0].Id).To(Equal(int64(232)))
Expect(logs[0].ID).To(Equal(int64(232)))
Expect(logs[1].Values["to"]).To(Equal(sender2.String()))
Expect(logs[1].Values["from"]).To(Equal(sender1.String()))
Expect(logs[1].Values["value"]).To(Equal(value.String()))
Expect(logs[1].Id).To(Equal(int64(232)))
Expect(logs[1].ID).To(Equal(int64(232)))
})
It("Keeps track of addresses it sees if they will be used for method polling", func() {

6
pkg/contract_watcher/header/fetcher/fetcher.go
View File

@ -24,6 +24,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// Fetcher is the fetching interface
type Fetcher interface {
FetchLogs(contractAddresses []string, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
}
@ -32,13 +33,14 @@ type fetcher struct {
blockChain core.BlockChain
}
func NewFetcher(blockchain core.BlockChain) *fetcher {
// NewFetcher returns a new Fetcher
func NewFetcher(blockchain core.BlockChain) Fetcher {
return &fetcher{
blockChain: blockchain,
}
}
// Checks all topic0s, on all addresses, fetching matching logs for the given header
// FetchLogs checks all topic0s, on all addresses, fetching matching logs for the given header
func (fetcher *fetcher) FetchLogs(contractAddresses []string, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
addresses := hexStringsToAddresses(contractAddresses)
blockHash := common.HexToHash(header.Hash)

32
pkg/contract_watcher/header/repository/header_repository.go
View File

@ -20,7 +20,6 @@ import (
"fmt"
"github.com/hashicorp/golang-lru"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -29,6 +28,7 @@ import (
const columnCacheSize = 1000
// HeaderRepository interfaces with the header and checked_headers tables
type HeaderRepository interface {
AddCheckColumn(id string) error
AddCheckColumns(ids []string) error
@ -46,7 +46,8 @@ type headerRepository struct {
columns *lru.Cache // Cache created columns to minimize db connections
}
func NewHeaderRepository(db *postgres.DB) *headerRepository {
// NewHeaderRepository returns a new HeaderRepository
func NewHeaderRepository(db *postgres.DB) HeaderRepository {
ccs, _ := lru.New(columnCacheSize)
return &headerRepository{
db: db,
@ -54,7 +55,7 @@ func NewHeaderRepository(db *postgres.DB) *headerRepository {
}
}
// Adds a checked_header column for the provided column id
// AddCheckColumn adds a checked_header column for the provided column id
func (r *headerRepository) AddCheckColumn(id string) error {
// Check cache to see if column already exists before querying pg
_, ok := r.columns.Get(id)
@ -75,7 +76,7 @@ func (r *headerRepository) AddCheckColumn(id string) error {
return nil
}
// Adds a checked_header column for all of the provided column ids
// AddCheckColumns adds a checked_header column for all of the provided column ids
func (r *headerRepository) AddCheckColumns(ids []string) error {
var err error
baseQuery := "ALTER TABLE public.checked_headers"
@ -99,7 +100,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error {
return err
}
// Marks the header checked for the provided column id
// MarkHeaderChecked marks the header checked for the provided column id
func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`)
VALUES ($1, $2)
@ -108,7 +109,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
return err
}
// Marks the header checked for all of the provided column ids
// MarkHeaderCheckedForAll marks the header checked for all of the provided column ids
func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
pgStr := "INSERT INTO public.checked_headers (header_id, "
for _, id := range ids {
@ -127,7 +128,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
return err
}
// Marks all of the provided headers checked for each of the provided column ids
// MarkHeadersCheckedForAll marks all of the provided headers checked for each of the provided column ids
func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
tx, err := r.db.Beginx()
if err != nil {
@ -160,7 +161,7 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [
return err
}
// Returns missing headers for the provided checked_headers column id
// MissingHeaders returns missing headers for the provided checked_headers column id
func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) {
var result []core.Header
var query string
@ -186,7 +187,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
return continuousHeaders(result), err
}
// Returns missing headers for all of the provided checked_headers column ids
// MissingHeadersForAll returns missing headers for all of the provided checked_headers column ids
func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
var result []core.Header
var query string
@ -214,7 +215,7 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock
return continuousHeaders(result), err
}
// Returns headers that have been checked for all of the provided event ids but not for the provided method ids
// MissingMethodsCheckedEventsIntersection returns headers that have been checked for all of the provided event ids but not for the provided method ids
func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
var result []core.Header
var query string
@ -264,16 +265,7 @@ func continuousHeaders(headers []core.Header) []core.Header {
return headers
}
// Check the repositories column id cache for a value
// CheckCache checks the repositories column id cache for a value
func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
return r.columns.Get(key)
}
// Used to mark a header checked as part of some external transaction so as to group into one commit
func (r *headerRepository) MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET `+eventID+` = checked_headers.`+eventID+` + 1`, headerID, 1)
return err
}

2
pkg/contract_watcher/header/repository/header_repository_test.go
View File

@ -18,7 +18,6 @@ package repository_test
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -26,6 +25,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)

2
pkg/contract_watcher/header/repository/repository_suite_test.go
View File

@ -17,12 +17,12 @@
package repository_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestRepository(t *testing.T) {

9
pkg/contract_watcher/header/retriever/block_retriever.go
View File

@ -20,7 +20,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// Block retriever is used to retrieve the first block for a given contract and the most recent block
// BlockRetriever is used to retrieve the first block for a given contract and the most recent block
// It requires a vDB synced database with blocks, transactions, receipts, and logs
type BlockRetriever interface {
RetrieveFirstBlock() (int64, error)
@ -31,13 +31,14 @@ type blockRetriever struct {
db *postgres.DB
}
func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
// NewBlockRetriever returns a new BlockRetriever
func NewBlockRetriever(db *postgres.DB) BlockRetriever {
return &blockRetriever{
db: db,
}
}
// Retrieve block number of earliest header in repo
// RetrieveFirstBlock retrieves block number of earliest header in repo
func (r *blockRetriever) RetrieveFirstBlock() (int64, error) {
var firstBlock int
err := r.db.Get(
@ -48,7 +49,7 @@ func (r *blockRetriever) RetrieveFirstBlock() (int64, error) {
return int64(firstBlock), err
}
// Retrieve block number of latest header in repo
// RetrieveMostRecentBlock retrieves block number of latest header in repo
func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
var lastBlock int
err := r.db.Get(

2
pkg/contract_watcher/header/retriever/retriever_suite_test.go
View File

@ -17,12 +17,12 @@
package retriever_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestRetriever(t *testing.T) {

33
pkg/contract_watcher/header/transformer/transformer.go
View File

@ -40,6 +40,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// Transformer is the top level struct for transforming watched contract data
// Requires a header synced vDB (headers) and a running eth node (or infura)
type Transformer struct {
// Database interfaces
@ -76,7 +77,7 @@ type Transformer struct {
// 3. Init
// 4. Execute
// Transformer takes in config for blockchain, database, and network id
// NewTransformer takes in a contract config, blockchain, and database, and returns a new Transformer
func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer {
return &Transformer{
@ -92,6 +93,7 @@ func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.
}
}
// Init initialized the Transformer
// Use after creating and setting transformer
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
@ -126,6 +128,7 @@ func (tr *Transformer) Init() error {
firstBlock, retrieveErr := tr.Retriever.RetrieveFirstBlock()
if retrieveErr != nil {
if retrieveErr == sql.ErrNoRows {
logrus.Error(fmt.Errorf("error retrieving first block: %s", retrieveErr.Error()))
firstBlock = 0
} else {
return fmt.Errorf("error retrieving first block: %s", retrieveErr.Error())
@ -175,14 +178,14 @@ func (tr *Transformer) Init() error {
// Create checked_headers columns for each event id and append to list of all event ids
tr.sortedEventIds[con.Address] = make([]string, 0, len(con.Events))
for _, event := range con.Events {
eventId := strings.ToLower(event.Name + "_" + con.Address)
addColumnErr := tr.HeaderRepository.AddCheckColumn(eventId)
eventID := strings.ToLower(event.Name + "_" + con.Address)
addColumnErr := tr.HeaderRepository.AddCheckColumn(eventID)
if addColumnErr != nil {
return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
}
// Keep track of this event id; sorted and unsorted
tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventId)
tr.eventIds = append(tr.eventIds, eventId)
tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventID)
tr.eventIds = append(tr.eventIds, eventID)
// Append this event sig to the filters
tr.eventFilters = append(tr.eventFilters, event.Sig())
}
@ -190,12 +193,12 @@ func (tr *Transformer) Init() error {
// Create checked_headers columns for each method id and append list of all method ids
tr.sortedMethodIds[con.Address] = make([]string, 0, len(con.Methods))
for _, m := range con.Methods {
methodId := strings.ToLower(m.Name + "_" + con.Address)
addColumnErr := tr.HeaderRepository.AddCheckColumn(methodId)
methodID := strings.ToLower(m.Name + "_" + con.Address)
addColumnErr := tr.HeaderRepository.AddCheckColumn(methodID)
if addColumnErr != nil {
return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
}
tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodId)
tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodID)
}
// Update start to the lowest block
@ -207,6 +210,7 @@ func (tr *Transformer) Init() error {
return nil
}
// Execute runs the transformation processes
func (tr *Transformer) Execute() error {
if len(tr.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts")
@ -248,7 +252,6 @@ func (tr *Transformer) Execute() error {
continue
}
// Sort logs by the contract they belong to
for _, log := range allLogs {
addr := strings.ToLower(log.Address.Hex())
sortedLogs[addr] = append(sortedLogs[addr], log)
@ -273,16 +276,10 @@ func (tr *Transformer) Execute() error {
for eventName, logs := range convertedLogs {
// If logs for this event are empty, mark them checked at this header and continue
if len(logs) < 1 {
eventId := strings.ToLower(eventName + "_" + con.Address)
markCheckedErr := tr.HeaderRepository.MarkHeaderChecked(header.Id, eventId)
if markCheckedErr != nil {
return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
}
logrus.Tracef("no logs found for event %s on contract %s at block %d, continuing", eventName, conAddr, header.BlockNumber)
continue
}
// If logs aren't empty, persist them
// Header is marked checked in the transactions
persistErr := tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name)
if persistErr != nil {
return fmt.Errorf("error persisting logs: %s", persistErr.Error())
@ -290,6 +287,11 @@ func (tr *Transformer) Execute() error {
}
}
markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, tr.eventIds)
if markCheckedErr != nil {
return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
}
// Poll contracts at this block height
pollingErr := tr.methodPolling(header, tr.sortedMethodIds)
if pollingErr != nil {
@ -328,6 +330,7 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str
return nil
}
// GetConfig returns the transformers config; satisfies the transformer interface
func (tr *Transformer) GetConfig() config.ContractConfig {
return tr.Config
}

2
pkg/contract_watcher/header/transformer/transformer_suite_test.go
View File

@ -17,12 +17,12 @@
package transformer_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestTransformer(t *testing.T) {

1
pkg/contract_watcher/header/transformer/transformer_test.go
View File

@ -18,6 +18,7 @@ package transformer_test
import (
"database/sql"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

27
pkg/contract_watcher/shared/constants/constants.go
View File

File diff suppressed because one or more lines are too long

10
pkg/contract_watcher/shared/constants/interface.go
View File

@ -20,11 +20,10 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil"
)
// Basic abi needed to check which interfaces are adhered to
// SupportsInterfaceABI is the basic abi needed to check which interfaces are adhered to
var SupportsInterfaceABI = `[{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]`
// Individual event interfaces for constructing ABI from
var SupportsInterface = `{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}`
var AddrChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"a","type":"address"}],"name":"AddrChanged","type":"event"}`
var ContentChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes32"}],"name":"ContentChanged","type":"event"}`
var NameChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"name","type":"string"}],"name":"NameChanged","type":"event"}`
@ -34,11 +33,10 @@ var TextChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"
var MultihashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"MultihashChanged","type":"event"}`
var ContenthashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"ContenthashChanged","type":"event"}`
var StartingBlock = int64(3648359)
// Resolver interface signatures
type Interface int
// Interface enums
const (
MetaSig Interface = iota
AddrChangeSig
@ -51,6 +49,7 @@ const (
ContentHashChangeSig
)
// Hex returns the hex signature for an interface
func (e Interface) Hex() string {
strings := [...]string{
"0x01ffc9a7",
@ -71,6 +70,7 @@ func (e Interface) Hex() string {
return strings[e]
}
// Bytes returns the bytes signature for an interface
func (e Interface) Bytes() [4]uint8 {
if e < MetaSig || e > ContentHashChangeSig {
return [4]byte{}
@ -86,6 +86,7 @@ func (e Interface) Bytes() [4]uint8 {
return byArray
}
// EventSig returns the event signature for an interface
func (e Interface) EventSig() string {
strings := [...]string{
"",
@ -106,6 +107,7 @@ func (e Interface) EventSig() string {
return strings[e]
}
// MethodSig returns the method signature for an interface
func (e Interface) MethodSig() string {
strings := [...]string{
"supportsInterface(bytes4)",

14
pkg/contract_watcher/shared/contract/contract.go
View File

@ -48,6 +48,7 @@ type Contract struct {
Piping bool // Whether or not to pipe method results forward as arguments to subsequent methods
}
// Init initializes a contract object
// If we will be calling methods that use addr, hash, or byte arrays
// as arguments then we initialize maps to hold these types of values
func (c Contract) Init() *Contract {
@ -66,7 +67,7 @@ func (c Contract) Init() *Contract {
return &c
}
// Use contract info to generate event filters - full sync contract watcher only
// GenerateFilters uses contract info to generate event filters - full sync contract watcher only
func (c *Contract) GenerateFilters() error {
c.Filters = map[string]filters.LogFilter{}
@ -87,7 +88,7 @@ func (c *Contract) GenerateFilters() error {
return nil
}
// Returns true if address is in list of arguments to
// WantedEventArg returns true if address is in list of arguments to
// filter events for or if no filtering is specified
func (c *Contract) WantedEventArg(arg string) bool {
if c.FilterArgs == nil {
@ -101,7 +102,7 @@ func (c *Contract) WantedEventArg(arg string) bool {
return false
}
// Returns true if address is in list of arguments to
// WantedMethodArg returns true if address is in list of arguments to
// poll methods with or if no filtering is specified
func (c *Contract) WantedMethodArg(arg interface{}) bool {
if c.MethodArgs == nil {
@ -121,7 +122,7 @@ func (c *Contract) WantedMethodArg(arg interface{}) bool {
return false
}
// Returns true if any mapping value matches filtered for address or if no filter exists
// PassesEventFilter returns true if any mapping value matches filtered for address or if no filter exists
// Used to check if an event log name-value mapping should be filtered or not
func (c *Contract) PassesEventFilter(args map[string]string) bool {
for _, arg := range args {
@ -133,7 +134,7 @@ func (c *Contract) PassesEventFilter(args map[string]string) bool {
return false
}
// Add event emitted address to our list if it passes filter and method polling is on
// AddEmittedAddr adds event emitted addresses to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
for _, addr := range addresses {
if c.WantedMethodArg(addr) && c.Methods != nil {
@ -142,7 +143,7 @@ func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
}
}
// Add event emitted hash to our list if it passes filter and method polling is on
// AddEmittedHash adds event emitted hashes to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedHash(hashes ...interface{}) {
for _, hash := range hashes {
if c.WantedMethodArg(hash) && c.Methods != nil {
@ -151,6 +152,7 @@ func (c *Contract) AddEmittedHash(hashes ...interface{}) {
}
}
// StringifyArg resolves a method argument type to string type
func StringifyArg(arg interface{}) (str string) {
switch arg.(type) {
case string:

14
pkg/contract_watcher/shared/fetcher/fetcher.go
View File

@ -29,7 +29,7 @@ import (
// Fetcher serves as the lower level data fetcher that calls the underlying
// blockchain's FetchConctractData method for a given return type
// Interface definition for a Fetcher
// FetcherInterface is the interface definition for a fetcher
type FetcherInterface interface {
FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error)
FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error)
@ -56,14 +56,14 @@ type fetcherError struct {
fetchMethod string
}
// Fetcher error method
// Error method
func (fe *fetcherError) Error() string {
return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err)
}
// Generic Fetcher methods used by Getters to call contract methods
// Method used to fetch big.Int value from contract
// FetchBigInt is the method used to fetch big.Int value from contract
func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) {
var result = new(big.Int)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@ -75,7 +75,7 @@ func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockN
return *result, nil
}
// Method used to fetch bool value from contract
// FetchBool is the method used to fetch bool value from contract
func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) {
var result = new(bool)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@ -87,7 +87,7 @@ func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNum
return *result, nil
}
// Method used to fetch address value from contract
// FetchAddress is the method used to fetch address value from contract
func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Address, error) {
var result = new(common.Address)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@ -99,7 +99,7 @@ func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, block
return *result, nil
}
// Method used to fetch string value from contract
// FetchString is the method used to fetch string value from contract
func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (string, error) {
var result = new(string)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@ -111,7 +111,7 @@ func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockN
return *result, nil
}
// Method used to fetch hash value from contract
// FetchHash is the method used to fetch hash value from contract
func (f Fetcher) FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error) {
var result = new(common.Hash)
err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)

11
pkg/contract_watcher/shared/getter/interface_getter.go
View File

@ -18,13 +18,15 @@ package getter
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// InterfaceGetter is used to derive the interface of a contract
type InterfaceGetter interface {
GetABI(resolverAddr string, blockNumber int64) string
GetABI(resolverAddr string, blockNumber int64) (string, error)
GetBlockChain() core.BlockChain
}
@ -32,7 +34,8 @@ type interfaceGetter struct {
fetcher.Fetcher
}
func NewInterfaceGetter(blockChain core.BlockChain) *interfaceGetter {
// NewInterfaceGetter returns a new InterfaceGetter
func NewInterfaceGetter(blockChain core.BlockChain) InterfaceGetter {
return &interfaceGetter{
Fetcher: fetcher.Fetcher{
BlockChain: blockChain,
@ -40,7 +43,7 @@ func NewInterfaceGetter(blockChain core.BlockChain) *interfaceGetter {
}
}
// Used to construct a custom ABI based on the results from calling supportsInterface
// GetABI is used to construct a custom ABI based on the results from calling supportsInterface
func (g *interfaceGetter) GetABI(resolverAddr string, blockNumber int64) (string, error) {
a := constants.SupportsInterfaceABI
args := make([]interface{}, 1)
@ -104,7 +107,7 @@ func (g *interfaceGetter) getSupportsInterface(contractAbi, contractAddress stri
return g.Fetcher.FetchBool("supportsInterface", contractAbi, contractAddress, blockNumber, methodArgs)
}
// Method to retrieve the Getter's blockchain
// GetBlockChain is a method to retrieve the Getter's blockchain
func (g *interfaceGetter) GetBlockChain() core.BlockChain {
return g.Fetcher.BlockChain
}

3
pkg/contract_watcher/shared/helpers/helpers.go
View File

@ -27,6 +27,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// ConvertToLog converts a watched event to a log
func ConvertToLog(watchedEvent core.WatchedEvent) types.Log {
allTopics := []string{watchedEvent.Topic0, watchedEvent.Topic1, watchedEvent.Topic2, watchedEvent.Topic3}
var nonNilTopics []string
@ -56,12 +57,14 @@ func createTopics(topics ...string) []common.Hash {
return topicsArray
}
// BigFromString creates a big.Int from a string
func BigFromString(n string) *big.Int {
b := new(big.Int)
b.SetString(n, 10)
return b
}
// GenerateSignature returns the keccak256 hash hex of a string
func GenerateSignature(s string) string {
eventSignature := []byte(s)
hash := crypto.Keccak256Hash(eventSignature)

3
pkg/contract_watcher/shared/helpers/test_helpers/database.go
View File

@ -294,8 +294,7 @@ func TearDown(db *postgres.DB) {
_, err = tx.Exec(`CREATE TABLE checked_headers (
id SERIAL PRIMARY KEY,
header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
check_count INTEGER NOT NULL DEFAULT 1);`)
header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)

17
pkg/contract_watcher/shared/parser/parser.go
View File

@ -45,7 +45,8 @@ type parser struct {
parsedAbi abi.ABI
}
func NewParser(network string) *parser {
// NewParser returns a new Parser
func NewParser(network string) Parser {
url := eth.GenURL(network)
return &parser{
@ -53,15 +54,17 @@ func NewParser(network string) *parser {
}
}
// Abi returns the parser's configured abi string
func (p *parser) Abi() string {
return p.abi
}
// ParsedAbi returns the parser's parsed abi
func (p *parser) ParsedAbi() abi.ABI {
return p.parsedAbi
}
// Retrieves and parses the abi string
// Parse retrieves and parses the abi string
// for the given contract address
func (p *parser) Parse(contractAddr string) error {
// If the abi is one our locally stored abis, fetch
@ -84,7 +87,7 @@ func (p *parser) Parse(contractAddr string) error {
return err
}
// Loads and parses an abi from a given abi string
// ParseAbiStr loads and parses an abi from a given abi string
func (p *parser) ParseAbiStr(abiStr string) error {
var err error
p.abi = abiStr
@ -94,14 +97,14 @@ func (p *parser) ParseAbiStr(abiStr string) error {
}
func (p *parser) lookUp(contractAddr string) (string, error) {
if v, ok := constants.Abis[common.HexToAddress(contractAddr)]; ok {
if v, ok := constants.ABIs[common.HexToAddress(contractAddr)]; ok {
return v, nil
}
return "", errors.New("ABI not present in lookup table")
}
// Returns only specified methods, if they meet the criteria
// GetSelectMethods returns only specified methods, if they meet the criteria
// Returns as array with methods in same order they were specified
// Nil or empty wanted array => no events are returned
func (p *parser) GetSelectMethods(wanted []string) []types.Method {
@ -121,7 +124,7 @@ func (p *parser) GetSelectMethods(wanted []string) []types.Method {
return methods
}
// Returns wanted methods
// GetMethods returns wanted methods
// Empty wanted array => all methods are returned
// Nil wanted array => no methods are returned
func (p *parser) GetMethods(wanted []string) []types.Method {
@ -139,7 +142,7 @@ func (p *parser) GetMethods(wanted []string) []types.Method {
return methods
}
// Returns wanted events as map of types.Events
// GetEvents returns wanted events as map of types.Events
// Empty wanted array => all events are returned
// Nil wanted array => no events are returned
func (p *parser) GetEvents(wanted []string) map[string]types.Event {

20
pkg/contract_watcher/shared/poller/poller.go
View File

@ -33,6 +33,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// Poller is the interface for polling public contract methods
type Poller interface {
PollContract(con contract.Contract, lastBlock int64) error
PollContractAt(con contract.Contract, blockNumber int64) error
@ -45,13 +46,15 @@ type poller struct {
contract contract.Contract
}
func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *poller {
// NewPoller returns a new Poller
func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) Poller {
return &poller{
MethodRepository: repository.NewMethodRepository(db, mode),
bc: blockChain,
}
}
// PollContract polls a contract's public methods from the contracts starting block to specified last block
func (p *poller) PollContract(con contract.Contract, lastBlock int64) error {
for i := con.StartingBlock; i <= lastBlock; i++ {
if err := p.PollContractAt(con, i); err != nil {
@ -62,6 +65,7 @@ func (p *poller) PollContract(con contract.Contract, lastBlock int64) error {
return nil
}
// PollContractAt polls a contract's public getter methods at the specified block height
func (p *poller) PollContractAt(con contract.Contract, blockNumber int64) error {
p.contract = con
for _, m := range con.Methods {
@ -98,7 +102,7 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error {
var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, bn)
if err != nil {
return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
return fmt.Errorf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
}
strOut, err := stringify(out)
if err != nil {
@ -112,7 +116,7 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error {
// Persist result immediately
err = p.PersistResults([]types.Result{result}, m, p.contract.Address, p.contract.Name)
if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
return fmt.Errorf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
}
return nil
@ -148,7 +152,7 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error {
var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
if err != nil {
return errors.New(fmt.Sprintf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
return fmt.Errorf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
}
strOut, err := stringify(out)
if err != nil {
@ -164,7 +168,7 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error {
// Persist result set as batch
err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
return fmt.Errorf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
}
return nil
@ -212,7 +216,7 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error {
var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
if err != nil {
return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
return fmt.Errorf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
}
strOut, err := stringify(out)
if err != nil {
@ -228,13 +232,13 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error {
err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
return fmt.Errorf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
}
return nil
}
// This is just a wrapper around the poller blockchain's FetchContractData method
// FetchContractData is just a wrapper around the poller blockchain's FetchContractData method
func (p *poller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
return p.bc.FetchContractData(contractAbi, contractAddress, method, methodArgs, result, blockNumber)
}

29
pkg/contract_watcher/shared/repository/event_repository.go
View File

@ -24,7 +24,6 @@ import (
"github.com/hashicorp/golang-lru"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
@ -35,7 +34,7 @@ const (
eventCacheSize = 1000
)
// Event repository is used to persist event data into custom tables
// EventRepository is used to persist event data into custom tables
type EventRepository interface {
PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error
CreateEventTable(contractAddr string, event types.Event) (bool, error)
@ -51,7 +50,8 @@ type eventRepository struct {
tables *lru.Cache // Cache names of recently used tables to minimize db connections
}
func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository {
// NewEventRepository returns a new EventRepository
func NewEventRepository(db *postgres.DB, mode types.Mode) EventRepository {
ccs, _ := lru.New(contractCacheSize)
ecs, _ := lru.New(eventCacheSize)
return &eventRepository{
@ -62,7 +62,7 @@ func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository {
}
}
// Creates a schema for the contract if needed
// PersistLogs creates a schema for the contract if needed
// Creates table for the watched contract event if needed
// Persists converted event log data into this custom table
func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
@ -112,7 +112,7 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type
// Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string
data := make([]interface{}, 0, 5+el)
data = append(data,
event.Id,
event.ID,
contractName,
event.Raw,
event.LogIndex,
@ -143,17 +143,6 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type
}
}
// Mark header as checked for this eventId
eventId := strings.ToLower(eventInfo.Name + "_" + contractAddr)
markCheckedErr := repository.MarkContractWatcherHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block
if markCheckedErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
logrus.Warnf("error rolling back transaction while marking header checked: %s", rollbackErr.Error())
}
return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
}
return tx.Commit()
}
@ -171,7 +160,7 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.
data := make([]interface{}, 0, 4+el)
data = append(data,
event.Id,
event.ID,
contractName,
event.Block,
event.Tx)
@ -201,7 +190,7 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.
return tx.Commit()
}
// Checks for event table and creates it if it does not already exist
// CreateEventTable checks for event table and creates it if it does not already exist
// Returns true if it created a new table; returns false if table already existed
func (r *eventRepository) CreateEventTable(contractAddr string, event types.Event) (bool, error) {
tableID := fmt.Sprintf("%s_%s.%s_event", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(event.Name))
@ -270,7 +259,7 @@ func (r *eventRepository) checkForTable(contractAddr string, eventName string) (
return exists, err
}
// Checks for contract schema and creates it if it does not already exist
// CreateContractSchema checks for contract schema and creates it if it does not already exist
// Returns true if it created a new schema; returns false if schema already existed
func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error) {
if contractAddr == "" {
@ -316,10 +305,12 @@ func (r *eventRepository) checkForSchema(contractAddr string) (bool, error) {
return exists, err
}
// CheckSchemaCache is used to query the schema name cache
func (r *eventRepository) CheckSchemaCache(key string) (interface{}, bool) {
return r.schemas.Get(key)
}
// CheckTableCache is used to query the table name cache
func (r *eventRepository) CheckTableCache(key string) (interface{}, bool) {
return r.tables.Get(key)
}

5
pkg/contract_watcher/shared/repository/event_repository_test.go
View File

@ -355,11 +355,6 @@ var _ = Describe("Repository", func() {
Expect(count).To(Equal(2))
})
It("Fails if the persisted event does not have a corresponding eventID column in the checked_headers table", func() {
err = dataStore.PersistLogs(logs, event, con.Address, con.Name)
Expect(err).To(HaveOccurred())
})
It("Fails with empty log", func() {
err = dataStore.PersistLogs([]types.Log{}, event, con.Address, con.Name)
Expect(err).To(HaveOccurred())

12
pkg/contract_watcher/shared/repository/method_repository.go
View File

@ -30,6 +30,7 @@ import (
const methodCacheSize = 1000
// MethodRepository is used to persist public getter method data
type MethodRepository interface {
PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error
CreateMethodTable(contractAddr string, method types.Method) (bool, error)
@ -45,7 +46,8 @@ type methodRepository struct {
tables *lru.Cache // Cache names of recently used tables to minimize db connections
}
func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository {
// NewMethodRepository returns a new MethodRepository
func NewMethodRepository(db *postgres.DB, mode types.Mode) MethodRepository {
ccs, _ := lru.New(contractCacheSize)
mcs, _ := lru.New(methodCacheSize)
return &methodRepository{
@ -56,7 +58,7 @@ func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository {
}
}
// Creates a schema for the contract if needed
// PersistResults creates a schema for the contract if needed
// Creates table for the contract method if needed
// Persists method polling data into this custom table
func (r *methodRepository) PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
@ -124,7 +126,7 @@ func (r *methodRepository) persistResults(results []types.Result, methodInfo typ
return tx.Commit()
}
// Checks for event table and creates it if it does not already exist
// CreateMethodTable checks for event table and creates it if it does not already exist
func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Method) (bool, error) {
tableID := fmt.Sprintf("%s_%s.%s_method", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name))
@ -177,7 +179,7 @@ func (r *methodRepository) checkForTable(contractAddr string, methodName string)
return exists, err
}
// Checks for contract schema and creates it if it does not already exist
// CreateContractSchema checks for contract schema and creates it if it does not already exist
func (r *methodRepository) CreateContractSchema(contractAddr string) (bool, error) {
if contractAddr == "" {
return false, errors.New("error: no contract address specified")
@ -222,10 +224,12 @@ func (r *methodRepository) checkForSchema(contractAddr string) (bool, error) {
return exists, err
}
// CheckSchemaCache is used to query the schema name cache
func (r *methodRepository) CheckSchemaCache(key string) (interface{}, bool) {
return r.schemas.Get(key)
}
// CheckTableCache is used to query the table name cache
func (r *methodRepository) CheckTableCache(key string) (interface{}, bool) {
return r.tables.Get(key)
}

2
pkg/contract_watcher/shared/repository/repository_suite_test.go
View File

@ -17,12 +17,12 @@
package repository_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestRepository(t *testing.T) {

9
pkg/contract_watcher/shared/retriever/address_retriever.go
View File

@ -18,17 +18,17 @@ package retriever
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
"strings"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// Address retriever is used to retrieve the addresses associated with a contract
// AddressRetriever is used to retrieve the addresses associated with a contract
type AddressRetriever interface {
RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error)
}
@ -38,14 +38,15 @@ type addressRetriever struct {
mode types.Mode
}
func NewAddressRetriever(db *postgres.DB, mode types.Mode) (r *addressRetriever) {
// NewAddressRetriever returns a new AddressRetriever
func NewAddressRetriever(db *postgres.DB, mode types.Mode) AddressRetriever {
return &addressRetriever{
db: db,
mode: mode,
}
}
// Method to retrieve list of token-holding/contract-related addresses by iterating over available events
// RetrieveTokenHolderAddresses is used to retrieve list of token-holding/contract-related addresses by iterating over available events
// This generic method should work whether or not the argument/input names of the events meet the expected standard
// This could be generalized to iterate over ALL events and pull out any address arguments
func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) {

2
pkg/contract_watcher/shared/retriever/retriever_suite_test.go
View File

@ -17,12 +17,12 @@
package retriever_test
import (
"github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestRetriever(t *testing.T) {

9
pkg/contract_watcher/shared/types/event.go
View File

@ -25,20 +25,22 @@ import (
"github.com/ethereum/go-ethereum/crypto"
)
// Event is our custom event type
type Event struct {
Name string
Anonymous bool
Fields []Field
}
// Field is our custom event field type which associates a postgres type with the field
type Field struct {
abi.Argument // Name, Type, Indexed
PgType string // Holds type used when committing data held in this field to postgres
}
// Struct to hold instance of an event log data
// Log is used to hold instance of an event log data
type Log struct {
Id int64 // VulcanizeIdLog for full sync and header ID for header sync contract watcher
ID int64 // VulcanizeIdLog for full sync and header ID for header sync contract watcher
Values map[string]string // Map of event input names to their values
// Used for full sync only
@ -51,7 +53,7 @@ type Log struct {
Raw []byte // json.Unmarshalled byte array of geth/core/types.Log{}
}
// Unpack abi.Event into our custom Event struct
// NewEvent unpacks abi.Event into our custom Event struct
func NewEvent(e abi.Event) Event {
fields := make([]Field, len(e.Inputs))
for i, input := range e.Inputs {
@ -85,6 +87,7 @@ func NewEvent(e abi.Event) Event {
}
}
// Sig returns the hash signature for an event
func (e Event) Sig() common.Hash {
types := make([]string, len(e.Fields))

6
pkg/contract_watcher/shared/types/method.go
View File

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/crypto"
)
// Method is our custom method struct
type Method struct {
Name string
Const bool
@ -32,7 +33,7 @@ type Method struct {
Return []Field
}
// Struct to hold instance of result from method call with given inputs and block
// Result is used to hold instance of result from method call with given inputs and block
type Result struct {
Method
Inputs []interface{} // Will only use addresses
@ -41,7 +42,7 @@ type Result struct {
Block int64
}
// Unpack abi.Method into our custom Method struct
// NewMethod unpacks abi.Method into our custom Method struct
func NewMethod(m abi.Method) Method {
inputs := make([]Field, len(m.Inputs))
for i, input := range m.Inputs {
@ -99,6 +100,7 @@ func NewMethod(m abi.Method) Method {
}
}
// Sig returns the hash signature for the method
func (m Method) Sig() common.Hash {
types := make([]string, len(m.Args))
i := 0

29
pkg/contract_watcher/shared/types/mode.go
View File

@ -16,19 +16,21 @@
package types
import "fmt"
// Mode is used to explicitly represent the operating mode of the transformer
type Mode int
// Mode enums
const (
HeaderSync Mode = iota
FullSync
)
// IsValid returns true is the Mode is valid
func (mode Mode) IsValid() bool {
return mode >= HeaderSync && mode <= FullSync
}
// String returns the string representation of the mode
func (mode Mode) String() string {
switch mode {
case HeaderSync:
@ -39,26 +41,3 @@ func (mode Mode) String() string {
return "unknown"
}
}
func (mode Mode) MarshalText() ([]byte, error) {
switch mode {
case HeaderSync:
return []byte("header"), nil
case FullSync:
return []byte("full"), nil
default:
return nil, fmt.Errorf("contract watcher: unknown mode %d, want HeaderSync or FullSync", mode)
}
}
func (mode *Mode) UnmarshalText(text []byte) error {
switch string(text) {
case "header":
*mode = HeaderSync
case "full":
*mode = FullSync
default:
return fmt.Errorf(`contract watcher: unknown mode %q, want "header" or "full"`, text)
}
return nil
}