diff --git a/integration_test/contract_watcher_full_transformer_test.go b/integration_test/contract_watcher_full_transformer_test.go index 4e0cce01..0a606c43 100644 --- a/integration_test/contract_watcher_full_transformer_test.go +++ b/integration_test/contract_watcher_full_transformer_test.go @@ -18,7 +18,6 @@ package integration import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/config" "math/rand" "strings" "time" @@ -27,6 +26,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" diff --git a/integration_test/contract_watcher_header_sync_transformer_test.go b/integration_test/contract_watcher_header_sync_transformer_test.go index 4181b08e..08030134 100644 --- a/integration_test/contract_watcher_header_sync_transformer_test.go +++ b/integration_test/contract_watcher_header_sync_transformer_test.go @@ -40,8 +40,8 @@ var _ = Describe("contractWatcher headerSync transformer", func() { var blockChain core.BlockChain var headerRepository repositories.HeaderRepository var headerID int64 - var ensAddr = strings.ToLower(constants.EnsContractAddress) - var tusdAddr = strings.ToLower(constants.TusdContractAddress) + var ensAddr = strings.ToLower(constants.EnsContractAddress) // 0x314159265dd8dbb310642f98f50c066173c1259b + var tusdAddr = strings.ToLower(constants.TusdContractAddress) // 0x8dd5fbce2f6a956c3022ba3663759011dd51e73e BeforeEach(func() { db, blockChain = test_helpers.SetupDBandBC() @@ -377,6 +377,42 @@ var _ = Describe("contractWatcher headerSync transformer", func() { Expect(transferLog.Value).To(Equal("2800000000000000000000")) }) + It("Marks header checked for a contract that has no logs at that header", func() { + t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(t.Start).To(Equal(int64(6885702))) + + newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog) + Expect(err).ToNot(HaveOccurred()) + transferLog := test_helpers.HeaderSyncTransferLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog) + Expect(err).ToNot(HaveOccurred()) + Expect(transferLog.HeaderID).ToNot(Equal(newOwnerLog.HeaderID)) + + type checkedHeader struct { + ID int64 `db:"id"` + HeaderID int64 `db:"header_id"` + NewOwner int64 `db:"newowner_0x314159265dd8dbb310642f98f50c066173c1259b"` + Transfer int64 `db:"transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e"` + } + + transferCheckedHeader := new(checkedHeader) + err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", transferLog.HeaderID).StructScan(transferCheckedHeader) + Expect(err).ToNot(HaveOccurred()) + Expect(transferCheckedHeader.Transfer).To(Equal(int64(1))) + Expect(transferCheckedHeader.NewOwner).To(Equal(int64(1))) + + newOwnerCheckedHeader := new(checkedHeader) + err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", newOwnerLog.HeaderID).StructScan(newOwnerCheckedHeader) + Expect(err).ToNot(HaveOccurred()) + Expect(newOwnerCheckedHeader.NewOwner).To(Equal(int64(1))) + Expect(newOwnerCheckedHeader.Transfer).To(Equal(int64(1))) + }) + It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { var testConf config.ContractConfig testConf = test_helpers.ENSandTusdConfig diff --git a/libraries/shared/repository/repository.go b/libraries/shared/repository/repository.go deleted file mode 100644 index 155e27ae..00000000 --- a/libraries/shared/repository/repository.go +++ /dev/null @@ -1,27 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package repository - -import "github.com/jmoiron/sqlx" - -func MarkContractWatcherHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error { - _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) - VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1) - return err -} diff --git a/libraries/shared/repository/repository_test.go b/libraries/shared/repository/repository_test.go deleted file mode 100644 index 72a613af..00000000 --- a/libraries/shared/repository/repository_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package repository_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/libraries/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("", func() { - Describe("MarkContractWatcherHeaderCheckedInTransaction", func() { - var ( - checkedHeadersColumn string - db *postgres.DB - ) - - BeforeEach(func() { - db = test_config.NewTestDB(test_config.NewTestNode()) - test_config.CleanTestDB(db) - checkedHeadersColumn = "test_column_checked" - _, migrateErr := db.Exec(`ALTER TABLE public.checked_headers - ADD COLUMN ` + checkedHeadersColumn + ` integer`) - Expect(migrateErr).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - _, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn) - Expect(cleanupMigrateErr).NotTo(HaveOccurred()) - }) - - It("marks passed header as checked within a passed transaction", func() { - headerRepository := repositories.NewHeaderRepository(db) - headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) - Expect(headerErr).NotTo(HaveOccurred()) - tx, txErr := db.Beginx() - Expect(txErr).NotTo(HaveOccurred()) - - err := repository.MarkContractWatcherHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn) - Expect(err).NotTo(HaveOccurred()) - commitErr := tx.Commit() - Expect(commitErr).NotTo(HaveOccurred()) - var checkedCount int - fetchErr := db.Get(&checkedCount, `SELECT COUNT(*) FROM public.checked_headers WHERE header_id = $1`, headerID) - Expect(fetchErr).NotTo(HaveOccurred()) - Expect(checkedCount).To(Equal(1)) - }) - }) -}) diff --git a/pkg/contract_watcher/full/converter/converter.go b/pkg/contract_watcher/full/converter/converter.go index 09bae256..5d47199b 100644 --- a/pkg/contract_watcher/full/converter/converter.go +++ b/pkg/contract_watcher/full/converter/converter.go @@ -17,7 +17,6 @@ package converter import ( - "errors" "fmt" "math/big" "strconv" @@ -32,22 +31,24 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" ) -// Converter is used to convert watched event logs to +// ConverterInterface is used to convert watched event logs to // custom logs containing event input name => value maps type ConverterInterface interface { Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) Update(info *contract.Contract) } +// Converter is the underlying struct for the ConverterInterface type Converter struct { ContractInfo *contract.Contract } +// Update configures the converter for a specific contract func (c *Converter) Update(info *contract.Contract) { c.ContractInfo = info } -// Convert the given watched event log into a types.Log for the given event +// Convert converts the given watched event log into a types.Log for the given event func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) values := make(map[string]interface{}) @@ -88,14 +89,14 @@ func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) ( b := input.(byte) strValues[fieldName] = string(b) default: - return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input)) + return nil, fmt.Errorf("error: unhandled abi type %T", input) } } // Only hold onto logs that pass our address filter, if any if c.ContractInfo.PassesEventFilter(strValues) { eventLog := &types.Log{ - Id: watchedEvent.LogID, + ID: watchedEvent.LogID, Values: strValues, Block: watchedEvent.BlockNumber, Tx: watchedEvent.TxHash, diff --git a/pkg/contract_watcher/full/retriever/block_retriever.go b/pkg/contract_watcher/full/retriever/block_retriever.go index 24c41003..0003e4c3 100644 --- a/pkg/contract_watcher/full/retriever/block_retriever.go +++ b/pkg/contract_watcher/full/retriever/block_retriever.go @@ -18,11 +18,12 @@ package retriever import ( "database/sql" + "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -// Block retriever is used to retrieve the first block for a given contract and the most recent block +// BlockRetriever is used to retrieve the first block for a given contract and the most recent block // It requires a vDB synced database with blocks, transactions, receipts, and logs type BlockRetriever interface { RetrieveFirstBlock(contractAddr string) (int64, error) @@ -33,13 +34,15 @@ type blockRetriever struct { db *postgres.DB } -func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) { +// NewBlockRetriever returns a new BlockRetriever +func NewBlockRetriever(db *postgres.DB) BlockRetriever { return &blockRetriever{ db: db, } } -// Try both methods of finding the first block, with the receipt method taking precedence +// RetrieveFirstBlock fetches the block number for the earliest block in the db +// Tries both methods of finding the first block, with the receipt method taking precedence func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) { i, err := r.retrieveFirstBlockFromReceipts(contractAddr) if err != nil { @@ -55,7 +58,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai) func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) { var firstBlock int64 - addressId, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr) + addressID, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr) if getAddressErr != nil { return firstBlock, getAddressErr } @@ -66,7 +69,7 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in WHERE contract_address_id = $1 ORDER BY block_id ASC LIMIT 1)`, - addressId, + addressID, ) return firstBlock, err @@ -84,7 +87,7 @@ func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, return int64(firstBlock), err } -// Method to retrieve the most recent block in vDB +// RetrieveMostRecentBlock retrieves the most recent block number in vDB func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) { var lastBlock int64 err := r.db.Get( diff --git a/pkg/contract_watcher/full/retriever/block_retriever_test.go b/pkg/contract_watcher/full/retriever/block_retriever_test.go index 88b171f8..adf55bf9 100644 --- a/pkg/contract_watcher/full/retriever/block_retriever_test.go +++ b/pkg/contract_watcher/full/retriever/block_retriever_test.go @@ -17,10 +17,11 @@ package retriever_test import ( + "strings" + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "strings" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" diff --git a/pkg/contract_watcher/full/retriever/retriever_suite_test.go b/pkg/contract_watcher/full/retriever/retriever_suite_test.go index 005d2817..2f97ce0a 100644 --- a/pkg/contract_watcher/full/retriever/retriever_suite_test.go +++ b/pkg/contract_watcher/full/retriever/retriever_suite_test.go @@ -17,12 +17,12 @@ package retriever_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" ) func TestRetriever(t *testing.T) { diff --git a/pkg/contract_watcher/full/transformer/transformer.go b/pkg/contract_watcher/full/transformer/transformer.go index dbc1dced..005c1fc7 100644 --- a/pkg/contract_watcher/full/transformer/transformer.go +++ b/pkg/contract_watcher/full/transformer/transformer.go @@ -35,6 +35,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" ) +// Transformer is the top level struct for transforming watched contract data // Requires a fully synced vDB and a running eth node (or infura) type Transformer struct { // Database interfaces @@ -60,7 +61,7 @@ type Transformer struct { LastBlock int64 } -// Transformer takes in config for blockchain, database, and network id +// NewTransformer takes in contract config, blockchain, and database, and returns a new Transformer func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer { return &Transformer{ Poller: poller.NewPoller(BC, DB, types.FullSync), @@ -75,6 +76,7 @@ func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres. } } +// Init initializes the transformer // Use after creating and setting transformer // Loops over all of the addr => filter sets // Uses parser to pull event info from abi @@ -167,6 +169,7 @@ func (tr *Transformer) Init() error { return nil } +// Execute runs the transformation processes // Iterates through stored, initialized contract objects // Iterates through contract's event filters, grabbing watched event logs // Uses converter to convert logs into custom log type @@ -227,6 +230,7 @@ func (tr *Transformer) Execute() error { return nil } +// GetConfig returns the transformers config; satisfies the transformer interface func (tr *Transformer) GetConfig() config.ContractConfig { return tr.Config } diff --git a/pkg/contract_watcher/full/transformer/transformer_suite_test.go b/pkg/contract_watcher/full/transformer/transformer_suite_test.go index 42d0ffb0..aac31e85 100644 --- a/pkg/contract_watcher/full/transformer/transformer_suite_test.go +++ b/pkg/contract_watcher/full/transformer/transformer_suite_test.go @@ -17,12 +17,12 @@ package transformer_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" ) func TestTransformer(t *testing.T) { diff --git a/pkg/contract_watcher/header/converter/converter.go b/pkg/contract_watcher/header/converter/converter.go index afc04212..5e263ed2 100644 --- a/pkg/contract_watcher/header/converter/converter.go +++ b/pkg/contract_watcher/header/converter/converter.go @@ -18,7 +18,6 @@ package converter import ( "encoding/json" - "errors" "fmt" "math/big" "strconv" @@ -32,16 +31,19 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" ) +// ConverterInterface is the interface for converting geth logs to our custom log type type ConverterInterface interface { Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) Update(info *contract.Contract) } +// Converter is the underlying struct for the ConverterInterface type Converter struct { ContractInfo *contract.Contract } +// Update is used to configure the converter with a specific contract func (c *Converter) Update(info *contract.Contract) { c.ContractInfo = info } @@ -98,7 +100,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in strValues[fieldName] = converted.String() seenHashes = append(seenHashes, converted) default: - return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input)) + return nil, fmt.Errorf("error: unhandled abi type %T", input) } } @@ -114,7 +116,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in Values: strValues, Raw: raw, TransactionIndex: log.TxIndex, - Id: headerID, + ID: headerID, }) // Cache emitted values if their caching is turned on @@ -130,7 +132,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in return returnLogs, nil } -// Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs +// ConvertBatch converts the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) { boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) eventsToLogs := make(map[string][]types.Log) @@ -182,7 +184,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E strValues[fieldName] = converted.String() seenHashes = append(seenHashes, converted) default: - return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input)) + return nil, fmt.Errorf("error: unhandled abi type %T", input) } } @@ -198,7 +200,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E Values: strValues, Raw: raw, TransactionIndex: log.TxIndex, - Id: headerID, + ID: headerID, }) // Cache emitted values that pass the argument filter if their caching is turned on diff --git a/pkg/contract_watcher/header/converter/converter_test.go b/pkg/contract_watcher/header/converter/converter_test.go index 72cfa59a..9c7d6f68 100644 --- a/pkg/contract_watcher/header/converter/converter_test.go +++ b/pkg/contract_watcher/header/converter/converter_test.go @@ -72,11 +72,11 @@ var _ = Describe("Converter", func() { Expect(logs[0].Values["to"]).To(Equal(sender1.String())) Expect(logs[0].Values["from"]).To(Equal(sender2.String())) Expect(logs[0].Values["value"]).To(Equal(value.String())) - Expect(logs[0].Id).To(Equal(int64(232))) + Expect(logs[0].ID).To(Equal(int64(232))) Expect(logs[1].Values["to"]).To(Equal(sender2.String())) Expect(logs[1].Values["from"]).To(Equal(sender1.String())) Expect(logs[1].Values["value"]).To(Equal(value.String())) - Expect(logs[1].Id).To(Equal(int64(232))) + Expect(logs[1].ID).To(Equal(int64(232))) }) It("Keeps track of addresses it sees if they will be used for method polling", func() { diff --git a/pkg/contract_watcher/header/fetcher/fetcher.go b/pkg/contract_watcher/header/fetcher/fetcher.go index bfecf807..afa9aade 100644 --- a/pkg/contract_watcher/header/fetcher/fetcher.go +++ b/pkg/contract_watcher/header/fetcher/fetcher.go @@ -24,6 +24,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" ) +// Fetcher is the fetching interface type Fetcher interface { FetchLogs(contractAddresses []string, topics []common.Hash, missingHeader core.Header) ([]types.Log, error) } @@ -32,13 +33,14 @@ type fetcher struct { blockChain core.BlockChain } -func NewFetcher(blockchain core.BlockChain) *fetcher { +// NewFetcher returns a new Fetcher +func NewFetcher(blockchain core.BlockChain) Fetcher { return &fetcher{ blockChain: blockchain, } } -// Checks all topic0s, on all addresses, fetching matching logs for the given header +// FetchLogs checks all topic0s, on all addresses, fetching matching logs for the given header func (fetcher *fetcher) FetchLogs(contractAddresses []string, topic0s []common.Hash, header core.Header) ([]types.Log, error) { addresses := hexStringsToAddresses(contractAddresses) blockHash := common.HexToHash(header.Hash) diff --git a/pkg/contract_watcher/header/repository/header_repository.go b/pkg/contract_watcher/header/repository/header_repository.go index 4ad817f0..4a3fd8d7 100644 --- a/pkg/contract_watcher/header/repository/header_repository.go +++ b/pkg/contract_watcher/header/repository/header_repository.go @@ -20,7 +20,6 @@ import ( "fmt" "github.com/hashicorp/golang-lru" - "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -29,6 +28,7 @@ import ( const columnCacheSize = 1000 +// HeaderRepository interfaces with the header and checked_headers tables type HeaderRepository interface { AddCheckColumn(id string) error AddCheckColumns(ids []string) error @@ -46,7 +46,8 @@ type headerRepository struct { columns *lru.Cache // Cache created columns to minimize db connections } -func NewHeaderRepository(db *postgres.DB) *headerRepository { +// NewHeaderRepository returns a new HeaderRepository +func NewHeaderRepository(db *postgres.DB) HeaderRepository { ccs, _ := lru.New(columnCacheSize) return &headerRepository{ db: db, @@ -54,7 +55,7 @@ func NewHeaderRepository(db *postgres.DB) *headerRepository { } } -// Adds a checked_header column for the provided column id +// AddCheckColumn adds a checked_header column for the provided column id func (r *headerRepository) AddCheckColumn(id string) error { // Check cache to see if column already exists before querying pg _, ok := r.columns.Get(id) @@ -75,7 +76,7 @@ func (r *headerRepository) AddCheckColumn(id string) error { return nil } -// Adds a checked_header column for all of the provided column ids +// AddCheckColumns adds a checked_header column for all of the provided column ids func (r *headerRepository) AddCheckColumns(ids []string) error { var err error baseQuery := "ALTER TABLE public.checked_headers" @@ -99,7 +100,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error { return err } -// Marks the header checked for the provided column id +// MarkHeaderChecked marks the header checked for the provided column id func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error { _, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`) VALUES ($1, $2) @@ -108,7 +109,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error { return err } -// Marks the header checked for all of the provided column ids +// MarkHeaderCheckedForAll marks the header checked for all of the provided column ids func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error { pgStr := "INSERT INTO public.checked_headers (header_id, " for _, id := range ids { @@ -127,7 +128,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) return err } -// Marks all of the provided headers checked for each of the provided column ids +// MarkHeadersCheckedForAll marks all of the provided headers checked for each of the provided column ids func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error { tx, err := r.db.Beginx() if err != nil { @@ -160,7 +161,7 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [ return err } -// Returns missing headers for the provided checked_headers column id +// MissingHeaders returns missing headers for the provided checked_headers column id func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) { var result []core.Header var query string @@ -186,7 +187,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber return continuousHeaders(result), err } -// Returns missing headers for all of the provided checked_headers column ids +// MissingHeadersForAll returns missing headers for all of the provided checked_headers column ids func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) { var result []core.Header var query string @@ -214,7 +215,7 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock return continuousHeaders(result), err } -// Returns headers that have been checked for all of the provided event ids but not for the provided method ids +// MissingMethodsCheckedEventsIntersection returns headers that have been checked for all of the provided event ids but not for the provided method ids func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) { var result []core.Header var query string @@ -264,16 +265,7 @@ func continuousHeaders(headers []core.Header) []core.Header { return headers } -// Check the repositories column id cache for a value +// CheckCache checks the repositories column id cache for a value func (r *headerRepository) CheckCache(key string) (interface{}, bool) { return r.columns.Get(key) } - -// Used to mark a header checked as part of some external transaction so as to group into one commit -func (r *headerRepository) MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error { - _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) - VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET `+eventID+` = checked_headers.`+eventID+` + 1`, headerID, 1) - return err -} diff --git a/pkg/contract_watcher/header/repository/header_repository_test.go b/pkg/contract_watcher/header/repository/header_repository_test.go index 327afb59..9084e40f 100644 --- a/pkg/contract_watcher/header/repository/header_repository_test.go +++ b/pkg/contract_watcher/header/repository/header_repository_test.go @@ -18,7 +18,6 @@ package repository_test import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -26,6 +25,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" ) diff --git a/pkg/contract_watcher/header/repository/repository_suite_test.go b/pkg/contract_watcher/header/repository/repository_suite_test.go index 6e8de467..87726ebd 100644 --- a/pkg/contract_watcher/header/repository/repository_suite_test.go +++ b/pkg/contract_watcher/header/repository/repository_suite_test.go @@ -17,12 +17,12 @@ package repository_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" ) func TestRepository(t *testing.T) { diff --git a/pkg/contract_watcher/header/retriever/block_retriever.go b/pkg/contract_watcher/header/retriever/block_retriever.go index 47691013..8362d483 100644 --- a/pkg/contract_watcher/header/retriever/block_retriever.go +++ b/pkg/contract_watcher/header/retriever/block_retriever.go @@ -20,7 +20,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -// Block retriever is used to retrieve the first block for a given contract and the most recent block +// BlockRetriever is used to retrieve the first block for a given contract and the most recent block // It requires a vDB synced database with blocks, transactions, receipts, and logs type BlockRetriever interface { RetrieveFirstBlock() (int64, error) @@ -31,13 +31,14 @@ type blockRetriever struct { db *postgres.DB } -func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) { +// NewBlockRetriever returns a new BlockRetriever +func NewBlockRetriever(db *postgres.DB) BlockRetriever { return &blockRetriever{ db: db, } } -// Retrieve block number of earliest header in repo +// RetrieveFirstBlock retrieves block number of earliest header in repo func (r *blockRetriever) RetrieveFirstBlock() (int64, error) { var firstBlock int err := r.db.Get( @@ -48,7 +49,7 @@ func (r *blockRetriever) RetrieveFirstBlock() (int64, error) { return int64(firstBlock), err } -// Retrieve block number of latest header in repo +// RetrieveMostRecentBlock retrieves block number of latest header in repo func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) { var lastBlock int err := r.db.Get( diff --git a/pkg/contract_watcher/header/retriever/retriever_suite_test.go b/pkg/contract_watcher/header/retriever/retriever_suite_test.go index 72bc1657..f6d4967e 100644 --- a/pkg/contract_watcher/header/retriever/retriever_suite_test.go +++ b/pkg/contract_watcher/header/retriever/retriever_suite_test.go @@ -17,12 +17,12 @@ package retriever_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" ) func TestRetriever(t *testing.T) { diff --git a/pkg/contract_watcher/header/transformer/transformer.go b/pkg/contract_watcher/header/transformer/transformer.go index 34b3ddc1..f3fe0188 100644 --- a/pkg/contract_watcher/header/transformer/transformer.go +++ b/pkg/contract_watcher/header/transformer/transformer.go @@ -40,6 +40,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) +// Transformer is the top level struct for transforming watched contract data // Requires a header synced vDB (headers) and a running eth node (or infura) type Transformer struct { // Database interfaces @@ -76,7 +77,7 @@ type Transformer struct { // 3. Init // 4. Execute -// Transformer takes in config for blockchain, database, and network id +// NewTransformer takes in a contract config, blockchain, and database, and returns a new Transformer func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer { return &Transformer{ @@ -92,6 +93,7 @@ func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres. } } +// Init initialized the Transformer // Use after creating and setting transformer // Loops over all of the addr => filter sets // Uses parser to pull event info from abi @@ -126,6 +128,7 @@ func (tr *Transformer) Init() error { firstBlock, retrieveErr := tr.Retriever.RetrieveFirstBlock() if retrieveErr != nil { if retrieveErr == sql.ErrNoRows { + logrus.Error(fmt.Errorf("error retrieving first block: %s", retrieveErr.Error())) firstBlock = 0 } else { return fmt.Errorf("error retrieving first block: %s", retrieveErr.Error()) @@ -175,14 +178,14 @@ func (tr *Transformer) Init() error { // Create checked_headers columns for each event id and append to list of all event ids tr.sortedEventIds[con.Address] = make([]string, 0, len(con.Events)) for _, event := range con.Events { - eventId := strings.ToLower(event.Name + "_" + con.Address) - addColumnErr := tr.HeaderRepository.AddCheckColumn(eventId) + eventID := strings.ToLower(event.Name + "_" + con.Address) + addColumnErr := tr.HeaderRepository.AddCheckColumn(eventID) if addColumnErr != nil { return fmt.Errorf("error adding check column: %s", addColumnErr.Error()) } // Keep track of this event id; sorted and unsorted - tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventId) - tr.eventIds = append(tr.eventIds, eventId) + tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventID) + tr.eventIds = append(tr.eventIds, eventID) // Append this event sig to the filters tr.eventFilters = append(tr.eventFilters, event.Sig()) } @@ -190,12 +193,12 @@ func (tr *Transformer) Init() error { // Create checked_headers columns for each method id and append list of all method ids tr.sortedMethodIds[con.Address] = make([]string, 0, len(con.Methods)) for _, m := range con.Methods { - methodId := strings.ToLower(m.Name + "_" + con.Address) - addColumnErr := tr.HeaderRepository.AddCheckColumn(methodId) + methodID := strings.ToLower(m.Name + "_" + con.Address) + addColumnErr := tr.HeaderRepository.AddCheckColumn(methodID) if addColumnErr != nil { return fmt.Errorf("error adding check column: %s", addColumnErr.Error()) } - tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodId) + tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodID) } // Update start to the lowest block @@ -207,6 +210,7 @@ func (tr *Transformer) Init() error { return nil } +// Execute runs the transformation processes func (tr *Transformer) Execute() error { if len(tr.Contracts) == 0 { return errors.New("error: transformer has no initialized contracts") @@ -248,7 +252,6 @@ func (tr *Transformer) Execute() error { continue } - // Sort logs by the contract they belong to for _, log := range allLogs { addr := strings.ToLower(log.Address.Hex()) sortedLogs[addr] = append(sortedLogs[addr], log) @@ -273,16 +276,10 @@ func (tr *Transformer) Execute() error { for eventName, logs := range convertedLogs { // If logs for this event are empty, mark them checked at this header and continue if len(logs) < 1 { - eventId := strings.ToLower(eventName + "_" + con.Address) - markCheckedErr := tr.HeaderRepository.MarkHeaderChecked(header.Id, eventId) - if markCheckedErr != nil { - return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error()) - } logrus.Tracef("no logs found for event %s on contract %s at block %d, continuing", eventName, conAddr, header.BlockNumber) continue } // If logs aren't empty, persist them - // Header is marked checked in the transactions persistErr := tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name) if persistErr != nil { return fmt.Errorf("error persisting logs: %s", persistErr.Error()) @@ -290,6 +287,11 @@ func (tr *Transformer) Execute() error { } } + markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, tr.eventIds) + if markCheckedErr != nil { + return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error()) + } + // Poll contracts at this block height pollingErr := tr.methodPolling(header, tr.sortedMethodIds) if pollingErr != nil { @@ -328,6 +330,7 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str return nil } +// GetConfig returns the transformers config; satisfies the transformer interface func (tr *Transformer) GetConfig() config.ContractConfig { return tr.Config } diff --git a/pkg/contract_watcher/header/transformer/transformer_suite_test.go b/pkg/contract_watcher/header/transformer/transformer_suite_test.go index 7d799896..56bcc01c 100644 --- a/pkg/contract_watcher/header/transformer/transformer_suite_test.go +++ b/pkg/contract_watcher/header/transformer/transformer_suite_test.go @@ -17,12 +17,12 @@ package transformer_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" ) func TestTransformer(t *testing.T) { diff --git a/pkg/contract_watcher/header/transformer/transformer_test.go b/pkg/contract_watcher/header/transformer/transformer_test.go index be1e2db2..bd448000 100644 --- a/pkg/contract_watcher/header/transformer/transformer_test.go +++ b/pkg/contract_watcher/header/transformer/transformer_test.go @@ -18,6 +18,7 @@ package transformer_test import ( "database/sql" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/contract_watcher/shared/constants/constants.go b/pkg/contract_watcher/shared/constants/constants.go index 86c6cd9a..b47d22ae 100644 --- a/pkg/contract_watcher/shared/constants/constants.go +++ b/pkg/contract_watcher/shared/constants/constants.go @@ -34,6 +34,7 @@ const ( NewOwnerEvent Event = 4 ) +// String returns the string name for an event func (e Event) String() string { strings := [...]string{ "Transfer", @@ -50,6 +51,7 @@ func (e Event) String() string { return strings[e] } +// Signature returns the keccak256 signature for an event func (e Event) Signature() string { strings := [...]string{ helpers.GenerateSignature("Transfer(address,address,uint256)"), @@ -74,10 +76,6 @@ var MarketPlaceContractAddress = "0x8e5660b4Ab70168b5a6fEeA0e0315cb49c8Cd539" var MolochContractAddress = "0x1fd169A4f5c59ACf79d0Fd5d91D1201EF1Bce9f1" var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5" -// Contract Owner -var DaiContractOwner = "0x0000000000000000000000000000000000000000" -var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f" - // Contract Abis var DaiAbiString = `[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"stop","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"mint","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wad","type":"uint256"}],"name":"burn","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"name_","type":"bytes32"}],"name":"setName","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"src","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"stopped","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"burn","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wad","type":"uint256"}],"name":"mint","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transfer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"push","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"move","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"start","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"src","type":"address"},{"name":"guy","type":"address"}],"name":"allowance","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"wad","type":"uint256"}],"name":"pull","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"symbol_","type":"bytes32"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Burn","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"dst","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Transfer","type":"event"}]` @@ -89,8 +87,8 @@ var MarketPlaceAbiString = `[{"constant":false,"inputs":[{"name":"_ownerCutPerMi var MolochAbiString = `[{"constant":true,"inputs":[],"name":"processingReward","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"memberAddress","type":"address"},{"name":"proposalIndex","type":"uint256"}],"name":"getMemberProposalVote","outputs":[{"name":"","type":"uint8"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getCurrentPeriod","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"members","outputs":[{"name":"delegateKey","type":"address"},{"name":"shares","type":"uint256"},{"name":"exists","type":"bool"},{"name":"highestIndexYesVote","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"totalSharesRequested","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"newDelegateKey","type":"address"}],"name":"updateDelegateKey","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalShares","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"uint256"}],"name":"proposalQueue","outputs":[{"name":"proposer","type":"address"},{"name":"applicant","type":"address"},{"name":"sharesRequested","type":"uint256"},{"name":"startingPeriod","type":"uint256"},{"name":"yesVotes","type":"uint256"},{"name":"noVotes","type":"uint256"},{"name":"processed","type":"bool"},{"name":"didPass","type":"bool"},{"name":"aborted","type":"bool"},{"name":"tokenTribute","type":"uint256"},{"name":"details","type":"string"},{"name":"maxTotalSharesAtYesVote","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"memberAddressByDelegateKey","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"gracePeriodLength","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"abortWindow","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getProposalQueueLength","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"summoningTime","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"votingPeriodLength","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"sharesToBurn","type":"uint256"}],"name":"ragequit","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"proposalDeposit","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"startingPeriod","type":"uint256"}],"name":"hasVotingPeriodExpired","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"applicant","type":"address"},{"name":"tokenTribute","type":"uint256"},{"name":"sharesRequested","type":"uint256"},{"name":"details","type":"string"}],"name":"submitProposal","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"proposalIndex","type":"uint256"},{"name":"uintVote","type":"uint8"}],"name":"submitVote","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"highestIndexYesVote","type":"uint256"}],"name":"canRagequit","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"guildBank","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"dilutionBound","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"periodDuration","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"approvedToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"proposalIndex","type":"uint256"}],"name":"abort","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"proposalIndex","type":"uint256"}],"name":"processProposal","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"summoner","type":"address"},{"name":"_approvedToken","type":"address"},{"name":"_periodDuration","type":"uint256"},{"name":"_votingPeriodLength","type":"uint256"},{"name":"_gracePeriodLength","type":"uint256"},{"name":"_abortWindow","type":"uint256"},{"name":"_proposalDeposit","type":"uint256"},{"name":"_dilutionBound","type":"uint256"},{"name":"_processingReward","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":false,"name":"proposalIndex","type":"uint256"},{"indexed":true,"name":"delegateKey","type":"address"},{"indexed":true,"name":"memberAddress","type":"address"},{"indexed":true,"name":"applicant","type":"address"},{"indexed":false,"name":"tokenTribute","type":"uint256"},{"indexed":false,"name":"sharesRequested","type":"uint256"}],"name":"SubmitProposal","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"proposalIndex","type":"uint256"},{"indexed":true,"name":"delegateKey","type":"address"},{"indexed":true,"name":"memberAddress","type":"address"},{"indexed":false,"name":"uintVote","type":"uint8"}],"name":"SubmitVote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"proposalIndex","type":"uint256"},{"indexed":true,"name":"applicant","type":"address"},{"indexed":true,"name":"memberAddress","type":"address"},{"indexed":false,"name":"tokenTribute","type":"uint256"},{"indexed":false,"name":"sharesRequested","type":"uint256"},{"indexed":false,"name":"didPass","type":"bool"}],"name":"ProcessProposal","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"memberAddress","type":"address"},{"indexed":false,"name":"sharesToBurn","type":"uint256"}],"name":"Ragequit","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"proposalIndex","type":"uint256"},{"indexed":false,"name":"applicantAddress","type":"address"}],"name":"Abort","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"memberAddress","type":"address"},{"indexed":false,"name":"newDelegateKey","type":"address"}],"name":"UpdateDelegateKey","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"summoner","type":"address"},{"indexed":false,"name":"shares","type":"uint256"}],"name":"SummonComplete","type":"event"}]` -// Look-up table for ABI strings -var Abis = map[common.Address]string{ +// ABIs is a look-up table for ABI strings +var ABIs = map[common.Address]string{ common.HexToAddress("0x314159265dD8dbb310642f98f50C066173C1259b"): ENSAbiString, common.HexToAddress("0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"): TusdAbiString, common.HexToAddress("0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"): DaiAbiString, @@ -98,23 +96,6 @@ var Abis = map[common.Address]string{ // Filters // To add additional filter parameters, filter by other Topics e.g. for a Transfer event filter Topics[1] to filter for a specific 'from' address -var DaiERC20Filters = []filters.LogFilter{ - { - Name: TransferEvent.String(), - FromBlock: 4752008, - ToBlock: -1, - Address: DaiContractAddress, - Topics: core.Topics{TransferEvent.Signature()}, - }, - { - Name: ApprovalEvent.String(), - FromBlock: 4752008, - ToBlock: -1, - Address: DaiContractAddress, - Topics: core.Topics{ApprovalEvent.Signature()}, - }, -} - var TusdGenericFilters = []filters.LogFilter{ { Name: BurnEvent.String(), diff --git a/pkg/contract_watcher/shared/constants/interface.go b/pkg/contract_watcher/shared/constants/interface.go index 5e383f28..7063bb99 100644 --- a/pkg/contract_watcher/shared/constants/interface.go +++ b/pkg/contract_watcher/shared/constants/interface.go @@ -20,11 +20,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" ) -// Basic abi needed to check which interfaces are adhered to +// SupportsInterfaceABI is the basic abi needed to check which interfaces are adhered to var SupportsInterfaceABI = `[{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]` // Individual event interfaces for constructing ABI from -var SupportsInterface = `{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}` var AddrChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"a","type":"address"}],"name":"AddrChanged","type":"event"}` var ContentChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes32"}],"name":"ContentChanged","type":"event"}` var NameChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"name","type":"string"}],"name":"NameChanged","type":"event"}` @@ -34,11 +33,10 @@ var TextChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":" var MultihashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"MultihashChanged","type":"event"}` var ContenthashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"ContenthashChanged","type":"event"}` -var StartingBlock = int64(3648359) - // Resolver interface signatures type Interface int +// Interface enums const ( MetaSig Interface = iota AddrChangeSig @@ -51,6 +49,7 @@ const ( ContentHashChangeSig ) +// Hex returns the hex signature for an interface func (e Interface) Hex() string { strings := [...]string{ "0x01ffc9a7", @@ -71,6 +70,7 @@ func (e Interface) Hex() string { return strings[e] } +// Bytes returns the bytes signature for an interface func (e Interface) Bytes() [4]uint8 { if e < MetaSig || e > ContentHashChangeSig { return [4]byte{} @@ -86,6 +86,7 @@ func (e Interface) Bytes() [4]uint8 { return byArray } +// EventSig returns the event signature for an interface func (e Interface) EventSig() string { strings := [...]string{ "", @@ -106,6 +107,7 @@ func (e Interface) EventSig() string { return strings[e] } +// MethodSig returns the method signature for an interface func (e Interface) MethodSig() string { strings := [...]string{ "supportsInterface(bytes4)", diff --git a/pkg/contract_watcher/shared/contract/contract.go b/pkg/contract_watcher/shared/contract/contract.go index 6b3eb0ff..1bb5ce3d 100644 --- a/pkg/contract_watcher/shared/contract/contract.go +++ b/pkg/contract_watcher/shared/contract/contract.go @@ -48,6 +48,7 @@ type Contract struct { Piping bool // Whether or not to pipe method results forward as arguments to subsequent methods } +// Init initializes a contract object // If we will be calling methods that use addr, hash, or byte arrays // as arguments then we initialize maps to hold these types of values func (c Contract) Init() *Contract { @@ -66,7 +67,7 @@ func (c Contract) Init() *Contract { return &c } -// Use contract info to generate event filters - full sync contract watcher only +// GenerateFilters uses contract info to generate event filters - full sync contract watcher only func (c *Contract) GenerateFilters() error { c.Filters = map[string]filters.LogFilter{} @@ -87,7 +88,7 @@ func (c *Contract) GenerateFilters() error { return nil } -// Returns true if address is in list of arguments to +// WantedEventArg returns true if address is in list of arguments to // filter events for or if no filtering is specified func (c *Contract) WantedEventArg(arg string) bool { if c.FilterArgs == nil { @@ -101,7 +102,7 @@ func (c *Contract) WantedEventArg(arg string) bool { return false } -// Returns true if address is in list of arguments to +// WantedMethodArg returns true if address is in list of arguments to // poll methods with or if no filtering is specified func (c *Contract) WantedMethodArg(arg interface{}) bool { if c.MethodArgs == nil { @@ -121,7 +122,7 @@ func (c *Contract) WantedMethodArg(arg interface{}) bool { return false } -// Returns true if any mapping value matches filtered for address or if no filter exists +// PassesEventFilter returns true if any mapping value matches filtered for address or if no filter exists // Used to check if an event log name-value mapping should be filtered or not func (c *Contract) PassesEventFilter(args map[string]string) bool { for _, arg := range args { @@ -133,7 +134,7 @@ func (c *Contract) PassesEventFilter(args map[string]string) bool { return false } -// Add event emitted address to our list if it passes filter and method polling is on +// AddEmittedAddr adds event emitted addresses to our list if it passes filter and method polling is on func (c *Contract) AddEmittedAddr(addresses ...interface{}) { for _, addr := range addresses { if c.WantedMethodArg(addr) && c.Methods != nil { @@ -142,7 +143,7 @@ func (c *Contract) AddEmittedAddr(addresses ...interface{}) { } } -// Add event emitted hash to our list if it passes filter and method polling is on +// AddEmittedHash adds event emitted hashes to our list if it passes filter and method polling is on func (c *Contract) AddEmittedHash(hashes ...interface{}) { for _, hash := range hashes { if c.WantedMethodArg(hash) && c.Methods != nil { @@ -151,6 +152,7 @@ func (c *Contract) AddEmittedHash(hashes ...interface{}) { } } +// StringifyArg resolves a method argument type to string type func StringifyArg(arg interface{}) (str string) { switch arg.(type) { case string: diff --git a/pkg/contract_watcher/shared/fetcher/fetcher.go b/pkg/contract_watcher/shared/fetcher/fetcher.go index 7507628f..db4c6561 100644 --- a/pkg/contract_watcher/shared/fetcher/fetcher.go +++ b/pkg/contract_watcher/shared/fetcher/fetcher.go @@ -29,7 +29,7 @@ import ( // Fetcher serves as the lower level data fetcher that calls the underlying // blockchain's FetchConctractData method for a given return type -// Interface definition for a Fetcher +// FetcherInterface is the interface definition for a fetcher type FetcherInterface interface { FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) @@ -56,14 +56,14 @@ type fetcherError struct { fetchMethod string } -// Fetcher error method +// Error method func (fe *fetcherError) Error() string { return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err) } // Generic Fetcher methods used by Getters to call contract methods -// Method used to fetch big.Int value from contract +// FetchBigInt is the method used to fetch big.Int value from contract func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) { var result = new(big.Int) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) @@ -75,7 +75,7 @@ func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockN return *result, nil } -// Method used to fetch bool value from contract +// FetchBool is the method used to fetch bool value from contract func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) { var result = new(bool) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) @@ -87,7 +87,7 @@ func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNum return *result, nil } -// Method used to fetch address value from contract +// FetchAddress is the method used to fetch address value from contract func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Address, error) { var result = new(common.Address) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) @@ -99,7 +99,7 @@ func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, block return *result, nil } -// Method used to fetch string value from contract +// FetchString is the method used to fetch string value from contract func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (string, error) { var result = new(string) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) @@ -111,7 +111,7 @@ func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockN return *result, nil } -// Method used to fetch hash value from contract +// FetchHash is the method used to fetch hash value from contract func (f Fetcher) FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error) { var result = new(common.Hash) err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber) diff --git a/pkg/contract_watcher/shared/getter/interface_getter.go b/pkg/contract_watcher/shared/getter/interface_getter.go index 5cf39865..273940e9 100644 --- a/pkg/contract_watcher/shared/getter/interface_getter.go +++ b/pkg/contract_watcher/shared/getter/interface_getter.go @@ -18,13 +18,15 @@ package getter import ( "fmt" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/fetcher" "github.com/vulcanize/vulcanizedb/pkg/core" ) +// InterfaceGetter is used to derive the interface of a contract type InterfaceGetter interface { - GetABI(resolverAddr string, blockNumber int64) string + GetABI(resolverAddr string, blockNumber int64) (string, error) GetBlockChain() core.BlockChain } @@ -32,7 +34,8 @@ type interfaceGetter struct { fetcher.Fetcher } -func NewInterfaceGetter(blockChain core.BlockChain) *interfaceGetter { +// NewInterfaceGetter returns a new InterfaceGetter +func NewInterfaceGetter(blockChain core.BlockChain) InterfaceGetter { return &interfaceGetter{ Fetcher: fetcher.Fetcher{ BlockChain: blockChain, @@ -40,7 +43,7 @@ func NewInterfaceGetter(blockChain core.BlockChain) *interfaceGetter { } } -// Used to construct a custom ABI based on the results from calling supportsInterface +// GetABI is used to construct a custom ABI based on the results from calling supportsInterface func (g *interfaceGetter) GetABI(resolverAddr string, blockNumber int64) (string, error) { a := constants.SupportsInterfaceABI args := make([]interface{}, 1) @@ -104,7 +107,7 @@ func (g *interfaceGetter) getSupportsInterface(contractAbi, contractAddress stri return g.Fetcher.FetchBool("supportsInterface", contractAbi, contractAddress, blockNumber, methodArgs) } -// Method to retrieve the Getter's blockchain +// GetBlockChain is a method to retrieve the Getter's blockchain func (g *interfaceGetter) GetBlockChain() core.BlockChain { return g.Fetcher.BlockChain } diff --git a/pkg/contract_watcher/shared/helpers/helpers.go b/pkg/contract_watcher/shared/helpers/helpers.go index 9fb8dfae..6312210f 100644 --- a/pkg/contract_watcher/shared/helpers/helpers.go +++ b/pkg/contract_watcher/shared/helpers/helpers.go @@ -27,6 +27,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" ) +// ConvertToLog converts a watched event to a log func ConvertToLog(watchedEvent core.WatchedEvent) types.Log { allTopics := []string{watchedEvent.Topic0, watchedEvent.Topic1, watchedEvent.Topic2, watchedEvent.Topic3} var nonNilTopics []string @@ -56,12 +57,14 @@ func createTopics(topics ...string) []common.Hash { return topicsArray } +// BigFromString creates a big.Int from a string func BigFromString(n string) *big.Int { b := new(big.Int) b.SetString(n, 10) return b } +// GenerateSignature returns the keccak256 hash hex of a string func GenerateSignature(s string) string { eventSignature := []byte(s) hash := crypto.Keccak256Hash(eventSignature) diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go index 352ad79b..1988060e 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -294,8 +294,7 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`CREATE TABLE checked_headers ( id SERIAL PRIMARY KEY, - header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - check_count INTEGER NOT NULL DEFAULT 1);`) + header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`) Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`) diff --git a/pkg/contract_watcher/shared/parser/parser.go b/pkg/contract_watcher/shared/parser/parser.go index a8247b52..c55c82dd 100644 --- a/pkg/contract_watcher/shared/parser/parser.go +++ b/pkg/contract_watcher/shared/parser/parser.go @@ -45,7 +45,8 @@ type parser struct { parsedAbi abi.ABI } -func NewParser(network string) *parser { +// NewParser returns a new Parser +func NewParser(network string) Parser { url := eth.GenURL(network) return &parser{ @@ -53,15 +54,17 @@ func NewParser(network string) *parser { } } +// Abi returns the parser's configured abi string func (p *parser) Abi() string { return p.abi } +// ParsedAbi returns the parser's parsed abi func (p *parser) ParsedAbi() abi.ABI { return p.parsedAbi } -// Retrieves and parses the abi string +// Parse retrieves and parses the abi string // for the given contract address func (p *parser) Parse(contractAddr string) error { // If the abi is one our locally stored abis, fetch @@ -84,7 +87,7 @@ func (p *parser) Parse(contractAddr string) error { return err } -// Loads and parses an abi from a given abi string +// ParseAbiStr loads and parses an abi from a given abi string func (p *parser) ParseAbiStr(abiStr string) error { var err error p.abi = abiStr @@ -94,14 +97,14 @@ func (p *parser) ParseAbiStr(abiStr string) error { } func (p *parser) lookUp(contractAddr string) (string, error) { - if v, ok := constants.Abis[common.HexToAddress(contractAddr)]; ok { + if v, ok := constants.ABIs[common.HexToAddress(contractAddr)]; ok { return v, nil } return "", errors.New("ABI not present in lookup table") } -// Returns only specified methods, if they meet the criteria +// GetSelectMethods returns only specified methods, if they meet the criteria // Returns as array with methods in same order they were specified // Nil or empty wanted array => no events are returned func (p *parser) GetSelectMethods(wanted []string) []types.Method { @@ -121,7 +124,7 @@ func (p *parser) GetSelectMethods(wanted []string) []types.Method { return methods } -// Returns wanted methods +// GetMethods returns wanted methods // Empty wanted array => all methods are returned // Nil wanted array => no methods are returned func (p *parser) GetMethods(wanted []string) []types.Method { @@ -139,7 +142,7 @@ func (p *parser) GetMethods(wanted []string) []types.Method { return methods } -// Returns wanted events as map of types.Events +// GetEvents returns wanted events as map of types.Events // Empty wanted array => all events are returned // Nil wanted array => no events are returned func (p *parser) GetEvents(wanted []string) map[string]types.Event { diff --git a/pkg/contract_watcher/shared/poller/poller.go b/pkg/contract_watcher/shared/poller/poller.go index b3f1c51b..38717377 100644 --- a/pkg/contract_watcher/shared/poller/poller.go +++ b/pkg/contract_watcher/shared/poller/poller.go @@ -33,6 +33,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) +// Poller is the interface for polling public contract methods type Poller interface { PollContract(con contract.Contract, lastBlock int64) error PollContractAt(con contract.Contract, blockNumber int64) error @@ -45,13 +46,15 @@ type poller struct { contract contract.Contract } -func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *poller { +// NewPoller returns a new Poller +func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) Poller { return &poller{ MethodRepository: repository.NewMethodRepository(db, mode), bc: blockChain, } } +// PollContract polls a contract's public methods from the contracts starting block to specified last block func (p *poller) PollContract(con contract.Contract, lastBlock int64) error { for i := con.StartingBlock; i <= lastBlock; i++ { if err := p.PollContractAt(con, i); err != nil { @@ -62,6 +65,7 @@ func (p *poller) PollContract(con contract.Contract, lastBlock int64) error { return nil } +// PollContractAt polls a contract's public getter methods at the specified block height func (p *poller) PollContractAt(con contract.Contract, blockNumber int64) error { p.contract = con for _, m := range con.Methods { @@ -98,7 +102,7 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error { var out interface{} err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, bn) if err != nil { - return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + return fmt.Errorf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err) } strOut, err := stringify(out) if err != nil { @@ -112,7 +116,7 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error { // Persist result immediately err = p.PersistResults([]types.Result{result}, m, p.contract.Address, p.contract.Name) if err != nil { - return errors.New(fmt.Sprintf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + return fmt.Errorf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err) } return nil @@ -148,7 +152,7 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error { var out interface{} err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn) if err != nil { - return errors.New(fmt.Sprintf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + return fmt.Errorf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err) } strOut, err := stringify(out) if err != nil { @@ -164,7 +168,7 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error { // Persist result set as batch err := p.PersistResults(results, m, p.contract.Address, p.contract.Name) if err != nil { - return errors.New(fmt.Sprintf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + return fmt.Errorf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err) } return nil @@ -212,7 +216,7 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error { var out interface{} err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn) if err != nil { - return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + return fmt.Errorf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err) } strOut, err := stringify(out) if err != nil { @@ -228,13 +232,13 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error { err := p.PersistResults(results, m, p.contract.Address, p.contract.Name) if err != nil { - return errors.New(fmt.Sprintf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + return fmt.Errorf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err) } return nil } -// This is just a wrapper around the poller blockchain's FetchContractData method +// FetchContractData is just a wrapper around the poller blockchain's FetchContractData method func (p *poller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error { return p.bc.FetchContractData(contractAbi, contractAddress, method, methodArgs, result, blockNumber) } diff --git a/pkg/contract_watcher/shared/repository/event_repository.go b/pkg/contract_watcher/shared/repository/event_repository.go index 759c3889..ab5756df 100644 --- a/pkg/contract_watcher/shared/repository/event_repository.go +++ b/pkg/contract_watcher/shared/repository/event_repository.go @@ -24,7 +24,6 @@ import ( "github.com/hashicorp/golang-lru" "github.com/sirupsen/logrus" - "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -35,7 +34,7 @@ const ( eventCacheSize = 1000 ) -// Event repository is used to persist event data into custom tables +// EventRepository is used to persist event data into custom tables type EventRepository interface { PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error CreateEventTable(contractAddr string, event types.Event) (bool, error) @@ -51,7 +50,8 @@ type eventRepository struct { tables *lru.Cache // Cache names of recently used tables to minimize db connections } -func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository { +// NewEventRepository returns a new EventRepository +func NewEventRepository(db *postgres.DB, mode types.Mode) EventRepository { ccs, _ := lru.New(contractCacheSize) ecs, _ := lru.New(eventCacheSize) return &eventRepository{ @@ -62,7 +62,7 @@ func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository { } } -// Creates a schema for the contract if needed +// PersistLogs creates a schema for the contract if needed // Creates table for the watched contract event if needed // Persists converted event log data into this custom table func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { @@ -112,7 +112,7 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type // Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string data := make([]interface{}, 0, 5+el) data = append(data, - event.Id, + event.ID, contractName, event.Raw, event.LogIndex, @@ -143,17 +143,6 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type } } - // Mark header as checked for this eventId - eventId := strings.ToLower(eventInfo.Name + "_" + contractAddr) - markCheckedErr := repository.MarkContractWatcherHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block - if markCheckedErr != nil { - rollbackErr := tx.Rollback() - if rollbackErr != nil { - logrus.Warnf("error rolling back transaction while marking header checked: %s", rollbackErr.Error()) - } - return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error()) - } - return tx.Commit() } @@ -171,7 +160,7 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types. data := make([]interface{}, 0, 4+el) data = append(data, - event.Id, + event.ID, contractName, event.Block, event.Tx) @@ -201,7 +190,7 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types. return tx.Commit() } -// Checks for event table and creates it if it does not already exist +// CreateEventTable checks for event table and creates it if it does not already exist // Returns true if it created a new table; returns false if table already existed func (r *eventRepository) CreateEventTable(contractAddr string, event types.Event) (bool, error) { tableID := fmt.Sprintf("%s_%s.%s_event", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(event.Name)) @@ -270,7 +259,7 @@ func (r *eventRepository) checkForTable(contractAddr string, eventName string) ( return exists, err } -// Checks for contract schema and creates it if it does not already exist +// CreateContractSchema checks for contract schema and creates it if it does not already exist // Returns true if it created a new schema; returns false if schema already existed func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error) { if contractAddr == "" { @@ -316,10 +305,12 @@ func (r *eventRepository) checkForSchema(contractAddr string) (bool, error) { return exists, err } +// CheckSchemaCache is used to query the schema name cache func (r *eventRepository) CheckSchemaCache(key string) (interface{}, bool) { return r.schemas.Get(key) } +// CheckTableCache is used to query the table name cache func (r *eventRepository) CheckTableCache(key string) (interface{}, bool) { return r.tables.Get(key) } diff --git a/pkg/contract_watcher/shared/repository/event_repository_test.go b/pkg/contract_watcher/shared/repository/event_repository_test.go index 40f1a783..bc61d776 100644 --- a/pkg/contract_watcher/shared/repository/event_repository_test.go +++ b/pkg/contract_watcher/shared/repository/event_repository_test.go @@ -355,11 +355,6 @@ var _ = Describe("Repository", func() { Expect(count).To(Equal(2)) }) - It("Fails if the persisted event does not have a corresponding eventID column in the checked_headers table", func() { - err = dataStore.PersistLogs(logs, event, con.Address, con.Name) - Expect(err).To(HaveOccurred()) - }) - It("Fails with empty log", func() { err = dataStore.PersistLogs([]types.Log{}, event, con.Address, con.Name) Expect(err).To(HaveOccurred()) diff --git a/pkg/contract_watcher/shared/repository/method_repository.go b/pkg/contract_watcher/shared/repository/method_repository.go index 346de5a6..c8776b67 100644 --- a/pkg/contract_watcher/shared/repository/method_repository.go +++ b/pkg/contract_watcher/shared/repository/method_repository.go @@ -30,6 +30,7 @@ import ( const methodCacheSize = 1000 +// MethodRepository is used to persist public getter method data type MethodRepository interface { PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error CreateMethodTable(contractAddr string, method types.Method) (bool, error) @@ -45,7 +46,8 @@ type methodRepository struct { tables *lru.Cache // Cache names of recently used tables to minimize db connections } -func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository { +// NewMethodRepository returns a new MethodRepository +func NewMethodRepository(db *postgres.DB, mode types.Mode) MethodRepository { ccs, _ := lru.New(contractCacheSize) mcs, _ := lru.New(methodCacheSize) return &methodRepository{ @@ -56,7 +58,7 @@ func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository { } } -// Creates a schema for the contract if needed +// PersistResults creates a schema for the contract if needed // Creates table for the contract method if needed // Persists method polling data into this custom table func (r *methodRepository) PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error { @@ -124,7 +126,7 @@ func (r *methodRepository) persistResults(results []types.Result, methodInfo typ return tx.Commit() } -// Checks for event table and creates it if it does not already exist +// CreateMethodTable checks for event table and creates it if it does not already exist func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Method) (bool, error) { tableID := fmt.Sprintf("%s_%s.%s_method", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name)) @@ -177,7 +179,7 @@ func (r *methodRepository) checkForTable(contractAddr string, methodName string) return exists, err } -// Checks for contract schema and creates it if it does not already exist +// CreateContractSchema checks for contract schema and creates it if it does not already exist func (r *methodRepository) CreateContractSchema(contractAddr string) (bool, error) { if contractAddr == "" { return false, errors.New("error: no contract address specified") @@ -222,10 +224,12 @@ func (r *methodRepository) checkForSchema(contractAddr string) (bool, error) { return exists, err } +// CheckSchemaCache is used to query the schema name cache func (r *methodRepository) CheckSchemaCache(key string) (interface{}, bool) { return r.schemas.Get(key) } +// CheckTableCache is used to query the table name cache func (r *methodRepository) CheckTableCache(key string) (interface{}, bool) { return r.tables.Get(key) } diff --git a/pkg/contract_watcher/shared/repository/repository_suite_test.go b/pkg/contract_watcher/shared/repository/repository_suite_test.go index 1ffa6ded..707e1917 100644 --- a/pkg/contract_watcher/shared/repository/repository_suite_test.go +++ b/pkg/contract_watcher/shared/repository/repository_suite_test.go @@ -17,12 +17,12 @@ package repository_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" ) func TestRepository(t *testing.T) { diff --git a/pkg/contract_watcher/shared/retriever/address_retriever.go b/pkg/contract_watcher/shared/retriever/address_retriever.go index 6acdf72b..8afb2b98 100644 --- a/pkg/contract_watcher/shared/retriever/address_retriever.go +++ b/pkg/contract_watcher/shared/retriever/address_retriever.go @@ -18,17 +18,17 @@ package retriever import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "strings" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -// Address retriever is used to retrieve the addresses associated with a contract +// AddressRetriever is used to retrieve the addresses associated with a contract type AddressRetriever interface { RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) } @@ -38,14 +38,15 @@ type addressRetriever struct { mode types.Mode } -func NewAddressRetriever(db *postgres.DB, mode types.Mode) (r *addressRetriever) { +// NewAddressRetriever returns a new AddressRetriever +func NewAddressRetriever(db *postgres.DB, mode types.Mode) AddressRetriever { return &addressRetriever{ db: db, mode: mode, } } -// Method to retrieve list of token-holding/contract-related addresses by iterating over available events +// RetrieveTokenHolderAddresses is used to retrieve list of token-holding/contract-related addresses by iterating over available events // This generic method should work whether or not the argument/input names of the events meet the expected standard // This could be generalized to iterate over ALL events and pull out any address arguments func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) { diff --git a/pkg/contract_watcher/shared/retriever/retriever_suite_test.go b/pkg/contract_watcher/shared/retriever/retriever_suite_test.go index aff7fab1..6056bbfc 100644 --- a/pkg/contract_watcher/shared/retriever/retriever_suite_test.go +++ b/pkg/contract_watcher/shared/retriever/retriever_suite_test.go @@ -17,12 +17,12 @@ package retriever_test import ( - "github.com/sirupsen/logrus" "io/ioutil" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" ) func TestRetriever(t *testing.T) { diff --git a/pkg/contract_watcher/shared/types/event.go b/pkg/contract_watcher/shared/types/event.go index dfca596d..d5474828 100644 --- a/pkg/contract_watcher/shared/types/event.go +++ b/pkg/contract_watcher/shared/types/event.go @@ -25,20 +25,22 @@ import ( "github.com/ethereum/go-ethereum/crypto" ) +// Event is our custom event type type Event struct { Name string Anonymous bool Fields []Field } +// Field is our custom event field type which associates a postgres type with the field type Field struct { abi.Argument // Name, Type, Indexed PgType string // Holds type used when committing data held in this field to postgres } -// Struct to hold instance of an event log data +// Log is used to hold instance of an event log data type Log struct { - Id int64 // VulcanizeIdLog for full sync and header ID for header sync contract watcher + ID int64 // VulcanizeIdLog for full sync and header ID for header sync contract watcher Values map[string]string // Map of event input names to their values // Used for full sync only @@ -51,7 +53,7 @@ type Log struct { Raw []byte // json.Unmarshalled byte array of geth/core/types.Log{} } -// Unpack abi.Event into our custom Event struct +// NewEvent unpacks abi.Event into our custom Event struct func NewEvent(e abi.Event) Event { fields := make([]Field, len(e.Inputs)) for i, input := range e.Inputs { @@ -85,6 +87,7 @@ func NewEvent(e abi.Event) Event { } } +// Sig returns the hash signature for an event func (e Event) Sig() common.Hash { types := make([]string, len(e.Fields)) diff --git a/pkg/contract_watcher/shared/types/method.go b/pkg/contract_watcher/shared/types/method.go index 7b962b17..293b3594 100644 --- a/pkg/contract_watcher/shared/types/method.go +++ b/pkg/contract_watcher/shared/types/method.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" ) +// Method is our custom method struct type Method struct { Name string Const bool @@ -32,7 +33,7 @@ type Method struct { Return []Field } -// Struct to hold instance of result from method call with given inputs and block +// Result is used to hold instance of result from method call with given inputs and block type Result struct { Method Inputs []interface{} // Will only use addresses @@ -41,7 +42,7 @@ type Result struct { Block int64 } -// Unpack abi.Method into our custom Method struct +// NewMethod unpacks abi.Method into our custom Method struct func NewMethod(m abi.Method) Method { inputs := make([]Field, len(m.Inputs)) for i, input := range m.Inputs { @@ -99,6 +100,7 @@ func NewMethod(m abi.Method) Method { } } +// Sig returns the hash signature for the method func (m Method) Sig() common.Hash { types := make([]string, len(m.Args)) i := 0 diff --git a/pkg/contract_watcher/shared/types/mode.go b/pkg/contract_watcher/shared/types/mode.go index 07d58dc6..667e66f2 100644 --- a/pkg/contract_watcher/shared/types/mode.go +++ b/pkg/contract_watcher/shared/types/mode.go @@ -16,19 +16,21 @@ package types -import "fmt" - +// Mode is used to explicitly represent the operating mode of the transformer type Mode int +// Mode enums const ( HeaderSync Mode = iota FullSync ) +// IsValid returns true is the Mode is valid func (mode Mode) IsValid() bool { return mode >= HeaderSync && mode <= FullSync } +// String returns the string representation of the mode func (mode Mode) String() string { switch mode { case HeaderSync: @@ -39,26 +41,3 @@ func (mode Mode) String() string { return "unknown" } } - -func (mode Mode) MarshalText() ([]byte, error) { - switch mode { - case HeaderSync: - return []byte("header"), nil - case FullSync: - return []byte("full"), nil - default: - return nil, fmt.Errorf("contract watcher: unknown mode %d, want HeaderSync or FullSync", mode) - } -} - -func (mode *Mode) UnmarshalText(text []byte) error { - switch string(text) { - case "header": - *mode = HeaderSync - case "full": - *mode = FullSync - default: - return fmt.Errorf(`contract watcher: unknown mode %q, want "header" or "full"`, text) - } - return nil -}