diff --git a/cmd/erc20.go b/cmd/erc20.go index 9a9cacde..71d86200 100644 --- a/cmd/erc20.go +++ b/cmd/erc20.go @@ -22,7 +22,10 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/spf13/cobra" + "github.com/vulcanize/vulcanizedb/examples/constants" + "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/event_triggered/dai" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" + "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/geth" @@ -35,7 +38,8 @@ import ( var erc20Cmd = &cobra.Command{ Use: "erc20", Short: "Fetches and persists token supply", - Long: `Fetches the totalSupply for the configured token from each block and persists it in Vulcanize DB. + Long: `Fetches transfer and approval events, totalSupply, allowances, and +balances for the configured token from each block and persists it in Vulcanize DB. vulcanizedb erc20 --config environments/public Expects an ethereum node to be running and requires a .toml config file: @@ -70,12 +74,24 @@ func watchERC20s() { if err != nil { log.Fatal("Failed to initialize database.") } + + con := generic.DaiConfig + con.Filters = constants.DaiERC20Filters watcher := shared.Watcher{ DB: *db, Blockchain: blockChain, } - watcher.AddTransformers(every_block.TransformerInitializers()) + // It is important that the event transformer is executed before the every_block transformer + // because the events are used to generate the token holder address list that is used to + // collect balances and allowances at every block + transformers := append(dai.DaiEventTriggeredTransformerInitializer(), every_block.ERC20EveryBlockTransformerInitializer()...) + + err = watcher.AddTransformers(transformers, con) + if err != nil { + log.Fatal(err) + } + for range ticker.C { watcher.Execute() } diff --git a/examples/erc20_watcher/event_triggered/dai/converter.go b/examples/erc20_watcher/event_triggered/dai/converter.go index 06f75145..50bc2347 100644 --- a/examples/erc20_watcher/event_triggered/dai/converter.go +++ b/examples/erc20_watcher/event_triggered/dai/converter.go @@ -20,8 +20,8 @@ import ( "github.com/vulcanize/vulcanizedb/examples/constants" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/event_triggered" - "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/examples/generic/helpers" + "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/geth" ) @@ -37,10 +37,10 @@ type ERC20ConverterInterface interface { } type ERC20Converter struct { - config generic.ContractConfig + config shared.ContractConfig } -func NewERC20Converter(config generic.ContractConfig) (*ERC20Converter, error) { +func NewERC20Converter(config shared.ContractConfig) (*ERC20Converter, error) { var err error config.ParsedAbi, err = geth.ParseAbi(config.Abi) diff --git a/examples/erc20_watcher/event_triggered/dai/integration_test.go b/examples/erc20_watcher/event_triggered/dai/integration_test.go index b8af844f..d05852df 100644 --- a/examples/erc20_watcher/event_triggered/dai/integration_test.go +++ b/examples/erc20_watcher/event_triggered/dai/integration_test.go @@ -71,6 +71,7 @@ var logs = []core.Log{ var _ = Describe("Integration test with vulcanizedb", func() { var db *postgres.DB + var blk core.BlockChain BeforeEach(func() { db = test_helpers.SetupIntegrationDB(db, logs) @@ -81,7 +82,7 @@ var _ = Describe("Integration test with vulcanizedb", func() { }) It("creates token_transfers entry for each Transfer event received", func() { - transformer, err := dai.NewTransformer(db, generic.DaiConfig) + transformer, err := dai.NewTransformer(db, blk, generic.DaiConfig) Expect(err).ToNot(HaveOccurred()) transformer.Execute() @@ -107,7 +108,7 @@ var _ = Describe("Integration test with vulcanizedb", func() { }) It("creates token_approvals entry for each Approval event received", func() { - transformer, err := dai.NewTransformer(db, generic.DaiConfig) + transformer, err := dai.NewTransformer(db, blk, generic.DaiConfig) Expect(err).ToNot(HaveOccurred()) transformer.Execute() diff --git a/examples/erc20_watcher/event_triggered/dai/transformer.go b/examples/erc20_watcher/event_triggered/dai/transformer.go index 3f3c3e1a..deaa07e0 100644 --- a/examples/erc20_watcher/event_triggered/dai/transformer.go +++ b/examples/erc20_watcher/event_triggered/dai/transformer.go @@ -20,11 +20,12 @@ import ( "github.com/vulcanize/vulcanizedb/examples/constants" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/event_triggered" - "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/libraries/shared" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/filters" ) type ERC20EventTransformer struct { @@ -32,34 +33,38 @@ type ERC20EventTransformer struct { WatchedEventRepository datastore.WatchedEventRepository FilterRepository datastore.FilterRepository Repository event_triggered.ERC20EventDatastore + Filters []filters.LogFilter } -func NewTransformer(db *postgres.DB, config generic.ContractConfig) (shared.Transformer, error) { +func NewTransformer(db *postgres.DB, blockchain core.BlockChain, con shared.ContractConfig) (shared.Transformer, error) { var transformer shared.Transformer - cnvtr, err := NewERC20Converter(config) + cnvtr, err := NewERC20Converter(con) if err != nil { - return transformer, err + return nil, err } wer := repositories.WatchedEventRepository{DB: db} fr := repositories.FilterRepository{DB: db} lkr := event_triggered.ERC20EventRepository{DB: db} + transformer = ERC20EventTransformer{ Converter: cnvtr, WatchedEventRepository: wer, FilterRepository: fr, Repository: lkr, + Filters: con.Filters, } - for _, filter := range constants.DaiERC20Filters { + for _, filter := range con.Filters { fr.CreateFilter(filter) } + return transformer, nil } func (tr ERC20EventTransformer) Execute() error { - for _, filter := range constants.DaiERC20Filters { + for _, filter := range tr.Filters { watchedEvents, err := tr.WatchedEventRepository.GetWatchedEvents(filter.Name) if err != nil { log.Println(fmt.Sprintf("Error fetching events for %s:", filter.Name), err) @@ -84,5 +89,6 @@ func (tr ERC20EventTransformer) Execute() error { } } } + return nil } diff --git a/examples/erc20_watcher/event_triggered/dai/transformer_test.go b/examples/erc20_watcher/event_triggered/dai/transformer_test.go index 0f35e025..722eadd2 100644 --- a/examples/erc20_watcher/event_triggered/dai/transformer_test.go +++ b/examples/erc20_watcher/event_triggered/dai/transformer_test.go @@ -71,12 +71,14 @@ var _ = Describe("Mock ERC20 transformer", func() { watchedEventsRepo.SetWatchedEvents(fakeWatchedEvents) mockEventRepo = mocks.MockEventRepo{} filterRepo = mocks.MockFilterRepository{} + filters := constants.DaiERC20Filters transformer = dai.ERC20EventTransformer{ Converter: &mockERC20Converter, WatchedEventRepository: &watchedEventsRepo, FilterRepository: filterRepo, Repository: &mockEventRepo, + Filters: filters, } }) diff --git a/examples/erc20_watcher/event_triggered/dai/transformers.go b/examples/erc20_watcher/event_triggered/dai/transformers.go new file mode 100644 index 00000000..d93e9ed9 --- /dev/null +++ b/examples/erc20_watcher/event_triggered/dai/transformers.go @@ -0,0 +1,25 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dai + +import ( + "github.com/vulcanize/vulcanizedb/libraries/shared" +) + +func DaiEventTriggeredTransformerInitializer() []shared.TransformerInitializer { + return []shared.TransformerInitializer{ + NewTransformer, + } +} diff --git a/examples/erc20_watcher/every_block/integration_test.go b/examples/erc20_watcher/every_block/integration_test.go index 862be926..e093e0a8 100644 --- a/examples/erc20_watcher/every_block/integration_test.go +++ b/examples/erc20_watcher/every_block/integration_test.go @@ -61,12 +61,12 @@ var _ = Describe("Everyblock transformers", func() { }) It("creates a token_supply record for each block in the given range", func() { - initializer := every_block.ERC20TokenTransformerInitializer{Config: generic.DaiConfig} - transformer := initializer.NewERC20TokenTransformer(db, blockChain) + transformer, err := every_block.NewERC20TokenTransformer(db, blockChain, generic.DaiConfig) + Expect(err).ToNot(HaveOccurred()) transformer.Execute() var tokenSupplyCount int - err := db.QueryRow(`SELECT COUNT(*) FROM token_supply where block_id = $1`, blockId).Scan(&tokenSupplyCount) + err = db.QueryRow(`SELECT COUNT(*) FROM token_supply where block_id = $1`, blockId).Scan(&tokenSupplyCount) Expect(err).ToNot(HaveOccurred()) Expect(tokenSupplyCount).To(Equal(1)) diff --git a/examples/erc20_watcher/every_block/transformer.go b/examples/erc20_watcher/every_block/transformer.go index c1aa3ed2..f5be8aba 100644 --- a/examples/erc20_watcher/every_block/transformer.go +++ b/examples/erc20_watcher/every_block/transformer.go @@ -21,7 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" - "github.com/vulcanize/vulcanizedb/examples/generic" + "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -30,30 +30,27 @@ import ( type ERC20Transformer struct { Getter ERC20GetterInterface Repository ERC20TokenDatastore - Retriever generic.TokenHolderRetriever - Config generic.ContractConfig + Retriever erc20_watcher.TokenHolderRetriever + Config shared.ContractConfig } -func (t *ERC20Transformer) SetConfiguration(config generic.ContractConfig) { +func (t *ERC20Transformer) SetConfiguration(config shared.ContractConfig) { t.Config = config } -type ERC20TokenTransformerInitializer struct { - Config generic.ContractConfig -} - -func (i ERC20TokenTransformerInitializer) NewERC20TokenTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer { +func NewERC20TokenTransformer(db *postgres.DB, blockchain core.BlockChain, con shared.ContractConfig) (shared.Transformer, error) { getter := NewGetter(blockchain) repository := ERC20TokenRepository{DB: db} - retriever := generic.NewTokenHolderRetriever(db, i.Config.Address) + retriever := erc20_watcher.NewTokenHolderRetriever(db, con.Address) + transformer := ERC20Transformer{ Getter: &getter, Repository: &repository, Retriever: retriever, - Config: i.Config, + Config: con, } - return transformer + return transformer, nil } const ( diff --git a/examples/erc20_watcher/every_block/transformer_test.go b/examples/erc20_watcher/every_block/transformer_test.go index d3590607..0caa88a4 100644 --- a/examples/erc20_watcher/every_block/transformer_test.go +++ b/examples/erc20_watcher/every_block/transformer_test.go @@ -23,14 +23,15 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/examples/constants" + "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/examples/mocks" "github.com/vulcanize/vulcanizedb/examples/test_helpers" + "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/fakes" ) -var testContractConfig = generic.ContractConfig{ +var testContractConfig = shared.ContractConfig{ Address: constants.DaiContractAddress, Abi: constants.DaiAbiString, FirstBlock: int64(4752008), @@ -59,7 +60,7 @@ var _ = Describe("Everyblock transformer", func() { repository = mocks.ERC20TokenRepository{} repository.SetMissingSupplyBlocks([]int64{config.FirstBlock}) db := test_helpers.CreateNewDatabase() - rt := generic.NewTokenHolderRetriever(db, config.Address) + rt := erc20_watcher.NewTokenHolderRetriever(db, config.Address) //setting the mock repository to return the first block as the missing blocks transformer = every_block.ERC20Transformer{ diff --git a/examples/erc20_watcher/every_block/transformers.go b/examples/erc20_watcher/every_block/transformers.go index ae06f1ad..58df35a5 100644 --- a/examples/erc20_watcher/every_block/transformers.go +++ b/examples/erc20_watcher/every_block/transformers.go @@ -15,14 +15,11 @@ package every_block import ( - "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/libraries/shared" ) -func TransformerInitializers() []shared.TransformerInitializer { - config := generic.DaiConfig - initializer := ERC20TokenTransformerInitializer{config} +func ERC20EveryBlockTransformerInitializer() []shared.TransformerInitializer { return []shared.TransformerInitializer{ - initializer.NewERC20TokenTransformer, + NewERC20TokenTransformer, } } diff --git a/examples/erc20_watcher/retriever.go b/examples/erc20_watcher/retriever.go new file mode 100644 index 00000000..eddcf225 --- /dev/null +++ b/examples/erc20_watcher/retriever.go @@ -0,0 +1,183 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package erc20_watcher + +import ( + "fmt" + "log" + + "github.com/ethereum/go-ethereum/common" + + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +// ERC20 retriever is used to iterate over addresses involved in token +// transfers and approvals to generate a list of token holder addresses + +type TokenHolderRetrieverInterface interface { + RetrieveTokenHolderAddresses() (map[common.Address]bool, error) + retrieveTokenSenders() ([]string, error) + retrieveTokenReceivers() ([]string, error) + retrieveTokenOwners() ([]string, error) + retrieveTokenSpenders() ([]string, error) +} + +type TokenHolderRetriever struct { + Database *postgres.DB + ContractAddress string +} + +type retrieverError struct { + err string + msg string + address string +} + +// Retriever error method +func (re *retrieverError) Error() string { + return fmt.Sprintf(re.msg, re.address, re.err) +} + +// Used to create a new retriever error for a given error and fetch method +func newRetrieverError(err error, msg string, address string) error { + e := retrieverError{err.Error(), msg, address} + log.Println(e.Error()) + return &e +} + +// Constant error definitions +const ( + GetSendersError = "Error fetching token senders from contract %s: %s" + GetReceiversError = "Error fetching token receivers from contract %s: %s" + GetOwnersError = "Error fetching token owners from contract %s: %s" + GetSpendersError = "Error fetching token spenders from contract %s: %s" +) + +func NewTokenHolderRetriever(db *postgres.DB, address string) TokenHolderRetriever { + return TokenHolderRetriever{ + Database: db, + ContractAddress: address, + } +} + +func (rt TokenHolderRetriever) retrieveTokenSenders() ([]string, error) { + + senders := make([]string, 0) + + err := rt.Database.DB.Select( + &senders, + `SELECT from_address FROM token_transfers + WHERE token_address = $1`, + rt.ContractAddress, + ) + if err != nil { + return []string{}, newRetrieverError(err, GetSendersError, rt.ContractAddress) + } + + return senders, nil +} + +func (rt TokenHolderRetriever) retrieveTokenReceivers() ([]string, error) { + + receivers := make([]string, 0) + + err := rt.Database.DB.Select( + &receivers, + `SELECT to_address FROM token_transfers + WHERE token_address = $1`, + rt.ContractAddress, + ) + if err != nil { + return []string{}, newRetrieverError(err, GetReceiversError, rt.ContractAddress) + } + return receivers, err +} + +func (rt TokenHolderRetriever) retrieveTokenOwners() ([]string, error) { + + owners := make([]string, 0) + + err := rt.Database.DB.Select( + &owners, + `SELECT owner FROM token_approvals + WHERE token_address = $1`, + rt.ContractAddress, + ) + if err != nil { + return []string{}, newRetrieverError(err, GetOwnersError, rt.ContractAddress) + } + + return owners, nil +} + +func (rt TokenHolderRetriever) retrieveTokenSpenders() ([]string, error) { + + spenders := make([]string, 0) + + err := rt.Database.DB.Select( + &spenders, + `SELECT spender FROM token_approvals + WHERE token_address = $1`, + rt.ContractAddress, + ) + if err != nil { + return []string{}, newRetrieverError(err, GetSpendersError, rt.ContractAddress) + } + + return spenders, nil +} + +func (rt TokenHolderRetriever) RetrieveTokenHolderAddresses() (map[common.Address]bool, error) { + + senders, err := rt.retrieveTokenSenders() + if err != nil { + return nil, err + } + + receivers, err := rt.retrieveTokenReceivers() + if err != nil { + return nil, err + } + + owners, err := rt.retrieveTokenOwners() + if err != nil { + return nil, err + } + + spenders, err := rt.retrieveTokenSenders() + if err != nil { + return nil, err + } + + contractAddresses := make(map[common.Address]bool) + + for _, addr := range senders { + contractAddresses[common.HexToAddress(addr)] = true + } + + for _, addr := range receivers { + contractAddresses[common.HexToAddress(addr)] = true + } + + for _, addr := range owners { + contractAddresses[common.HexToAddress(addr)] = true + } + + for _, addr := range spenders { + contractAddresses[common.HexToAddress(addr)] = true + } + + return contractAddresses, nil +} diff --git a/examples/generic/config.go b/examples/generic/config.go index f5d96e32..7419e10e 100644 --- a/examples/generic/config.go +++ b/examples/generic/config.go @@ -15,34 +15,25 @@ package generic import ( - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/vulcanize/vulcanizedb/examples/constants" + "github.com/vulcanize/vulcanizedb/libraries/shared" ) -type ContractConfig struct { - Address string - Owner string - Abi string - ParsedAbi abi.ABI - FirstBlock int64 - LastBlock int64 - Name string -} - -var DaiConfig = ContractConfig{ +var DaiConfig = shared.ContractConfig{ Address: constants.DaiContractAddress, - Owner: constants.DaiContractOwner, Abi: constants.DaiAbiString, FirstBlock: int64(4752008), LastBlock: -1, Name: "Dai", + Filters: constants.DaiERC20Filters, } -var TusdConfig = ContractConfig{ +var TusdConfig = shared.ContractConfig{ Address: constants.TusdContractAddress, Owner: constants.TusdContractOwner, Abi: constants.TusdAbiString, FirstBlock: int64(5197514), LastBlock: -1, Name: "Tusd", + Filters: constants.TusdGenericFilters, } diff --git a/examples/generic/event_triggered/tusd/converter.go b/examples/generic/event_triggered/tusd/converter.go index ab6c8aa3..5e3d77ef 100644 --- a/examples/generic/event_triggered/tusd/converter.go +++ b/examples/generic/event_triggered/tusd/converter.go @@ -19,9 +19,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/examples/constants" - "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/examples/generic/event_triggered" "github.com/vulcanize/vulcanizedb/examples/generic/helpers" + "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/geth" ) @@ -37,10 +37,10 @@ type GenericConverterInterface interface { } type GenericConverter struct { - config generic.ContractConfig + config shared.ContractConfig } -func NewGenericConverter(config generic.ContractConfig) (*GenericConverter, error) { +func NewGenericConverter(config shared.ContractConfig) (*GenericConverter, error) { var err error config.ParsedAbi, err = geth.ParseAbi(config.Abi) diff --git a/examples/generic/event_triggered/tusd/integration_test.go b/examples/generic/event_triggered/tusd/integration_test.go index 1d0627ee..919e1dcd 100644 --- a/examples/generic/event_triggered/tusd/integration_test.go +++ b/examples/generic/event_triggered/tusd/integration_test.go @@ -71,6 +71,7 @@ var logs = []core.Log{ var _ = Describe("Integration test with vulcanizedb", func() { var db *postgres.DB + var blk core.BlockChain BeforeEach(func() { db = test_helpers.SetupIntegrationDB(db, logs) @@ -81,7 +82,7 @@ var _ = Describe("Integration test with vulcanizedb", func() { }) It("creates token_burns entry for each Burn event received", func() { - transformer, err := tusd.NewTransformer(db, generic.TusdConfig) + transformer, err := tusd.NewTransformer(db, blk, generic.TusdConfig) Expect(err).ToNot(HaveOccurred()) transformer.Execute() @@ -106,7 +107,7 @@ var _ = Describe("Integration test with vulcanizedb", func() { }) It("creates token_mints entry for each Mint event received", func() { - transformer, err := tusd.NewTransformer(db, generic.TusdConfig) + transformer, err := tusd.NewTransformer(db, blk, generic.TusdConfig) Expect(err).ToNot(HaveOccurred()) transformer.Execute() diff --git a/examples/generic/event_triggered/tusd/transformer.go b/examples/generic/event_triggered/tusd/transformer.go index 93a9b2fc..1195e004 100644 --- a/examples/generic/event_triggered/tusd/transformer.go +++ b/examples/generic/event_triggered/tusd/transformer.go @@ -19,12 +19,13 @@ import ( "log" "github.com/vulcanize/vulcanizedb/examples/constants" - "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/examples/generic/event_triggered" "github.com/vulcanize/vulcanizedb/libraries/shared" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/filters" ) type GenericTransformer struct { @@ -32,34 +33,38 @@ type GenericTransformer struct { WatchedEventRepository datastore.WatchedEventRepository FilterRepository datastore.FilterRepository Repository event_triggered.GenericEventDatastore + Filters []filters.LogFilter } -func NewTransformer(db *postgres.DB, config generic.ContractConfig) (shared.Transformer, error) { +func NewTransformer(db *postgres.DB, blockchain core.BlockChain, con shared.ContractConfig) (shared.Transformer, error) { var transformer shared.Transformer - cnvtr, err := NewGenericConverter(config) + cnvtr, err := NewGenericConverter(con) if err != nil { - return transformer, err + return nil, err } wer := repositories.WatchedEventRepository{DB: db} fr := repositories.FilterRepository{DB: db} lkr := event_triggered.GenericEventRepository{DB: db} + transformer = GenericTransformer{ Converter: cnvtr, WatchedEventRepository: wer, FilterRepository: fr, Repository: lkr, + Filters: con.Filters, } - for _, filter := range constants.TusdGenericFilters { + for _, filter := range con.Filters { fr.CreateFilter(filter) } + return transformer, nil } func (tr GenericTransformer) Execute() error { - for _, filter := range constants.TusdGenericFilters { + for _, filter := range tr.Filters { watchedEvents, err := tr.WatchedEventRepository.GetWatchedEvents(filter.Name) if err != nil { log.Println(fmt.Sprintf("Error fetching events for %s:", filter.Name), err) @@ -70,7 +75,7 @@ func (tr GenericTransformer) Execute() error { entity, err := tr.Converter.ToBurnEntity(*we) model := tr.Converter.ToBurnModel(entity) if err != nil { - log.Printf("Error persisting data for Dai Burns (watchedEvent.LogID %d):\n %s", we.LogID, err) + log.Printf("Error persisting data for TrueUSD Burns (watchedEvent.LogID %d):\n %s", we.LogID, err) } tr.Repository.CreateBurn(model, we.LogID) } @@ -78,11 +83,12 @@ func (tr GenericTransformer) Execute() error { entity, err := tr.Converter.ToMintEntity(*we) model := tr.Converter.ToMintModel(entity) if err != nil { - log.Printf("Error persisting data for Dai Mints (watchedEvent.LogID %d):\n %s", we.LogID, err) + log.Printf("Error persisting data for TrueUSD Mints (watchedEvent.LogID %d):\n %s", we.LogID, err) } tr.Repository.CreateMint(model, we.LogID) } } } + return nil } diff --git a/examples/generic/event_triggered/tusd/transformer_test.go b/examples/generic/event_triggered/tusd/transformer_test.go index 4591a405..0e94960d 100644 --- a/examples/generic/event_triggered/tusd/transformer_test.go +++ b/examples/generic/event_triggered/tusd/transformer_test.go @@ -71,12 +71,14 @@ var _ = Describe("Mock ERC20 transformer", func() { watchedEventsRepo.SetWatchedEvents(fakeWatchedEvents) mockEventRepo = mocks.MockEventRepo{} filterRepo = mocks.MockFilterRepository{} + filters := constants.TusdGenericFilters transformer = tusd.GenericTransformer{ Converter: &mockERC20Converter, WatchedEventRepository: &watchedEventsRepo, FilterRepository: filterRepo, Repository: &mockEventRepo, + Filters: filters, } }) diff --git a/examples/generic/event_triggered/tusd/transformers.go b/examples/generic/event_triggered/tusd/transformers.go new file mode 100644 index 00000000..d5612ef3 --- /dev/null +++ b/examples/generic/event_triggered/tusd/transformers.go @@ -0,0 +1,25 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tusd + +import ( + "github.com/vulcanize/vulcanizedb/libraries/shared" +) + +func TusdEventTriggeredTransformerInitializer() []shared.TransformerInitializer { + return []shared.TransformerInitializer{ + NewTransformer, + } +} diff --git a/examples/generic/retriever.go b/examples/generic/retriever.go index 4ff5a1e6..e7a56ee1 100644 --- a/examples/generic/retriever.go +++ b/examples/generic/retriever.go @@ -23,8 +23,8 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -// Retriever is used to iterate over addresses going into or out of a contract -// address in an attempt to generate a list of token holder addresses +// Generic retriever is used to iterate over addresses involved in token +// transfers, approvals, mints, and burns to generate a list of token holder addresses type TokenHolderRetrieverInterface interface { RetrieveTokenHolderAddresses() (map[common.Address]bool, error) @@ -32,6 +32,8 @@ type TokenHolderRetrieverInterface interface { retrieveTokenReceivers() ([]string, error) retrieveTokenOwners() ([]string, error) retrieveTokenSpenders() ([]string, error) + retrieveTokenMintees() ([]string, error) + retrieveTokenBurners() ([]string, error) } type TokenHolderRetriever struct { diff --git a/libraries/shared/contract_config.go b/libraries/shared/contract_config.go new file mode 100644 index 00000000..8206c321 --- /dev/null +++ b/libraries/shared/contract_config.go @@ -0,0 +1,17 @@ +package shared + +import ( + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/vulcanize/vulcanizedb/pkg/filters" +) + +type ContractConfig struct { + Address string + Owner string + Abi string + ParsedAbi abi.ABI + FirstBlock int64 + LastBlock int64 + Name string + Filters []filters.LogFilter +} diff --git a/libraries/shared/transformer_interface.go b/libraries/shared/transformer_interface.go index a2fa6675..565e5813 100644 --- a/libraries/shared/transformer_interface.go +++ b/libraries/shared/transformer_interface.go @@ -10,7 +10,7 @@ type Transformer interface { Execute() error } -type TransformerInitializer func(db *postgres.DB, blockchain core.BlockChain) Transformer +type TransformerInitializer func(db *postgres.DB, blockchain core.BlockChain, con ContractConfig) (Transformer, error) func HexToInt64(byteString string) int64 { value := common.HexToHash(byteString) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 1a5cd668..ca3f0a00 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -11,11 +11,16 @@ type Watcher struct { Blockchain core.BlockChain } -func (watcher *Watcher) AddTransformers(us []TransformerInitializer) { +func (watcher *Watcher) AddTransformers(us []TransformerInitializer, con ContractConfig) error { for _, transformerInitializer := range us { - transformer := transformerInitializer(&watcher.DB, watcher.Blockchain) + transformer, err := transformerInitializer(&watcher.DB, watcher.Blockchain, con) + if err != nil { + return err + } watcher.Transformers = append(watcher.Transformers, transformer) } + + return nil } func (watcher *Watcher) Execute() error { diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index 7add85d5..c72415b7 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -23,15 +23,16 @@ func (mh *MockTransformer) Execute() error { return nil } -func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer { - return &MockTransformer{} +func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain, con shared.ContractConfig) (shared.Transformer, error) { + return &MockTransformer{}, nil } var _ = Describe("Watcher", func() { It("Adds transformers", func() { watcher := shared.Watcher{} + con := shared.ContractConfig{} - watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer}) + watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer}, con) Expect(len(watcher.Transformers)).To(Equal(1)) Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{})) @@ -39,9 +40,10 @@ var _ = Describe("Watcher", func() { It("Adds transformers from multiple sources", func() { watcher := shared.Watcher{} + con := shared.ContractConfig{} - watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer}) - watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer}) + watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer}, con) + watcher.AddTransformers([]shared.TransformerInitializer{fakeTransformerInitializer}, con) Expect(len(watcher.Transformers)).To(Equal(2)) })