From dc43547612dd6cd20e75fe09cec879bcff8fc414 Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 12 Dec 2018 15:41:29 +0100 Subject: [PATCH] Improve process of adding transformers --- cmd/backfillMakerLogs.go | 7 +- cmd/continuousLogSync.go | 130 ++++++++++++++++--------- libraries/shared/watcher.go | 47 +++++---- pkg/transformers/shared/log_chunker.go | 30 +++--- pkg/transformers/shared/log_fetcher.go | 4 +- 5 files changed, 134 insertions(+), 84 deletions(-) diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 0c424d4a..56986c85 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -47,11 +47,12 @@ func backfillMakerLogs() { log.Fatal("Failed to initialize database.") } - repository := shared2.Repository{} + repository := &shared2.Repository{} fetcher := shared2.NewFetcher(blockChain) - watcher := shared.NewWatcher(db, fetcher, repository, transformers.TransformerConfigs()) + chunker := shared2.NewLogChunker() + watcher := shared.NewWatcher(db, fetcher, repository, chunker) - watcher.AddTransformers(transformers.TransformerInitializers()) + watcher.AddTransformers(transformers.TransformerInitializers(), transformers.TransformerConfigs()) watcher.Execute() } diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 786ea5ea..9a70cbe4 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -59,71 +59,109 @@ func syncMakerLogs() { } fetcher := shared2.NewFetcher(blockChain) + repository := &shared2.Repository{} + chunker := shared2.NewLogChunker() - watcher := shared.Watcher{ - DB: db, - Fetcher: fetcher, - } + initializers, configs := getTransformerSubset(transformerNames) + chunker.AddConfigs(configs) - transformerInititalizers := getTransformerInititalizers(transformerNames) - watcher.AddTransformers(transformerInititalizers) + watcher := shared.NewWatcher(db, fetcher, repository, chunker) + watcher.AddTransformers(initializers, configs) for range ticker.C { watcher.Execute() } } -func getTransformerInititalizers(transformerNames []string) []shared2.TransformerInitializer { - transformerInitializerMap := buildTransformerInitializerMap() - var transformerInitializers []shared2.TransformerInitializer +func getTransformerSubset(transformerNames []string) ([]shared2.TransformerInitializer, []shared2.TransformerConfig) { + var initializers []shared2.TransformerInitializer + var configs []shared2.TransformerConfig if transformerNames[0] == "all" { - for _, v := range transformerInitializerMap { - transformerInitializers = append(transformerInitializers, v) - } + initializers = transformers.TransformerInitializers() + configs = transformers.TransformerConfigs() } else { + initializerMap := buildTransformerInitializerMap() + configMap := buildTransformerConfigMap() + for _, transformerName := range transformerNames { - initializer := transformerInitializerMap[transformerName] - transformerInitializers = append(transformerInitializers, initializer) + initializers = append(initializers, initializerMap[transformerName]) + configs = append(configs, configMap[transformerName]) } } - - return transformerInitializers + return initializers, configs } func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { - transformerInitializerMap := make(map[string]shared2.TransformerInitializer) + initializerMap := make(map[string]shared2.TransformerInitializer) - transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer - transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer - transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer - transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer - transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer - transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer - transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer + initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer + initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer + initializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer + initializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer + initializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer + initializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer + initializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer + initializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer + initializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer + initializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer + initializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer + initializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer + initializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer + initializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer + initializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer + initializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer + initializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer + initializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer + initializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer + initializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer + initializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer + initializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer + initializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer + initializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer + initializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer + initializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer - return transformerInitializerMap + return initializerMap +} + +func buildTransformerConfigMap() map[string]shared2.TransformerConfig { + configMap := make(map[string]shared2.TransformerConfig) + + configMap[constants.BiteLabel] = transformers.BiteTransformer.Config + configMap[constants.BiteLabel] = transformers.BiteTransformer.Config + configMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.Config + configMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.Config + configMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.Config + configMap[constants.DealLabel] = transformers.DealTransformer.Config + configMap[constants.DentLabel] = transformers.DentTransformer.Config + configMap[constants.DripDripLabel] = transformers.DripDripTransformer.Config + configMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.Config + configMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.Config + configMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.Config + configMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.Config + configMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.Config + configMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.Config + configMap[constants.FrobLabel] = transformers.FrobTransformer.Config + configMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.Config + configMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.Config + configMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.Config + configMap[constants.TendLabel] = transformers.TendTransformer.Config + configMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.Config + configMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.Config + configMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.Config + configMap[constants.VatHealLabel] = transformers.VatHealTransformer.Config + configMap[constants.VatInitLabel] = transformers.VatInitTransformer.Config + configMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.Config + configMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.Config + configMap[constants.VatTollLabel] = transformers.VatTollTransformer.Config + configMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.Config + configMap[constants.VowFlogLabel] = transformers.FlogTransformer.Config + + return configMap } func init() { diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 207b0cc0..e5c96851 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -18,41 +18,46 @@ type Watcher struct { Transformers []shared.Transformer DB *postgres.DB Fetcher shared.LogFetcher - Chunker shared.LogChunker + Chunker shared.Chunker Addresses []common.Address Topics []common.Hash Repository WatcherRepository } -func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository, - transformerConfigs []shared.TransformerConfig) Watcher { +func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository, chunker shared.Chunker) Watcher { + return Watcher{ + DB: db, + Fetcher: fetcher, + Chunker: chunker, + Repository: repository, + } +} + +// Adds transformers to the watcher, each needs an initializer and the associated config. +// This also changes the configuration of the chunker, so that it will consider the new transformers. +func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer, configs []shared.TransformerConfig) { + if len(initializers) != len(configs) { + panic("Mismatch in number of transformers initializers and configs!") + } + + for _, initializer := range initializers { + transformer := initializer(watcher.DB) + watcher.Transformers = append(watcher.Transformers, transformer) + } + var contractAddresses []common.Address var topic0s []common.Hash - for _, config := range transformerConfigs { + for _, config := range configs { for _, address := range config.ContractAddresses { contractAddresses = append(contractAddresses, common.HexToAddress(address)) } topic0s = append(topic0s, common.HexToHash(config.Topic)) } - chunker := shared.NewLogChunker(transformerConfigs) - - return Watcher{ - DB: db, - Fetcher: fetcher, - Chunker: chunker, - Addresses: contractAddresses, - Topics: topic0s, - Repository: repository, - } -} - -func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { - for _, transformerInitializer := range us { - transformer := transformerInitializer(watcher.DB) - watcher.Transformers = append(watcher.Transformers, transformer) - } + watcher.Addresses = append(watcher.Addresses, contractAddresses...) + watcher.Topics = append(watcher.Topics, topic0s...) + watcher.Chunker.AddConfigs(configs) } func (watcher *Watcher) Execute() error { diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go index a7ebcc17..dea03a5b 100644 --- a/pkg/transformers/shared/log_chunker.go +++ b/pkg/transformers/shared/log_chunker.go @@ -19,31 +19,37 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) +type Chunker interface { + AddConfigs(transformerConfigs []TransformerConfig) + ChunkLogs(logs []types.Log) map[string][]types.Log +} + type LogChunker struct { AddressToNames map[string][]string NameToTopic0 map[string]common.Hash } -// Initialises a chunker by creating efficient lookup maps -func NewLogChunker(transformerConfigs []TransformerConfig) LogChunker { - addressToNames := map[string][]string{} - nameToTopic0 := map[string]common.Hash{} +// Returns a new log chunker with initialised maps. +// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer. +func NewLogChunker() *LogChunker { + return &LogChunker{ + AddressToNames: map[string][]string{}, + NameToTopic0: map[string]common.Hash{}, + } +} +// Configures the chunker by adding more addreses and topics to consider. +func (chunker *LogChunker) AddConfigs(transformerConfigs []TransformerConfig) { for _, config := range transformerConfigs { for _, address := range config.ContractAddresses { - addressToNames[address] = append(addressToNames[address], config.TransformerName) - nameToTopic0[config.TransformerName] = common.HexToHash(config.Topic) + chunker.AddressToNames[address] = append(chunker.AddressToNames[address], config.TransformerName) + chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic) } } - - return LogChunker{ - AddressToNames: addressToNames, - NameToTopic0: nameToTopic0, - } } // Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers -func (chunker LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log { +func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log { chunks := map[string][]types.Log{} for _, log := range logs { // Topic0 is not unique to each transformer, also need to consider the contract address diff --git a/pkg/transformers/shared/log_fetcher.go b/pkg/transformers/shared/log_fetcher.go index 9d0e9415..11081a24 100644 --- a/pkg/transformers/shared/log_fetcher.go +++ b/pkg/transformers/shared/log_fetcher.go @@ -30,8 +30,8 @@ type Fetcher struct { blockChain core.BlockChain } -func NewFetcher(blockchain core.BlockChain) Fetcher { - return Fetcher{ +func NewFetcher(blockchain core.BlockChain) *Fetcher { + return &Fetcher{ blockChain: blockchain, } }