diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 9c23af1b..9f081ea3 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -92,34 +92,34 @@ func getTransformerInititalizers(transformerNames []string) []shared2.Transforme func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { transformerInitializerMap := make(map[string]shared2.TransformerInitializer) - transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformerInitializer - transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformerInitializer - transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformerInitializer - transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformerInitializer - transformerInitializerMap[constants.DealLabel] = transformers.DealTransformerInitializer - transformerInitializerMap[constants.DentLabel] = transformers.DentTransformerInitializer - transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformerInitializer - transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformerInitializer - transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformerInitializer - transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromerInitializer - transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformerInitializer - transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformerInitializer - transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformerInitializer - transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformerInitializer - transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformerInitializer - transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformerInitializer - transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformerInitializer - transformerInitializerMap[constants.TendLabel] = transformers.TendTransformerInitializer - transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformerInitializer - transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformerInitializer - transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformerInitializer - transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformerInitializer - transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformerInitializer - transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformerInitializer - transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformerInitializer - transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformerInitializer - transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformerInitializer - transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformerInitializer + transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer + transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer + transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer + transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer + transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer + transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer + transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer return transformerInitializerMap } diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index cd566ddf..2820ad1d 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -1,9 +1,10 @@ package shared import ( - "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) @@ -11,6 +12,35 @@ type Watcher struct { Transformers []shared.Transformer DB postgres.DB Blockchain core.BlockChain + Fetcher shared.LogFetcher + Chunker shared.LogChunker + Addresses []common.Address + Topics []common.Hash +} + +func NewWatcher(db postgres.DB, bc core.BlockChain) Watcher { + transformerConfigs := transformers.TransformerConfigs() + var contractAddresses []common.Address + var topic0s []common.Hash + + for _, config := range transformerConfigs { + for _, address := range config.ContractAddresses { + contractAddresses = append(contractAddresses, common.HexToAddress(address)) + } + topic0s = append(topic0s, common.HexToHash(config.Topic)) + } + + chunker := shared.NewLogChunker(transformerConfigs) + fetcher := shared.NewFetcher(bc) + + return Watcher{ + DB: db, + Blockchain: bc, + Fetcher: fetcher, + Chunker: chunker, + Addresses: contractAddresses, + Topics: topic0s, + } } func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { @@ -23,14 +53,24 @@ func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { func (watcher *Watcher) Execute() error { // TODO Solve checkedHeadersColumn issue // TODO Handle start and end numbers in transformers? - var missingHeaders []core.Header + missingHeaders, err := shared.MissingHeaders(0, -1, &watcher.DB, "") - // TODO Get contract addresses and topic0s - var logs []types.Log + for _, header := range missingHeaders { + // TODO Extend FetchLogs for doing several blocks at a time + logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header) + if err != nil { + // TODO Handle fetch error in watcher + return err + } - var err error - for _, transformer := range watcher.Transformers { - err = transformer.Execute(logs, missingHeaders) + chunkedLogs := watcher.Chunker.ChunkLogs(logs) + + for _, transformer := range watcher.Transformers { + // TODO delegate log chunks to respective transformers + // Need to get the transformer name... :/ + logChunk := chunkedLogs["transformerName"] + err = transformer.Execute(logChunk, missingHeaders) + } } return err } diff --git a/pkg/transformers/shared/log_fetcher.go b/pkg/transformers/shared/log_fetcher.go index 0e6ad7ce..47f6f77d 100644 --- a/pkg/transformers/shared/log_fetcher.go +++ b/pkg/transformers/shared/log_fetcher.go @@ -22,8 +22,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" ) +// TODO Check if Fetcher can be simplified with aggregate logic + type LogFetcher interface { - FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) + FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error) } type SettableLogFetcher interface { @@ -45,23 +47,21 @@ func NewFetcher(blockchain core.BlockChain) Fetcher { } } -func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) { - addresses := hexStringsToAddresses(contractAddresses) +// Checks all topic0s, on all addresses, fetching matching logs for the given header +func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) { blockHash := common.HexToHash(header.Hash) query := ethereum.FilterQuery{ BlockHash: &blockHash, Addresses: addresses, - Topics: topics, - } - return fetcher.blockChain.GetEthLogsWithCustomQuery(query) -} - -func hexStringsToAddresses(hexStrings []string) []common.Address { - var addresses []common.Address - for _, hexString := range hexStrings { - address := common.HexToAddress(hexString) - addresses = append(addresses, address) + // Search for _any_ of the topics in topic0 position; see docs on `FilterQuery` + Topics: [][]common.Hash{topic0s}, } - return addresses + logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query) + if err != nil { + // TODO review aggregate fetching error handling + return []types.Log{}, err + } + + return logs, nil } diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 73e82d6f..ae036206 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -47,205 +47,235 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/vow_flog" ) +// Custom event transformers var ( - BiteTransformerInitializer = factories.Transformer{ + BiteTransformer = factories.Transformer{ Config: bite.BiteConfig, Converter: &bite.BiteConverter{}, Repository: &bite.BiteRepository{}, - }.NewTransformer + } - CatFileChopLumpTransformerInitializer = factories.LogNoteTransformer{ - Config: chop_lump.CatFileChopLumpConfig, - Converter: &chop_lump.CatFileChopLumpConverter{}, - Repository: &chop_lump.CatFileChopLumpRepository{}, - }.NewLogNoteTransformer - - CatFileFlipTransformerInitializer = factories.LogNoteTransformer{ - Config: flip.CatFileFlipConfig, - Converter: &flip.CatFileFlipConverter{}, - Repository: &flip.CatFileFlipRepository{}, - }.NewLogNoteTransformer - - CatFilePitVowTransformerInitializer = factories.LogNoteTransformer{ - Config: pit_vow.CatFilePitVowConfig, - Converter: &pit_vow.CatFilePitVowConverter{}, - Repository: &pit_vow.CatFilePitVowRepository{}, - }.NewLogNoteTransformer - - DealTransformerInitializer = factories.LogNoteTransformer{ - Config: deal.DealConfig, - Converter: &deal.DealConverter{}, - Repository: &deal.DealRepository{}, - }.NewLogNoteTransformer - - DentTransformerInitializer = factories.LogNoteTransformer{ - Config: dent.DentConfig, - Converter: &dent.DentConverter{}, - Repository: &dent.DentRepository{}, - }.NewLogNoteTransformer - - DripDripTransformerInitializer = factories.LogNoteTransformer{ - Config: drip_drip.DripDripConfig, - Converter: &drip_drip.DripDripConverter{}, - Repository: &drip_drip.DripDripRepository{}, - }.NewLogNoteTransformer - - DripFileIlkTransformerInitializer = factories.LogNoteTransformer{ - Config: ilk2.DripFileIlkConfig, - Converter: &ilk2.DripFileIlkConverter{}, - Repository: &ilk2.DripFileIlkRepository{}, - }.NewLogNoteTransformer - - DripFileRepoTransformerInitializer = factories.LogNoteTransformer{ - Config: repo.DripFileRepoConfig, - Converter: &repo.DripFileRepoConverter{}, - Repository: &repo.DripFileRepoRepository{}, - }.NewLogNoteTransformer - - DripFileVowTransfromerInitializer = factories.LogNoteTransformer{ - Config: vow.DripFileVowConfig, - Converter: &vow.DripFileVowConverter{}, - Repository: &vow.DripFileVowRepository{}, - }.NewLogNoteTransformer - - FlapKickTransformerInitializer = factories.Transformer{ + FlapKickTransformer = factories.Transformer{ Config: flap_kick.FlapKickConfig, Converter: &flap_kick.FlapKickConverter{}, Repository: &flap_kick.FlapKickRepository{}, - }.NewTransformer + } - FlipKickTransformerInitializer = factories.Transformer{ + FlipKickTransformer = factories.Transformer{ Config: flip_kick.FlipKickConfig, Converter: &flip_kick.FlipKickConverter{}, Repository: &flip_kick.FlipKickRepository{}, - }.NewTransformer + } - FlogTransformerInitializer = factories.LogNoteTransformer{ - Config: vow_flog.VowFlogConfig, - Converter: &vow_flog.VowFlogConverter{}, - Repository: &vow_flog.VowFlogRepository{}, - }.NewLogNoteTransformer - - FrobTransformerInitializer = factories.Transformer{ + FrobTransformer = factories.Transformer{ Config: frob.FrobConfig, Converter: &frob.FrobConverter{}, Repository: &frob.FrobRepository{}, - }.NewTransformer + } - FlopKickTransformerInitializer = factories.Transformer{ + FlopKickTransformer = factories.Transformer{ Config: flop_kick.Config, Converter: &flop_kick.FlopKickConverter{}, Repository: &flop_kick.FlopKickRepository{}, - }.NewTransformer + } - PitFileDebtCeilingTransformerInitializer = factories.LogNoteTransformer{ + customEventTransformers = []factories.Transformer { + BiteTransformer, + FlapKickTransformer, + FlipKickTransformer, + FrobTransformer, + FlopKickTransformer, + } +) + +// LogNote transformers +var ( + CatFileChopLumpTransformer = factories.LogNoteTransformer{ + Config: chop_lump.CatFileChopLumpConfig, + Converter: &chop_lump.CatFileChopLumpConverter{}, + Repository: &chop_lump.CatFileChopLumpRepository{}, + } + + CatFileFlipTransformer = factories.LogNoteTransformer{ + Config: flip.CatFileFlipConfig, + Converter: &flip.CatFileFlipConverter{}, + Repository: &flip.CatFileFlipRepository{}, + } + + CatFilePitVowTransformer = factories.LogNoteTransformer{ + Config: pit_vow.CatFilePitVowConfig, + Converter: &pit_vow.CatFilePitVowConverter{}, + Repository: &pit_vow.CatFilePitVowRepository{}, + } + + DealTransformer = factories.LogNoteTransformer{ + Config: deal.DealConfig, + Converter: &deal.DealConverter{}, + Repository: &deal.DealRepository{}, + } + + DentTransformer = factories.LogNoteTransformer{ + Config: dent.DentConfig, + Converter: &dent.DentConverter{}, + Repository: &dent.DentRepository{}, + } + + DripDripTransformer = factories.LogNoteTransformer{ + Config: drip_drip.DripDripConfig, + Converter: &drip_drip.DripDripConverter{}, + Repository: &drip_drip.DripDripRepository{}, + } + + DripFileIlkTransformer = factories.LogNoteTransformer{ + Config: ilk2.DripFileIlkConfig, + Converter: &ilk2.DripFileIlkConverter{}, + Repository: &ilk2.DripFileIlkRepository{}, + } + + DripFileRepoTransformer = factories.LogNoteTransformer{ + Config: repo.DripFileRepoConfig, + Converter: &repo.DripFileRepoConverter{}, + Repository: &repo.DripFileRepoRepository{}, + } + + DripFileVowTransfromer = factories.LogNoteTransformer{ + Config: vow.DripFileVowConfig, + Converter: &vow.DripFileVowConverter{}, + Repository: &vow.DripFileVowRepository{}, + } + + FlogTransformer = factories.LogNoteTransformer{ + Config: vow_flog.VowFlogConfig, + Converter: &vow_flog.VowFlogConverter{}, + Repository: &vow_flog.VowFlogRepository{}, + } + + PitFileDebtCeilingTransformer = factories.LogNoteTransformer{ Config: debt_ceiling.DebtCeilingFileConfig, Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, - }.NewLogNoteTransformer + } - PitFileIlkTransformerInitializer = factories.LogNoteTransformer{ + PitFileIlkTransformer = factories.LogNoteTransformer{ Config: ilk.IlkFileConfig, Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, - }.NewLogNoteTransformer + } - PriceFeedTransformerInitializer = factories.LogNoteTransformer{ + PriceFeedTransformer = factories.LogNoteTransformer{ Config: price_feeds.PriceFeedConfig, Converter: &price_feeds.PriceFeedConverter{}, Repository: &price_feeds.PriceFeedRepository{}, - }.NewLogNoteTransformer + } - TendTransformerInitializer = factories.LogNoteTransformer{ + TendTransformer = factories.LogNoteTransformer{ Config: tend.TendConfig, Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, - }.NewLogNoteTransformer + } - VatInitTransformerInitializer = factories.LogNoteTransformer{ + VatInitTransformer = factories.LogNoteTransformer{ Config: vat_init.VatInitConfig, Converter: &vat_init.VatInitConverter{}, Repository: &vat_init.VatInitRepository{}, - }.NewLogNoteTransformer + } - VatGrabTransformerInitializer = factories.LogNoteTransformer{ + VatGrabTransformer = factories.LogNoteTransformer{ Config: vat_grab.VatGrabConfig, Converter: &vat_grab.VatGrabConverter{}, Repository: &vat_grab.VatGrabRepository{}, - }.NewLogNoteTransformer + } - VatFoldTransformerInitializer = factories.LogNoteTransformer{ + VatFoldTransformer = factories.LogNoteTransformer{ Config: vat_fold.VatFoldConfig, Converter: &vat_fold.VatFoldConverter{}, Repository: &vat_fold.VatFoldRepository{}, - }.NewLogNoteTransformer + } - VatHealTransformerInitializer = factories.LogNoteTransformer{ + VatHealTransformer = factories.LogNoteTransformer{ Config: vat_heal.VatHealConfig, Converter: &vat_heal.VatHealConverter{}, Repository: &vat_heal.VatHealRepository{}, - }.NewLogNoteTransformer + } - VatMoveTransformerInitializer = factories.LogNoteTransformer{ + VatMoveTransformer = factories.LogNoteTransformer{ Config: vat_move.VatMoveConfig, Converter: &vat_move.VatMoveConverter{}, Repository: &vat_move.VatMoveRepository{}, - }.NewLogNoteTransformer + } - VatSlipTransformerInitializer = factories.LogNoteTransformer{ + VatSlipTransformer = factories.LogNoteTransformer{ Config: vat_slip.VatSlipConfig, Converter: &vat_slip.VatSlipConverter{}, Repository: &vat_slip.VatSlipRepository{}, - }.NewLogNoteTransformer + } - VatTollTransformerInitializer = factories.LogNoteTransformer{ + VatTollTransformer = factories.LogNoteTransformer{ Config: vat_toll.VatTollConfig, Converter: &vat_toll.VatTollConverter{}, Repository: &vat_toll.VatTollRepository{}, - }.NewLogNoteTransformer + } - VatTuneTransformerInitializer = factories.LogNoteTransformer{ + VatTuneTransformer = factories.LogNoteTransformer{ Config: vat_tune.VatTuneConfig, Converter: &vat_tune.VatTuneConverter{}, Repository: &vat_tune.VatTuneRepository{}, - }.NewLogNoteTransformer + } - VatFluxTransformerInitializer = factories.LogNoteTransformer{ + VatFluxTransformer = factories.LogNoteTransformer{ Config: vat_flux.VatFluxConfig, Converter: &vat_flux.VatFluxConverter{}, Repository: &vat_flux.VatFluxRepository{}, - }.NewLogNoteTransformer + } + + logNoteTransformers = []factories.LogNoteTransformer { + CatFileChopLumpTransformer, + CatFileFlipTransformer, + CatFilePitVowTransformer, + DealTransformer, + DentTransformer, + DripDripTransformer, + DripFileIlkTransformer, + DripFileRepoTransformer, + DripFileVowTransfromer, + FlogTransformer, + PitFileDebtCeilingTransformer, + PitFileIlkTransformer, + PriceFeedTransformer, + TendTransformer, + VatInitTransformer, + VatGrabTransformer, + VatFoldTransformer, + VatHealTransformer, + VatMoveTransformer, + VatSlipTransformer, + VatTollTransformer, + VatTuneTransformer, + VatFluxTransformer, + } ) -func TransformerInitializers() []shared.TransformerInitializer { - return []shared.TransformerInitializer{ - BiteTransformerInitializer, - CatFileChopLumpTransformerInitializer, - CatFileFlipTransformerInitializer, - CatFilePitVowTransformerInitializer, - DealTransformerInitializer, - DentTransformerInitializer, - DripDripTransformerInitializer, - DripFileIlkTransformerInitializer, - DripFileVowTransfromerInitializer, - DripFileRepoTransformerInitializer, - FlapKickTransformerInitializer, - FlipKickTransformerInitializer, - FlogTransformerInitializer, - FlopKickTransformerInitializer, - FrobTransformerInitializer, - PitFileDebtCeilingTransformerInitializer, - PitFileIlkTransformerInitializer, - PriceFeedTransformerInitializer, - TendTransformerInitializer, - VatGrabTransformerInitializer, - VatInitTransformerInitializer, - VatMoveTransformerInitializer, - VatHealTransformerInitializer, - VatFoldTransformerInitializer, - VatSlipTransformerInitializer, - VatTollTransformerInitializer, - VatTuneTransformerInitializer, - VatFluxTransformerInitializer, +// `TransformerInitializers` returns a list of functions, that given a db pointer +// will return a `shared.Transformer` +func TransformerInitializers() (initializers []shared.TransformerInitializer) { + for _, transformer := range logNoteTransformers { + initializers = append(initializers, transformer.NewLogNoteTransformer) } + + for _, transformer := range customEventTransformers { + initializers = append(initializers, transformer.NewTransformer) + } + return +} + +// `TransformerConfigs` returns the config structs for all available transformers +func TransformerConfigs() (allConfigs []shared.TransformerConfig) { + for _, transformer := range logNoteTransformers { + allConfigs = append(allConfigs, transformer.Config) + } + + for _, transformer := range logNoteTransformers { + allConfigs = append(allConfigs, transformer.Config) + } + return }