WIP continue on aggregate fetching architecture

This commit is contained in:
Edvard 2018-11-30 17:28:52 +01:00
parent b5dab1e83b
commit c26736dc9e
4 changed files with 245 additions and 175 deletions

View File

@ -92,34 +92,34 @@ func getTransformerInititalizers(transformerNames []string) []shared2.Transforme
func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer {
transformerInitializerMap := make(map[string]shared2.TransformerInitializer)
transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformerInitializer
transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformerInitializer
transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformerInitializer
transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformerInitializer
transformerInitializerMap[constants.DealLabel] = transformers.DealTransformerInitializer
transformerInitializerMap[constants.DentLabel] = transformers.DentTransformerInitializer
transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformerInitializer
transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformerInitializer
transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformerInitializer
transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromerInitializer
transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformerInitializer
transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformerInitializer
transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformerInitializer
transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformerInitializer
transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformerInitializer
transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformerInitializer
transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformerInitializer
transformerInitializerMap[constants.TendLabel] = transformers.TendTransformerInitializer
transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformerInitializer
transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformerInitializer
transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformerInitializer
transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformerInitializer
transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformerInitializer
transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformerInitializer
transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformerInitializer
transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformerInitializer
transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformerInitializer
transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformerInitializer
transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer
transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer
transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer
transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer
transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer
transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer
transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer
return transformerInitializerMap
}

View File

@ -1,9 +1,10 @@
package shared
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
@ -11,6 +12,35 @@ type Watcher struct {
Transformers []shared.Transformer
DB postgres.DB
Blockchain core.BlockChain
Fetcher shared.LogFetcher
Chunker shared.LogChunker
Addresses []common.Address
Topics []common.Hash
}
func NewWatcher(db postgres.DB, bc core.BlockChain) Watcher {
transformerConfigs := transformers.TransformerConfigs()
var contractAddresses []common.Address
var topic0s []common.Hash
for _, config := range transformerConfigs {
for _, address := range config.ContractAddresses {
contractAddresses = append(contractAddresses, common.HexToAddress(address))
}
topic0s = append(topic0s, common.HexToHash(config.Topic))
}
chunker := shared.NewLogChunker(transformerConfigs)
fetcher := shared.NewFetcher(bc)
return Watcher{
DB: db,
Blockchain: bc,
Fetcher: fetcher,
Chunker: chunker,
Addresses: contractAddresses,
Topics: topic0s,
}
}
func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) {
@ -23,14 +53,24 @@ func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) {
func (watcher *Watcher) Execute() error {
// TODO Solve checkedHeadersColumn issue
// TODO Handle start and end numbers in transformers?
var missingHeaders []core.Header
missingHeaders, err := shared.MissingHeaders(0, -1, &watcher.DB, "")
// TODO Get contract addresses and topic0s
var logs []types.Log
for _, header := range missingHeaders {
// TODO Extend FetchLogs for doing several blocks at a time
logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header)
if err != nil {
// TODO Handle fetch error in watcher
return err
}
var err error
for _, transformer := range watcher.Transformers {
err = transformer.Execute(logs, missingHeaders)
chunkedLogs := watcher.Chunker.ChunkLogs(logs)
for _, transformer := range watcher.Transformers {
// TODO delegate log chunks to respective transformers
// Need to get the transformer name... :/
logChunk := chunkedLogs["transformerName"]
err = transformer.Execute(logChunk, missingHeaders)
}
}
return err
}

View File

@ -22,8 +22,10 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// TODO Check if Fetcher can be simplified with aggregate logic
type LogFetcher interface {
FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error)
FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
}
type SettableLogFetcher interface {
@ -45,23 +47,21 @@ func NewFetcher(blockchain core.BlockChain) Fetcher {
}
}
func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
addresses := hexStringsToAddresses(contractAddresses)
// Checks all topic0s, on all addresses, fetching matching logs for the given header
func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
blockHash := common.HexToHash(header.Hash)
query := ethereum.FilterQuery{
BlockHash: &blockHash,
Addresses: addresses,
Topics: topics,
}
return fetcher.blockChain.GetEthLogsWithCustomQuery(query)
}
func hexStringsToAddresses(hexStrings []string) []common.Address {
var addresses []common.Address
for _, hexString := range hexStrings {
address := common.HexToAddress(hexString)
addresses = append(addresses, address)
// Search for _any_ of the topics in topic0 position; see docs on `FilterQuery`
Topics: [][]common.Hash{topic0s},
}
return addresses
logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
if err != nil {
// TODO review aggregate fetching error handling
return []types.Log{}, err
}
return logs, nil
}

View File

@ -47,205 +47,235 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/vow_flog"
)
// Custom event transformers
var (
BiteTransformerInitializer = factories.Transformer{
BiteTransformer = factories.Transformer{
Config: bite.BiteConfig,
Converter: &bite.BiteConverter{},
Repository: &bite.BiteRepository{},
}.NewTransformer
}
CatFileChopLumpTransformerInitializer = factories.LogNoteTransformer{
Config: chop_lump.CatFileChopLumpConfig,
Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{},
}.NewLogNoteTransformer
CatFileFlipTransformerInitializer = factories.LogNoteTransformer{
Config: flip.CatFileFlipConfig,
Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{},
}.NewLogNoteTransformer
CatFilePitVowTransformerInitializer = factories.LogNoteTransformer{
Config: pit_vow.CatFilePitVowConfig,
Converter: &pit_vow.CatFilePitVowConverter{},
Repository: &pit_vow.CatFilePitVowRepository{},
}.NewLogNoteTransformer
DealTransformerInitializer = factories.LogNoteTransformer{
Config: deal.DealConfig,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
}.NewLogNoteTransformer
DentTransformerInitializer = factories.LogNoteTransformer{
Config: dent.DentConfig,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
}.NewLogNoteTransformer
DripDripTransformerInitializer = factories.LogNoteTransformer{
Config: drip_drip.DripDripConfig,
Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{},
}.NewLogNoteTransformer
DripFileIlkTransformerInitializer = factories.LogNoteTransformer{
Config: ilk2.DripFileIlkConfig,
Converter: &ilk2.DripFileIlkConverter{},
Repository: &ilk2.DripFileIlkRepository{},
}.NewLogNoteTransformer
DripFileRepoTransformerInitializer = factories.LogNoteTransformer{
Config: repo.DripFileRepoConfig,
Converter: &repo.DripFileRepoConverter{},
Repository: &repo.DripFileRepoRepository{},
}.NewLogNoteTransformer
DripFileVowTransfromerInitializer = factories.LogNoteTransformer{
Config: vow.DripFileVowConfig,
Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{},
}.NewLogNoteTransformer
FlapKickTransformerInitializer = factories.Transformer{
FlapKickTransformer = factories.Transformer{
Config: flap_kick.FlapKickConfig,
Converter: &flap_kick.FlapKickConverter{},
Repository: &flap_kick.FlapKickRepository{},
}.NewTransformer
}
FlipKickTransformerInitializer = factories.Transformer{
FlipKickTransformer = factories.Transformer{
Config: flip_kick.FlipKickConfig,
Converter: &flip_kick.FlipKickConverter{},
Repository: &flip_kick.FlipKickRepository{},
}.NewTransformer
}
FlogTransformerInitializer = factories.LogNoteTransformer{
Config: vow_flog.VowFlogConfig,
Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{},
}.NewLogNoteTransformer
FrobTransformerInitializer = factories.Transformer{
FrobTransformer = factories.Transformer{
Config: frob.FrobConfig,
Converter: &frob.FrobConverter{},
Repository: &frob.FrobRepository{},
}.NewTransformer
}
FlopKickTransformerInitializer = factories.Transformer{
FlopKickTransformer = factories.Transformer{
Config: flop_kick.Config,
Converter: &flop_kick.FlopKickConverter{},
Repository: &flop_kick.FlopKickRepository{},
}.NewTransformer
}
PitFileDebtCeilingTransformerInitializer = factories.LogNoteTransformer{
customEventTransformers = []factories.Transformer {
BiteTransformer,
FlapKickTransformer,
FlipKickTransformer,
FrobTransformer,
FlopKickTransformer,
}
)
// LogNote transformers
var (
CatFileChopLumpTransformer = factories.LogNoteTransformer{
Config: chop_lump.CatFileChopLumpConfig,
Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{},
}
CatFileFlipTransformer = factories.LogNoteTransformer{
Config: flip.CatFileFlipConfig,
Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{},
}
CatFilePitVowTransformer = factories.LogNoteTransformer{
Config: pit_vow.CatFilePitVowConfig,
Converter: &pit_vow.CatFilePitVowConverter{},
Repository: &pit_vow.CatFilePitVowRepository{},
}
DealTransformer = factories.LogNoteTransformer{
Config: deal.DealConfig,
Converter: &deal.DealConverter{},
Repository: &deal.DealRepository{},
}
DentTransformer = factories.LogNoteTransformer{
Config: dent.DentConfig,
Converter: &dent.DentConverter{},
Repository: &dent.DentRepository{},
}
DripDripTransformer = factories.LogNoteTransformer{
Config: drip_drip.DripDripConfig,
Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{},
}
DripFileIlkTransformer = factories.LogNoteTransformer{
Config: ilk2.DripFileIlkConfig,
Converter: &ilk2.DripFileIlkConverter{},
Repository: &ilk2.DripFileIlkRepository{},
}
DripFileRepoTransformer = factories.LogNoteTransformer{
Config: repo.DripFileRepoConfig,
Converter: &repo.DripFileRepoConverter{},
Repository: &repo.DripFileRepoRepository{},
}
DripFileVowTransfromer = factories.LogNoteTransformer{
Config: vow.DripFileVowConfig,
Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{},
}
FlogTransformer = factories.LogNoteTransformer{
Config: vow_flog.VowFlogConfig,
Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{},
}
PitFileDebtCeilingTransformer = factories.LogNoteTransformer{
Config: debt_ceiling.DebtCeilingFileConfig,
Converter: &debt_ceiling.PitFileDebtCeilingConverter{},
Repository: &debt_ceiling.PitFileDebtCeilingRepository{},
}.NewLogNoteTransformer
}
PitFileIlkTransformerInitializer = factories.LogNoteTransformer{
PitFileIlkTransformer = factories.LogNoteTransformer{
Config: ilk.IlkFileConfig,
Converter: &ilk.PitFileIlkConverter{},
Repository: &ilk.PitFileIlkRepository{},
}.NewLogNoteTransformer
}
PriceFeedTransformerInitializer = factories.LogNoteTransformer{
PriceFeedTransformer = factories.LogNoteTransformer{
Config: price_feeds.PriceFeedConfig,
Converter: &price_feeds.PriceFeedConverter{},
Repository: &price_feeds.PriceFeedRepository{},
}.NewLogNoteTransformer
}
TendTransformerInitializer = factories.LogNoteTransformer{
TendTransformer = factories.LogNoteTransformer{
Config: tend.TendConfig,
Converter: &tend.TendConverter{},
Repository: &tend.TendRepository{},
}.NewLogNoteTransformer
}
VatInitTransformerInitializer = factories.LogNoteTransformer{
VatInitTransformer = factories.LogNoteTransformer{
Config: vat_init.VatInitConfig,
Converter: &vat_init.VatInitConverter{},
Repository: &vat_init.VatInitRepository{},
}.NewLogNoteTransformer
}
VatGrabTransformerInitializer = factories.LogNoteTransformer{
VatGrabTransformer = factories.LogNoteTransformer{
Config: vat_grab.VatGrabConfig,
Converter: &vat_grab.VatGrabConverter{},
Repository: &vat_grab.VatGrabRepository{},
}.NewLogNoteTransformer
}
VatFoldTransformerInitializer = factories.LogNoteTransformer{
VatFoldTransformer = factories.LogNoteTransformer{
Config: vat_fold.VatFoldConfig,
Converter: &vat_fold.VatFoldConverter{},
Repository: &vat_fold.VatFoldRepository{},
}.NewLogNoteTransformer
}
VatHealTransformerInitializer = factories.LogNoteTransformer{
VatHealTransformer = factories.LogNoteTransformer{
Config: vat_heal.VatHealConfig,
Converter: &vat_heal.VatHealConverter{},
Repository: &vat_heal.VatHealRepository{},
}.NewLogNoteTransformer
}
VatMoveTransformerInitializer = factories.LogNoteTransformer{
VatMoveTransformer = factories.LogNoteTransformer{
Config: vat_move.VatMoveConfig,
Converter: &vat_move.VatMoveConverter{},
Repository: &vat_move.VatMoveRepository{},
}.NewLogNoteTransformer
}
VatSlipTransformerInitializer = factories.LogNoteTransformer{
VatSlipTransformer = factories.LogNoteTransformer{
Config: vat_slip.VatSlipConfig,
Converter: &vat_slip.VatSlipConverter{},
Repository: &vat_slip.VatSlipRepository{},
}.NewLogNoteTransformer
}
VatTollTransformerInitializer = factories.LogNoteTransformer{
VatTollTransformer = factories.LogNoteTransformer{
Config: vat_toll.VatTollConfig,
Converter: &vat_toll.VatTollConverter{},
Repository: &vat_toll.VatTollRepository{},
}.NewLogNoteTransformer
}
VatTuneTransformerInitializer = factories.LogNoteTransformer{
VatTuneTransformer = factories.LogNoteTransformer{
Config: vat_tune.VatTuneConfig,
Converter: &vat_tune.VatTuneConverter{},
Repository: &vat_tune.VatTuneRepository{},
}.NewLogNoteTransformer
}
VatFluxTransformerInitializer = factories.LogNoteTransformer{
VatFluxTransformer = factories.LogNoteTransformer{
Config: vat_flux.VatFluxConfig,
Converter: &vat_flux.VatFluxConverter{},
Repository: &vat_flux.VatFluxRepository{},
}.NewLogNoteTransformer
}
logNoteTransformers = []factories.LogNoteTransformer {
CatFileChopLumpTransformer,
CatFileFlipTransformer,
CatFilePitVowTransformer,
DealTransformer,
DentTransformer,
DripDripTransformer,
DripFileIlkTransformer,
DripFileRepoTransformer,
DripFileVowTransfromer,
FlogTransformer,
PitFileDebtCeilingTransformer,
PitFileIlkTransformer,
PriceFeedTransformer,
TendTransformer,
VatInitTransformer,
VatGrabTransformer,
VatFoldTransformer,
VatHealTransformer,
VatMoveTransformer,
VatSlipTransformer,
VatTollTransformer,
VatTuneTransformer,
VatFluxTransformer,
}
)
func TransformerInitializers() []shared.TransformerInitializer {
return []shared.TransformerInitializer{
BiteTransformerInitializer,
CatFileChopLumpTransformerInitializer,
CatFileFlipTransformerInitializer,
CatFilePitVowTransformerInitializer,
DealTransformerInitializer,
DentTransformerInitializer,
DripDripTransformerInitializer,
DripFileIlkTransformerInitializer,
DripFileVowTransfromerInitializer,
DripFileRepoTransformerInitializer,
FlapKickTransformerInitializer,
FlipKickTransformerInitializer,
FlogTransformerInitializer,
FlopKickTransformerInitializer,
FrobTransformerInitializer,
PitFileDebtCeilingTransformerInitializer,
PitFileIlkTransformerInitializer,
PriceFeedTransformerInitializer,
TendTransformerInitializer,
VatGrabTransformerInitializer,
VatInitTransformerInitializer,
VatMoveTransformerInitializer,
VatHealTransformerInitializer,
VatFoldTransformerInitializer,
VatSlipTransformerInitializer,
VatTollTransformerInitializer,
VatTuneTransformerInitializer,
VatFluxTransformerInitializer,
// `TransformerInitializers` returns a list of functions, that given a db pointer
// will return a `shared.Transformer`
func TransformerInitializers() (initializers []shared.TransformerInitializer) {
for _, transformer := range logNoteTransformers {
initializers = append(initializers, transformer.NewLogNoteTransformer)
}
for _, transformer := range customEventTransformers {
initializers = append(initializers, transformer.NewTransformer)
}
return
}
// `TransformerConfigs` returns the config structs for all available transformers
func TransformerConfigs() (allConfigs []shared.TransformerConfig) {
for _, transformer := range logNoteTransformers {
allConfigs = append(allConfigs, transformer.Config)
}
for _, transformer := range logNoteTransformers {
allConfigs = append(allConfigs, transformer.Config)
}
return
}