deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Improve process of adding transformers

Browse Source
This commit is contained in:
Edvard 2018-12-12 15:41:29 +01:00
parent a86a2ec5c6
commit dc43547612
5 changed files with 134 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/backfillMakerLogs.go
View File

@ -47,11 +47,12 @@ func backfillMakerLogs() {
log.Fatal("Failed to initialize database.")
}
repository := shared2.Repository{}
repository := &shared2.Repository{}
fetcher := shared2.NewFetcher(blockChain)
watcher := shared.NewWatcher(db, fetcher, repository, transformers.TransformerConfigs())
chunker := shared2.NewLogChunker()
watcher := shared.NewWatcher(db, fetcher, repository, chunker)
watcher.AddTransformers(transformers.TransformerInitializers())
watcher.AddTransformers(transformers.TransformerInitializers(), transformers.TransformerConfigs())
watcher.Execute()
}

130
cmd/continuousLogSync.go
View File

@ -59,71 +59,109 @@ func syncMakerLogs() {
}
fetcher := shared2.NewFetcher(blockChain)
repository := &shared2.Repository{}
chunker := shared2.NewLogChunker()
watcher := shared.Watcher{
DB: db,
Fetcher: fetcher,
}
initializers, configs := getTransformerSubset(transformerNames)
chunker.AddConfigs(configs)
transformerInititalizers := getTransformerInititalizers(transformerNames)
watcher.AddTransformers(transformerInititalizers)
watcher := shared.NewWatcher(db, fetcher, repository, chunker)
watcher.AddTransformers(initializers, configs)
for range ticker.C {
watcher.Execute()
}
}
func getTransformerInititalizers(transformerNames []string) []shared2.TransformerInitializer {
transformerInitializerMap := buildTransformerInitializerMap()
var transformerInitializers []shared2.TransformerInitializer
func getTransformerSubset(transformerNames []string) ([]shared2.TransformerInitializer, []shared2.TransformerConfig) {
var initializers []shared2.TransformerInitializer
var configs []shared2.TransformerConfig
if transformerNames[0] == "all" {
for _, v := range transformerInitializerMap {
transformerInitializers = append(transformerInitializers, v)
}
initializers = transformers.TransformerInitializers()
configs = transformers.TransformerConfigs()
} else {
initializerMap := buildTransformerInitializerMap()
configMap := buildTransformerConfigMap()
for _, transformerName := range transformerNames {
initializer := transformerInitializerMap[transformerName]
transformerInitializers = append(transformerInitializers, initializer)
initializers = append(initializers, initializerMap[transformerName])
configs = append(configs, configMap[transformerName])
}
}
return transformerInitializers
return initializers, configs
}
func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer {
transformerInitializerMap := make(map[string]shared2.TransformerInitializer)
initializerMap := make(map[string]shared2.TransformerInitializer)
transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer
transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer
transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer
transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer
transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer
transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer
transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer
transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer
initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer
initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer
initializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer
initializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer
initializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer
initializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer
initializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer
initializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer
initializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer
initializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer
initializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer
initializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer
initializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer
initializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer
initializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer
initializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer
initializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer
initializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer
initializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer
initializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer
initializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer
initializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer
initializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer
initializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer
initializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer
initializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer
initializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer
initializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer
initializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer
return transformerInitializerMap
return initializerMap
}
func buildTransformerConfigMap() map[string]shared2.TransformerConfig {
configMap := make(map[string]shared2.TransformerConfig)
configMap[constants.BiteLabel] = transformers.BiteTransformer.Config
configMap[constants.BiteLabel] = transformers.BiteTransformer.Config
configMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.Config
configMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.Config
configMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.Config
configMap[constants.DealLabel] = transformers.DealTransformer.Config
configMap[constants.DentLabel] = transformers.DentTransformer.Config
configMap[constants.DripDripLabel] = transformers.DripDripTransformer.Config
configMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.Config
configMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.Config
configMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.Config
configMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.Config
configMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.Config
configMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.Config
configMap[constants.FrobLabel] = transformers.FrobTransformer.Config
configMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.Config
configMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.Config
configMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.Config
configMap[constants.TendLabel] = transformers.TendTransformer.Config
configMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.Config
configMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.Config
configMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.Config
configMap[constants.VatHealLabel] = transformers.VatHealTransformer.Config
configMap[constants.VatInitLabel] = transformers.VatInitTransformer.Config
configMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.Config
configMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.Config
configMap[constants.VatTollLabel] = transformers.VatTollTransformer.Config
configMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.Config
configMap[constants.VowFlogLabel] = transformers.FlogTransformer.Config
return configMap
}
func init() {

47
libraries/shared/watcher.go
View File

@ -18,41 +18,46 @@ type Watcher struct {
Transformers []shared.Transformer
DB *postgres.DB
Fetcher shared.LogFetcher
Chunker shared.LogChunker
Chunker shared.Chunker
Addresses []common.Address
Topics []common.Hash
Repository WatcherRepository
}
func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository,
transformerConfigs []shared.TransformerConfig) Watcher {
func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository, chunker shared.Chunker) Watcher {
return Watcher{
DB: db,
Fetcher: fetcher,
Chunker: chunker,
Repository: repository,
}
}
// Adds transformers to the watcher, each needs an initializer and the associated config.
// This also changes the configuration of the chunker, so that it will consider the new transformers.
func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer, configs []shared.TransformerConfig) {
if len(initializers) != len(configs) {
panic("Mismatch in number of transformers initializers and configs!")
}
for _, initializer := range initializers {
transformer := initializer(watcher.DB)
watcher.Transformers = append(watcher.Transformers, transformer)
}
var contractAddresses []common.Address
var topic0s []common.Hash
for _, config := range transformerConfigs {
for _, config := range configs {
for _, address := range config.ContractAddresses {
contractAddresses = append(contractAddresses, common.HexToAddress(address))
}
topic0s = append(topic0s, common.HexToHash(config.Topic))
}
chunker := shared.NewLogChunker(transformerConfigs)
return Watcher{
DB: db,
Fetcher: fetcher,
Chunker: chunker,
Addresses: contractAddresses,
Topics: topic0s,
Repository: repository,
}
}
func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) {
for _, transformerInitializer := range us {
transformer := transformerInitializer(watcher.DB)
watcher.Transformers = append(watcher.Transformers, transformer)
}
watcher.Addresses = append(watcher.Addresses, contractAddresses...)
watcher.Topics = append(watcher.Topics, topic0s...)
watcher.Chunker.AddConfigs(configs)
}
func (watcher *Watcher) Execute() error {

30
pkg/transformers/shared/log_chunker.go
View File

@ -19,31 +19,37 @@ import (
"github.com/ethereum/go-ethereum/core/types"
)
type Chunker interface {
AddConfigs(transformerConfigs []TransformerConfig)
ChunkLogs(logs []types.Log) map[string][]types.Log
}
type LogChunker struct {
AddressToNames map[string][]string
NameToTopic0 map[string]common.Hash
}
// Initialises a chunker by creating efficient lookup maps
func NewLogChunker(transformerConfigs []TransformerConfig) LogChunker {
addressToNames := map[string][]string{}
nameToTopic0 := map[string]common.Hash{}
// Returns a new log chunker with initialised maps.
// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer.
func NewLogChunker() *LogChunker {
return &LogChunker{
AddressToNames: map[string][]string{},
NameToTopic0: map[string]common.Hash{},
}
}
// Configures the chunker by adding more addreses and topics to consider.
func (chunker *LogChunker) AddConfigs(transformerConfigs []TransformerConfig) {
for _, config := range transformerConfigs {
for _, address := range config.ContractAddresses {
addressToNames[address] = append(addressToNames[address], config.TransformerName)
nameToTopic0[config.TransformerName] = common.HexToHash(config.Topic)
chunker.AddressToNames[address] = append(chunker.AddressToNames[address], config.TransformerName)
chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic)
}
}
return LogChunker{
AddressToNames: addressToNames,
NameToTopic0: nameToTopic0,
}
}
// Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers
func (chunker LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log {
func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log {
chunks := map[string][]types.Log{}
for _, log := range logs {
// Topic0 is not unique to each transformer, also need to consider the contract address

4
pkg/transformers/shared/log_fetcher.go
View File

@ -30,8 +30,8 @@ type Fetcher struct {
blockChain core.BlockChain
}
func NewFetcher(blockchain core.BlockChain) Fetcher {
return Fetcher{
func NewFetcher(blockchain core.BlockChain) *Fetcher {
return &Fetcher{
blockChain: blockchain,
}
}