cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

work on porting storage watcher; watcher type is defined for

Browse Source 
transformers in the config, storage transformers are exported from
plugin like event transformers
This commit is contained in:
Ian Norden 2019-02-07 15:35:25 -06:00
parent b51dcb55de
commit 03a7379617
5 changed files with 485 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
cmd/composeAndExecute.go
View File

@ -16,10 +16,12 @@
package cmd package cmd
import ( import (
"errors"
"fmt" "fmt"
"log" "log"
"os" "os"
"plugin" "plugin"
syn "sync"
"time" "time"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -28,6 +30,7 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/fs"
p2 "github.com/vulcanize/vulcanizedb/pkg/plugin" p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
"github.com/vulcanize/vulcanizedb/utils" "github.com/vulcanize/vulcanizedb/utils"
@ -56,9 +59,14 @@ var composeAndExecuteCmd = &cobra.Command{
transformer2 = "github.com/path/to/transformer2" transformer2 = "github.com/path/to/transformer2"
transformer3 = "github.com/path/to/transformer3" transformer3 = "github.com/path/to/transformer3"
transformer4 = "github.com/different/path/to/transformer1" transformer4 = "github.com/different/path/to/transformer1"
[exporter.types]
transformer1 = "eth_event"
transformer2 = "eth_event"
transformer3 = "eth_event"
transformer4 = "eth_storage"
[exporter.repositories] [exporter.repositories]
transformers = "github.com/path/to" transformers = "github.com/path/to"
transformer4 = "github.com/different/path transformer4 = "github.com/different/path"
[exporter.migrations] [exporter.migrations]
transformers = "db/migrations" transformers = "db/migrations"
transformer4 = "to/db/migrations" transformer4 = "to/db/migrations"
@ -66,26 +74,31 @@ var composeAndExecuteCmd = &cobra.Command{
Note: If any of the imported transformer need additional Note: If any of the imported transformer need additional
config variables do not forget to include those as well config variables do not forget to include those as well
This information is used to write and build a .so with an arbitrary transformer This information is used to write and build a go plugin with a transformer
set composed from the transformer imports specified in the config file set composed from the transformer imports specified in the config file
This .so is loaded as a plugin and the set of transformer initializers is This plugin is loaded and the set of transformer initializers is exported
loaded into and executed over by a generic watcher`, from it and loaded into and executed over by the appropriate watcher.
The type of watcher that the transformer works with is specified using the
exporter.types config variable as shown above. Currently there are watchers
of event data from an eth node (eth_event) and storage data from an eth node
(eth_storage). Soon there will be watchers for ipfs (ipfs_event and ipfs_storage).
Transformers of different types can be ran together in the same command using a
single config file or in separate command instances using different config files
Specify config location when executing the command:
./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
composeAndExecute() composeAndExecute()
}, },
} }
func composeAndExecute() { func composeAndExecute() {
// generate code to build the plugin according to the config file // Build plugin generator config
genConfig = config.Plugin{ prepConfig()
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
FileName: viper.GetString("exporter.name"),
Save: viper.GetBool("exporter.save"),
Initializers: viper.GetStringMapString("exporter.transformers"),
Dependencies: viper.GetStringMapString("exporter.repositories"),
Migrations: viper.GetStringMapString("exporter.migrations"),
}
// Generate code to build the plugin according to the config file
fmt.Println("generating plugin") fmt.Println("generating plugin")
generator, err := p2.NewGenerator(genConfig, databaseConfig) generator, err := p2.NewGenerator(genConfig, databaseConfig)
if err != nil { if err != nil {
@ -127,17 +140,44 @@ func composeAndExecute() {
os.Exit(1) os.Exit(1)
} }
// Use the Exporters export method to load the TransformerInitializer set // Use the Exporters export method to load the TransformerInitializer and StorageTransformerInitializer sets
initializers := exporter.Export() ethEventInitializers, ethStorageInitializers := exporter.Export()
// Setup bc and db objects // Setup bc and db objects
blockChain := getBlockChain() blockChain := getBlockChain()
db := utils.LoadPostgres(databaseConfig, blockChain.Node()) db := utils.LoadPostgres(databaseConfig, blockChain.Node())
// Create a watcher and load the TransformerInitializer set into it // Execute over transformer sets returned by the exporter
w := watcher.NewWatcher(&db, blockChain) // Use WaitGroup to wait on both goroutines
w.AddTransformers(initializers) var wg syn.WaitGroup
if len(ethEventInitializers) > 0 {
w := watcher.NewWatcher(&db, blockChain)
w.AddTransformers(ethEventInitializers)
wg.Add(1)
go watchEthEvents(&w, &wg)
}
if len(ethStorageInitializers) > 0 {
tailer := fs.FileTailer{Path: storageDiffsPath}
w := watcher.NewStorageWatcher(tailer, &db)
w.AddTransformers(ethStorageInitializers)
wg.Add(1)
go watchEthStorage(&w, &wg)
}
wg.Wait()
}
type Exporter interface {
Export() ([]transformer.TransformerInitializer, []transformer.StorageTransformerInitializer)
}
func init() {
rootCmd.AddCommand(composeAndExecuteCmd)
composeAndExecuteCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start transformer execution from")
}
func watchEthEvents(w *watcher.Watcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the TransformerInitializer set using the watcher // Execute over the TransformerInitializer set using the watcher
fmt.Println("executing transformers") fmt.Println("executing transformers")
ticker := time.NewTicker(pollingInterval) ticker := time.NewTicker(pollingInterval)
@ -150,11 +190,39 @@ func composeAndExecute() {
} }
} }
type Exporter interface { func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
Export() []transformer.TransformerInitializer defer wg.Done()
// Execute over the TransformerInitializer set using the watcher
fmt.Println("executing transformers")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
err := w.Execute()
if err != nil {
// TODO Handle watcher errors in composeAndExecute
}
}
} }
func init() { func prepConfig() {
rootCmd.AddCommand(composeAndExecuteCmd) fmt.Println("configuring plugin")
composeAndExecuteCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start transformer execution from") types := viper.GetStringMapString("exporter.types")
genTypes := map[string]config.PluginType{}
for transformerName, transformerType := range types {
genType := config.GetPluginType(transformerType)
if genType == config.UnknownTransformerType {
log.Fatal(errors.New(`unknown transformer type in exporter config
accepted types are "eth_event", "eth_storage", "ipfs_event" and "ipfs_storage"`))
}
genTypes[transformerName] = genType
}
genConfig = config.Plugin{
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
FileName: viper.GetString("exporter.name"),
Save: viper.GetBool("exporter.save"),
Initializers: viper.GetStringMapString("exporter.transformers"),
Dependencies: viper.GetStringMapString("exporter.repositories"),
Migrations: viper.GetStringMapString("exporter.migrations"),
Types: genTypes,
}
} }

92
environments/compose.toml
View File

@ -12,39 +12,71 @@
name = "maker_vdb_staging" name = "maker_vdb_staging"
[exporter] [exporter]
name = "exporter" name = "eventTransformerExporter"
save = false save = false
[exporter.transformers] [exporter.transformers]
bite = "github.com/vulcanize/mcd_transformers/transformers/bite/initializer" bite = "github.com/vulcanize/mcd_transformers/transformers/bite/initializer"
cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump/initializer" cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump/initializer"
cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip/initializer" cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip/initializer"
cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow/initializer" cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow/initializer"
deal = "github.com/vulcanize/mcd_transformers/transformers/deal/initializer" deal = "github.com/vulcanize/mcd_transformers/transformers/deal/initializer"
dent = "github.com/vulcanize/mcd_transformers/transformers/dent/initializer" dent = "github.com/vulcanize/mcd_transformers/transformers/dent/initializer"
drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip/initializer" drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip/initializer"
drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk/initializer" drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk/initializer"
drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo/initializer" drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo/initializer"
drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow/initializer" drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow/initializer"
flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick/initializer" flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick/initializer"
flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick/initializer" flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick/initializer"
flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick/initializer" flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick/initializer"
frob = "github.com/vulcanize/mcd_transformers/transformers/frob/initializer" frob = "github.com/vulcanize/mcd_transformers/transformers/frob/initializer"
pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling/initializer" pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling/initializer"
pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk/initializer" pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk/initializer"
price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds/initializer" price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds/initializer"
tend = "github.com/vulcanize/mcd_transformers/transformers/tend/initializer" tend = "github.com/vulcanize/mcd_transformers/transformers/tend/initializer"
vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux/initializer" vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux/initializer"
vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold/initializer" vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold/initializer"
vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab/initializer" vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab/initializer"
vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal/initializer" vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal/initializer"
vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init/initializer" vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init/initializer"
vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move/initializer" vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move/initializer"
vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip/initializer" vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip/initializer"
vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll/initializer" vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll/initializer"
vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune/initializer" vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune/initializer"
vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog/initializer" vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog/initializer"
[exporter.types]
bite = "eth_event"
cat_chop_lump = "eth_event"
cat_flip = "eth_event"
cat_pit_vow = "eth_event"
deal = "eth_event"
dent = "eth_event"
drip_drip = "eth_event"
drip_file_ilk = "eth_event"
drip_file_repo = "eth_event"
drip_file_vow = "eth_event"
flap_kick = "eth_event"
flip_kick = "eth_event"
flop_kick = "eth_event"
frob = "eth_event"
pit_file_debt_ceiling = "eth_event"
pit_file_ilk = "eth_event"
price_feeds = "eth_event"
tend = "eth_event"
vat_flux = "eth_event"
vat_fold = "eth_event"
vat_grab = "eth_event"
vat_heal = "eth_event"
vat_init = "eth_event"
vat_move = "eth_event"
vat_slip = "eth_event"
vat_toll = "eth_event"
vat_tune = "eth_event"
vow_flog = "eth_event"
[exporter.repositories] [exporter.repositories]
mcd_transformers = "github.com/vulcanize/mcd_transformers" mcd__event_transformers = "github.com/vulcanize/mcd_transformers"
[filesystem]
storageDiffsPath = "INSERT-PATH-TO-STORAGE-DIFFS"
[contract] [contract]
[contract.address] [contract.address]

49
pkg/config/plugin.go
View File

@ -26,9 +26,10 @@ import (
) )
type Plugin struct { type Plugin struct {
Initializers map[string]string // Map of import aliases to transformer initializer paths Initializers map[string]string // Map of import aliases to transformer initializer paths
Dependencies map[string]string // Map of vendor dep names to their repositories Dependencies map[string]string // Map of vendor dep names to their repositories
Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations
Types map[string]PluginType // Map of import aliases to their transformer initializer type (e.g. eth-event vs eth-storage)
FilePath string FilePath string
FileName string FileName string
Save bool Save bool
@ -64,3 +65,45 @@ func (c *Plugin) GetMigrationsPaths() ([]string, error) {
return paths, nil return paths, nil
} }
type PluginType int
const (
UnknownTransformerType PluginType = iota + 1
EthEvent
EthStorage
IpfsEvent
IpfsStorage
)
func (pt PluginType) String() string {
names := [...]string{
"eth_event",
"eth_storage",
"ipfs_event",
"ipfs_storage",
}
if pt > IpfsStorage || pt < EthEvent {
return "Unknown"
}
return names[pt]
}
func GetPluginType(str string) PluginType {
types := [...]PluginType{
EthEvent,
EthStorage,
IpfsEvent,
IpfsStorage,
}
for _, ty := range types {
if ty.String() == str && ty.String() != "Unknown" {
return ty
}
}
return UnknownTransformerType
}

323
pkg/plugin/generator_test.go
View File

@ -16,7 +16,7 @@
package plugin_test package plugin_test
/* comment out til mcd_transformers is updated /*
import ( import (
"plugin" "plugin"
@ -30,6 +30,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fs"
p2 "github.com/vulcanize/vulcanizedb/pkg/plugin" p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
"github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers"
@ -40,11 +41,51 @@ var genConfig = config.Plugin{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer", "bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer", "deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
}, },
Types: map[string]config.PluginType{
"bite": config.EthEvent,
"deal": config.EthEvent,
},
Dependencies: map[string]string{ Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers", "mcd_transformers": "github.com/vulcanize/mcd_transformers",
}, },
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"}, //Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
FileName: "externalTestTransformerSet", FileName: "testEventTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var genStorageConfig = config.Plugin{
Initializers: map[string]string{
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
},
Types: map[string]config.PluginType{
"pit": config.EthStorage,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var combinedConfig = config.Plugin{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
},
Types: map[string]config.PluginType{
"bite": config.EthEvent,
"deal": config.EthEvent,
"pit": config.EthStorage,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false, Save: false,
} }
@ -56,7 +97,7 @@ var dbConfig = config.Database{
} }
type Exporter interface { type Exporter interface {
Export() []transformer.TransformerInitializer Export() ([]transformer.TransformerInitializer, []transformer.StorageTransformerInitializer)
} }
var _ = Describe("Generator test", func() { var _ = Describe("Generator test", func() {
@ -70,84 +111,222 @@ var _ = Describe("Generator test", func() {
viper.SetConfigName("compose") viper.SetConfigName("compose")
viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/") viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
BeforeEach(func() { Describe("Event Transformers only", func() {
goPath, soPath, err = genConfig.GetPluginPaths() BeforeEach(func() {
Expect(err).ToNot(HaveOccurred()) goPath, soPath, err = genConfig.GetPluginPaths()
g, err = p2.NewGenerator(genConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
err := helpers.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter") g, err = p2.NewGenerator(genConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
err := helpers.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
}) })
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() { Describe("GenerateTransformerPlugin", func() {
db, bc = test_helpers.SetupDBandBC() It("It bundles the specified TransformerInitializers into a Exporter object and creates .so", func() {
defer test_helpers.TearDown(db) plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers, store := exporter.Export()
Expect(len(initializers)).To(Equal(2))
Expect(len(store)).To(Equal(0))
})
hr = repositories.NewHeaderRepository(db) It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
header1, err := bc.GetHeaderByNumber(9377319) db, bc = test_helpers.SetupDBandBC()
Expect(err).ToNot(HaveOccurred()) defer test_helpers.TearDown(db)
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
plug, err := plugin.Open(soPath) hr = repositories.NewHeaderRepository(db)
Expect(err).ToNot(HaveOccurred()) header1, err := bc.GetHeaderByNumber(9377319)
symExporter, err := plug.Lookup("Exporter") Expect(err).ToNot(HaveOccurred())
Expect(err).ToNot(HaveOccurred()) headerID, err = hr.CreateOrUpdateHeader(header1)
exporter, ok := symExporter.(Exporter) Expect(err).ToNot(HaveOccurred())
Expect(ok).To(Equal(true))
initializers := exporter.Export()
w := watcher.NewWatcher(db, bc) plug, err := plugin.Open(soPath)
w.AddTransformers(initializers) Expect(err).ToNot(HaveOccurred())
err = w.Execute() symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers, _ := exporter.Export()
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
Ilk string
Urn string
Ink string
Art string
IArt string
Tab string
NFlip string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
})
})
Describe("Storage Transformers only", func() {
BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(genStorageConfig, dbConfig)
type model struct {
Ilk string
Urn string
Ink string
Art string
IArt string
Tab string
NFlip string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH")) err = g.GenerateExporterPlugin()
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(err).ToNot(HaveOccurred())
Expect(returned.Ink).To(Equal("80000000000000000000")) })
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992")) AfterEach(func() {
Expect(returned.Tab).To(Equal("11000000000000000000000")) err := helpers.ClearFiles(goPath, soPath)
Expect(returned.NFlip).To(Equal("7")) Expect(err).ToNot(HaveOccurred())
Expect(returned.TransactionIndex).To(Equal(uint(1))) })
Expect(returned.LogIndex).To(Equal(uint(4))) Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified StorageTransformerInitializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
event, initializers := exporter.Export()
Expect(len(initializers)).To(Equal(1))
Expect(len(event)).To(Equal(0))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of StorageTransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
_, initializers := exporter.Export()
tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")}
w := watcher.NewStorageWatcher(tailer, db)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
})
})
})
Describe("Event and Storage Transformers in same instance", func() {
BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(combinedConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
err := helpers.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified TransformerInitializers and StorageTransformerInitializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
eventInitializers, storageInitializers := exporter.Export()
Expect(len(eventInitializers)).To(Equal(2))
Expect(len(storageInitializers)).To(Equal(1))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers and StorageTransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
eventInitializers, storageInitializers := exporter.Export()
ew := watcher.NewWatcher(db, bc)
ew.AddTransformers(eventInitializers)
err = ew.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
Ilk string
Urn string
Ink string
Art string
IArt string
Tab string
NFlip string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")}
sw := watcher.NewStorageWatcher(tailer, db)
sw.AddTransformers(storageInitializers)
err = sw.Execute()
Expect(err).ToNot(HaveOccurred())
})
}) })
}) })
}) })
*/ */

43
pkg/plugin/writer/writer.go
View File

@ -58,21 +58,23 @@ func (w *writer) WritePlugin() error {
f.ImportAlias(imp, alias) f.ImportAlias(imp, alias)
} }
// Collect TransformerInitializer names // Collect initializer code
importedInitializers := make([]Code, 0, len(w.GenConfig.Initializers)) ethEventInitializers, ethStorageInitializers, _, _ := w.sortTransformers()
for _, path := range w.GenConfig.Initializers {
importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer"))
}
// Create Exporter variable with method to export the set of the imported TransformerInitializers // Create Exporter variable with method to export the set of the imported storage and event transformer initializers
f.Type().Id("exporter").String() f.Type().Id("exporter").String()
f.Var().Id("Exporter").Id("exporter") f.Var().Id("Exporter").Id("exporter")
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual( f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Block( "TransformerInitializer").Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Block(
Return(Index().Qual( Return(Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers "TransformerInitializer").Values(ethEventInitializers...)),
Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Values(ethStorageInitializers...)) // Exports the collected initializers
// Write code to destination file // Write code to destination file
err = f.Save(goFile) err = f.Save(goFile)
@ -82,6 +84,31 @@ func (w *writer) WritePlugin() error {
return nil return nil
} }
func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
// Collect code for various initializers
importedEthEventInitializers := make([]Code, 0)
importerEthStorageInitializers := make([]Code, 0)
importedIpfsEventInitializers := make([]Code, 0)
importerIpfsStorageInitializers := make([]Code, 0)
for name, path := range w.GenConfig.Initializers {
switch w.GenConfig.Types[name] {
case config.EthEvent:
importedEthEventInitializers = append(importedEthEventInitializers, Qual(path, "TransformerInitializer"))
case config.EthStorage:
importerEthStorageInitializers = append(importerEthStorageInitializers, Qual(path, "StorageTransformerInitializer"))
case config.IpfsEvent:
//importedIpfsEventInitializers = append(importedIpfsEventInitializers, Qual(path, "IpfsEventTransformerInitializer"))
case config.IpfsStorage:
//importerIpfsStorageInitializers = append(importerIpfsStorageInitializers, Qual(path, "IpfsStorageTransformerInitializer"))
}
}
return importedEthEventInitializers,
importerEthStorageInitializers,
importedIpfsEventInitializers,
importerIpfsStorageInitializers
}
func (w *writer) setupFilePath() (string, error) { func (w *writer) setupFilePath() (string, error) {
goFile, soFile, err := w.GenConfig.GetPluginPaths() goFile, soFile, err := w.GenConfig.GetPluginPaths()
if err != nil { if err != nil {