deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

goose changes; update plugin readme; use rel paths for transformers in config; more commenting in pkg/plugin; fix PluginWriter to work with different transformer types; reopen generator tests

Browse Source
This commit is contained in:
Ian Norden 2019-02-11 04:08:24 -06:00
parent 90e67d6da9
commit 2868cf2f73
10 changed files with 212 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/composeAndExecute.go
View File

@ -56,18 +56,18 @@ var composeAndExecuteCmd = &cobra.Command{
[exporter] [exporter]
name = "exporter" name = "exporter"
[exporter.transformers] [exporter.transformers]
transformer1 = "github.com/path/to/transformer1" transformer1 = "path/to/transformer1"
transformer2 = "github.com/path/to/transformer2" transformer2 = "path/to/transformer2"
transformer3 = "github.com/path/to/transformer3" transformer3 = "path/to/transformer3"
transformer4 = "github.com/different/path/to/transformer1" transformer4 = "path/to/transformer4"
[exporter.types] [exporter.types]
transformer1 = "eth_event" transformer1 = "eth_event"
transformer2 = "eth_event" transformer2 = "eth_event"
transformer3 = "eth_event" transformer3 = "eth_event"
transformer4 = "eth_storage" transformer4 = "eth_storage"
[exporter.repositories] [exporter.repositories]
transformers = "github.com/path/to" transformers = "github.com/account/repo"
transformer4 = "github.com/different/path" transformer4 = "github.com/account2/repo2"
[exporter.migrations] [exporter.migrations]
transformers = "db/migrations" transformers = "db/migrations"
transformer4 = "to/db/migrations" transformer4 = "to/db/migrations"

56
environments/compose.toml
View File

@ -15,34 +15,34 @@
name = "eventTransformerExporter" name = "eventTransformerExporter"
save = false save = false
[exporter.transformers] [exporter.transformers]
bite = "github.com/vulcanize/mcd_transformers/transformers/bite/initializer" bite = "transformers/bite/initializer"
cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump/initializer" cat_chop_lump = "transformers/cat_file/chop_lump/initializer"
cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip/initializer" cat_flip = "transformers/cat_file/flip/initializer"
cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow/initializer" cat_pit_vow = "transformers/cat_file/pit_vow/initializer"
deal = "github.com/vulcanize/mcd_transformers/transformers/deal/initializer" deal = "transformers/deal/initializer"
dent = "github.com/vulcanize/mcd_transformers/transformers/dent/initializer" dent = "transformers/dent/initializer"
drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip/initializer" drip_drip = "transformers/drip_drip/initializer"
drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk/initializer" drip_file_ilk = "transformers/drip_file/ilk/initializer"
drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo/initializer" drip_file_repo = "transformers/drip_file/repo/initializer"
drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow/initializer" drip_file_vow = "transformers/drip_file/vow/initializer"
flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick/initializer" flap_kick = "transformers/flap_kick/initializer"
flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick/initializer" flip_kick = "transformers/flip_kick/initializer"
flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick/initializer" flop_kick = "transformers/flop_kick/initializer"
frob = "github.com/vulcanize/mcd_transformers/transformers/frob/initializer" frob = "transformers/frob/initializer"
pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling/initializer" pit_file_debt_ceiling = "transformers/pit_file/debt_ceiling/initializer"
pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk/initializer" pit_file_ilk = "transformers/pit_file/ilk/initializer"
price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds/initializer" price_feeds = "transformers/price_feeds/initializer"
tend = "github.com/vulcanize/mcd_transformers/transformers/tend/initializer" tend = "transformers/tend/initializer"
vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux/initializer" vat_flux = "transformers/vat_flux/initializer"
vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold/initializer" vat_fold = "transformers/vat_fold/initializer"
vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab/initializer" vat_grab = "transformers/vat_grab/initializer"
vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal/initializer" vat_heal = "transformers/vat_heal/initializer"
vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init/initializer" vat_init = "transformers/vat_init/initializer"
vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move/initializer" vat_move = "transformers/vat_move/initializer"
vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip/initializer" vat_slip = "transformers/vat_slip/initializer"
vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll/initializer" vat_toll = "transformers/vat_toll/initializer"
vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune/initializer" vat_tune = "transformers/vat_tune/initializer"
vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog/initializer" vow_flog = "transformers/vow_flog/initializer"
[exporter.types] [exporter.types]
bite = "eth_event" bite = "eth_event"
cat_chop_lump = "eth_event" cat_chop_lump = "eth_event"

23
pkg/plugin/builder/builder.go
View File

@ -28,6 +28,9 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
) )
// Interface for compile Go code written by the
// PluginWriter into a shared object (.so file)
// which can be used loaded as a plugin
type PluginBuilder interface { type PluginBuilder interface {
BuildPlugin() error BuildPlugin() error
CleanUp() error CleanUp() error
@ -35,11 +38,12 @@ type PluginBuilder interface {
type builder struct { type builder struct {
GenConfig config.Plugin GenConfig config.Plugin
tmpVenDirs []string tmpVenDirs []string // Keep track of temp vendor directories
goFile string goFile string // Keep track of goFile name
} }
func NewPluginBuilder(gc config.Plugin, dbc config.Database) *builder { // Requires populated plugin config
func NewPluginBuilder(gc config.Plugin) *builder {
return &builder{ return &builder{
GenConfig: gc, GenConfig: gc,
tmpVenDirs: make([]string, 0, len(gc.Dependencies)), tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
@ -70,6 +74,7 @@ func (b *builder) BuildPlugin() error {
} }
// Sets up temporary vendor libs needed for plugin build // Sets up temporary vendor libs needed for plugin build
// This is to work around a conflict between plugins and vendoring (https://github.com/golang/go/issues/20481)
func (b *builder) setupBuildEnv() error { func (b *builder) setupBuildEnv() error {
// TODO: Less hacky way of handling plugin build deps // TODO: Less hacky way of handling plugin build deps
vendorPath, err := helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor") vendorPath, err := helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor")
@ -79,6 +84,11 @@ func (b *builder) setupBuildEnv() error {
// Import transformer dependencies so that we can build our plugin // Import transformer dependencies so that we can build our plugin
for name, importPath := range b.GenConfig.Dependencies { for name, importPath := range b.GenConfig.Dependencies {
// Use dependency paths in config to form git ssh string
// TODO: Change this to https once we are no longer working private transformer repos
// Right now since vulcanize/mcd_transformers is a private repo we
// are using ssh and uploading ssh key to travis for testing
// This is slower and more involved than using https urls
index := strings.Index(importPath, "/") index := strings.Index(importPath, "/")
gitPath := importPath[:index] + ":" + importPath[index+1:] gitPath := importPath[:index] + ":" + importPath[index+1:]
importURL := "git@" + gitPath + ".git" importURL := "git@" + gitPath + ".git"
@ -87,18 +97,21 @@ func (b *builder) setupBuildEnv() error {
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error())) return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
} }
err := os.RemoveAll(filepath.Join(depPath, "vendor/")) err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil { if err != nil {
return err return err
} }
// Keep track of this vendor directory to clear later
b.tmpVenDirs = append(b.tmpVenDirs, depPath) b.tmpVenDirs = append(b.tmpVenDirs, depPath)
} }
return nil return nil
} }
// Used to clear all of the tmp vendor libs used to build the plugin
// Also clears the go file if saving it has not been specified in the config
// Do not call until after the MigrationManager has performed its operations
// as it needs to pull the db migrations from the tmpVenDirs
func (b *builder) CleanUp() error { func (b *builder) CleanUp() error {
if !b.GenConfig.Save { if !b.GenConfig.Save {
err := helpers.ClearFiles(b.goFile) err := helpers.ClearFiles(b.goFile)

10
pkg/plugin/generator.go
View File

@ -27,6 +27,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/plugin/writer" "github.com/vulcanize/vulcanizedb/pkg/plugin/writer"
) )
// Generator is the top-level interface for creating transformer plugins
type Generator interface { type Generator interface {
GenerateExporterPlugin() error GenerateExporterPlugin() error
} }
@ -37,6 +38,7 @@ type generator struct {
manager.MigrationManager manager.MigrationManager
} }
// Creates a new generator from a plugin and database config
func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) { func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
if len(gc.Initializers) < 1 { if len(gc.Initializers) < 1 {
return nil, errors.New("generator needs to be configured with TransformerInitializer import paths") return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
@ -49,21 +51,27 @@ func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
} }
return &generator{ return &generator{
PluginWriter: writer.NewPluginWriter(gc), PluginWriter: writer.NewPluginWriter(gc),
PluginBuilder: builder.NewPluginBuilder(gc, dbc), PluginBuilder: builder.NewPluginBuilder(gc),
MigrationManager: manager.NewMigrationManager(gc, dbc), MigrationManager: manager.NewMigrationManager(gc, dbc),
}, nil }, nil
} }
// Generates plugin for the transformer initializers specified in the generator config
// Writes plugin code => Sets up build environment => Builds .so file => Performs db migrations for the plugin transformers => Clean up
func (g *generator) GenerateExporterPlugin() error { func (g *generator) GenerateExporterPlugin() error {
// Use plugin writer interface to write the plugin code
err := g.PluginWriter.WritePlugin() err := g.PluginWriter.WritePlugin()
if err != nil { if err != nil {
return err return err
} }
// Clean up temporary files and directories when we are done
defer g.PluginBuilder.CleanUp() defer g.PluginBuilder.CleanUp()
// Use plugin builder interface to setup build environment and compile .go file into a .so file
err = g.PluginBuilder.BuildPlugin() err = g.PluginBuilder.BuildPlugin()
if err != nil { if err != nil {
return err return err
} }
// Perform db migrations for the transformers
return g.MigrationManager.RunMigrations() return g.MigrationManager.RunMigrations()
} }

2
pkg/plugin/generator_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestRepository(t *testing.T) { func TestRepository(t *testing.T) {
RegisterFailHandler(Fail) RegisterFailHandler(Fail)
RunSpecs(t, "Gen Suite Test") RunSpecs(t, "Plugin Suite Test")
} }
var _ = BeforeSuite(func() { var _ = BeforeSuite(func() {

51
pkg/plugin/generator_test.go
View File

@ -16,7 +16,6 @@
package plugin_test package plugin_test
/*
import ( import (
"plugin" "plugin"
@ -24,6 +23,7 @@ import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
@ -48,7 +48,7 @@ var genConfig = config.Plugin{
Dependencies: map[string]string{ Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers", "mcd_transformers": "github.com/vulcanize/mcd_transformers",
}, },
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"}, Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testEventTransformerSet", FileName: "testEventTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false, Save: false,
@ -57,14 +57,16 @@ var genConfig = config.Plugin{
var genStorageConfig = config.Plugin{ var genStorageConfig = config.Plugin{
Initializers: map[string]string{ Initializers: map[string]string{
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer", "pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
}, },
Types: map[string]config.PluginType{ Types: map[string]config.PluginType{
"pit": config.EthStorage, "pit": config.EthStorage,
"vat": config.EthStorage,
}, },
Dependencies: map[string]string{ Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers", "mcd_transformers": "github.com/vulcanize/mcd_transformers",
}, },
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"}, Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testStorageTransformerSet", FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false, Save: false,
@ -75,17 +77,19 @@ var combinedConfig = config.Plugin{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer", "bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer", "deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer", "pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
}, },
Types: map[string]config.PluginType{ Types: map[string]config.PluginType{
"bite": config.EthEvent, "bite": config.EthEvent,
"deal": config.EthEvent, "deal": config.EthEvent,
"pit": config.EthStorage, "pit": config.EthStorage,
"vat": config.EthStorage,
}, },
Dependencies: map[string]string{ Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers", "mcd_transformers": "github.com/vulcanize/mcd_transformers",
}, },
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"}, Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testStorageTransformerSet", FileName: "testComboTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false, Save: false,
} }
@ -156,9 +160,9 @@ var _ = Describe("Generator test", func() {
Expect(ok).To(Equal(true)) Expect(ok).To(Equal(true))
initializers, _ := exporter.Export() initializers, _ := exporter.Export()
w := watcher.NewWatcher(db, bc) w := watcher.NewEventWatcher(db, bc)
w.AddTransformers(initializers) w.AddTransformers(initializers)
err = w.Execute() err = w.Execute(constants.HeaderMissing)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
type model struct { type model struct {
@ -180,8 +184,8 @@ var _ = Describe("Generator test", func() {
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID) err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH")) Expect(returned.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(returned.Urn).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb"))
Expect(returned.Ink).To(Equal("80000000000000000000")) Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000")) Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992")) Expect(returned.IArt).To(Equal("12496609999999999999992"))
@ -195,7 +199,7 @@ var _ = Describe("Generator test", func() {
Describe("Storage Transformers only", func() { Describe("Storage Transformers only", func() {
BeforeEach(func() { BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths() goPath, soPath, err = genStorageConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(genStorageConfig, dbConfig) g, err = p2.NewGenerator(genStorageConfig, dbConfig)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -216,12 +220,12 @@ var _ = Describe("Generator test", func() {
exporter, ok := symExporter.(Exporter) exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true)) Expect(ok).To(Equal(true))
event, initializers := exporter.Export() event, initializers := exporter.Export()
Expect(len(initializers)).To(Equal(1)) Expect(len(initializers)).To(Equal(2))
Expect(len(event)).To(Equal(0)) Expect(len(event)).To(Equal(0))
}) })
It("Loads our generated Exporter and uses it to import an arbitrary set of StorageTransformerInitializers that we can execute over", func() { It("Loads our generated Exporter and uses it to import an arbitrary set of StorageTransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC() db, _ = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db) defer test_helpers.TearDown(db)
plug, err := plugin.Open(soPath) plug, err := plugin.Open(soPath)
@ -235,15 +239,16 @@ var _ = Describe("Generator test", func() {
tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")} tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")}
w := watcher.NewStorageWatcher(tailer, db) w := watcher.NewStorageWatcher(tailer, db)
w.AddTransformers(initializers) w.AddTransformers(initializers)
err = w.Execute() // This blocks right now, need to make test file to read from
Expect(err).ToNot(HaveOccurred()) //err = w.Execute()
//Expect(err).ToNot(HaveOccurred())
}) })
}) })
}) })
Describe("Event and Storage Transformers in same instance", func() { Describe("Event and Storage Transformers in same instance", func() {
BeforeEach(func() { BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths() goPath, soPath, err = combinedConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(combinedConfig, dbConfig) g, err = p2.NewGenerator(combinedConfig, dbConfig)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -265,7 +270,7 @@ var _ = Describe("Generator test", func() {
Expect(ok).To(Equal(true)) Expect(ok).To(Equal(true))
eventInitializers, storageInitializers := exporter.Export() eventInitializers, storageInitializers := exporter.Export()
Expect(len(eventInitializers)).To(Equal(2)) Expect(len(eventInitializers)).To(Equal(2))
Expect(len(storageInitializers)).To(Equal(1)) Expect(len(storageInitializers)).To(Equal(2))
}) })
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers and StorageTransformerInitializers that we can execute over", func() { It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers and StorageTransformerInitializers that we can execute over", func() {
@ -286,9 +291,9 @@ var _ = Describe("Generator test", func() {
Expect(ok).To(Equal(true)) Expect(ok).To(Equal(true))
eventInitializers, storageInitializers := exporter.Export() eventInitializers, storageInitializers := exporter.Export()
ew := watcher.NewWatcher(db, bc) ew := watcher.NewEventWatcher(db, bc)
ew.AddTransformers(eventInitializers) ew.AddTransformers(eventInitializers)
err = ew.Execute() err = ew.Execute(constants.HeaderMissing)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
type model struct { type model struct {
@ -310,8 +315,8 @@ var _ = Describe("Generator test", func() {
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID) err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH")) Expect(returned.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(returned.Urn).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb"))
Expect(returned.Ink).To(Equal("80000000000000000000")) Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000")) Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992")) Expect(returned.IArt).To(Equal("12496609999999999999992"))
@ -323,10 +328,10 @@ var _ = Describe("Generator test", func() {
tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")} tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")}
sw := watcher.NewStorageWatcher(tailer, db) sw := watcher.NewStorageWatcher(tailer, db)
sw.AddTransformers(storageInitializers) sw.AddTransformers(storageInitializers)
err = sw.Execute() // This blocks right now, need to make test file to read from
Expect(err).ToNot(HaveOccurred()) //err = w.Execute()
//Expect(err).ToNot(HaveOccurred())
}) })
}) })
}) })
}) })
*/

42
pkg/plugin/manager/manager.go
View File

@ -19,16 +19,15 @@ package manager
import ( import (
"errors" "errors"
"fmt" "fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
"io/ioutil" "io/ioutil"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strconv"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
) )
// Interface for managing the db migrations for plugin transformers
type MigrationManager interface { type MigrationManager interface {
RunMigrations() error RunMigrations() error
} }
@ -39,6 +38,7 @@ type manager struct {
tmpMigDir string tmpMigDir string
} }
// Manager requires both filled in generator and database configs
func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager { func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
return &manager{ return &manager{
GenConfig: gc, GenConfig: gc,
@ -47,7 +47,7 @@ func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
} }
func (m *manager) RunMigrations() error { func (m *manager) RunMigrations() error {
// Get paths to db migrations // Get paths to db migrations from the plugin config
paths, err := m.GenConfig.GetMigrationsPaths() paths, err := m.GenConfig.GetMigrationsPaths()
if err != nil { if err != nil {
return err return err
@ -55,27 +55,22 @@ func (m *manager) RunMigrations() error {
if len(paths) < 1 { if len(paths) < 1 {
return nil return nil
} }
// Init directory for temporary copies of migrations
// Init directory for temporary copies
err = m.setupMigrationEnv() err = m.setupMigrationEnv()
if err != nil { if err != nil {
return err return err
} }
defer m.cleanUp() defer m.cleanUp()
// Creates copies of migrations for all the plugin's transformers in a tmp dir
// Create temporary copies of migrations to the temporary migrationDir
// These tmps are identical except they have had `1` added in front of their unix_timestamps
// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
// But will still be ran in the same order relative to one another
// TODO: Less hacky way of handing migrations
err = m.createMigrationCopies(paths) err = m.createMigrationCopies(paths)
if err != nil { if err != nil {
return err return err
} }
// Run the copied migrations with goose
// Run the copied migrations
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name) pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
err = exec.Command("migrate", "-path", m.tmpMigDir, "-database", pgStr, "up").Run() cmd := exec.Command("goose", "postgres", pgStr, "up")
cmd.Dir = m.tmpMigDir
err = cmd.Run()
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("db migrations for plugin transformers failed: %s", err.Error())) return errors.New(fmt.Sprintf("db migrations for plugin transformers failed: %s", err.Error()))
} }
@ -83,8 +78,8 @@ func (m *manager) RunMigrations() error {
return nil return nil
} }
// Setup a temporary directory to hold transformer db migrations
func (m *manager) setupMigrationEnv() error { func (m *manager) setupMigrationEnv() error {
// Initialize temp directory for transformer migrations
var err error var err error
m.tmpMigDir, err = helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations") m.tmpMigDir, err = helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil { if err != nil {
@ -102,23 +97,22 @@ func (m *manager) setupMigrationEnv() error {
return nil return nil
} }
// Create copies of db migrations from vendored libs
func (m *manager) createMigrationCopies(paths []string) error { func (m *manager) createMigrationCopies(paths []string) error {
// Iterate through migration paths to find migration directory
for _, path := range paths { for _, path := range paths {
dir, err := ioutil.ReadDir(path) dir, err := ioutil.ReadDir(path)
if err != nil { if err != nil {
return err return err
} }
// For each file in the directory check if it is a migration
for _, file := range dir { for _, file := range dir {
if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum if file.IsDir() || filepath.Ext(file.Name()) != ".sql" {
continue
}
_, err := strconv.Atoi(file.Name()[:10])
if err != nil {
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name())
continue continue
} }
src := filepath.Join(path, file.Name()) src := filepath.Join(path, file.Name())
dst := filepath.Join(m.tmpMigDir, "1"+file.Name()) dst := filepath.Join(m.tmpMigDir, file.Name())
// and if it is make a copy of it to our tmp migration directory
err = helpers.CopyFile(src, dst) err = helpers.CopyFile(src, dst)
if err != nil { if err != nil {
return err return err

16
pkg/plugin/test_helpers/database.go
View File

@ -1,3 +1,19 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package test_helpers package test_helpers
import ( import (

38
pkg/plugin/writer/writer.go
View File

@ -26,6 +26,9 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
) )
// Interface for writing a .go file for a simple
// plugin that exports the set of transformer
// initializers specified in the config
type PluginWriter interface { type PluginWriter interface {
WritePlugin() error WritePlugin() error
} }
@ -34,13 +37,14 @@ type writer struct {
GenConfig config.Plugin GenConfig config.Plugin
} }
// Requires populated plugin config
func NewPluginWriter(gc config.Plugin) *writer { func NewPluginWriter(gc config.Plugin) *writer {
return &writer{ return &writer{
GenConfig: gc, GenConfig: gc,
} }
} }
// Generates the plugin code // Generates the plugin code according to config specification
func (w *writer) WritePlugin() error { func (w *writer) WritePlugin() error {
// Setup plugin file paths // Setup plugin file paths
goFile, err := w.setupFilePath() goFile, err := w.setupFilePath()
@ -52,10 +56,10 @@ func (w *writer) WritePlugin() error {
f := NewFile("main") f := NewFile("main")
f.HeaderComment("This is a plugin generated to export the configured transformer initializers") f.HeaderComment("This is a plugin generated to export the configured transformer initializers")
// Import TransformerInitializers specified in config // Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface") f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, imp := range w.GenConfig.Initializers { for alias, relPath := range w.GenConfig.Initializers {
f.ImportAlias(imp, alias) f.ImportAlias(w.makePath(alias, relPath), alias)
} }
// Collect initializer code // Collect initializer code
@ -64,17 +68,16 @@ func (w *writer) WritePlugin() error {
// Create Exporter variable with method to export the set of the imported storage and event transformer initializers // Create Exporter variable with method to export the set of the imported storage and event transformer initializers
f.Type().Id("exporter").String() f.Type().Id("exporter").String()
f.Var().Id("Exporter").Id("exporter") f.Var().Id("Exporter").Id("exporter")
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual( f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Parens(List(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer", Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "TransformerInitializer"),
"TransformerInitializer").Index().Qual( Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "StorageTransformerInitializer"),
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer", )).Block(Return(
"StorageTransformerInitializer").Block(
Return(Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(ethEventInitializers...)),
Index().Qual( Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Values(ethStorageInitializers...)) // Exports the collected initializers "TransformerInitializer").Values(ethEventInitializers...),
Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers
// Write code to destination file // Write code to destination file
err = f.Save(goFile) err = f.Save(goFile)
@ -84,8 +87,8 @@ func (w *writer) WritePlugin() error {
return nil return nil
} }
// Collect code for various types of initializers
func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) { func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
// Collect code for various initializers
importedEthEventInitializers := make([]Code, 0) importedEthEventInitializers := make([]Code, 0)
importerEthStorageInitializers := make([]Code, 0) importerEthStorageInitializers := make([]Code, 0)
importedIpfsEventInitializers := make([]Code, 0) importedIpfsEventInitializers := make([]Code, 0)
@ -109,6 +112,13 @@ func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
importerIpfsStorageInitializers importerIpfsStorageInitializers
} }
// Concat relative path with its repo's root path
func (w *writer) makePath(alias, relPath string) string {
pathRoot := w.GenConfig.Dependencies[alias]
return pathRoot + "/" + relPath
}
// Setup the .go, clear old ones if present
func (w *writer) setupFilePath() (string, error) { func (w *writer) setupFilePath() (string, error) {
goFile, soFile, err := w.GenConfig.GetPluginPaths() goFile, soFile, err := w.GenConfig.GetPluginPaths()
if err != nil { if err != nil {

61
plugins/README.md
View File

@ -19,18 +19,22 @@ The config file requires, at a minimum, the below fields:
[exporter] [exporter]
name = "exporter" name = "exporter"
[exporter.transformers] [exporter.transformers]
transformer1 = "github.com/path/to/transformer1" transformer1 = "path/to/transformer1"
transformer2 = "github.com/path/to/transformer2" transformer2 = "path/to/transformer2"
transformer3 = "github.com/path/to/transformer3" transformer3 = "path/to/transformer3"
transformer4 = "github.com/different/path/to/transformer1" transformer4 = "path/to/transformer4"
[exporter.types]
transformer1 = "eth_event"
transformer2 = "eth_event"
transformer3 = "eth_event"
transformer4 = "eth_storage"
[exporter.repositories] [exporter.repositories]
transformers = "github.com/path/to" transformers = "github.com/account/repo"
transformer4 = "github.com/different/path" transformer4 = "github.com/account2/repo2"
[exporter.migrations] [exporter.migrations]
transformers = "db/migrations" transformers = "db/migrations"
transformer4 = "to/db/migrations" transformer4 = "to/db/migrations"
``` ```
- `exporter.transformers` are mappings of import aliases to paths to `TransformerInitializer`s - `exporter.transformers` are mappings of import aliases to paths to `TransformerInitializer`s
- Import aliases can be arbitrarily named but note that `interface1` is a reserved alias needed for the generic TransformerInitializer type - Import aliases can be arbitrarily named but note that `interface1` is a reserved alias needed for the generic TransformerInitializer type
- `exporter.repositores` are the paths to the repositories which contain the transformers - `exporter.repositores` are the paths to the repositories which contain the transformers
@ -38,7 +42,25 @@ The config file requires, at a minimum, the below fields:
- Migrations need to be located in the repos in `exporter.repositores` - Migrations need to be located in the repos in `exporter.repositores`
- Keys should match the keys for the corresponding repo - Keys should match the keys for the corresponding repo
If the individual transformers require additional configuration variables be sure to include them in the .toml file Note: If any of the imported transformer need additional
config variables do not forget to include those as well
This information is used to write and build a go plugin with a transformer
set composed from the transformer imports specified in the config file
This plugin is loaded and the set of transformer initializers is exported
from it and loaded into and executed over by the appropriate watcher
The type of watcher that the transformer works with is specified using the
exporter.types config variable as shown above
Currently there are watchers for event data from an eth node (eth_event)
and storage data from an eth node (eth_storage)
In the future there will be watchers for ipfs (ipfs_event and ipfs_storage)
Transformers of different types can be ran together in the same command using a
single config file or in separate command instances using different config files
Specify config location when executing the command:
`./vulcanizedb composeAndExecute --config=./environments/config_name.toml`
The general structure of a plugin .go file, and what we would see with the above config is shown below The general structure of a plugin .go file, and what we would see with the above config is shown below
@ -57,21 +79,24 @@ type exporter string
var Exporter exporter var Exporter exporter
func (e exporter) Export() []interface1.TransformerInitializer { func (e exporter) Export() []interface1.TransformerInitializer, []interface1.StorageTransformerInitializer {
return []interface1.TransformerInitializer{ return []interface1.TransformerInitializer{
transformer1.TransformerInitializer, transformer1.TransformerInitializer,
transformer2.TransformerInitializer, transformer2.TransformerInitializer,
transformer3.TransformerInitializer, transformer3.TransformerInitializer,
transformer4.TransformerInitializer, }, []interface1.StorageTransformerInitializer{
transformer4.StorageTransformerInitializer,
} }
} }
``` ```
As such, to plug in an external transformer we need to create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19) To plug in an external transformer we need to:
As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them * create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go)
Note: currently the transformers must also operate using this watcher's [execution mode](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/watcher.go#L80), in the future the watcher will become pluggable as well that exports a variable `TransformerInitializer` or `StorageTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33)
and [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/storage_transformer.go#L31), respectively
For each transformer we will also need to create db migrations to run against vulcanizeDB so that we can store the transformed data * design the transformers to work in the context of the [event](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go#L83)
The migrations needed for a specific transformer need to be included in the same repository as the transformer(s) that require them, and their relative paths in that repo must be specified in the config as discussed above or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53) watchers
* create db migrations to run against vulcanizeDB so that we can store the transformed data
NOTE: Due to a bug with plugin migrations, currently need to leave the `exporter.migrations` blank and manually run migrations before running composeAndExecute * store the db migrations required for a transformer in the same repository as the transformer(s) that require them
* specify their relative paths in that repo in the config, as discussed above
* NOTE: due to a bug with plugin migrations, currently need to leave the `exporter.migrations` blank and manually run migrations before running composeAndExecute