goose changes; update plugin readme; use rel paths for transformers in config; more commenting in pkg/plugin; fix PluginWriter to work with different transformer types; reopen generator tests
This commit is contained in:
Ian Norden
parent
90e67d6da9
commit
2868cf2f73
@ -56,18 +56,18 @@ var composeAndExecuteCmd = &cobra.Command{
|
||||
[exporter]
|
||||
name = "exporter"
|
||||
[exporter.transformers]
|
||||
transformer1 = "github.com/path/to/transformer1"
|
||||
transformer2 = "github.com/path/to/transformer2"
|
||||
transformer3 = "github.com/path/to/transformer3"
|
||||
transformer4 = "github.com/different/path/to/transformer1"
|
||||
transformer1 = "path/to/transformer1"
|
||||
transformer2 = "path/to/transformer2"
|
||||
transformer3 = "path/to/transformer3"
|
||||
transformer4 = "path/to/transformer4"
|
||||
[exporter.types]
|
||||
transformer1 = "eth_event"
|
||||
transformer2 = "eth_event"
|
||||
transformer3 = "eth_event"
|
||||
transformer4 = "eth_storage"
|
||||
[exporter.repositories]
|
||||
transformers = "github.com/path/to"
|
||||
transformer4 = "github.com/different/path"
|
||||
transformers = "github.com/account/repo"
|
||||
transformer4 = "github.com/account2/repo2"
|
||||
[exporter.migrations]
|
||||
transformers = "db/migrations"
|
||||
transformer4 = "to/db/migrations"
|
||||
|
||||
@ -15,34 +15,34 @@
|
||||
name = "eventTransformerExporter"
|
||||
save = false
|
||||
[exporter.transformers]
|
||||
bite = "github.com/vulcanize/mcd_transformers/transformers/bite/initializer"
|
||||
cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump/initializer"
|
||||
cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip/initializer"
|
||||
cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow/initializer"
|
||||
deal = "github.com/vulcanize/mcd_transformers/transformers/deal/initializer"
|
||||
dent = "github.com/vulcanize/mcd_transformers/transformers/dent/initializer"
|
||||
drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip/initializer"
|
||||
drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk/initializer"
|
||||
drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo/initializer"
|
||||
drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow/initializer"
|
||||
flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick/initializer"
|
||||
flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick/initializer"
|
||||
flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick/initializer"
|
||||
frob = "github.com/vulcanize/mcd_transformers/transformers/frob/initializer"
|
||||
pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling/initializer"
|
||||
pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk/initializer"
|
||||
price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds/initializer"
|
||||
tend = "github.com/vulcanize/mcd_transformers/transformers/tend/initializer"
|
||||
vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux/initializer"
|
||||
vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold/initializer"
|
||||
vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab/initializer"
|
||||
vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal/initializer"
|
||||
vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init/initializer"
|
||||
vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move/initializer"
|
||||
vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip/initializer"
|
||||
vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll/initializer"
|
||||
vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune/initializer"
|
||||
vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog/initializer"
|
||||
bite = "transformers/bite/initializer"
|
||||
cat_chop_lump = "transformers/cat_file/chop_lump/initializer"
|
||||
cat_flip = "transformers/cat_file/flip/initializer"
|
||||
cat_pit_vow = "transformers/cat_file/pit_vow/initializer"
|
||||
deal = "transformers/deal/initializer"
|
||||
dent = "transformers/dent/initializer"
|
||||
drip_drip = "transformers/drip_drip/initializer"
|
||||
drip_file_ilk = "transformers/drip_file/ilk/initializer"
|
||||
drip_file_repo = "transformers/drip_file/repo/initializer"
|
||||
drip_file_vow = "transformers/drip_file/vow/initializer"
|
||||
flap_kick = "transformers/flap_kick/initializer"
|
||||
flip_kick = "transformers/flip_kick/initializer"
|
||||
flop_kick = "transformers/flop_kick/initializer"
|
||||
frob = "transformers/frob/initializer"
|
||||
pit_file_debt_ceiling = "transformers/pit_file/debt_ceiling/initializer"
|
||||
pit_file_ilk = "transformers/pit_file/ilk/initializer"
|
||||
price_feeds = "transformers/price_feeds/initializer"
|
||||
tend = "transformers/tend/initializer"
|
||||
vat_flux = "transformers/vat_flux/initializer"
|
||||
vat_fold = "transformers/vat_fold/initializer"
|
||||
vat_grab = "transformers/vat_grab/initializer"
|
||||
vat_heal = "transformers/vat_heal/initializer"
|
||||
vat_init = "transformers/vat_init/initializer"
|
||||
vat_move = "transformers/vat_move/initializer"
|
||||
vat_slip = "transformers/vat_slip/initializer"
|
||||
vat_toll = "transformers/vat_toll/initializer"
|
||||
vat_tune = "transformers/vat_tune/initializer"
|
||||
vow_flog = "transformers/vow_flog/initializer"
|
||||
[exporter.types]
|
||||
bite = "eth_event"
|
||||
cat_chop_lump = "eth_event"
|
||||
|
||||
@ -28,6 +28,9 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
|
||||
)
|
||||
|
||||
// Interface for compile Go code written by the
|
||||
// PluginWriter into a shared object (.so file)
|
||||
// which can be used loaded as a plugin
|
||||
type PluginBuilder interface {
|
||||
BuildPlugin() error
|
||||
CleanUp() error
|
||||
@ -35,11 +38,12 @@ type PluginBuilder interface {
|
||||
|
||||
type builder struct {
|
||||
GenConfig config.Plugin
|
||||
tmpVenDirs []string
|
||||
goFile string
|
||||
tmpVenDirs []string // Keep track of temp vendor directories
|
||||
goFile string // Keep track of goFile name
|
||||
}
|
||||
|
||||
func NewPluginBuilder(gc config.Plugin, dbc config.Database) *builder {
|
||||
// Requires populated plugin config
|
||||
func NewPluginBuilder(gc config.Plugin) *builder {
|
||||
return &builder{
|
||||
GenConfig: gc,
|
||||
tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
|
||||
@ -70,6 +74,7 @@ func (b *builder) BuildPlugin() error {
|
||||
}
|
||||
|
||||
// Sets up temporary vendor libs needed for plugin build
|
||||
// This is to work around a conflict between plugins and vendoring (https://github.com/golang/go/issues/20481)
|
||||
func (b *builder) setupBuildEnv() error {
|
||||
// TODO: Less hacky way of handling plugin build deps
|
||||
vendorPath, err := helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor")
|
||||
@ -79,6 +84,11 @@ func (b *builder) setupBuildEnv() error {
|
||||
|
||||
// Import transformer dependencies so that we can build our plugin
|
||||
for name, importPath := range b.GenConfig.Dependencies {
|
||||
// Use dependency paths in config to form git ssh string
|
||||
// TODO: Change this to https once we are no longer working private transformer repos
|
||||
// Right now since vulcanize/mcd_transformers is a private repo we
|
||||
// are using ssh and uploading ssh key to travis for testing
|
||||
// This is slower and more involved than using https urls
|
||||
index := strings.Index(importPath, "/")
|
||||
gitPath := importPath[:index] + ":" + importPath[index+1:]
|
||||
importURL := "git@" + gitPath + ".git"
|
||||
@ -87,18 +97,21 @@ func (b *builder) setupBuildEnv() error {
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
|
||||
}
|
||||
|
||||
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Keep track of this vendor directory to clear later
|
||||
b.tmpVenDirs = append(b.tmpVenDirs, depPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Used to clear all of the tmp vendor libs used to build the plugin
|
||||
// Also clears the go file if saving it has not been specified in the config
|
||||
// Do not call until after the MigrationManager has performed its operations
|
||||
// as it needs to pull the db migrations from the tmpVenDirs
|
||||
func (b *builder) CleanUp() error {
|
||||
if !b.GenConfig.Save {
|
||||
err := helpers.ClearFiles(b.goFile)
|
||||
|
||||
@ -27,6 +27,7 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/writer"
|
||||
)
|
||||
|
||||
// Generator is the top-level interface for creating transformer plugins
|
||||
type Generator interface {
|
||||
GenerateExporterPlugin() error
|
||||
}
|
||||
@ -37,6 +38,7 @@ type generator struct {
|
||||
manager.MigrationManager
|
||||
}
|
||||
|
||||
// Creates a new generator from a plugin and database config
|
||||
func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
|
||||
if len(gc.Initializers) < 1 {
|
||||
return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
|
||||
@ -49,21 +51,27 @@ func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
|
||||
}
|
||||
return &generator{
|
||||
PluginWriter: writer.NewPluginWriter(gc),
|
||||
PluginBuilder: builder.NewPluginBuilder(gc, dbc),
|
||||
PluginBuilder: builder.NewPluginBuilder(gc),
|
||||
MigrationManager: manager.NewMigrationManager(gc, dbc),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Generates plugin for the transformer initializers specified in the generator config
|
||||
// Writes plugin code => Sets up build environment => Builds .so file => Performs db migrations for the plugin transformers => Clean up
|
||||
func (g *generator) GenerateExporterPlugin() error {
|
||||
// Use plugin writer interface to write the plugin code
|
||||
err := g.PluginWriter.WritePlugin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Clean up temporary files and directories when we are done
|
||||
defer g.PluginBuilder.CleanUp()
|
||||
// Use plugin builder interface to setup build environment and compile .go file into a .so file
|
||||
err = g.PluginBuilder.BuildPlugin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Perform db migrations for the transformers
|
||||
return g.MigrationManager.RunMigrations()
|
||||
}
|
||||
|
||||
@ -27,7 +27,7 @@ import (
|
||||
|
||||
func TestRepository(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Gen Suite Test")
|
||||
RunSpecs(t, "Plugin Suite Test")
|
||||
}
|
||||
|
||||
var _ = BeforeSuite(func() {
|
||||
|
||||
@ -16,7 +16,6 @@
|
||||
|
||||
package plugin_test
|
||||
|
||||
/*
|
||||
import (
|
||||
"plugin"
|
||||
|
||||
@ -24,6 +23,7 @@ import (
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/config"
|
||||
@ -48,26 +48,28 @@ var genConfig = config.Plugin{
|
||||
Dependencies: map[string]string{
|
||||
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
|
||||
FileName: "testEventTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
|
||||
FileName: "testEventTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
}
|
||||
|
||||
var genStorageConfig = config.Plugin{
|
||||
Initializers: map[string]string{
|
||||
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
|
||||
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
|
||||
},
|
||||
Types: map[string]config.PluginType{
|
||||
"pit": config.EthStorage,
|
||||
"vat": config.EthStorage,
|
||||
},
|
||||
Dependencies: map[string]string{
|
||||
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
|
||||
FileName: "testStorageTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
|
||||
FileName: "testStorageTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
}
|
||||
|
||||
var combinedConfig = config.Plugin{
|
||||
@ -75,19 +77,21 @@ var combinedConfig = config.Plugin{
|
||||
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
|
||||
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
|
||||
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
|
||||
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
|
||||
},
|
||||
Types: map[string]config.PluginType{
|
||||
"bite": config.EthEvent,
|
||||
"deal": config.EthEvent,
|
||||
"pit": config.EthStorage,
|
||||
"vat": config.EthStorage,
|
||||
},
|
||||
Dependencies: map[string]string{
|
||||
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
|
||||
FileName: "testStorageTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
|
||||
FileName: "testComboTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
}
|
||||
|
||||
var dbConfig = config.Database{
|
||||
@ -156,9 +160,9 @@ var _ = Describe("Generator test", func() {
|
||||
Expect(ok).To(Equal(true))
|
||||
initializers, _ := exporter.Export()
|
||||
|
||||
w := watcher.NewWatcher(db, bc)
|
||||
w := watcher.NewEventWatcher(db, bc)
|
||||
w.AddTransformers(initializers)
|
||||
err = w.Execute()
|
||||
err = w.Execute(constants.HeaderMissing)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
type model struct {
|
||||
@ -180,8 +184,8 @@ var _ = Describe("Generator test", func() {
|
||||
|
||||
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(returned.Ilk).To(Equal("ETH"))
|
||||
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
|
||||
Expect(returned.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
|
||||
Expect(returned.Urn).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb"))
|
||||
Expect(returned.Ink).To(Equal("80000000000000000000"))
|
||||
Expect(returned.Art).To(Equal("11000000000000000000000"))
|
||||
Expect(returned.IArt).To(Equal("12496609999999999999992"))
|
||||
@ -195,7 +199,7 @@ var _ = Describe("Generator test", func() {
|
||||
|
||||
Describe("Storage Transformers only", func() {
|
||||
BeforeEach(func() {
|
||||
goPath, soPath, err = genConfig.GetPluginPaths()
|
||||
goPath, soPath, err = genStorageConfig.GetPluginPaths()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
g, err = p2.NewGenerator(genStorageConfig, dbConfig)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@ -216,12 +220,12 @@ var _ = Describe("Generator test", func() {
|
||||
exporter, ok := symExporter.(Exporter)
|
||||
Expect(ok).To(Equal(true))
|
||||
event, initializers := exporter.Export()
|
||||
Expect(len(initializers)).To(Equal(1))
|
||||
Expect(len(initializers)).To(Equal(2))
|
||||
Expect(len(event)).To(Equal(0))
|
||||
})
|
||||
|
||||
It("Loads our generated Exporter and uses it to import an arbitrary set of StorageTransformerInitializers that we can execute over", func() {
|
||||
db, bc = test_helpers.SetupDBandBC()
|
||||
db, _ = test_helpers.SetupDBandBC()
|
||||
defer test_helpers.TearDown(db)
|
||||
|
||||
plug, err := plugin.Open(soPath)
|
||||
@ -235,15 +239,16 @@ var _ = Describe("Generator test", func() {
|
||||
tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")}
|
||||
w := watcher.NewStorageWatcher(tailer, db)
|
||||
w.AddTransformers(initializers)
|
||||
err = w.Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// This blocks right now, need to make test file to read from
|
||||
//err = w.Execute()
|
||||
//Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Event and Storage Transformers in same instance", func() {
|
||||
BeforeEach(func() {
|
||||
goPath, soPath, err = genConfig.GetPluginPaths()
|
||||
goPath, soPath, err = combinedConfig.GetPluginPaths()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
g, err = p2.NewGenerator(combinedConfig, dbConfig)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@ -265,7 +270,7 @@ var _ = Describe("Generator test", func() {
|
||||
Expect(ok).To(Equal(true))
|
||||
eventInitializers, storageInitializers := exporter.Export()
|
||||
Expect(len(eventInitializers)).To(Equal(2))
|
||||
Expect(len(storageInitializers)).To(Equal(1))
|
||||
Expect(len(storageInitializers)).To(Equal(2))
|
||||
})
|
||||
|
||||
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers and StorageTransformerInitializers that we can execute over", func() {
|
||||
@ -286,9 +291,9 @@ var _ = Describe("Generator test", func() {
|
||||
Expect(ok).To(Equal(true))
|
||||
eventInitializers, storageInitializers := exporter.Export()
|
||||
|
||||
ew := watcher.NewWatcher(db, bc)
|
||||
ew := watcher.NewEventWatcher(db, bc)
|
||||
ew.AddTransformers(eventInitializers)
|
||||
err = ew.Execute()
|
||||
err = ew.Execute(constants.HeaderMissing)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
type model struct {
|
||||
@ -310,8 +315,8 @@ var _ = Describe("Generator test", func() {
|
||||
|
||||
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(returned.Ilk).To(Equal("ETH"))
|
||||
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
|
||||
Expect(returned.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
|
||||
Expect(returned.Urn).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb"))
|
||||
Expect(returned.Ink).To(Equal("80000000000000000000"))
|
||||
Expect(returned.Art).To(Equal("11000000000000000000000"))
|
||||
Expect(returned.IArt).To(Equal("12496609999999999999992"))
|
||||
@ -323,10 +328,10 @@ var _ = Describe("Generator test", func() {
|
||||
tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")}
|
||||
sw := watcher.NewStorageWatcher(tailer, db)
|
||||
sw.AddTransformers(storageInitializers)
|
||||
err = sw.Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// This blocks right now, need to make test file to read from
|
||||
//err = w.Execute()
|
||||
//Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
*/
|
||||
|
||||
@ -19,16 +19,15 @@ package manager
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/config"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/config"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
|
||||
)
|
||||
|
||||
// Interface for managing the db migrations for plugin transformers
|
||||
type MigrationManager interface {
|
||||
RunMigrations() error
|
||||
}
|
||||
@ -39,6 +38,7 @@ type manager struct {
|
||||
tmpMigDir string
|
||||
}
|
||||
|
||||
// Manager requires both filled in generator and database configs
|
||||
func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
|
||||
return &manager{
|
||||
GenConfig: gc,
|
||||
@ -47,7 +47,7 @@ func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
|
||||
}
|
||||
|
||||
func (m *manager) RunMigrations() error {
|
||||
// Get paths to db migrations
|
||||
// Get paths to db migrations from the plugin config
|
||||
paths, err := m.GenConfig.GetMigrationsPaths()
|
||||
if err != nil {
|
||||
return err
|
||||
@ -55,27 +55,22 @@ func (m *manager) RunMigrations() error {
|
||||
if len(paths) < 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Init directory for temporary copies
|
||||
// Init directory for temporary copies of migrations
|
||||
err = m.setupMigrationEnv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer m.cleanUp()
|
||||
|
||||
// Create temporary copies of migrations to the temporary migrationDir
|
||||
// These tmps are identical except they have had `1` added in front of their unix_timestamps
|
||||
// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
|
||||
// But will still be ran in the same order relative to one another
|
||||
// TODO: Less hacky way of handing migrations
|
||||
// Creates copies of migrations for all the plugin's transformers in a tmp dir
|
||||
err = m.createMigrationCopies(paths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Run the copied migrations
|
||||
// Run the copied migrations with goose
|
||||
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
|
||||
err = exec.Command("migrate", "-path", m.tmpMigDir, "-database", pgStr, "up").Run()
|
||||
cmd := exec.Command("goose", "postgres", pgStr, "up")
|
||||
cmd.Dir = m.tmpMigDir
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("db migrations for plugin transformers failed: %s", err.Error()))
|
||||
}
|
||||
@ -83,8 +78,8 @@ func (m *manager) RunMigrations() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Setup a temporary directory to hold transformer db migrations
|
||||
func (m *manager) setupMigrationEnv() error {
|
||||
// Initialize temp directory for transformer migrations
|
||||
var err error
|
||||
m.tmpMigDir, err = helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
|
||||
if err != nil {
|
||||
@ -102,23 +97,22 @@ func (m *manager) setupMigrationEnv() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create copies of db migrations from vendored libs
|
||||
func (m *manager) createMigrationCopies(paths []string) error {
|
||||
// Iterate through migration paths to find migration directory
|
||||
for _, path := range paths {
|
||||
dir, err := ioutil.ReadDir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// For each file in the directory check if it is a migration
|
||||
for _, file := range dir {
|
||||
if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum
|
||||
continue
|
||||
}
|
||||
_, err := strconv.Atoi(file.Name()[:10])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name())
|
||||
if file.IsDir() || filepath.Ext(file.Name()) != ".sql" {
|
||||
continue
|
||||
}
|
||||
src := filepath.Join(path, file.Name())
|
||||
dst := filepath.Join(m.tmpMigDir, "1"+file.Name())
|
||||
dst := filepath.Join(m.tmpMigDir, file.Name())
|
||||
// and if it is make a copy of it to our tmp migration directory
|
||||
err = helpers.CopyFile(src, dst)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@ -1,3 +1,19 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2018 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package test_helpers
|
||||
|
||||
import (
|
||||
|
||||
@ -26,6 +26,9 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
|
||||
)
|
||||
|
||||
// Interface for writing a .go file for a simple
|
||||
// plugin that exports the set of transformer
|
||||
// initializers specified in the config
|
||||
type PluginWriter interface {
|
||||
WritePlugin() error
|
||||
}
|
||||
@ -34,13 +37,14 @@ type writer struct {
|
||||
GenConfig config.Plugin
|
||||
}
|
||||
|
||||
// Requires populated plugin config
|
||||
func NewPluginWriter(gc config.Plugin) *writer {
|
||||
return &writer{
|
||||
GenConfig: gc,
|
||||
}
|
||||
}
|
||||
|
||||
// Generates the plugin code
|
||||
// Generates the plugin code according to config specification
|
||||
func (w *writer) WritePlugin() error {
|
||||
// Setup plugin file paths
|
||||
goFile, err := w.setupFilePath()
|
||||
@ -52,10 +56,10 @@ func (w *writer) WritePlugin() error {
|
||||
f := NewFile("main")
|
||||
f.HeaderComment("This is a plugin generated to export the configured transformer initializers")
|
||||
|
||||
// Import TransformerInitializers specified in config
|
||||
// Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config
|
||||
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
|
||||
for alias, imp := range w.GenConfig.Initializers {
|
||||
f.ImportAlias(imp, alias)
|
||||
for alias, relPath := range w.GenConfig.Initializers {
|
||||
f.ImportAlias(w.makePath(alias, relPath), alias)
|
||||
}
|
||||
|
||||
// Collect initializer code
|
||||
@ -64,17 +68,16 @@ func (w *writer) WritePlugin() error {
|
||||
// Create Exporter variable with method to export the set of the imported storage and event transformer initializers
|
||||
f.Type().Id("exporter").String()
|
||||
f.Var().Id("Exporter").Id("exporter")
|
||||
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual(
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
|
||||
"TransformerInitializer").Index().Qual(
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
|
||||
"StorageTransformerInitializer").Block(
|
||||
Return(Index().Qual(
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
|
||||
"TransformerInitializer").Values(ethEventInitializers...)),
|
||||
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Parens(List(
|
||||
Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "TransformerInitializer"),
|
||||
Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "StorageTransformerInitializer"),
|
||||
)).Block(Return(
|
||||
Index().Qual(
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
|
||||
"StorageTransformerInitializer").Values(ethStorageInitializers...)) // Exports the collected initializers
|
||||
"TransformerInitializer").Values(ethEventInitializers...),
|
||||
Index().Qual(
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
|
||||
"StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers
|
||||
|
||||
// Write code to destination file
|
||||
err = f.Save(goFile)
|
||||
@ -84,8 +87,8 @@ func (w *writer) WritePlugin() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Collect code for various types of initializers
|
||||
func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
|
||||
// Collect code for various initializers
|
||||
importedEthEventInitializers := make([]Code, 0)
|
||||
importerEthStorageInitializers := make([]Code, 0)
|
||||
importedIpfsEventInitializers := make([]Code, 0)
|
||||
@ -109,6 +112,13 @@ func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
|
||||
importerIpfsStorageInitializers
|
||||
}
|
||||
|
||||
// Concat relative path with its repo's root path
|
||||
func (w *writer) makePath(alias, relPath string) string {
|
||||
pathRoot := w.GenConfig.Dependencies[alias]
|
||||
return pathRoot + "/" + relPath
|
||||
}
|
||||
|
||||
// Setup the .go, clear old ones if present
|
||||
func (w *writer) setupFilePath() (string, error) {
|
||||
goFile, soFile, err := w.GenConfig.GetPluginPaths()
|
||||
if err != nil {
|
||||
|
||||
@ -7,38 +7,60 @@ The config file requires, at a minimum, the below fields:
|
||||
|
||||
```toml
|
||||
[database]
|
||||
name = "vulcanize_public"
|
||||
hostname = "localhost"
|
||||
user = "vulcanize"
|
||||
password = "vulcanize"
|
||||
port = 5432
|
||||
name = "vulcanize_public"
|
||||
hostname = "localhost"
|
||||
user = "vulcanize"
|
||||
password = "vulcanize"
|
||||
port = 5432
|
||||
|
||||
[client]
|
||||
ipcPath = "http://kovan0.vulcanize.io:8545"
|
||||
ipcPath = "http://kovan0.vulcanize.io:8545"
|
||||
|
||||
[exporter]
|
||||
name = "exporter"
|
||||
[exporter.transformers]
|
||||
transformer1 = "github.com/path/to/transformer1"
|
||||
transformer2 = "github.com/path/to/transformer2"
|
||||
transformer3 = "github.com/path/to/transformer3"
|
||||
transformer4 = "github.com/different/path/to/transformer1"
|
||||
[exporter.repositories]
|
||||
transformers = "github.com/path/to"
|
||||
transformer4 = "github.com/different/path"
|
||||
[exporter.migrations]
|
||||
transformers = "db/migrations"
|
||||
transformer4 = "to/db/migrations"
|
||||
name = "exporter"
|
||||
[exporter.transformers]
|
||||
transformer1 = "path/to/transformer1"
|
||||
transformer2 = "path/to/transformer2"
|
||||
transformer3 = "path/to/transformer3"
|
||||
transformer4 = "path/to/transformer4"
|
||||
[exporter.types]
|
||||
transformer1 = "eth_event"
|
||||
transformer2 = "eth_event"
|
||||
transformer3 = "eth_event"
|
||||
transformer4 = "eth_storage"
|
||||
[exporter.repositories]
|
||||
transformers = "github.com/account/repo"
|
||||
transformer4 = "github.com/account2/repo2"
|
||||
[exporter.migrations]
|
||||
transformers = "db/migrations"
|
||||
transformer4 = "to/db/migrations"
|
||||
```
|
||||
|
||||
- `exporter.transformers` are mappings of import aliases to paths to `TransformerInitializer`s
|
||||
- Import aliases can be arbitrarily named but note that `interface1` is a reserved alias needed for the generic TransformerInitializer type
|
||||
- `exporter.repositores` are the paths to the repositories which contain the transformers
|
||||
- `exporter.migrations` are the relative paths to the db migrations found within the `exporter.repositores`
|
||||
- Migrations need to be located in the repos in `exporter.repositores`
|
||||
- Keys should match the keys for the corresponding repo
|
||||
|
||||
Note: If any of the imported transformer need additional
|
||||
config variables do not forget to include those as well
|
||||
|
||||
If the individual transformers require additional configuration variables be sure to include them in the .toml file
|
||||
This information is used to write and build a go plugin with a transformer
|
||||
set composed from the transformer imports specified in the config file
|
||||
This plugin is loaded and the set of transformer initializers is exported
|
||||
from it and loaded into and executed over by the appropriate watcher
|
||||
|
||||
The type of watcher that the transformer works with is specified using the
|
||||
exporter.types config variable as shown above
|
||||
Currently there are watchers for event data from an eth node (eth_event)
|
||||
and storage data from an eth node (eth_storage)
|
||||
In the future there will be watchers for ipfs (ipfs_event and ipfs_storage)
|
||||
|
||||
Transformers of different types can be ran together in the same command using a
|
||||
single config file or in separate command instances using different config files
|
||||
|
||||
Specify config location when executing the command:
|
||||
`./vulcanizedb composeAndExecute --config=./environments/config_name.toml`
|
||||
|
||||
The general structure of a plugin .go file, and what we would see with the above config is shown below
|
||||
|
||||
@ -57,21 +79,24 @@ type exporter string
|
||||
|
||||
var Exporter exporter
|
||||
|
||||
func (e exporter) Export() []interface1.TransformerInitializer {
|
||||
func (e exporter) Export() []interface1.TransformerInitializer, []interface1.StorageTransformerInitializer {
|
||||
return []interface1.TransformerInitializer{
|
||||
transformer1.TransformerInitializer,
|
||||
transformer2.TransformerInitializer,
|
||||
transformer3.TransformerInitializer,
|
||||
transformer4.TransformerInitializer,
|
||||
}
|
||||
}, []interface1.StorageTransformerInitializer{
|
||||
transformer4.StorageTransformerInitializer,
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
As such, to plug in an external transformer we need to create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19)
|
||||
As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them
|
||||
Note: currently the transformers must also operate using this watcher's [execution mode](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/watcher.go#L80), in the future the watcher will become pluggable as well
|
||||
|
||||
For each transformer we will also need to create db migrations to run against vulcanizeDB so that we can store the transformed data
|
||||
The migrations needed for a specific transformer need to be included in the same repository as the transformer(s) that require them, and their relative paths in that repo must be specified in the config as discussed above
|
||||
|
||||
NOTE: Due to a bug with plugin migrations, currently need to leave the `exporter.migrations` blank and manually run migrations before running composeAndExecute
|
||||
To plug in an external transformer we need to:
|
||||
* create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go)
|
||||
that exports a variable `TransformerInitializer` or `StorageTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33)
|
||||
and [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/storage_transformer.go#L31), respectively
|
||||
* design the transformers to work in the context of the [event](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go#L83)
|
||||
or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53) watchers
|
||||
* create db migrations to run against vulcanizeDB so that we can store the transformed data
|
||||
* store the db migrations required for a transformer in the same repository as the transformer(s) that require them
|
||||
* specify their relative paths in that repo in the config, as discussed above
|
||||
* NOTE: due to a bug with plugin migrations, currently need to leave the `exporter.migrations` blank and manually run migrations before running composeAndExecute
|
||||
Loading…
Reference in New Issue
Block a user