From 2868cf2f73362fd0be6b4d233534ed0306933c22 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Feb 2019 04:08:24 -0600 Subject: [PATCH] goose changes; update plugin readme; use rel paths for transformers in config; more commenting in pkg/plugin; fix PluginWriter to work with different transformer types; reopen generator tests --- cmd/composeAndExecute.go | 12 ++-- environments/compose.toml | 56 +++++++++---------- pkg/plugin/builder/builder.go | 23 ++++++-- pkg/plugin/generator.go | 10 +++- pkg/plugin/generator_suite_test.go | 2 +- pkg/plugin/generator_test.go | 67 ++++++++++++---------- pkg/plugin/manager/manager.go | 42 ++++++-------- pkg/plugin/test_helpers/database.go | 16 ++++++ pkg/plugin/writer/writer.go | 38 ++++++++----- plugins/README.md | 87 +++++++++++++++++++---------- 10 files changed, 212 insertions(+), 141 deletions(-) diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index 09172fe5..eceb9bb4 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -56,18 +56,18 @@ var composeAndExecuteCmd = &cobra.Command{ [exporter] name = "exporter" [exporter.transformers] - transformer1 = "github.com/path/to/transformer1" - transformer2 = "github.com/path/to/transformer2" - transformer3 = "github.com/path/to/transformer3" - transformer4 = "github.com/different/path/to/transformer1" + transformer1 = "path/to/transformer1" + transformer2 = "path/to/transformer2" + transformer3 = "path/to/transformer3" + transformer4 = "path/to/transformer4" [exporter.types] transformer1 = "eth_event" transformer2 = "eth_event" transformer3 = "eth_event" transformer4 = "eth_storage" [exporter.repositories] - transformers = "github.com/path/to" - transformer4 = "github.com/different/path" + transformers = "github.com/account/repo" + transformer4 = "github.com/account2/repo2" [exporter.migrations] transformers = "db/migrations" transformer4 = "to/db/migrations" diff --git a/environments/compose.toml b/environments/compose.toml index 966f855c..2933bb45 100644 --- a/environments/compose.toml +++ b/environments/compose.toml @@ -15,34 +15,34 @@ name = "eventTransformerExporter" save = false [exporter.transformers] - bite = "github.com/vulcanize/mcd_transformers/transformers/bite/initializer" - cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump/initializer" - cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip/initializer" - cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow/initializer" - deal = "github.com/vulcanize/mcd_transformers/transformers/deal/initializer" - dent = "github.com/vulcanize/mcd_transformers/transformers/dent/initializer" - drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip/initializer" - drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk/initializer" - drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo/initializer" - drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow/initializer" - flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick/initializer" - flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick/initializer" - flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick/initializer" - frob = "github.com/vulcanize/mcd_transformers/transformers/frob/initializer" - pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling/initializer" - pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk/initializer" - price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds/initializer" - tend = "github.com/vulcanize/mcd_transformers/transformers/tend/initializer" - vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux/initializer" - vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold/initializer" - vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab/initializer" - vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal/initializer" - vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init/initializer" - vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move/initializer" - vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip/initializer" - vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll/initializer" - vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune/initializer" - vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog/initializer" + bite = "transformers/bite/initializer" + cat_chop_lump = "transformers/cat_file/chop_lump/initializer" + cat_flip = "transformers/cat_file/flip/initializer" + cat_pit_vow = "transformers/cat_file/pit_vow/initializer" + deal = "transformers/deal/initializer" + dent = "transformers/dent/initializer" + drip_drip = "transformers/drip_drip/initializer" + drip_file_ilk = "transformers/drip_file/ilk/initializer" + drip_file_repo = "transformers/drip_file/repo/initializer" + drip_file_vow = "transformers/drip_file/vow/initializer" + flap_kick = "transformers/flap_kick/initializer" + flip_kick = "transformers/flip_kick/initializer" + flop_kick = "transformers/flop_kick/initializer" + frob = "transformers/frob/initializer" + pit_file_debt_ceiling = "transformers/pit_file/debt_ceiling/initializer" + pit_file_ilk = "transformers/pit_file/ilk/initializer" + price_feeds = "transformers/price_feeds/initializer" + tend = "transformers/tend/initializer" + vat_flux = "transformers/vat_flux/initializer" + vat_fold = "transformers/vat_fold/initializer" + vat_grab = "transformers/vat_grab/initializer" + vat_heal = "transformers/vat_heal/initializer" + vat_init = "transformers/vat_init/initializer" + vat_move = "transformers/vat_move/initializer" + vat_slip = "transformers/vat_slip/initializer" + vat_toll = "transformers/vat_toll/initializer" + vat_tune = "transformers/vat_tune/initializer" + vow_flog = "transformers/vow_flog/initializer" [exporter.types] bite = "eth_event" cat_chop_lump = "eth_event" diff --git a/pkg/plugin/builder/builder.go b/pkg/plugin/builder/builder.go index 4f1876ca..fa54c693 100644 --- a/pkg/plugin/builder/builder.go +++ b/pkg/plugin/builder/builder.go @@ -28,6 +28,9 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" ) +// Interface for compile Go code written by the +// PluginWriter into a shared object (.so file) +// which can be used loaded as a plugin type PluginBuilder interface { BuildPlugin() error CleanUp() error @@ -35,11 +38,12 @@ type PluginBuilder interface { type builder struct { GenConfig config.Plugin - tmpVenDirs []string - goFile string + tmpVenDirs []string // Keep track of temp vendor directories + goFile string // Keep track of goFile name } -func NewPluginBuilder(gc config.Plugin, dbc config.Database) *builder { +// Requires populated plugin config +func NewPluginBuilder(gc config.Plugin) *builder { return &builder{ GenConfig: gc, tmpVenDirs: make([]string, 0, len(gc.Dependencies)), @@ -70,6 +74,7 @@ func (b *builder) BuildPlugin() error { } // Sets up temporary vendor libs needed for plugin build +// This is to work around a conflict between plugins and vendoring (https://github.com/golang/go/issues/20481) func (b *builder) setupBuildEnv() error { // TODO: Less hacky way of handling plugin build deps vendorPath, err := helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor") @@ -79,6 +84,11 @@ func (b *builder) setupBuildEnv() error { // Import transformer dependencies so that we can build our plugin for name, importPath := range b.GenConfig.Dependencies { + // Use dependency paths in config to form git ssh string + // TODO: Change this to https once we are no longer working private transformer repos + // Right now since vulcanize/mcd_transformers is a private repo we + // are using ssh and uploading ssh key to travis for testing + // This is slower and more involved than using https urls index := strings.Index(importPath, "/") gitPath := importPath[:index] + ":" + importPath[index+1:] importURL := "git@" + gitPath + ".git" @@ -87,18 +97,21 @@ func (b *builder) setupBuildEnv() error { if err != nil { return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error())) } - err := os.RemoveAll(filepath.Join(depPath, "vendor/")) if err != nil { return err } - + // Keep track of this vendor directory to clear later b.tmpVenDirs = append(b.tmpVenDirs, depPath) } return nil } +// Used to clear all of the tmp vendor libs used to build the plugin +// Also clears the go file if saving it has not been specified in the config +// Do not call until after the MigrationManager has performed its operations +// as it needs to pull the db migrations from the tmpVenDirs func (b *builder) CleanUp() error { if !b.GenConfig.Save { err := helpers.ClearFiles(b.goFile) diff --git a/pkg/plugin/generator.go b/pkg/plugin/generator.go index 870e5611..a5882ba6 100644 --- a/pkg/plugin/generator.go +++ b/pkg/plugin/generator.go @@ -27,6 +27,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/plugin/writer" ) +// Generator is the top-level interface for creating transformer plugins type Generator interface { GenerateExporterPlugin() error } @@ -37,6 +38,7 @@ type generator struct { manager.MigrationManager } +// Creates a new generator from a plugin and database config func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) { if len(gc.Initializers) < 1 { return nil, errors.New("generator needs to be configured with TransformerInitializer import paths") @@ -49,21 +51,27 @@ func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) { } return &generator{ PluginWriter: writer.NewPluginWriter(gc), - PluginBuilder: builder.NewPluginBuilder(gc, dbc), + PluginBuilder: builder.NewPluginBuilder(gc), MigrationManager: manager.NewMigrationManager(gc, dbc), }, nil } +// Generates plugin for the transformer initializers specified in the generator config +// Writes plugin code => Sets up build environment => Builds .so file => Performs db migrations for the plugin transformers => Clean up func (g *generator) GenerateExporterPlugin() error { + // Use plugin writer interface to write the plugin code err := g.PluginWriter.WritePlugin() if err != nil { return err } + // Clean up temporary files and directories when we are done defer g.PluginBuilder.CleanUp() + // Use plugin builder interface to setup build environment and compile .go file into a .so file err = g.PluginBuilder.BuildPlugin() if err != nil { return err } + // Perform db migrations for the transformers return g.MigrationManager.RunMigrations() } diff --git a/pkg/plugin/generator_suite_test.go b/pkg/plugin/generator_suite_test.go index 0a44bdb3..828be5dd 100644 --- a/pkg/plugin/generator_suite_test.go +++ b/pkg/plugin/generator_suite_test.go @@ -27,7 +27,7 @@ import ( func TestRepository(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Gen Suite Test") + RunSpecs(t, "Plugin Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/plugin/generator_test.go b/pkg/plugin/generator_test.go index a9018e55..bd9f374d 100644 --- a/pkg/plugin/generator_test.go +++ b/pkg/plugin/generator_test.go @@ -16,7 +16,6 @@ package plugin_test -/* import ( "plugin" @@ -24,6 +23,7 @@ import ( . "github.com/onsi/gomega" "github.com/spf13/viper" + "github.com/vulcanize/vulcanizedb/libraries/shared/constants" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher" "github.com/vulcanize/vulcanizedb/pkg/config" @@ -48,26 +48,28 @@ var genConfig = config.Plugin{ Dependencies: map[string]string{ "mcd_transformers": "github.com/vulcanize/mcd_transformers", }, - //Migrations: map[string]string{"mcd_transformers" : "db/migrations"}, - FileName: "testEventTransformerSet", - FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", - Save: false, + Migrations: map[string]string{"mcd_transformers": "db/migrations"}, + FileName: "testEventTransformerSet", + FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", + Save: false, } var genStorageConfig = config.Plugin{ Initializers: map[string]string{ "pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer", + "vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer", }, Types: map[string]config.PluginType{ "pit": config.EthStorage, + "vat": config.EthStorage, }, Dependencies: map[string]string{ "mcd_transformers": "github.com/vulcanize/mcd_transformers", }, - //Migrations: map[string]string{"mcd_transformers" : "db/migrations"}, - FileName: "testStorageTransformerSet", - FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", - Save: false, + Migrations: map[string]string{"mcd_transformers": "db/migrations"}, + FileName: "testStorageTransformerSet", + FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", + Save: false, } var combinedConfig = config.Plugin{ @@ -75,19 +77,21 @@ var combinedConfig = config.Plugin{ "bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer", "deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer", "pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer", + "vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer", }, Types: map[string]config.PluginType{ "bite": config.EthEvent, "deal": config.EthEvent, "pit": config.EthStorage, + "vat": config.EthStorage, }, Dependencies: map[string]string{ "mcd_transformers": "github.com/vulcanize/mcd_transformers", }, - //Migrations: map[string]string{"mcd_transformers" : "db/migrations"}, - FileName: "testStorageTransformerSet", - FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", - Save: false, + Migrations: map[string]string{"mcd_transformers": "db/migrations"}, + FileName: "testComboTransformerSet", + FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", + Save: false, } var dbConfig = config.Database{ @@ -156,9 +160,9 @@ var _ = Describe("Generator test", func() { Expect(ok).To(Equal(true)) initializers, _ := exporter.Export() - w := watcher.NewWatcher(db, bc) + w := watcher.NewEventWatcher(db, bc) w.AddTransformers(initializers) - err = w.Execute() + err = w.Execute(constants.HeaderMissing) Expect(err).ToNot(HaveOccurred()) type model struct { @@ -180,8 +184,8 @@ var _ = Describe("Generator test", func() { err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID) Expect(err).ToNot(HaveOccurred()) - Expect(returned.Ilk).To(Equal("ETH")) - Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) + Expect(returned.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + Expect(returned.Urn).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb")) Expect(returned.Ink).To(Equal("80000000000000000000")) Expect(returned.Art).To(Equal("11000000000000000000000")) Expect(returned.IArt).To(Equal("12496609999999999999992")) @@ -195,7 +199,7 @@ var _ = Describe("Generator test", func() { Describe("Storage Transformers only", func() { BeforeEach(func() { - goPath, soPath, err = genConfig.GetPluginPaths() + goPath, soPath, err = genStorageConfig.GetPluginPaths() Expect(err).ToNot(HaveOccurred()) g, err = p2.NewGenerator(genStorageConfig, dbConfig) Expect(err).ToNot(HaveOccurred()) @@ -216,12 +220,12 @@ var _ = Describe("Generator test", func() { exporter, ok := symExporter.(Exporter) Expect(ok).To(Equal(true)) event, initializers := exporter.Export() - Expect(len(initializers)).To(Equal(1)) + Expect(len(initializers)).To(Equal(2)) Expect(len(event)).To(Equal(0)) }) It("Loads our generated Exporter and uses it to import an arbitrary set of StorageTransformerInitializers that we can execute over", func() { - db, bc = test_helpers.SetupDBandBC() + db, _ = test_helpers.SetupDBandBC() defer test_helpers.TearDown(db) plug, err := plugin.Open(soPath) @@ -235,15 +239,16 @@ var _ = Describe("Generator test", func() { tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")} w := watcher.NewStorageWatcher(tailer, db) w.AddTransformers(initializers) - err = w.Execute() - Expect(err).ToNot(HaveOccurred()) + // This blocks right now, need to make test file to read from + //err = w.Execute() + //Expect(err).ToNot(HaveOccurred()) }) }) }) Describe("Event and Storage Transformers in same instance", func() { BeforeEach(func() { - goPath, soPath, err = genConfig.GetPluginPaths() + goPath, soPath, err = combinedConfig.GetPluginPaths() Expect(err).ToNot(HaveOccurred()) g, err = p2.NewGenerator(combinedConfig, dbConfig) Expect(err).ToNot(HaveOccurred()) @@ -265,7 +270,7 @@ var _ = Describe("Generator test", func() { Expect(ok).To(Equal(true)) eventInitializers, storageInitializers := exporter.Export() Expect(len(eventInitializers)).To(Equal(2)) - Expect(len(storageInitializers)).To(Equal(1)) + Expect(len(storageInitializers)).To(Equal(2)) }) It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers and StorageTransformerInitializers that we can execute over", func() { @@ -286,9 +291,9 @@ var _ = Describe("Generator test", func() { Expect(ok).To(Equal(true)) eventInitializers, storageInitializers := exporter.Export() - ew := watcher.NewWatcher(db, bc) + ew := watcher.NewEventWatcher(db, bc) ew.AddTransformers(eventInitializers) - err = ew.Execute() + err = ew.Execute(constants.HeaderMissing) Expect(err).ToNot(HaveOccurred()) type model struct { @@ -310,8 +315,8 @@ var _ = Describe("Generator test", func() { err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID) Expect(err).ToNot(HaveOccurred()) - Expect(returned.Ilk).To(Equal("ETH")) - Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) + Expect(returned.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + Expect(returned.Urn).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb")) Expect(returned.Ink).To(Equal("80000000000000000000")) Expect(returned.Art).To(Equal("11000000000000000000000")) Expect(returned.IArt).To(Equal("12496609999999999999992")) @@ -323,10 +328,10 @@ var _ = Describe("Generator test", func() { tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")} sw := watcher.NewStorageWatcher(tailer, db) sw.AddTransformers(storageInitializers) - err = sw.Execute() - Expect(err).ToNot(HaveOccurred()) + // This blocks right now, need to make test file to read from + //err = w.Execute() + //Expect(err).ToNot(HaveOccurred()) }) }) }) }) -*/ diff --git a/pkg/plugin/manager/manager.go b/pkg/plugin/manager/manager.go index 3927448f..66b38b84 100644 --- a/pkg/plugin/manager/manager.go +++ b/pkg/plugin/manager/manager.go @@ -19,16 +19,15 @@ package manager import ( "errors" "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" "io/ioutil" "os" "os/exec" "path/filepath" - "strconv" - - "github.com/vulcanize/vulcanizedb/pkg/config" - "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" ) +// Interface for managing the db migrations for plugin transformers type MigrationManager interface { RunMigrations() error } @@ -39,6 +38,7 @@ type manager struct { tmpMigDir string } +// Manager requires both filled in generator and database configs func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager { return &manager{ GenConfig: gc, @@ -47,7 +47,7 @@ func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager { } func (m *manager) RunMigrations() error { - // Get paths to db migrations + // Get paths to db migrations from the plugin config paths, err := m.GenConfig.GetMigrationsPaths() if err != nil { return err @@ -55,27 +55,22 @@ func (m *manager) RunMigrations() error { if len(paths) < 1 { return nil } - - // Init directory for temporary copies + // Init directory for temporary copies of migrations err = m.setupMigrationEnv() if err != nil { return err } defer m.cleanUp() - - // Create temporary copies of migrations to the temporary migrationDir - // These tmps are identical except they have had `1` added in front of their unix_timestamps - // As such, they will be ran on top of all core migrations (at least, for the next ~317 years) - // But will still be ran in the same order relative to one another - // TODO: Less hacky way of handing migrations + // Creates copies of migrations for all the plugin's transformers in a tmp dir err = m.createMigrationCopies(paths) if err != nil { return err } - - // Run the copied migrations + // Run the copied migrations with goose pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name) - err = exec.Command("migrate", "-path", m.tmpMigDir, "-database", pgStr, "up").Run() + cmd := exec.Command("goose", "postgres", pgStr, "up") + cmd.Dir = m.tmpMigDir + err = cmd.Run() if err != nil { return errors.New(fmt.Sprintf("db migrations for plugin transformers failed: %s", err.Error())) } @@ -83,8 +78,8 @@ func (m *manager) RunMigrations() error { return nil } +// Setup a temporary directory to hold transformer db migrations func (m *manager) setupMigrationEnv() error { - // Initialize temp directory for transformer migrations var err error m.tmpMigDir, err = helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations") if err != nil { @@ -102,23 +97,22 @@ func (m *manager) setupMigrationEnv() error { return nil } +// Create copies of db migrations from vendored libs func (m *manager) createMigrationCopies(paths []string) error { + // Iterate through migration paths to find migration directory for _, path := range paths { dir, err := ioutil.ReadDir(path) if err != nil { return err } + // For each file in the directory check if it is a migration for _, file := range dir { - if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum - continue - } - _, err := strconv.Atoi(file.Name()[:10]) - if err != nil { - fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name()) + if file.IsDir() || filepath.Ext(file.Name()) != ".sql" { continue } src := filepath.Join(path, file.Name()) - dst := filepath.Join(m.tmpMigDir, "1"+file.Name()) + dst := filepath.Join(m.tmpMigDir, file.Name()) + // and if it is make a copy of it to our tmp migration directory err = helpers.CopyFile(src, dst) if err != nil { return err diff --git a/pkg/plugin/test_helpers/database.go b/pkg/plugin/test_helpers/database.go index 66d6cc0e..0ec05f8e 100644 --- a/pkg/plugin/test_helpers/database.go +++ b/pkg/plugin/test_helpers/database.go @@ -1,3 +1,19 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package test_helpers import ( diff --git a/pkg/plugin/writer/writer.go b/pkg/plugin/writer/writer.go index b199eb0c..9d9766ff 100644 --- a/pkg/plugin/writer/writer.go +++ b/pkg/plugin/writer/writer.go @@ -26,6 +26,9 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" ) +// Interface for writing a .go file for a simple +// plugin that exports the set of transformer +// initializers specified in the config type PluginWriter interface { WritePlugin() error } @@ -34,13 +37,14 @@ type writer struct { GenConfig config.Plugin } +// Requires populated plugin config func NewPluginWriter(gc config.Plugin) *writer { return &writer{ GenConfig: gc, } } -// Generates the plugin code +// Generates the plugin code according to config specification func (w *writer) WritePlugin() error { // Setup plugin file paths goFile, err := w.setupFilePath() @@ -52,10 +56,10 @@ func (w *writer) WritePlugin() error { f := NewFile("main") f.HeaderComment("This is a plugin generated to export the configured transformer initializers") - // Import TransformerInitializers specified in config + // Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface") - for alias, imp := range w.GenConfig.Initializers { - f.ImportAlias(imp, alias) + for alias, relPath := range w.GenConfig.Initializers { + f.ImportAlias(w.makePath(alias, relPath), alias) } // Collect initializer code @@ -64,17 +68,16 @@ func (w *writer) WritePlugin() error { // Create Exporter variable with method to export the set of the imported storage and event transformer initializers f.Type().Id("exporter").String() f.Var().Id("Exporter").Id("exporter") - f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual( - "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "TransformerInitializer").Index().Qual( - "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "StorageTransformerInitializer").Block( - Return(Index().Qual( - "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "TransformerInitializer").Values(ethEventInitializers...)), + f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Parens(List( + Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "TransformerInitializer"), + Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "StorageTransformerInitializer"), + )).Block(Return( Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "StorageTransformerInitializer").Values(ethStorageInitializers...)) // Exports the collected initializers + "TransformerInitializer").Values(ethEventInitializers...), + Index().Qual( + "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", + "StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers // Write code to destination file err = f.Save(goFile) @@ -84,8 +87,8 @@ func (w *writer) WritePlugin() error { return nil } +// Collect code for various types of initializers func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) { - // Collect code for various initializers importedEthEventInitializers := make([]Code, 0) importerEthStorageInitializers := make([]Code, 0) importedIpfsEventInitializers := make([]Code, 0) @@ -109,6 +112,13 @@ func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) { importerIpfsStorageInitializers } +// Concat relative path with its repo's root path +func (w *writer) makePath(alias, relPath string) string { + pathRoot := w.GenConfig.Dependencies[alias] + return pathRoot + "/" + relPath +} + +// Setup the .go, clear old ones if present func (w *writer) setupFilePath() (string, error) { goFile, soFile, err := w.GenConfig.GetPluginPaths() if err != nil { diff --git a/plugins/README.md b/plugins/README.md index 3a8b4f08..ee5655cc 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -7,38 +7,60 @@ The config file requires, at a minimum, the below fields: ```toml [database] - name = "vulcanize_public" - hostname = "localhost" - user = "vulcanize" - password = "vulcanize" - port = 5432 + name = "vulcanize_public" + hostname = "localhost" + user = "vulcanize" + password = "vulcanize" + port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "http://kovan0.vulcanize.io:8545" [exporter] - name = "exporter" - [exporter.transformers] - transformer1 = "github.com/path/to/transformer1" - transformer2 = "github.com/path/to/transformer2" - transformer3 = "github.com/path/to/transformer3" - transformer4 = "github.com/different/path/to/transformer1" - [exporter.repositories] - transformers = "github.com/path/to" - transformer4 = "github.com/different/path" - [exporter.migrations] - transformers = "db/migrations" - transformer4 = "to/db/migrations" + name = "exporter" + [exporter.transformers] + transformer1 = "path/to/transformer1" + transformer2 = "path/to/transformer2" + transformer3 = "path/to/transformer3" + transformer4 = "path/to/transformer4" +[exporter.types] + transformer1 = "eth_event" + transformer2 = "eth_event" + transformer3 = "eth_event" + transformer4 = "eth_storage" + [exporter.repositories] + transformers = "github.com/account/repo" + transformer4 = "github.com/account2/repo2" + [exporter.migrations] + transformers = "db/migrations" + transformer4 = "to/db/migrations" ``` - - `exporter.transformers` are mappings of import aliases to paths to `TransformerInitializer`s - Import aliases can be arbitrarily named but note that `interface1` is a reserved alias needed for the generic TransformerInitializer type - `exporter.repositores` are the paths to the repositories which contain the transformers - `exporter.migrations` are the relative paths to the db migrations found within the `exporter.repositores` - Migrations need to be located in the repos in `exporter.repositores` - Keys should match the keys for the corresponding repo + +Note: If any of the imported transformer need additional +config variables do not forget to include those as well -If the individual transformers require additional configuration variables be sure to include them in the .toml file +This information is used to write and build a go plugin with a transformer +set composed from the transformer imports specified in the config file +This plugin is loaded and the set of transformer initializers is exported +from it and loaded into and executed over by the appropriate watcher + +The type of watcher that the transformer works with is specified using the +exporter.types config variable as shown above +Currently there are watchers for event data from an eth node (eth_event) +and storage data from an eth node (eth_storage) +In the future there will be watchers for ipfs (ipfs_event and ipfs_storage) + +Transformers of different types can be ran together in the same command using a +single config file or in separate command instances using different config files + +Specify config location when executing the command: +`./vulcanizedb composeAndExecute --config=./environments/config_name.toml` The general structure of a plugin .go file, and what we would see with the above config is shown below @@ -57,21 +79,24 @@ type exporter string var Exporter exporter -func (e exporter) Export() []interface1.TransformerInitializer { +func (e exporter) Export() []interface1.TransformerInitializer, []interface1.StorageTransformerInitializer { return []interface1.TransformerInitializer{ transformer1.TransformerInitializer, transformer2.TransformerInitializer, transformer3.TransformerInitializer, - transformer4.TransformerInitializer, - } + }, []interface1.StorageTransformerInitializer{ + transformer4.StorageTransformerInitializer, + } } ``` -As such, to plug in an external transformer we need to create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19) -As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them -Note: currently the transformers must also operate using this watcher's [execution mode](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/watcher.go#L80), in the future the watcher will become pluggable as well - -For each transformer we will also need to create db migrations to run against vulcanizeDB so that we can store the transformed data -The migrations needed for a specific transformer need to be included in the same repository as the transformer(s) that require them, and their relative paths in that repo must be specified in the config as discussed above - -NOTE: Due to a bug with plugin migrations, currently need to leave the `exporter.migrations` blank and manually run migrations before running composeAndExecute \ No newline at end of file +To plug in an external transformer we need to: +* create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) +that exports a variable `TransformerInitializer` or `StorageTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33) +and [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/storage_transformer.go#L31), respectively +* design the transformers to work in the context of the [event](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go#L83) +or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53) watchers +* create db migrations to run against vulcanizeDB so that we can store the transformed data + * store the db migrations required for a transformer in the same repository as the transformer(s) that require them + * specify their relative paths in that repo in the config, as discussed above + * NOTE: due to a bug with plugin migrations, currently need to leave the `exporter.migrations` blank and manually run migrations before running composeAndExecute \ No newline at end of file