From a322372713b938da145814fc9062331900c5628e Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sat, 2 Feb 2019 16:15:09 -0600 Subject: [PATCH] refactoring plugin generating code --- cmd/composeAndExecute.go | 27 +- cmd/root.go | 3 +- environments/compose.toml | 56 ++-- pkg/autogen/generator.go | 243 ------------------ pkg/{autogen/config.go => config/plugin.go} | 14 +- pkg/plugin/builder/builder.go | 118 +++++++++ pkg/plugin/generator.go | 69 +++++ .../generator_suite_test.go | 4 +- pkg/{autogen => plugin}/generator_test.go | 30 ++- pkg/plugin/helpers/helpers.go | 61 +++++ pkg/plugin/manager/manager.go | 134 ++++++++++ .../test_helpers/database.go | 0 .../test_helpers/test/README.md | 0 pkg/plugin/writer/writer.go | 92 +++++++ plugins/README.md | 2 +- utils/utils.go | 57 +--- 16 files changed, 549 insertions(+), 361 deletions(-) delete mode 100644 pkg/autogen/generator.go rename pkg/{autogen/config.go => config/plugin.go} (85%) create mode 100644 pkg/plugin/builder/builder.go create mode 100644 pkg/plugin/generator.go rename pkg/{autogen => plugin}/generator_suite_test.go (94%) rename pkg/{autogen => plugin}/generator_test.go (87%) create mode 100644 pkg/plugin/helpers/helpers.go create mode 100644 pkg/plugin/manager/manager.go rename pkg/{autogen => plugin}/test_helpers/database.go (100%) rename pkg/{autogen => plugin}/test_helpers/test/README.md (100%) create mode 100644 pkg/plugin/writer/writer.go diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index 7201fb78..6ab9ecb8 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -27,7 +27,9 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher" - "github.com/vulcanize/vulcanizedb/pkg/autogen" + "github.com/vulcanize/vulcanizedb/pkg/config" + p2 "github.com/vulcanize/vulcanizedb/pkg/plugin" + "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" "github.com/vulcanize/vulcanizedb/utils" ) @@ -75,7 +77,7 @@ loaded into and executed over by a generic watcher`, func composeAndExecute() { // generate code to build the plugin according to the config file - autogenConfig = autogen.Config{ + genConfig = config.Plugin{ FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins", FileName: viper.GetString("exporter.name"), Save: viper.GetBool("exporter.save"), @@ -85,25 +87,28 @@ func composeAndExecute() { } fmt.Println("generating plugin") - generator := autogen.NewGenerator(autogenConfig, databaseConfig) - err := generator.GenerateExporterPlugin() + generator, err := p2.NewGenerator(genConfig, databaseConfig) if err != nil { - fmt.Println("generating plugin failed") + log.Fatal(err) + } + err = generator.GenerateExporterPlugin() + if err != nil { + fmt.Fprint(os.Stderr, "generating plugin failed") log.Fatal(err) } // Get the plugin path and load the plugin - _, pluginPath, err := autogenConfig.GetPluginPaths() + _, pluginPath, err := genConfig.GetPluginPaths() if err != nil { log.Fatal(err) } - if !autogenConfig.Save { - defer utils.ClearFiles(pluginPath) + if !genConfig.Save { + defer helpers.ClearFiles(pluginPath) } fmt.Println("opening plugin") plug, err := plugin.Open(pluginPath) if err != nil { - fmt.Println("opening pluggin failed") + fmt.Fprint(os.Stderr, "opening pluggin failed") log.Fatal(err) } @@ -111,14 +116,14 @@ func composeAndExecute() { fmt.Println("loading transformers from plugin") symExporter, err := plug.Lookup("Exporter") if err != nil { - fmt.Println("loading Exporter symbol failed") + fmt.Fprint(os.Stderr, "loading Exporter symbol failed") log.Fatal(err) } // Assert that the symbol is of type Exporter exporter, ok := symExporter.(Exporter) if !ok { - fmt.Println("plugged-in symbol not of type Exporter") + fmt.Fprint(os.Stderr, "plugged-in symbol not of type Exporter") os.Exit(1) } diff --git a/cmd/root.go b/cmd/root.go index 77c75f17..0cfafd6e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -28,7 +28,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/vulcanize/vulcanizedb/pkg/autogen" "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/geth/client" @@ -39,7 +38,7 @@ import ( var ( cfgFile string databaseConfig config.Database - autogenConfig autogen.Config + genConfig config.Plugin ipc string levelDbPath string startingBlockNumber int64 diff --git a/environments/compose.toml b/environments/compose.toml index ae84d490..91410c50 100644 --- a/environments/compose.toml +++ b/environments/compose.toml @@ -15,34 +15,34 @@ name = "exporter" save = false [exporter.transformers] - bite = "github.com/vulcanize/mcd_transformers/transformers/bite" - cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump" - cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip" - cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow" - deal = "github.com/vulcanize/mcd_transformers/transformers/deal" - dent = "github.com/vulcanize/mcd_transformers/transformers/dent" - drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip" - drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk" - drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo" - drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow" - flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick" - flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick" - flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick" - frob = "github.com/vulcanize/mcd_transformers/transformers/frob" - pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling" - pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk" - price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds" - tend = "github.com/vulcanize/mcd_transformers/transformers/tend" - vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux" - vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold" - vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab" - vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal" - vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init" - vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move" - vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip" - vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll" - vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune" - vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog" + bite = "github.com/vulcanize/mcd_transformers/transformers/bite/initializer" + cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump/initializer" + cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip/initializer" + cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow/initializer" + deal = "github.com/vulcanize/mcd_transformers/transformers/deal/initializer" + dent = "github.com/vulcanize/mcd_transformers/transformers/dent/initializer" + drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip/initializer" + drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk/initializer" + drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo/initializer" + drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow/initializer" + flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick/initializer" + flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick/initializer" + flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick/initializer" + frob = "github.com/vulcanize/mcd_transformers/transformers/frob/initializer" + pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling/initializer" + pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk/initializer" + price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds/initializer" + tend = "github.com/vulcanize/mcd_transformers/transformers/tend/initializer" + vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux/initializer" + vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold/initializer" + vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab/initializer" + vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal/initializer" + vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init/initializer" + vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move/initializer" + vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip/initializer" + vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll/initializer" + vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune/initializer" + vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog/initializer" [exporter.repositories] mcd_transformers = "github.com/vulcanize/mcd_transformers" diff --git a/pkg/autogen/generator.go b/pkg/autogen/generator.go deleted file mode 100644 index 7e5ce5f2..00000000 --- a/pkg/autogen/generator.go +++ /dev/null @@ -1,243 +0,0 @@ -// VulcanizeDB -// Copyright © 2018 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package autogen - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - . "github.com/dave/jennifer/jen" - - "github.com/vulcanize/vulcanizedb/pkg/config" - "github.com/vulcanize/vulcanizedb/utils" -) - -type Generator interface { - GenerateExporterPlugin() error -} - -type generator struct { - GenConfig *Config - DBConfig config.Database - tmpMigDir string - tmpVenDirs []string -} - -func NewGenerator(gc Config, dbc config.Database) *generator { - return &generator{ - GenConfig: &gc, - DBConfig: dbc, - } -} - -func (g *generator) GenerateExporterPlugin() error { - if len(g.GenConfig.Initializers) < 1 { - return errors.New("generator needs to be configured with TransformerInitializer import paths") - } - if len(g.GenConfig.Dependencies) < 1 { - return errors.New("generator needs to be configured with root repository path(s)") - } - if len(g.GenConfig.Migrations) < 1 { - fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided\r\n") - } - - // Get plugin file paths - goFile, soFile, err := g.GenConfig.GetPluginPaths() - if err != nil { - return err - } - - // Generate Exporter code - err = g.generateCode(goFile, soFile) - if err != nil { - return err - } - - // Setup temp vendor lib and migrations directories - err = g.setupTempDirs() - if err != nil { - return err - } - - // Clear tmp files and directories when we exit - defer g.cleanUp(goFile) - - // Build the .go file into a .so plugin - err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run() - if err != nil { - return errors.New(fmt.Sprintf("unable to build .so file: %s", err.Error())) - } - - // Run migrations only after successfully building .so file - return g.runMigrations() -} - -// Generates the plugin code -func (g *generator) generateCode(goFile, soFile string) error { - // Clear .go and .so files of the same name if they exist - err := utils.ClearFiles(goFile, soFile) - if err != nil { - return err - } - // Begin code generation - f := NewFile("main") - f.HeaderComment("This exporter is generated to export the configured transformer initializers") - - // Import TransformerInitializers specified in config - f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface") - for alias, imp := range g.GenConfig.Initializers { - f.ImportAlias(imp, alias) - } - - // Collect TransformerInitializer names - importedInitializers := make([]Code, 0, len(g.GenConfig.Initializers)) - for _, path := range g.GenConfig.Initializers { - importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer")) - } - - // Create Exporter variable with method to export the set of the imported TransformerInitializers - f.Type().Id("exporter").String() - f.Var().Id("Exporter").Id("exporter") - f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual( - "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "TransformerInitializer").Block( - Return(Index().Qual( - "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers - - // Write code to destination file - return f.Save(goFile) -} - -// Sets up temporary vendor libs and migration directories -func (g *generator) setupTempDirs() error { - // TODO: Less hacky way of handling plugin build deps - dirPath, err := utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/") - if err != nil { - return err - } - vendorPath := filepath.Join(dirPath, "vendor") - - // Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards - g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies)) - // Import transformer dependencies so that we build our plugin - for name, importPath := range g.GenConfig.Dependencies { - index := strings.Index(importPath, "/") - gitPath := importPath[:index] + ":" + importPath[index+1:] - importURL := "git@" + gitPath + ".git" - depPath := filepath.Join(vendorPath, importPath) - err = exec.Command("git", "clone", importURL, depPath).Run() - if err != nil { - return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error())) - } - - err := os.RemoveAll(filepath.Join(depPath, "vendor/")) - if err != nil { - return err - } - - g.tmpVenDirs = append(g.tmpVenDirs, depPath) - } - - // Initialize temp directory for transformer migrations - g.tmpMigDir, err = utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations") - if err != nil { - return err - } - err = os.RemoveAll(g.tmpMigDir) - if err != nil { - return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", g.tmpMigDir)) - } - - return os.Mkdir(g.tmpMigDir, os.FileMode(0777)) -} - -func (g *generator) runMigrations() error { - // Get paths to db migrations - paths, err := g.GenConfig.GetMigrationsPaths() - if err != nil { - return err - } - if len(paths) < 1 { - return nil - } - - // Create temporary copies of migrations to the temporary migrationDir - // These tmps are identical except they have had `1` added in front of their unix_timestamps - // As such, they will be ran on top of all core migrations (at least, for the next ~317 years) - // But will still be ran in the same order relative to one another - // TODO: Less hacky way of handing migrations - err = g.createMigrationCopies(paths) - if err != nil { - return err - } - - // Run the copied migrations - pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", g.DBConfig.Hostname, g.DBConfig.Port, g.DBConfig.Name) - return exec.Command("migrate", "-path", g.tmpMigDir, "-database", pgStr, "up").Run() -} - -func (g *generator) createMigrationCopies(paths []string) error { - for _, path := range paths { - dir, err := ioutil.ReadDir(path) - if err != nil { - return err - } - for _, file := range dir { - if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum - continue - } - _, err := strconv.Atoi(file.Name()[:10]) - if err != nil { - fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name()) - continue - } - src := filepath.Join(path, file.Name()) - dst := filepath.Join(g.tmpMigDir, "1"+file.Name()) - err = utils.CopyFile(src, dst) - if err != nil { - return err - } - } - } - - return nil -} - -func (g *generator) cleanUp(goFile string) error { - if !g.GenConfig.Save { - err := utils.ClearFiles(goFile) - if err != nil { - return err - } - } - - for _, venDir := range g.tmpVenDirs { - err := os.RemoveAll(venDir) - if err != nil { - return err - } - } - - return os.RemoveAll(g.tmpMigDir) -} diff --git a/pkg/autogen/config.go b/pkg/config/plugin.go similarity index 85% rename from pkg/autogen/config.go rename to pkg/config/plugin.go index 0bdfed1e..1f054638 100644 --- a/pkg/autogen/config.go +++ b/pkg/config/plugin.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package autogen +package config import ( "errors" @@ -22,10 +22,10 @@ import ( "path/filepath" "strings" - "github.com/vulcanize/vulcanizedb/utils" + "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" ) -type Config struct { +type Plugin struct { Initializers map[string]string // Map of import aliases to transformer initializer paths Dependencies map[string]string // Map of vendor dep names to their repositories Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations @@ -34,8 +34,8 @@ type Config struct { Save bool } -func (c *Config) GetPluginPaths() (string, string, error) { - path, err := utils.CleanPath(c.FilePath) +func (c *Plugin) GetPluginPaths() (string, string, error) { + path, err := helpers.CleanPath(c.FilePath) if err != nil { return "", "", err } @@ -47,7 +47,7 @@ func (c *Config) GetPluginPaths() (string, string, error) { return goFile, soFile, nil } -func (c *Config) GetMigrationsPaths() ([]string, error) { +func (c *Plugin) GetMigrationsPaths() ([]string, error) { paths := make([]string, 0, len(c.Migrations)) for key, relPath := range c.Migrations { repo, ok := c.Dependencies[key] @@ -55,7 +55,7 @@ func (c *Config) GetMigrationsPaths() ([]string, error) { return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath)) } path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath) - cleanPath, err := utils.CleanPath(path) + cleanPath, err := helpers.CleanPath(path) if err != nil { return nil, err } diff --git a/pkg/plugin/builder/builder.go b/pkg/plugin/builder/builder.go new file mode 100644 index 00000000..4f1876ca --- /dev/null +++ b/pkg/plugin/builder/builder.go @@ -0,0 +1,118 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package builder + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" +) + +type PluginBuilder interface { + BuildPlugin() error + CleanUp() error +} + +type builder struct { + GenConfig config.Plugin + tmpVenDirs []string + goFile string +} + +func NewPluginBuilder(gc config.Plugin, dbc config.Database) *builder { + return &builder{ + GenConfig: gc, + tmpVenDirs: make([]string, 0, len(gc.Dependencies)), + } +} + +func (b *builder) BuildPlugin() error { + // Get plugin .go and .so file paths + var err error + var soFile string + b.goFile, soFile, err = b.GenConfig.GetPluginPaths() + if err != nil { + return err + } + + // setup env to build plugin + err = b.setupBuildEnv() + if err != nil { + return err + } + + // Build the .go file into a .so plugin + err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run() + if err != nil { + return errors.New(fmt.Sprintf("unable to build .so file: %s", err.Error())) + } + return nil +} + +// Sets up temporary vendor libs needed for plugin build +func (b *builder) setupBuildEnv() error { + // TODO: Less hacky way of handling plugin build deps + vendorPath, err := helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor") + if err != nil { + return err + } + + // Import transformer dependencies so that we can build our plugin + for name, importPath := range b.GenConfig.Dependencies { + index := strings.Index(importPath, "/") + gitPath := importPath[:index] + ":" + importPath[index+1:] + importURL := "git@" + gitPath + ".git" + depPath := filepath.Join(vendorPath, importPath) + err = exec.Command("git", "clone", importURL, depPath).Run() + if err != nil { + return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error())) + } + + err := os.RemoveAll(filepath.Join(depPath, "vendor/")) + if err != nil { + return err + } + + b.tmpVenDirs = append(b.tmpVenDirs, depPath) + } + + return nil +} + +func (b *builder) CleanUp() error { + if !b.GenConfig.Save { + err := helpers.ClearFiles(b.goFile) + if err != nil { + return err + } + } + + for _, venDir := range b.tmpVenDirs { + err := os.RemoveAll(venDir) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/plugin/generator.go b/pkg/plugin/generator.go new file mode 100644 index 00000000..870e5611 --- /dev/null +++ b/pkg/plugin/generator.go @@ -0,0 +1,69 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package plugin + +import ( + "errors" + "fmt" + "os" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/plugin/builder" + "github.com/vulcanize/vulcanizedb/pkg/plugin/manager" + "github.com/vulcanize/vulcanizedb/pkg/plugin/writer" +) + +type Generator interface { + GenerateExporterPlugin() error +} + +type generator struct { + writer.PluginWriter + builder.PluginBuilder + manager.MigrationManager +} + +func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) { + if len(gc.Initializers) < 1 { + return nil, errors.New("generator needs to be configured with TransformerInitializer import paths") + } + if len(gc.Dependencies) < 1 { + return nil, errors.New("generator needs to be configured with root repository path(s)") + } + if len(gc.Migrations) < 1 { + fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided for the plugin transformers\r\n") + } + return &generator{ + PluginWriter: writer.NewPluginWriter(gc), + PluginBuilder: builder.NewPluginBuilder(gc, dbc), + MigrationManager: manager.NewMigrationManager(gc, dbc), + }, nil +} + +func (g *generator) GenerateExporterPlugin() error { + err := g.PluginWriter.WritePlugin() + if err != nil { + return err + } + defer g.PluginBuilder.CleanUp() + err = g.PluginBuilder.BuildPlugin() + if err != nil { + return err + } + + return g.MigrationManager.RunMigrations() +} diff --git a/pkg/autogen/generator_suite_test.go b/pkg/plugin/generator_suite_test.go similarity index 94% rename from pkg/autogen/generator_suite_test.go rename to pkg/plugin/generator_suite_test.go index 3ebcedca..0a44bdb3 100644 --- a/pkg/autogen/generator_suite_test.go +++ b/pkg/plugin/generator_suite_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package autogen_test +package plugin_test import ( "io/ioutil" @@ -27,7 +27,7 @@ import ( func TestRepository(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Autogen Suite Test") + RunSpecs(t, "Gen Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/autogen/generator_test.go b/pkg/plugin/generator_test.go similarity index 87% rename from pkg/autogen/generator_test.go rename to pkg/plugin/generator_test.go index 6736fed3..11466a63 100644 --- a/pkg/autogen/generator_test.go +++ b/pkg/plugin/generator_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package autogen_test +package plugin_test import ( "plugin" @@ -25,34 +25,41 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher" - "github.com/vulcanize/vulcanizedb/pkg/autogen" - "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/utils" + p2 "github.com/vulcanize/vulcanizedb/pkg/plugin" + "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" + "github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers" ) -var genConfig = autogen.Config{ +var genConfig = config.Plugin{ Initializers: map[string]string{ - "bite": "github.com/vulcanize/mcd_transformers/transformers/bite", - "deal": "github.com/vulcanize/mcd_transformers/transformers/deal", + "bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer", + "deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer", }, Dependencies: map[string]string{ "mcd_transformers": "github.com/vulcanize/mcd_transformers", }, + //Migrations: map[string]string{"mcd_transformers" : "db/migrations"}, FileName: "externalTestTransformerSet", - FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test", + FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", Save: false, } +var dbConfig = config.Database{ + Hostname: "localhost", + Port: 5432, + Name: "vulcanize_private", +} + type Exporter interface { Export() []transformer.TransformerInitializer } var _ = Describe("Generator test", func() { - var g autogen.Generator + var g p2.Generator var goPath, soPath string var err error var bc core.BlockChain @@ -65,13 +72,14 @@ var _ = Describe("Generator test", func() { BeforeEach(func() { goPath, soPath, err = genConfig.GetPluginPaths() Expect(err).ToNot(HaveOccurred()) - g = autogen.NewGenerator(genConfig, config.Database{}) + g, err = p2.NewGenerator(genConfig, dbConfig) + Expect(err).ToNot(HaveOccurred()) err = g.GenerateExporterPlugin() Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - err := utils.ClearFiles(goPath, soPath) + err := helpers.ClearFiles(goPath, soPath) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/plugin/helpers/helpers.go b/pkg/plugin/helpers/helpers.go new file mode 100644 index 00000000..391df2c7 --- /dev/null +++ b/pkg/plugin/helpers/helpers.go @@ -0,0 +1,61 @@ +package helpers + +import ( + "io" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/mitchellh/go-homedir" +) + +func CleanPath(str string) (string, error) { + path, err := homedir.Expand(filepath.Clean(str)) + if err != nil { + return "", err + } + if strings.Contains(path, "$GOPATH") { + env := os.Getenv("GOPATH") + spl := strings.Split(path, "$GOPATH")[1] + path = filepath.Join(env, spl) + } + + return path, nil +} + +func ClearFiles(files ...string) error { + for _, file := range files { + if _, err := os.Stat(file); err == nil { + err = os.Remove(file) + if err != nil { + return err + } + } else if os.IsNotExist(err) { + // fall through + } else { + return err + } + } + + return nil +} + +func CopyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL|os.O_WRONLY, os.FileMode(0666)) // Doesn't overwrite files + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + return out.Close() +} diff --git a/pkg/plugin/manager/manager.go b/pkg/plugin/manager/manager.go new file mode 100644 index 00000000..3927448f --- /dev/null +++ b/pkg/plugin/manager/manager.go @@ -0,0 +1,134 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package manager + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strconv" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" +) + +type MigrationManager interface { + RunMigrations() error +} + +type manager struct { + GenConfig config.Plugin + DBConfig config.Database + tmpMigDir string +} + +func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager { + return &manager{ + GenConfig: gc, + DBConfig: dbc, + } +} + +func (m *manager) RunMigrations() error { + // Get paths to db migrations + paths, err := m.GenConfig.GetMigrationsPaths() + if err != nil { + return err + } + if len(paths) < 1 { + return nil + } + + // Init directory for temporary copies + err = m.setupMigrationEnv() + if err != nil { + return err + } + defer m.cleanUp() + + // Create temporary copies of migrations to the temporary migrationDir + // These tmps are identical except they have had `1` added in front of their unix_timestamps + // As such, they will be ran on top of all core migrations (at least, for the next ~317 years) + // But will still be ran in the same order relative to one another + // TODO: Less hacky way of handing migrations + err = m.createMigrationCopies(paths) + if err != nil { + return err + } + + // Run the copied migrations + pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name) + err = exec.Command("migrate", "-path", m.tmpMigDir, "-database", pgStr, "up").Run() + if err != nil { + return errors.New(fmt.Sprintf("db migrations for plugin transformers failed: %s", err.Error())) + } + + return nil +} + +func (m *manager) setupMigrationEnv() error { + // Initialize temp directory for transformer migrations + var err error + m.tmpMigDir, err = helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations") + if err != nil { + return err + } + err = os.RemoveAll(m.tmpMigDir) + if err != nil { + return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir)) + } + err = os.Mkdir(m.tmpMigDir, os.FileMode(0777)) + if err != nil { + return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir)) + } + + return nil +} + +func (m *manager) createMigrationCopies(paths []string) error { + for _, path := range paths { + dir, err := ioutil.ReadDir(path) + if err != nil { + return err + } + for _, file := range dir { + if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum + continue + } + _, err := strconv.Atoi(file.Name()[:10]) + if err != nil { + fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name()) + continue + } + src := filepath.Join(path, file.Name()) + dst := filepath.Join(m.tmpMigDir, "1"+file.Name()) + err = helpers.CopyFile(src, dst) + if err != nil { + return err + } + } + } + + return nil +} + +func (m *manager) cleanUp() error { + return os.RemoveAll(m.tmpMigDir) +} diff --git a/pkg/autogen/test_helpers/database.go b/pkg/plugin/test_helpers/database.go similarity index 100% rename from pkg/autogen/test_helpers/database.go rename to pkg/plugin/test_helpers/database.go diff --git a/pkg/autogen/test_helpers/test/README.md b/pkg/plugin/test_helpers/test/README.md similarity index 100% rename from pkg/autogen/test_helpers/test/README.md rename to pkg/plugin/test_helpers/test/README.md diff --git a/pkg/plugin/writer/writer.go b/pkg/plugin/writer/writer.go new file mode 100644 index 00000000..98460c42 --- /dev/null +++ b/pkg/plugin/writer/writer.go @@ -0,0 +1,92 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package writer + +import ( + "errors" + "fmt" + + . "github.com/dave/jennifer/jen" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" +) + +type PluginWriter interface { + WritePlugin() error +} + +type writer struct { + GenConfig config.Plugin +} + +func NewPluginWriter(gc config.Plugin) *writer { + return &writer{ + GenConfig: gc, + } +} + +// Generates the plugin code +func (w *writer) WritePlugin() error { + // Setup plugin file paths + goFile, err := w.setupFilePath() + if err != nil { + return err + } + + // Begin code generation + f := NewFile("main") + f.HeaderComment("This is a plugin generated to export the configured transformer initializers") + + // Import TransformerInitializers specified in config + f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface") + for alias, imp := range w.GenConfig.Initializers { + f.ImportAlias(imp, alias) + } + + // Collect TransformerInitializer names + importedInitializers := make([]Code, 0, len(w.GenConfig.Initializers)) + for _, path := range w.GenConfig.Initializers { + importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer")) + } + + // Create Exporter variable with method to export the set of the imported TransformerInitializers + f.Type().Id("exporter").String() + f.Var().Id("Exporter").Id("exporter") + f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual( + "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", + "TransformerInitializer").Block( + Return(Index().Qual( + "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", + "TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers + + // Write code to destination file + err = f.Save(goFile) + if err != nil { + return errors.New(fmt.Sprintf("failed to save generated .go file: %s\r\n%s", goFile, err.Error())) + } + return nil +} + +func (w *writer) setupFilePath() (string, error) { + goFile, soFile, err := w.GenConfig.GetPluginPaths() + if err != nil { + return "", err + } + // Clear .go and .so files of the same name if they exist + return goFile, helpers.ClearFiles(goFile, soFile) +} diff --git a/plugins/README.md b/plugins/README.md index 4a1d8983..3a8b4f08 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -25,7 +25,7 @@ The config file requires, at a minimum, the below fields: transformer4 = "github.com/different/path/to/transformer1" [exporter.repositories] transformers = "github.com/path/to" - transformer4 = "github.com/different/path + transformer4 = "github.com/different/path" [exporter.migrations] transformers = "db/migrations" transformer4 = "to/db/migrations" diff --git a/utils/utils.go b/utils/utils.go index 7145f2c8..ae610cbf 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -17,15 +17,10 @@ package utils import ( - "io" + log "github.com/sirupsen/logrus" "math/big" "os" "path/filepath" - "strings" - "syscall" - - "github.com/mitchellh/go-homedir" - log "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -84,53 +79,3 @@ func RequestedBlockNumber(blockNumber *int64) *big.Int { } return _blockNumber } - -func CleanPath(str string) (string, error) { - path, err := homedir.Expand(filepath.Clean(str)) - if err != nil { - return "", err - } - if strings.Contains(path, "$GOPATH") { - env := os.Getenv("GOPATH") - spl := strings.Split(path, "$GOPATH")[1] - path = filepath.Join(env, spl) - } - - return path, nil -} - -func ClearFiles(files ...string) error { - for _, file := range files { - if _, err := os.Stat(file); err == nil { - err = os.Remove(file) - if err != nil { - return err - } - } else if os.IsNotExist(err) { - // fall through - } else { - return err - } - } - - return nil -} - -func CopyFile(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL|os.O_WRONLY, os.FileMode(0666)) // Doesn't overwrite files - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, in) - if err != nil { - return err - } - return out.Close() -}