From 55fa9b8364329dd669b697bbc6a7b44b275e86b3 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 30 Jan 2019 19:38:56 -0600 Subject: [PATCH] auto management of dependencies temporarily needed to build plugin and transformer migrations; new test showing it is working with external transformers runs properly when migrations were manually performed but still need to test automated migration management --- cmd/composeAndExecute.go | 43 ++++--- cmd/root.go | 8 +- environments/compose.toml | 20 ++-- pkg/autogen/config.go | 48 +++++++- pkg/autogen/generator.go | 210 +++++++++++++++++++++++++------- pkg/autogen/generator_test.go | 213 ++++++++++++++++++++++----------- plugins/README.md | 34 ++++-- plugins/example_maker_exporter | 43 +++++++ utils/utils.go | 55 +++++++++ 9 files changed, 529 insertions(+), 145 deletions(-) create mode 100644 plugins/example_maker_exporter diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index 4f647802..e0d1b18b 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -47,11 +47,19 @@ var composeAndExecuteCmd = &cobra.Command{ ipcPath = "http://kovan0.vulcanize.io:8545" [exporter] - filePath = "~/go/src/github.com/vulcanize/vulcanizedb/plugins" - fileName = "exporter" + filePath = "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/" + fileName = "exporter" [exporter.transformers] - transformerImport1 = "github.com/path_to/transformerInitializer1" - transformerImport2 = "github.com/path_to/transformerInitializer2" + transformer1 = "github.com/path/to/transformer1" + transformer2 = "github.com/path/to/transformer2" + transformer3 = "github.com/path/to/transformer3" + transformer4 = "github.com/different/path/to/transformer1" + [exporter.repositories] + transformers = "github.com/path/to" + transformer4 = "github.com/different/path + [exporter.migrations] + transformers = "db/migrations" + transformer4 = "to/db/migrations" Note: If any of the imported transformer need additional config variables do not forget to include those as well @@ -66,43 +74,50 @@ loaded into and executed over by a generic watcher`, } func composeAndExecute() { - generator := autogen.NewGenerator(autogenConfig) - err := generator.GenerateTransformerPlugin() + // generate code to build the plugin according to the config file + generator := autogen.NewGenerator(autogenConfig, databaseConfig) + err := generator.GenerateExporterPlugin() if err != nil { log.Fatal(err) } - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - - blockChain := getBlockChain() - db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - - _, pluginPath, err := autogen.GetPaths(autogenConfig) + // Get the plugin path and load the plugin + _, pluginPath, err := autogenConfig.GetPluginPaths() if err != nil { log.Fatal(err) } - plug, err := plugin.Open(pluginPath) if err != nil { log.Fatal(err) } + // Load the `Exporter` symbol from the plugin symExporter, err := plug.Lookup("Exporter") if err != nil { log.Fatal(err) } + // Assert that the symbol is of type Exporter exporter, ok := symExporter.(Exporter) if !ok { fmt.Println("plugged-in symbol not of type Exporter") os.Exit(1) } + // Use the Exporters export method to load the TransformerInitializer set initializers := exporter.Export() + + // Setup bc and db objects + blockChain := getBlockChain() + db := utils.LoadPostgres(databaseConfig, blockChain.Node()) + + // Create a watcher and load the TransformerInitializer set into it w := watcher.NewWatcher(&db, blockChain) w.AddTransformers(initializers) + // Execute over the TransformerInitializer set using the watcher + ticker := time.NewTicker(pollingInterval) + defer ticker.Stop() for range ticker.C { err := w.Execute() if err != nil { diff --git a/cmd/root.go b/cmd/root.go index af2ea008..8e2e0a74 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -77,9 +77,11 @@ func configure(cmd *cobra.Command, args []string) { Password: viper.GetString("database.password"), } autogenConfig = autogen.Config{ - FilePath: viper.GetString("exporter.filePath"), - FileName: viper.GetString("exporter.fileName"), - Imports: viper.GetStringMapString("exporter.transformers"), + FilePath: viper.GetString("exporter.filePath"), + FileName: viper.GetString("exporter.fileName"), + Initializers: viper.GetStringMapString("exporter.transformers"), + Dependencies: viper.GetStringMapString("exporter.repositories"), + Migrations: viper.GetStringMapString("exporter.migrations"), } viper.Set("database.config", databaseConfig) } diff --git a/environments/compose.toml b/environments/compose.toml index eb3b43d4..3d760048 100644 --- a/environments/compose.toml +++ b/environments/compose.toml @@ -1,24 +1,24 @@ [database] - name = "vulcanize_public" + name = "vulcanize_public" hostname = "localhost" - user = "vulcanize" + user = "vulcanize" password = "vulcanize" - port = 5432 + port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "http://kovan0.vulcanize.io:8545" [datadog] - name = "maker_vdb_staging" + name = "maker_vdb_staging" [exporter] filePath = "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/" fileName = "exporter" [exporter.transformers] bite = "github.com/vulcanize/mcd_transformers/transformers/bite" - cat_chop_lump = "github.com/vulcanize/maker_transformers/cat/chop_lump" - cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat/flip" - cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat/pit_vow" + cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump" + cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip" + cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow" deal = "github.com/vulcanize/mcd_transformers/transformers/deal" dent = "github.com/vulcanize/mcd_transformers/transformers/dent" drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip" @@ -43,6 +43,10 @@ vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll" vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune" vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog" + [exporter.repositories] + mcd_transformers = "github.com/vulcanize/mcd_transformers" + [exporter.migrations] + mcd_transformers = "db/migrations" [contract] [contract.address] diff --git a/pkg/autogen/config.go b/pkg/autogen/config.go index ed81472e..383270f5 100644 --- a/pkg/autogen/config.go +++ b/pkg/autogen/config.go @@ -16,8 +16,50 @@ package autogen +import ( + "errors" + "fmt" + "path/filepath" + "strings" + + "github.com/vulcanize/vulcanizedb/utils" +) + type Config struct { - Imports map[string]string // Map of import alias to import path - FilePath string - FileName string + Initializers map[string]string // Map of import aliases to transformer paths + Dependencies map[string]string // Map of vendor dep names to their repositories + Migrations map[string]string // Map of migration names to their paths within the vendored deps + FilePath string + FileName string +} + +func (c *Config) GetPluginPaths() (string, string, error) { + path, err := utils.CleanPath(c.FilePath) + if err != nil { + return "", "", err + } + + name := strings.Split(c.FileName, ".")[0] + goFile := filepath.Join(path, name+".go") + soFile := filepath.Join(path, name+".so") + + return goFile, soFile, nil +} + +func (c *Config) GetMigrationsPaths() ([]string, error) { + paths := make([]string, 0, len(c.Migrations)) + for key, relPath := range c.Migrations { + repo, ok := c.Dependencies[key] + if !ok { + return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath)) + } + path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath) + cleanPath, err := utils.CleanPath(path) + if err != nil { + return nil, err + } + paths = append(paths, cleanPath) + } + + return paths, nil } diff --git a/pkg/autogen/generator.go b/pkg/autogen/generator.go index ca6c39ae..21862bdb 100644 --- a/pkg/autogen/generator.go +++ b/pkg/autogen/generator.go @@ -18,65 +18,97 @@ package autogen import ( "errors" + "fmt" + "io/ioutil" "os" "os/exec" "path/filepath" - "strings" + "strconv" . "github.com/dave/jennifer/jen" - "github.com/mitchellh/go-homedir" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/utils" ) type Generator interface { - GenerateTransformerPlugin() error + GenerateExporterPlugin() error } type generator struct { - *Config + GenConfig *Config + DBConfig config.Database + tmpMigDir string + tmpVenDirs []string } -func NewGenerator(config Config) *generator { +func NewGenerator(gc Config, dbc config.Database) *generator { return &generator{ - Config: &config, + GenConfig: &gc, + DBConfig: dbc, } } -func (g *generator) GenerateTransformerPlugin() error { - if g.Config == nil { +func (g *generator) GenerateExporterPlugin() error { + if g.GenConfig == nil { return errors.New("generator needs a config file") } - if g.Config.FilePath == "" { + if g.GenConfig.FilePath == "" { return errors.New("generator is missing file path") } - if len(g.Config.Imports) < 1 { + if len(g.GenConfig.Initializers) < 1 { return errors.New("generator needs to be configured with imports") } - // Create file path - goFile, soFile, err := GetPaths(*g.Config) + // Get plugin file paths + goFile, soFile, err := g.GenConfig.GetPluginPaths() if err != nil { return err } - // Clear previous .go and .so files if they exist - err = ClearFiles(goFile, soFile) + // Clear .go and .so files of the same name if they exist (overwrite) + err = utils.ClearFiles(goFile, soFile) if err != nil { return err } + // Generate Exporter code + err = g.generateCode(goFile) + if err != nil { + return err + } + + // Setup temp vendor lib and migrations directories + err = g.setupTempDirs() + if err != nil { + return err + } + defer g.cleanUp() // Clear these up when we are done building our plugin + + // Build the .go file into a .so plugin + err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run() + if err != nil { + return err + } + // Run migrations only after successfully building .so file + return g.runMigrations() +} + +// Generates the plugin code +func (g *generator) generateCode(goFile string) error { // Begin code generation f := NewFile("main") f.HeaderComment("This exporter is generated to export the configured transformer initializers") // Import TransformerInitializers f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface") - for alias, imp := range g.Config.Imports { + for alias, imp := range g.GenConfig.Initializers { f.ImportAlias(imp, alias) } // Collect TransformerInitializer names - importedInitializers := make([]Code, 0, len(g.Config.Imports)) - for _, path := range g.Config.Imports { + importedInitializers := make([]Code, 0, len(g.GenConfig.Initializers)) + for _, path := range g.GenConfig.Initializers { importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer")) } @@ -90,49 +122,143 @@ func (g *generator) GenerateTransformerPlugin() error { "TransformerInitializer").Block( Return(Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "TransformerInitializer").Values(importedInitializers...))) + "TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers // Write code to destination file - err = f.Save(goFile) + return f.Save(goFile) +} + +func (g *generator) runMigrations() error { + // Get paths to db migrations + paths, err := g.GenConfig.GetMigrationsPaths() + if err != nil { + return err + } + if len(paths) < 1 { + return nil + } + + // Create temporary copies of migrations to the temporary migrationDir + // These tmps are identical except they have had `1` added in front of their unix_timestamps + // As such, they will be ran on top of all core migrations (at least, for the next ~317 years) + // But will still be ran in the same order relative to one another + // TODO: Less hacky way of handing migrations + err = g.createMigrationCopies(paths) if err != nil { return err } - // Build the .go file into a .so plugin - return exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run() + // Run the copied migrations + location := "file://" + g.tmpMigDir + pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable up", g.DBConfig.Hostname, g.DBConfig.Port, g.DBConfig.Name) + return exec.Command("migrate", "-source", location, pgStr).Run() } -func GetPaths(config Config) (string, string, error) { - path, err := homedir.Expand(filepath.Clean(config.FilePath)) +// Sets up temporary vendor libs and migration directories +func (g *generator) setupTempDirs() error { + // TODO: Less hacky way of handling plugin build deps + dirPath, err := utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/") if err != nil { - return "", "", err + return err } - if strings.Contains(path, "$GOPATH") { - env := os.Getenv("GOPATH") - spl := strings.Split(path, "$GOPATH")[1] - path = filepath.Join(env, spl) + vendorPath := filepath.Join(dirPath, "vendor/") + + /* + // Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards + g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies)) + // Import transformer dependencies so that we build our plugin + for _, importPath := range g.GenConfig.Dependencies { + importURL := "https://" + importPath + ".git" + depPath := filepath.Join(vendorPath, importPath) + err = exec.Command("git", "clone", importURL, depPath).Run() + if err != nil { + return err + } + err := os.RemoveAll(filepath.Join(depPath, "vendor/")) + if err != nil { + return err + } + g.tmpVenDirs = append(g.tmpVenDirs, depPath) + } + */ + + // Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards + g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies)) + for _, importPath := range g.GenConfig.Dependencies { + depPath := filepath.Join(vendorPath, importPath) + g.tmpVenDirs = append(g.tmpVenDirs, depPath) } - name := strings.Split(config.FileName, ".")[0] - goFile := filepath.Join(path, name+".go") - soFile := filepath.Join(path, name+".so") + // Dep ensure to make sure vendor pkgs are in place for building the plugin + err = exec.Command("dep", "ensure").Run() + if err != nil { + return errors.New("failed to vendor transformer packages required to build plugin") + } - return goFile, soFile, nil + // Git checkout our head-state vendor libraries + // This is necessary because we currently need to manual edit our vendored + // go-ethereum abi library to allow for unpacking in empty interfaces and maps + // This can be removed once the PRs against geth merged + err = exec.Command("git", "checkout", dirPath).Run() + if err != nil { + return errors.New("failed to checkout vendored go-ethereum lib") + } + + // Initialize temp directory for transformer migrations + g.tmpMigDir, err = utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations") + if err != nil { + return err + } + stat, err := os.Stat(g.tmpMigDir) + if err == nil { + if !stat.IsDir() { + return errors.New(fmt.Sprintf("file %s found where directory is expected", stat.Name())) + } + } else if os.IsNotExist(err) { + os.Mkdir(g.tmpMigDir, os.FileMode(0777)) + } else { + return err + } + + return nil } -func ClearFiles(files ...string) error { - for _, file := range files { - if _, err := os.Stat(file); err == nil { - err = os.Remove(file) - if err != nil { - return err - } - } else if os.IsNotExist(err) { - // fall through - } else { +func (g *generator) createMigrationCopies(paths []string) error { + for _, path := range paths { + dir, err := ioutil.ReadDir(path) + if err != nil { return err } + for _, file := range dir { + if file.IsDir() || len(file.Name()) < 15 { // (10 digit unix time stamp + x + .sql) is bare minimum + continue + } + _, err := strconv.Atoi(file.Name()[:10]) + if err != nil { + fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix", file.Name()) + continue + } + if filepath.Ext(file.Name()) == "sql" { + src := filepath.Join(path, file.Name()) + dst := filepath.Join(g.tmpMigDir, "1"+file.Name()) + err = utils.CopyFile(src, dst) + if err != nil { + return err + } + } + } } return nil } + +func (g *generator) cleanUp() error { + for _, venDir := range g.tmpVenDirs { + err := os.RemoveAll(venDir) + if err != nil { + return err + } + } + + return os.RemoveAll(g.tmpMigDir) +} diff --git a/pkg/autogen/generator_test.go b/pkg/autogen/generator_test.go index bc60d980..ebd33cb9 100644 --- a/pkg/autogen/generator_test.go +++ b/pkg/autogen/generator_test.go @@ -27,28 +27,33 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/watcher" "github.com/vulcanize/vulcanizedb/pkg/autogen" "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/bite" + "github.com/vulcanize/vulcanizedb/utils" ) -var testConfig = autogen.Config{ - Imports: map[string]string{ +var localConfig = autogen.Config{ + Initializers: map[string]string{ "bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite", "deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal", }, - FileName: "testTransformerSet", + FileName: "localTestTransformerSet", FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/", } -var targetConfig = autogen.Config{ - Imports: map[string]string{ - "bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite", - "deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal", +var externalConfig = autogen.Config{ + Initializers: map[string]string{ + "bite": "github.com/vulcanize/mcd_transformers/transformers/bite", + "deal": "github.com/vulcanize/mcd_transformers/transformers/deal", }, - FileName: "targetTransformerSet", - FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/target/", + Dependencies: map[string]string{ + "mcd_transformers": "github.com/vulcanize/mcd_transformers", + }, + FileName: "externalTestTransformerSet", + FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/", } type Exporter interface { @@ -66,73 +71,147 @@ var _ = Describe("Generator test", func() { viper.SetConfigName("compose") viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/") - BeforeEach(func() { - goPath, soPath, err = autogen.GetPaths(testConfig) - Expect(err).ToNot(HaveOccurred()) - g = autogen.NewGenerator(testConfig) - err = g.GenerateTransformerPlugin() - Expect(err).ToNot(HaveOccurred()) - }) - - AfterEach(func() { - err := autogen.ClearFiles(goPath, soPath) - Expect(err).ToNot(HaveOccurred()) - }) - - Describe("GenerateTransformerPlugin", func() { - It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() { - plug, err := plugin.Open(soPath) + Describe("Using local config", func() { + BeforeEach(func() { + goPath, soPath, err = localConfig.GetPluginPaths() Expect(err).ToNot(HaveOccurred()) - symExporter, err := plug.Lookup("Exporter") + g = autogen.NewGenerator(localConfig, config.Database{}) + err = g.GenerateExporterPlugin() Expect(err).ToNot(HaveOccurred()) - exporter, ok := symExporter.(Exporter) - Expect(ok).To(Equal(true)) - initializers := exporter.Export() - Expect(len(initializers)).To(Equal(2)) }) - It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() { - db, bc = test_helpers.SetupDBandBC() - defer test_helpers.TearDown(db) - - hr = repositories.NewHeaderRepository(db) - header1, err := bc.GetHeaderByNumber(9377319) - Expect(err).ToNot(HaveOccurred()) - headerID, err = hr.CreateOrUpdateHeader(header1) + AfterEach(func() { + err := utils.ClearFiles(goPath, soPath) Expect(err).ToNot(HaveOccurred()) + }) - plug, err := plugin.Open(soPath) - Expect(err).ToNot(HaveOccurred()) - symExporter, err := plug.Lookup("Exporter") - Expect(err).ToNot(HaveOccurred()) - exporter, ok := symExporter.(Exporter) - Expect(ok).To(Equal(true)) - initializers := exporter.Export() + Describe("GenerateTransformerPlugin", func() { + It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() { + plug, err := plugin.Open(soPath) + Expect(err).ToNot(HaveOccurred()) + symExporter, err := plug.Lookup("Exporter") + Expect(err).ToNot(HaveOccurred()) + exporter, ok := symExporter.(Exporter) + Expect(ok).To(Equal(true)) + initializers := exporter.Export() + Expect(len(initializers)).To(Equal(2)) + }) - w := watcher.NewWatcher(db, bc) - w.AddTransformers(initializers) - err = w.Execute() + It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() { + db, bc = test_helpers.SetupDBandBC() + defer test_helpers.TearDown(db) + + hr = repositories.NewHeaderRepository(db) + header1, err := bc.GetHeaderByNumber(9377319) + Expect(err).ToNot(HaveOccurred()) + headerID, err = hr.CreateOrUpdateHeader(header1) + Expect(err).ToNot(HaveOccurred()) + + plug, err := plugin.Open(soPath) + Expect(err).ToNot(HaveOccurred()) + symExporter, err := plug.Lookup("Exporter") + Expect(err).ToNot(HaveOccurred()) + exporter, ok := symExporter.(Exporter) + Expect(ok).To(Equal(true)) + initializers := exporter.Export() + + w := watcher.NewWatcher(db, bc) + w.AddTransformers(initializers) + err = w.Execute() + Expect(err).ToNot(HaveOccurred()) + + type model struct { + bite.BiteModel + Id int64 `db:"id"` + HeaderId int64 `db:"header_id"` + } + + returned := model{} + + err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID) + Expect(err).ToNot(HaveOccurred()) + Expect(returned.Ilk).To(Equal("ETH")) + Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) + Expect(returned.Ink).To(Equal("80000000000000000000")) + Expect(returned.Art).To(Equal("11000000000000000000000")) + Expect(returned.IArt).To(Equal("12496609999999999999992")) + Expect(returned.Tab).To(Equal("11000000000000000000000")) + Expect(returned.NFlip).To(Equal("7")) + Expect(returned.TransactionIndex).To(Equal(uint(1))) + Expect(returned.LogIndex).To(Equal(uint(4))) + }) + }) + }) + + Describe("Using external config", func() { + BeforeEach(func() { + goPath, soPath, err = externalConfig.GetPluginPaths() Expect(err).ToNot(HaveOccurred()) - - type model struct { - bite.BiteModel - Id int64 `db:"id"` - HeaderId int64 `db:"header_id"` - } - - returned := model{} - - err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID) + g = autogen.NewGenerator(externalConfig, config.Database{}) + err = g.GenerateExporterPlugin() Expect(err).ToNot(HaveOccurred()) - Expect(returned.Ilk).To(Equal("ETH")) - Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) - Expect(returned.Ink).To(Equal("80000000000000000000")) - Expect(returned.Art).To(Equal("11000000000000000000000")) - Expect(returned.IArt).To(Equal("12496609999999999999992")) - Expect(returned.Tab).To(Equal("11000000000000000000000")) - Expect(returned.NFlip).To(Equal("7")) - Expect(returned.TransactionIndex).To(Equal(uint(1))) - Expect(returned.LogIndex).To(Equal(uint(4))) + }) + + AfterEach(func() { + err := utils.ClearFiles(goPath, soPath) + Expect(err).ToNot(HaveOccurred()) + }) + + Describe("GenerateTransformerPlugin", func() { + It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() { + plug, err := plugin.Open(soPath) + Expect(err).ToNot(HaveOccurred()) + symExporter, err := plug.Lookup("Exporter") + Expect(err).ToNot(HaveOccurred()) + exporter, ok := symExporter.(Exporter) + Expect(ok).To(Equal(true)) + initializers := exporter.Export() + Expect(len(initializers)).To(Equal(2)) + }) + + It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() { + db, bc = test_helpers.SetupDBandBC() + defer test_helpers.TearDown(db) + + hr = repositories.NewHeaderRepository(db) + header1, err := bc.GetHeaderByNumber(9377319) + Expect(err).ToNot(HaveOccurred()) + headerID, err = hr.CreateOrUpdateHeader(header1) + Expect(err).ToNot(HaveOccurred()) + + plug, err := plugin.Open(soPath) + Expect(err).ToNot(HaveOccurred()) + symExporter, err := plug.Lookup("Exporter") + Expect(err).ToNot(HaveOccurred()) + exporter, ok := symExporter.(Exporter) + Expect(ok).To(Equal(true)) + initializers := exporter.Export() + + w := watcher.NewWatcher(db, bc) + w.AddTransformers(initializers) + err = w.Execute() + Expect(err).ToNot(HaveOccurred()) + + type model struct { + bite.BiteModel + Id int64 `db:"id"` + HeaderId int64 `db:"header_id"` + } + + returned := model{} + + err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID) + Expect(err).ToNot(HaveOccurred()) + Expect(returned.Ilk).To(Equal("ETH")) + Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) + Expect(returned.Ink).To(Equal("80000000000000000000")) + Expect(returned.Art).To(Equal("11000000000000000000000")) + Expect(returned.IArt).To(Equal("12496609999999999999992")) + Expect(returned.Tab).To(Equal("11000000000000000000000")) + Expect(returned.NFlip).To(Equal("7")) + Expect(returned.TransactionIndex).To(Equal(uint(1))) + Expect(returned.LogIndex).To(Equal(uint(4))) + }) }) }) }) diff --git a/plugins/README.md b/plugins/README.md index 390eaeca..0b3b440d 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -23,36 +23,54 @@ The config file requires, at a minimum, the below fields: transformer1 = "github.com/path/to/transformer1" transformer2 = "github.com/path/to/transformer2" transformer3 = "github.com/path/to/transformer3" + transformer4 = "github.com/different/path/to/transformer1" + [exporter.repositories] + transformers = "github.com/path/to" + transformer4 = "github.com/different/path + [exporter.migrations] + transformers = "db/migrations" + transformer4 = "to/db/migrations" ``` -In the above, the exporter.transformers are mappings of import aliases to their import paths +- `exporter.transformers` are mappings of import aliases to paths to `TransformerInitializer`s + - Import aliases can be arbitrarily named but note that `interface1` is a reserved alias needed for the generic TransformerInitializer type +- `exporter.repositores` are the paths to the repositories which contain the transformers +- `exporter.migrations` are the relative paths to the db migrations found within the `exporter.repositores` + - Migrations need to be located in the repos in `exporter.repositores` + - Keys should match the keys for the corresponding repo + If the individual transformers require additional configuration variables be sure to include them in the .toml file -The general structure of a plugin .go file, and what we would see with the above config is below -Note that `shared_transformer` is a reserved alias needed for the generic TransformerInitializer type: +The general structure of a plugin .go file, and what we would see with the above config is shown below ```go package main import ( - shared_transformer "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" + interface1 "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" transformer1 "github.com/path/to/transformer1" transformer2 "github.com/path/to/transformer2" transformer3 "github.com/path/to/transformer3" + transformer4 "github.com/different/path/to/transformer1" ) type exporter string var Exporter exporter -func (e exporter) Export() []shared_transformer.TransformerInitializer { - return []shared_transformer.TransformerInitializer{ +func (e exporter) Export() []interface1.TransformerInitializer { + return []interface1.TransformerInitializer{ transformer1.TransformerInitializer, transformer2.TransformerInitializer, transformer3.TransformerInitializer, + transformer4.TransformerInitializer, } } ``` -As such, to plug in an external transformer all we need to do is create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19) -As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them \ No newline at end of file +As such, to plug in an external transformer we need to create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19) +As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them +Note: currently the transformers must also operate using the watcher's [execution mode](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/watcher.go#L80) + +For each transformer we will also need to create db migrations to run against vulcanizeDB so that we can store the transformed data +The migrations needed for a specific transformer need to be included in the same repository as the transformers that require them, and their relative paths in that repo specified in the config as discussed above \ No newline at end of file diff --git a/plugins/example_maker_exporter b/plugins/example_maker_exporter new file mode 100644 index 00000000..b2702cfd --- /dev/null +++ b/plugins/example_maker_exporter @@ -0,0 +1,43 @@ +// This should be the output from running composeAndExecute with compose.toml + +package main + +import ( + cat_chop_lump "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump" + bite "github.com/vulcanize/mcd_transformers/transformers/bite" + cat_flip "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip" + cat_pit_vow "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow" + deal "github.com/vulcanize/mcd_transformers/transformers/deal" + dent "github.com/vulcanize/mcd_transformers/transformers/dent" + drip_drip "github.com/vulcanize/mcd_transformers/transformers/drip_drip" + drip_file_ilk "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk" + drip_file_repo "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo" + drip_file_vow "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow" + flap_kick "github.com/vulcanize/mcd_transformers/transformers/flap_kick" + flip_kick "github.com/vulcanize/mcd_transformers/transformers/flip_kick" + flop_kick "github.com/vulcanize/mcd_transformers/transformers/flop_kick" + frob "github.com/vulcanize/mcd_transformers/transformers/frob" + pit_file_debt_ceiling "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling" + pit_file_ilk "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk" + price_feeds "github.com/vulcanize/mcd_transformers/transformers/price_feeds" + tend "github.com/vulcanize/mcd_transformers/transformers/tend" + vat_flux "github.com/vulcanize/mcd_transformers/transformers/vat_flux" + vat_fold "github.com/vulcanize/mcd_transformers/transformers/vat_fold" + vat_grab "github.com/vulcanize/mcd_transformers/transformers/vat_grab" + vat_heal "github.com/vulcanize/mcd_transformers/transformers/vat_heal" + vat_init "github.com/vulcanize/mcd_transformers/transformers/vat_init" + vat_move "github.com/vulcanize/mcd_transformers/transformers/vat_move" + vat_slip "github.com/vulcanize/mcd_transformers/transformers/vat_slip" + vat_toll "github.com/vulcanize/mcd_transformers/transformers/vat_toll" + vat_tune "github.com/vulcanize/mcd_transformers/transformers/vat_tune" + vow_flog "github.com/vulcanize/mcd_transformers/transformers/vow_flog" + "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" +) + +type exporter string + +var Exporter exporter + +func (e exporter) Export() []transformer.TransformerInitializer { + return []transformer.TransformerInitializer{deal.TransformerInitializer, cat_chop_lump.TransformerInitializer, vat_slip.TransformerInitializer, bite.TransformerInitializer, vat_heal.TransformerInitializer, vat_toll.TransformerInitializer, price_feeds.TransformerInitializer, vat_init.TransformerInitializer, cat_pit_vow.TransformerInitializer, drip_drip.TransformerInitializer, vat_grab.TransformerInitializer, tend.TransformerInitializer, pit_file_ilk.TransformerInitializer, vat_fold.TransformerInitializer, vat_tune.TransformerInitializer, dent.TransformerInitializer, vow_flog.TransformerInitializer, flip_kick.TransformerInitializer, vat_flux.TransformerInitializer, frob.TransformerInitializer, flap_kick.TransformerInitializer, drip_file_repo.TransformerInitializer, flop_kick.TransformerInitializer, vat_move.TransformerInitializer, cat_flip.TransformerInitializer, drip_file_ilk.TransformerInitializer, drip_file_vow.TransformerInitializer, pit_file_debt_ceiling.TransformerInitializer} +} diff --git a/utils/utils.go b/utils/utils.go index 596d8ad3..3a862f51 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -17,10 +17,14 @@ package utils import ( + "io" "math/big" "os" "path/filepath" + "strings" + "syscall" + "github.com/mitchellh/go-homedir" log "github.com/sirupsen/logrus" "github.com/vulcanize/vulcanizedb/pkg/config" @@ -80,3 +84,54 @@ func RequestedBlockNumber(blockNumber *int64) *big.Int { } return _blockNumber } + +func CleanPath(str string) (string, error) { + path, err := homedir.Expand(filepath.Clean(str)) + if err != nil { + return "", err + } + if strings.Contains(path, "$GOPATH") { + env := os.Getenv("GOPATH") + spl := strings.Split(path, "$GOPATH")[1] + path = filepath.Join(env, spl) + } + + return path, nil +} + +func ClearFiles(files ...string) error { + for _, file := range files { + if _, err := os.Stat(file); err == nil { + err = os.Remove(file) + if err != nil { + return err + } + } else if os.IsNotExist(err) { + // fall through + } else { + return err + } + } + + return nil +} + +func CopyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL, os.FileMode(0666)) // Doesn't overwrite files + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + return out.Close() +}