auto management of dependencies temporarily needed to build plugin and transformer

migrations; new test showing it is working with external transformers runs
properly when migrations were manually performed but still need
to test automated migration management

cmd/composeAndExecute.go

@@ -47,11 +47,19 @@ var composeAndExecuteCmd = &cobra.Command{
     ipcPath = "http://kovan0.vulcanize.io:8545"
 
 [exporter]
-    filePath = "~/go/src/github.com/vulcanize/vulcanizedb/plugins"
-	fileName = "exporter"
+    filePath = "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/"
+    fileName = "exporter"
     [exporter.transformers]
-		transformerImport1 = "github.com/path_to/transformerInitializer1"
-		transformerImport2 = "github.com/path_to/transformerInitializer2"
+            transformer1 = "github.com/path/to/transformer1"
+            transformer2 = "github.com/path/to/transformer2"
+            transformer3 = "github.com/path/to/transformer3"
+            transformer4 = "github.com/different/path/to/transformer1"
+    [exporter.repositories]
+            transformers = "github.com/path/to"
+            transformer4 = "github.com/different/path
+    [exporter.migrations]
+            transformers = "db/migrations"
+            transformer4 = "to/db/migrations"
 
 Note: If any of the imported transformer need additional
 config variables do not forget to include those as well
@@ -66,43 +74,50 @@ loaded into and executed over by a generic watcher`,
 }
 
 func composeAndExecute() {
-	generator := autogen.NewGenerator(autogenConfig)
-	err := generator.GenerateTransformerPlugin()
+	// generate code to build the plugin according to the config file
+	generator := autogen.NewGenerator(autogenConfig, databaseConfig)
+	err := generator.GenerateExporterPlugin()
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	_, pluginPath, err := autogen.GetPaths(autogenConfig)
+	// Get the plugin path and load the plugin
+	_, pluginPath, err := autogenConfig.GetPluginPaths()
 	if err != nil {
 		log.Fatal(err)
 	}
-
 	plug, err := plugin.Open(pluginPath)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	// Load the `Exporter` symbol from the plugin
 	symExporter, err := plug.Lookup("Exporter")
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	// Assert that the symbol is of type Exporter
 	exporter, ok := symExporter.(Exporter)
 	if !ok {
 		fmt.Println("plugged-in symbol not of type Exporter")
 		os.Exit(1)
 	}
 
+	// Use the Exporters export method to load the TransformerInitializer set
 	initializers := exporter.Export()
+
+	// Setup bc and db objects
+	blockChain := getBlockChain()
+	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
+
+	// Create a watcher and load the TransformerInitializer set into it
 	w := watcher.NewWatcher(&db, blockChain)
 	w.AddTransformers(initializers)
 
+	// Execute over the TransformerInitializer set using the watcher
+	ticker := time.NewTicker(pollingInterval)
+	defer ticker.Stop()
 	for range ticker.C {
 		err := w.Execute()
 		if err != nil {

cmd/root.go

@@ -77,9 +77,11 @@ func configure(cmd *cobra.Command, args []string) {
 		Password: viper.GetString("database.password"),
 	}
 	autogenConfig = autogen.Config{
-		FilePath: viper.GetString("exporter.filePath"),
-		FileName: viper.GetString("exporter.fileName"),
-		Imports:  viper.GetStringMapString("exporter.transformers"),
+		FilePath:     viper.GetString("exporter.filePath"),
+		FileName:     viper.GetString("exporter.fileName"),
+		Initializers: viper.GetStringMapString("exporter.transformers"),
+		Dependencies: viper.GetStringMapString("exporter.repositories"),
+		Migrations:   viper.GetStringMapString("exporter.migrations"),
 	}
 	viper.Set("database.config", databaseConfig)
 }

environments/compose.toml

@@ -1,24 +1,24 @@
 [database]
-    name = "vulcanize_public"
+    name     = "vulcanize_public"
     hostname = "localhost"
-    user = "vulcanize"
+    user     = "vulcanize"
     password = "vulcanize"
-    port = 5432
+    port     = 5432
 
 [client]
-    ipcPath = "http://kovan0.vulcanize.io:8545"
+    ipcPath  = "http://kovan0.vulcanize.io:8545"
 
 [datadog]
-    name = "maker_vdb_staging"
+    name     = "maker_vdb_staging"
 
 [exporter]
     filePath = "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/"
     fileName = "exporter"
     [exporter.transformers]
             bite                  = "github.com/vulcanize/mcd_transformers/transformers/bite"
-            cat_chop_lump         = "github.com/vulcanize/maker_transformers/cat/chop_lump"
-            cat_flip              = "github.com/vulcanize/mcd_transformers/transformers/cat/flip"
-            cat_pit_vow           = "github.com/vulcanize/mcd_transformers/transformers/cat/pit_vow"
+            cat_chop_lump         = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump"
+            cat_flip              = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip"
+            cat_pit_vow           = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow"
             deal                  = "github.com/vulcanize/mcd_transformers/transformers/deal"
             dent                  = "github.com/vulcanize/mcd_transformers/transformers/dent"
             drip_drip             = "github.com/vulcanize/mcd_transformers/transformers/drip_drip"
@@ -43,6 +43,10 @@
             vat_toll              = "github.com/vulcanize/mcd_transformers/transformers/vat_toll"
             vat_tune              = "github.com/vulcanize/mcd_transformers/transformers/vat_tune"
             vow_flog              = "github.com/vulcanize/mcd_transformers/transformers/vow_flog"
+     [exporter.repositories]
+            mcd_transformers      = "github.com/vulcanize/mcd_transformers"
+     [exporter.migrations]
+            mcd_transformers      = "db/migrations"
 
 [contract]
     [contract.address]

pkg/autogen/config.go

@@ -16,8 +16,50 @@
 
 package autogen
 
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/vulcanize/vulcanizedb/utils"
+)
+
 type Config struct {
-	Imports  map[string]string // Map of import alias to import path
-	FilePath string
-	FileName string
+	Initializers map[string]string // Map of import aliases to transformer paths
+	Dependencies map[string]string // Map of vendor dep names to their repositories
+	Migrations   map[string]string // Map of migration names to their paths within the vendored deps
+	FilePath     string
+	FileName     string
+}
+
+func (c *Config) GetPluginPaths() (string, string, error) {
+	path, err := utils.CleanPath(c.FilePath)
+	if err != nil {
+		return "", "", err
+	}
+
+	name := strings.Split(c.FileName, ".")[0]
+	goFile := filepath.Join(path, name+".go")
+	soFile := filepath.Join(path, name+".so")
+
+	return goFile, soFile, nil
+}
+
+func (c *Config) GetMigrationsPaths() ([]string, error) {
+	paths := make([]string, 0, len(c.Migrations))
+	for key, relPath := range c.Migrations {
+		repo, ok := c.Dependencies[key]
+		if !ok {
+			return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath))
+		}
+		path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath)
+		cleanPath, err := utils.CleanPath(path)
+		if err != nil {
+			return nil, err
+		}
+		paths = append(paths, cleanPath)
+	}
+
+	return paths, nil
 }

pkg/autogen/generator.go

@@ -18,65 +18,97 @@ package autogen
 
 import (
 	"errors"
+	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
+	"strconv"
 
 	. "github.com/dave/jennifer/jen"
-	"github.com/mitchellh/go-homedir"
+
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/utils"
 )
 
 type Generator interface {
-	GenerateTransformerPlugin() error
+	GenerateExporterPlugin() error
 }
 
 type generator struct {
-	*Config
+	GenConfig  *Config
+	DBConfig   config.Database
+	tmpMigDir  string
+	tmpVenDirs []string
 }
 
-func NewGenerator(config Config) *generator {
+func NewGenerator(gc Config, dbc config.Database) *generator {
 	return &generator{
-		Config: &config,
+		GenConfig: &gc,
+		DBConfig:  dbc,
 	}
 }
 
-func (g *generator) GenerateTransformerPlugin() error {
-	if g.Config == nil {
+func (g *generator) GenerateExporterPlugin() error {
+	if g.GenConfig == nil {
 		return errors.New("generator needs a config file")
 	}
-	if g.Config.FilePath == "" {
+	if g.GenConfig.FilePath == "" {
 		return errors.New("generator is missing file path")
 	}
-	if len(g.Config.Imports) < 1 {
+	if len(g.GenConfig.Initializers) < 1 {
 		return errors.New("generator needs to be configured with imports")
 	}
 
-	// Create file path
-	goFile, soFile, err := GetPaths(*g.Config)
+	// Get plugin file paths
+	goFile, soFile, err := g.GenConfig.GetPluginPaths()
 	if err != nil {
 		return err
 	}
 
-	// Clear previous .go and .so files if they exist
-	err = ClearFiles(goFile, soFile)
+	// Clear .go and .so files of the same name if they exist (overwrite)
+	err = utils.ClearFiles(goFile, soFile)
 	if err != nil {
 		return err
 	}
 
+	// Generate Exporter code
+	err = g.generateCode(goFile)
+	if err != nil {
+		return err
+	}
+
+	// Setup temp vendor lib and migrations directories
+	err = g.setupTempDirs()
+	if err != nil {
+		return err
+	}
+	defer g.cleanUp() // Clear these up when we are done building our plugin
+
+	// Build the .go file into a .so plugin
+	err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run()
+	if err != nil {
+		return err
+	}
+	// Run migrations only after successfully building .so file
+	return g.runMigrations()
+}
+
+// Generates the plugin code
+func (g *generator) generateCode(goFile string) error {
 	// Begin code generation
 	f := NewFile("main")
 	f.HeaderComment("This exporter is generated to export the configured transformer initializers")
 
 	// Import TransformerInitializers
 	f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
-	for alias, imp := range g.Config.Imports {
+	for alias, imp := range g.GenConfig.Initializers {
 		f.ImportAlias(imp, alias)
 	}
 
 	// Collect TransformerInitializer names
-	importedInitializers := make([]Code, 0, len(g.Config.Imports))
-	for _, path := range g.Config.Imports {
+	importedInitializers := make([]Code, 0, len(g.GenConfig.Initializers))
+	for _, path := range g.GenConfig.Initializers {
 		importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer"))
 	}
 
@@ -90,49 +122,143 @@ func (g *generator) GenerateTransformerPlugin() error {
 		"TransformerInitializer").Block(
 		Return(Index().Qual(
 			"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
-			"TransformerInitializer").Values(importedInitializers...)))
+			"TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers
 
 	// Write code to destination file
-	err = f.Save(goFile)
+	return f.Save(goFile)
+}
+
+func (g *generator) runMigrations() error {
+	// Get paths to db migrations
+	paths, err := g.GenConfig.GetMigrationsPaths()
+	if err != nil {
+		return err
+	}
+	if len(paths) < 1 {
+		return nil
+	}
+
+	// Create temporary copies of migrations to the temporary migrationDir
+	// These tmps are identical except they have had `1` added in front of their unix_timestamps
+	// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
+	// But will still be ran in the same order relative to one another
+	// TODO: Less hacky way of handing migrations
+	err = g.createMigrationCopies(paths)
 	if err != nil {
 		return err
 	}
 
-	// Build the .go file into a .so plugin
-	return exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run()
+	// Run the copied migrations
+	location := "file://" + g.tmpMigDir
+	pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable up", g.DBConfig.Hostname, g.DBConfig.Port, g.DBConfig.Name)
+	return exec.Command("migrate", "-source", location, pgStr).Run()
 }
 
-func GetPaths(config Config) (string, string, error) {
-	path, err := homedir.Expand(filepath.Clean(config.FilePath))
+// Sets up temporary vendor libs and migration directories
+func (g *generator) setupTempDirs() error {
+	// TODO: Less hacky way of handling plugin build deps
+	dirPath, err := utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/")
 	if err != nil {
-		return "", "", err
+		return err
 	}
-	if strings.Contains(path, "$GOPATH") {
-		env := os.Getenv("GOPATH")
-		spl := strings.Split(path, "$GOPATH")[1]
-		path = filepath.Join(env, spl)
+	vendorPath := filepath.Join(dirPath, "vendor/")
+
+	/*
+	// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
+	g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
+	// Import transformer dependencies so that we build our plugin
+	for _, importPath := range g.GenConfig.Dependencies {
+		importURL := "https://" + importPath + ".git"
+		depPath := filepath.Join(vendorPath, importPath)
+		err = exec.Command("git", "clone", importURL, depPath).Run()
+		if err != nil {
+			return err
+		}
+		err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
+		if err != nil {
+			return err
+		}
+		g.tmpVenDirs = append(g.tmpVenDirs, depPath)
+	}
+	*/
+
+	// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
+	g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
+	for _, importPath := range g.GenConfig.Dependencies {
+		depPath := filepath.Join(vendorPath, importPath)
+		g.tmpVenDirs = append(g.tmpVenDirs, depPath)
 	}
 
-	name := strings.Split(config.FileName, ".")[0]
-	goFile := filepath.Join(path, name+".go")
-	soFile := filepath.Join(path, name+".so")
+	// Dep ensure to make sure vendor pkgs are in place for building the plugin
+	err = exec.Command("dep", "ensure").Run()
+	if err != nil {
+		return errors.New("failed to vendor transformer packages required to build plugin")
+	}
 
-	return goFile, soFile, nil
+	// Git checkout our head-state vendor libraries
+	// This is necessary because we currently need to manual edit our vendored
+	// go-ethereum abi library to allow for unpacking in empty interfaces and maps
+	// This can be removed once the PRs against geth merged
+	err = exec.Command("git", "checkout", dirPath).Run()
+	if err != nil {
+		return errors.New("failed to checkout vendored go-ethereum lib")
+	}
+
+	// Initialize temp directory for transformer migrations
+	g.tmpMigDir, err = utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
+	if err != nil {
+		return err
+	}
+	stat, err := os.Stat(g.tmpMigDir)
+	if err == nil {
+		if !stat.IsDir() {
+			return errors.New(fmt.Sprintf("file %s found where directory is expected", stat.Name()))
+		}
+	} else if os.IsNotExist(err) {
+		os.Mkdir(g.tmpMigDir, os.FileMode(0777))
+	} else {
+		return err
+	}
+
+	return nil
 }
 
-func ClearFiles(files ...string) error {
-	for _, file := range files {
-		if _, err := os.Stat(file); err == nil {
-			err = os.Remove(file)
-			if err != nil {
-				return err
-			}
-		} else if os.IsNotExist(err) {
-			// fall through
-		} else {
+func (g *generator) createMigrationCopies(paths []string) error {
+	for _, path := range paths {
+		dir, err := ioutil.ReadDir(path)
+		if err != nil {
 			return err
 		}
+		for _, file := range dir {
+			if file.IsDir() || len(file.Name()) < 15 { // (10 digit unix time stamp + x + .sql) is bare minimum
+				continue
+			}
+			_, err := strconv.Atoi(file.Name()[:10])
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix", file.Name())
+				continue
+			}
+			if filepath.Ext(file.Name()) == "sql" {
+				src := filepath.Join(path, file.Name())
+				dst := filepath.Join(g.tmpMigDir, "1"+file.Name())
+				err = utils.CopyFile(src, dst)
+				if err != nil {
+					return err
+				}
+			}
+		}
 	}
 
 	return nil
 }
+
+func (g *generator) cleanUp() error {
+	for _, venDir := range g.tmpVenDirs {
+		err := os.RemoveAll(venDir)
+		if err != nil {
+			return err
+		}
+	}
+
+	return os.RemoveAll(g.tmpMigDir)
+}

pkg/autogen/generator_test.go

@@ -27,28 +27,33 @@ import (
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/autogen"
 	"github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
+	"github.com/vulcanize/vulcanizedb/utils"
 )
 
-var testConfig = autogen.Config{
-	Imports: map[string]string{
+var localConfig = autogen.Config{
+	Initializers: map[string]string{
 		"bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite",
 		"deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal",
 	},
-	FileName: "testTransformerSet",
+	FileName: "localTestTransformerSet",
 	FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/",
 }
 
-var targetConfig = autogen.Config{
-	Imports: map[string]string{
-		"bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite",
-		"deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal",
+var externalConfig = autogen.Config{
+	Initializers: map[string]string{
+		"bite": "github.com/vulcanize/mcd_transformers/transformers/bite",
+		"deal": "github.com/vulcanize/mcd_transformers/transformers/deal",
 	},
-	FileName: "targetTransformerSet",
-	FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/target/",
+	Dependencies: map[string]string{
+		"mcd_transformers": "github.com/vulcanize/mcd_transformers",
+	},
+	FileName: "externalTestTransformerSet",
+	FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/",
 }
 
 type Exporter interface {
@@ -66,73 +71,147 @@ var _ = Describe("Generator test", func() {
 	viper.SetConfigName("compose")
 	viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
 
-	BeforeEach(func() {
-		goPath, soPath, err = autogen.GetPaths(testConfig)
-		Expect(err).ToNot(HaveOccurred())
-		g = autogen.NewGenerator(testConfig)
-		err = g.GenerateTransformerPlugin()
-		Expect(err).ToNot(HaveOccurred())
-	})
-
-	AfterEach(func() {
-		err := autogen.ClearFiles(goPath, soPath)
-		Expect(err).ToNot(HaveOccurred())
-	})
-
-	Describe("GenerateTransformerPlugin", func() {
-		It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
-			plug, err := plugin.Open(soPath)
+	Describe("Using local config", func() {
+		BeforeEach(func() {
+			goPath, soPath, err = localConfig.GetPluginPaths()
 			Expect(err).ToNot(HaveOccurred())
-			symExporter, err := plug.Lookup("Exporter")
+			g = autogen.NewGenerator(localConfig, config.Database{})
+			err = g.GenerateExporterPlugin()
 			Expect(err).ToNot(HaveOccurred())
-			exporter, ok := symExporter.(Exporter)
-			Expect(ok).To(Equal(true))
-			initializers := exporter.Export()
-			Expect(len(initializers)).To(Equal(2))
 		})
 
-		It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
-			db, bc = test_helpers.SetupDBandBC()
-			defer test_helpers.TearDown(db)
-
-			hr = repositories.NewHeaderRepository(db)
-			header1, err := bc.GetHeaderByNumber(9377319)
-			Expect(err).ToNot(HaveOccurred())
-			headerID, err = hr.CreateOrUpdateHeader(header1)
+		AfterEach(func() {
+			err := utils.ClearFiles(goPath, soPath)
 			Expect(err).ToNot(HaveOccurred())
+		})
 
-			plug, err := plugin.Open(soPath)
-			Expect(err).ToNot(HaveOccurred())
-			symExporter, err := plug.Lookup("Exporter")
-			Expect(err).ToNot(HaveOccurred())
-			exporter, ok := symExporter.(Exporter)
-			Expect(ok).To(Equal(true))
-			initializers := exporter.Export()
+		Describe("GenerateTransformerPlugin", func() {
+			It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
+				plug, err := plugin.Open(soPath)
+				Expect(err).ToNot(HaveOccurred())
+				symExporter, err := plug.Lookup("Exporter")
+				Expect(err).ToNot(HaveOccurred())
+				exporter, ok := symExporter.(Exporter)
+				Expect(ok).To(Equal(true))
+				initializers := exporter.Export()
+				Expect(len(initializers)).To(Equal(2))
+			})
 
-			w := watcher.NewWatcher(db, bc)
-			w.AddTransformers(initializers)
-			err = w.Execute()
+			It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
+				db, bc = test_helpers.SetupDBandBC()
+				defer test_helpers.TearDown(db)
+
+				hr = repositories.NewHeaderRepository(db)
+				header1, err := bc.GetHeaderByNumber(9377319)
+				Expect(err).ToNot(HaveOccurred())
+				headerID, err = hr.CreateOrUpdateHeader(header1)
+				Expect(err).ToNot(HaveOccurred())
+
+				plug, err := plugin.Open(soPath)
+				Expect(err).ToNot(HaveOccurred())
+				symExporter, err := plug.Lookup("Exporter")
+				Expect(err).ToNot(HaveOccurred())
+				exporter, ok := symExporter.(Exporter)
+				Expect(ok).To(Equal(true))
+				initializers := exporter.Export()
+
+				w := watcher.NewWatcher(db, bc)
+				w.AddTransformers(initializers)
+				err = w.Execute()
+				Expect(err).ToNot(HaveOccurred())
+
+				type model struct {
+					bite.BiteModel
+					Id       int64 `db:"id"`
+					HeaderId int64 `db:"header_id"`
+				}
+
+				returned := model{}
+
+				err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(returned.Ilk).To(Equal("ETH"))
+				Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
+				Expect(returned.Ink).To(Equal("80000000000000000000"))
+				Expect(returned.Art).To(Equal("11000000000000000000000"))
+				Expect(returned.IArt).To(Equal("12496609999999999999992"))
+				Expect(returned.Tab).To(Equal("11000000000000000000000"))
+				Expect(returned.NFlip).To(Equal("7"))
+				Expect(returned.TransactionIndex).To(Equal(uint(1)))
+				Expect(returned.LogIndex).To(Equal(uint(4)))
+			})
+		})
+	})
+
+	Describe("Using external config", func() {
+		BeforeEach(func() {
+			goPath, soPath, err = externalConfig.GetPluginPaths()
 			Expect(err).ToNot(HaveOccurred())
-
-			type model struct {
-				bite.BiteModel
-				Id       int64 `db:"id"`
-				HeaderId int64 `db:"header_id"`
-			}
-
-			returned := model{}
-
-			err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
+			g = autogen.NewGenerator(externalConfig, config.Database{})
+			err = g.GenerateExporterPlugin()
 			Expect(err).ToNot(HaveOccurred())
-			Expect(returned.Ilk).To(Equal("ETH"))
-			Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
-			Expect(returned.Ink).To(Equal("80000000000000000000"))
-			Expect(returned.Art).To(Equal("11000000000000000000000"))
-			Expect(returned.IArt).To(Equal("12496609999999999999992"))
-			Expect(returned.Tab).To(Equal("11000000000000000000000"))
-			Expect(returned.NFlip).To(Equal("7"))
-			Expect(returned.TransactionIndex).To(Equal(uint(1)))
-			Expect(returned.LogIndex).To(Equal(uint(4)))
+		})
+
+		AfterEach(func() {
+			err := utils.ClearFiles(goPath, soPath)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		Describe("GenerateTransformerPlugin", func() {
+			It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
+				plug, err := plugin.Open(soPath)
+				Expect(err).ToNot(HaveOccurred())
+				symExporter, err := plug.Lookup("Exporter")
+				Expect(err).ToNot(HaveOccurred())
+				exporter, ok := symExporter.(Exporter)
+				Expect(ok).To(Equal(true))
+				initializers := exporter.Export()
+				Expect(len(initializers)).To(Equal(2))
+			})
+
+			It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
+				db, bc = test_helpers.SetupDBandBC()
+				defer test_helpers.TearDown(db)
+
+				hr = repositories.NewHeaderRepository(db)
+				header1, err := bc.GetHeaderByNumber(9377319)
+				Expect(err).ToNot(HaveOccurred())
+				headerID, err = hr.CreateOrUpdateHeader(header1)
+				Expect(err).ToNot(HaveOccurred())
+
+				plug, err := plugin.Open(soPath)
+				Expect(err).ToNot(HaveOccurred())
+				symExporter, err := plug.Lookup("Exporter")
+				Expect(err).ToNot(HaveOccurred())
+				exporter, ok := symExporter.(Exporter)
+				Expect(ok).To(Equal(true))
+				initializers := exporter.Export()
+
+				w := watcher.NewWatcher(db, bc)
+				w.AddTransformers(initializers)
+				err = w.Execute()
+				Expect(err).ToNot(HaveOccurred())
+
+				type model struct {
+					bite.BiteModel
+					Id       int64 `db:"id"`
+					HeaderId int64 `db:"header_id"`
+				}
+
+				returned := model{}
+
+				err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(returned.Ilk).To(Equal("ETH"))
+				Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
+				Expect(returned.Ink).To(Equal("80000000000000000000"))
+				Expect(returned.Art).To(Equal("11000000000000000000000"))
+				Expect(returned.IArt).To(Equal("12496609999999999999992"))
+				Expect(returned.Tab).To(Equal("11000000000000000000000"))
+				Expect(returned.NFlip).To(Equal("7"))
+				Expect(returned.TransactionIndex).To(Equal(uint(1)))
+				Expect(returned.LogIndex).To(Equal(uint(4)))
+			})
 		})
 	})
 })

plugins/README.md

@@ -23,36 +23,54 @@ The config file requires, at a minimum, the below fields:
             transformer1 = "github.com/path/to/transformer1"
             transformer2 = "github.com/path/to/transformer2"
             transformer3 = "github.com/path/to/transformer3"
+            transformer4 = "github.com/different/path/to/transformer1"
+    [exporter.repositories]
+            transformers = "github.com/path/to"
+            transformer4 = "github.com/different/path
+    [exporter.migrations]
+            transformers = "db/migrations"
+            transformer4 = "to/db/migrations"
 ```
 
-In the above, the exporter.transformers are mappings of import aliases to their import paths 
+- `exporter.transformers` are mappings of import aliases to paths to `TransformerInitializer`s  
+    -  Import aliases can be arbitrarily named but note that `interface1` is a reserved alias needed for the generic TransformerInitializer type  
+- `exporter.repositores` are the paths to the repositories which contain the transformers   
+- `exporter.migrations` are the relative paths to the db migrations found within the `exporter.repositores`  
+    - Migrations need to be located in the repos in `exporter.repositores`  
+    - Keys should match the keys for the corresponding repo  
+
 If the individual transformers require additional configuration variables be sure to include them in the .toml file 
 
-The general structure of a plugin .go file, and what we would see with the above config is below   
-Note that `shared_transformer` is a reserved alias needed for the generic TransformerInitializer type:
+The general structure of a plugin .go file, and what we would see with the above config is shown below   
 
 ```go
 package main
 
 import (
-	shared_transformer "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	interface1 "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	transformer1 "github.com/path/to/transformer1"
 	transformer2 "github.com/path/to/transformer2"
 	transformer3 "github.com/path/to/transformer3"
+	transformer4 "github.com/different/path/to/transformer1"
 )
 
 type exporter string
 
 var Exporter exporter
 
-func (e exporter) Export() []shared_transformer.TransformerInitializer {
-	return []shared_transformer.TransformerInitializer{
+func (e exporter) Export() []interface1.TransformerInitializer {
+	return []interface1.TransformerInitializer{
 		transformer1.TransformerInitializer,
 		transformer2.TransformerInitializer,
 		transformer3.TransformerInitializer,
+		transformer4.TransformerInitializer,
 	}
 }
 ```
 
-As such, to plug in an external transformer all we need to do is create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19)   
-As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them  
+As such, to plug in an external transformer we need to create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19)   
+As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them   
+Note: currently the transformers must also operate using the watcher's [execution mode](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/watcher.go#L80)  
+
+For each transformer we will also need to create db migrations to run against vulcanizeDB so that we can store the transformed data  
+The migrations needed for a specific transformer need to be included in the same repository as the transformers that require them, and their relative paths in that repo specified in the config as discussed above

plugins/example_maker_exporter

@@ -0,0 +1,43 @@
+// This should be the output from running composeAndExecute with compose.toml
+
+package main
+
+import (
+	cat_chop_lump "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump"
+	bite "github.com/vulcanize/mcd_transformers/transformers/bite"
+	cat_flip "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip"
+	cat_pit_vow "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow"
+	deal "github.com/vulcanize/mcd_transformers/transformers/deal"
+	dent "github.com/vulcanize/mcd_transformers/transformers/dent"
+	drip_drip "github.com/vulcanize/mcd_transformers/transformers/drip_drip"
+	drip_file_ilk "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk"
+	drip_file_repo "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo"
+	drip_file_vow "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow"
+	flap_kick "github.com/vulcanize/mcd_transformers/transformers/flap_kick"
+	flip_kick "github.com/vulcanize/mcd_transformers/transformers/flip_kick"
+	flop_kick "github.com/vulcanize/mcd_transformers/transformers/flop_kick"
+	frob "github.com/vulcanize/mcd_transformers/transformers/frob"
+	pit_file_debt_ceiling "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling"
+	pit_file_ilk "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk"
+	price_feeds "github.com/vulcanize/mcd_transformers/transformers/price_feeds"
+	tend "github.com/vulcanize/mcd_transformers/transformers/tend"
+	vat_flux "github.com/vulcanize/mcd_transformers/transformers/vat_flux"
+	vat_fold "github.com/vulcanize/mcd_transformers/transformers/vat_fold"
+	vat_grab "github.com/vulcanize/mcd_transformers/transformers/vat_grab"
+	vat_heal "github.com/vulcanize/mcd_transformers/transformers/vat_heal"
+	vat_init "github.com/vulcanize/mcd_transformers/transformers/vat_init"
+	vat_move "github.com/vulcanize/mcd_transformers/transformers/vat_move"
+	vat_slip "github.com/vulcanize/mcd_transformers/transformers/vat_slip"
+	vat_toll "github.com/vulcanize/mcd_transformers/transformers/vat_toll"
+	vat_tune "github.com/vulcanize/mcd_transformers/transformers/vat_tune"
+	vow_flog "github.com/vulcanize/mcd_transformers/transformers/vow_flog"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+)
+
+type exporter string
+
+var Exporter exporter
+
+func (e exporter) Export() []transformer.TransformerInitializer {
+	return []transformer.TransformerInitializer{deal.TransformerInitializer, cat_chop_lump.TransformerInitializer, vat_slip.TransformerInitializer, bite.TransformerInitializer, vat_heal.TransformerInitializer, vat_toll.TransformerInitializer, price_feeds.TransformerInitializer, vat_init.TransformerInitializer, cat_pit_vow.TransformerInitializer, drip_drip.TransformerInitializer, vat_grab.TransformerInitializer, tend.TransformerInitializer, pit_file_ilk.TransformerInitializer, vat_fold.TransformerInitializer, vat_tune.TransformerInitializer, dent.TransformerInitializer, vow_flog.TransformerInitializer, flip_kick.TransformerInitializer, vat_flux.TransformerInitializer, frob.TransformerInitializer, flap_kick.TransformerInitializer, drip_file_repo.TransformerInitializer, flop_kick.TransformerInitializer, vat_move.TransformerInitializer, cat_flip.TransformerInitializer, drip_file_ilk.TransformerInitializer, drip_file_vow.TransformerInitializer, pit_file_debt_ceiling.TransformerInitializer}
+}

utils/utils.go

@@ -17,10 +17,14 @@
 package utils
 
 import (
+	"io"
 	"math/big"
 	"os"
 	"path/filepath"
+	"strings"
+	"syscall"
 
+	"github.com/mitchellh/go-homedir"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
@@ -80,3 +84,54 @@ func RequestedBlockNumber(blockNumber *int64) *big.Int {
 	}
 	return _blockNumber
 }
+
+func CleanPath(str string) (string, error) {
+	path, err := homedir.Expand(filepath.Clean(str))
+	if err != nil {
+		return "", err
+	}
+	if strings.Contains(path, "$GOPATH") {
+		env := os.Getenv("GOPATH")
+		spl := strings.Split(path, "$GOPATH")[1]
+		path = filepath.Join(env, spl)
+	}
+
+	return path, nil
+}
+
+func ClearFiles(files ...string) error {
+	for _, file := range files {
+		if _, err := os.Stat(file); err == nil {
+			err = os.Remove(file)
+			if err != nil {
+				return err
+			}
+		} else if os.IsNotExist(err) {
+			// fall through
+		} else {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func CopyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL, os.FileMode(0666)) // Doesn't overwrite files
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+	return out.Close()
+}