addressing review comments; still need to reorg migrations and add helper tests

Makefile

@@ -50,7 +50,7 @@ lint:
 test: | $(GINKGO) $(LINT)
 	go vet ./...
 	go fmt ./...
-	$(GINKGO) -r
+	$(GINKGO) -r --skipPackage=integration_tests,integration
 
 .PHONY: integrationtest
 integrationtest: | $(GINKGO) $(LINT)

cmd/composeAndExecute.go

@@ -213,23 +213,45 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
 
 func prepConfig() {
 	fmt.Println("configuring plugin")
-	types := viper.GetStringMapString("exporter.types")
+	names := viper.GetStringSlice("exporter.transformerNames")
-	genTypes := map[string]config.PluginType{}
+	transformers := make(map[string]config.Transformer)
-	for transformerName, transformerType := range types {
+	for _, name := range names {
-		genType := config.GetPluginType(transformerType)
+		transformer := viper.GetStringMapString("exporter." + name)
-		if genType == config.UnknownTransformerType {
+		_, ok := transformer["path"]
-			log.Fatal(errors.New(`unknown transformer type in exporter config
+		if !ok {
-accepted types are "eth_event", "eth_storage", "ipfs_event" and "ipfs_storage"`))
+			log.Fatal(fmt.Sprintf("%s transformer config is missing `path` value", name))
+		}
+		_, ok = transformer["repository"]
+		if !ok {
+			log.Fatal(fmt.Sprintf("%s transformer config is missing `repository` value", name))
+		}
+		_, ok = transformer["migrations"]
+		if !ok {
+			log.Fatal(fmt.Sprintf("%s transformer config is missing `migrations` value", name))
+		}
+		ty, ok := transformer["type"]
+		if !ok {
+			log.Fatal(fmt.Sprintf("%s transformer config is missing `type` value", name))
+		}
+
+		transformerType := config.GetTransformerType(ty)
+		if transformerType == config.UnknownTransformerType {
+			log.Fatal(errors.New(`unknown transformer type in exporter config
+accepted types are "eth_event", "eth_storage"`))
+		}
+
+		transformers[name] = config.Transformer{
+			Path:           transformer["path"],
+			Type:           transformerType,
+			RepositoryPath: transformer["repository"],
+			MigrationPath:  transformer["migrations"],
 		}
-		genTypes[transformerName] = genType
 	}
+
 	genConfig = config.Plugin{
+		Transformers: transformers,
 		FilePath:     "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
 		FileName:     viper.GetString("exporter.name"),
 		Save:         viper.GetBool("exporter.save"),
-		Initializers: viper.GetStringMapString("exporter.transformers"),
-		Dependencies: viper.GetStringMapString("exporter.repositories"),
-		Migrations:   viper.GetStringMapString("exporter.migrations"),
-		Types:        genTypes,
 	}
 }

cmd/root.go

@@ -95,7 +95,6 @@ func init() {
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
 	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
-	rootCmd.PersistentFlags().String("exporter-path", "~/go/src/github.com/vulcanize/vulcanizedb/plugins", "file path to transformer exporter plugin")
 	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
 
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
@@ -107,7 +106,6 @@ func init() {
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
 	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
-	viper.BindPFlag("exporter.filePath", rootCmd.PersistentFlags().Lookup("exporter-path"))
 	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
 }
 

environments/compose.toml

@@ -14,66 +14,177 @@
 [exporter]
     name     = "eventTransformerExporter"
     save     = false
-    [exporter.transformers]
+    transformerNames = [
-            bite                    = "transformers/bite/initializer"
+        "bite",
-            cat_chop_lump           = "transformers/cat_file/chop_lump/initializer"
+        "cat_chop_lump",
-            cat_flip                = "transformers/cat_file/flip/initializer"
+        "cat_flip",
-            cat_pit_vow             = "transformers/cat_file/pit_vow/initializer"
+        "cat_pit_vow",
-            deal                    = "transformers/deal/initializer"
+        "deal",
-            dent                    = "transformers/dent/initializer"
+        "dent",
-            drip_drip               = "transformers/drip_drip/initializer"
+        "drip_drip",
-            drip_file_ilk           = "transformers/drip_file/ilk/initializer"
+        "drip_file_ilk",
-            drip_file_repo          = "transformers/drip_file/repo/initializer"
+        "drop_file_repo",
-            drip_file_vow           = "transformers/drip_file/vow/initializer"
+        "drip_file_vow",
-            flap_kick               = "transformers/flap_kick/initializer"
+        "flap_kick",
-            flip_kick               = "transformers/flip_kick/initializer"
+        "flip_kick",
-            flop_kick               = "transformers/flop_kick/initializer"
+        "flop_kick",
-            frob                    = "transformers/frob/initializer"
+        "frob_kick",
-            pit_file_debt_ceiling   = "transformers/pit_file/debt_ceiling/initializer"
+        "frob",
-            pit_file_ilk            = "transformers/pit_file/ilk/initializer"
+        "pit_file_debt_ceiling",
-            price_feeds             = "transformers/price_feeds/initializer"
+        "pit_file_ilk",
-            tend                    = "transformers/tend/initializer"
+        "price_feeds",
-            vat_flux                = "transformers/vat_flux/initializer"
+        "tend",
-            vat_fold                = "transformers/vat_fold/initializer"
+        "vat_flux",
-            vat_grab                = "transformers/vat_grab/initializer"
+        "vat_fold",
-            vat_heal                = "transformers/vat_heal/initializer"
+        "vat_grab",
-            vat_init                = "transformers/vat_init/initializer"
+        "vat_heal",
-            vat_move                = "transformers/vat_move/initializer"
+        "vat_init",
-            vat_slip                = "transformers/vat_slip/initializer"
+        "vat_move",
-            vat_toll                = "transformers/vat_toll/initializer"
+        "vat_slip",
-            vat_tune                = "transformers/vat_tune/initializer"
+        "vat_toll",
-            vow_flog                = "transformers/vow_flog/initializer"
+        "vat_tune",
-    [exporter.types]
+        "vow_flog"
-            bite                    = "eth_event"
+    ]
-            cat_chop_lump           = "eth_event"
+    [exporter.bite]
-            cat_flip                = "eth_event"
+        path = "transformers/bite/initializer"
-            cat_pit_vow             = "eth_event"
+        type = "eth_event"
-            deal                    = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
-            dent                    = "eth_event"
+        migrations = "db/migrations"
-            drip_drip               = "eth_event"
+    [exporter.cat_chop_lump]
-            drip_file_ilk           = "eth_event"
+        path = "transformers/cat_file/chop_lump/initializer"
-            drip_file_repo          = "eth_event"
+        type = "eth_event"
-            drip_file_vow           = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
-            flap_kick               = "eth_event"
+        migrations = "db/migrations"
-            flip_kick               = "eth_event"
+    [exporter.cat_flip]
-            flop_kick               = "eth_event"
+        path = "transformers/cat_file/flip/initializer"
-            frob                    = "eth_event"
+        type = "eth_event"
-            pit_file_debt_ceiling   = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
-            pit_file_ilk            = "eth_event"
+        migrations = "db/migrations"
-            price_feeds             = "eth_event"
+    [exporter.cat_pit_vow]
-            tend                    = "eth_event"
+        path = "transformers/cat_file/pit_vow/initializer"
-            vat_flux                = "eth_event"
+        type = "eth_event"
-            vat_fold                = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
-            vat_grab                = "eth_event"
+        migrations = "db/migrations"
-            vat_heal                = "eth_event"
+    [exporter.deal]
-            vat_init                = "eth_event"
+        path = "transformers/deal/initializer"
-            vat_move                = "eth_event"
+        type = "eth_event"
-            vat_slip                = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
-            vat_toll                = "eth_event"
+        migrations = "db/migrations"
-            vat_tune                = "eth_event"
+    [exporter.dent]
-            vow_flog                = "eth_event"
+        path = "transformers/dent/initializer"
-     [exporter.repositories]
+        type = "eth_event"
-            mcd__event_transformers = "github.com/vulcanize/mcd_transformers"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.drip_drip]
+        path = "transformers/drip_drip/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.drip_file_ilk]
+        path = "transformers/drip_file/ilk/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.drip_file_repo]
+        path = "transformers/drip_file/repo/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.drip_file_vow]
+        path = "transformers/drip_file/vow/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.flap_kick]
+        path = "transformers/flap_kick/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.flip_kick]
+        path = "transformers/flip_kick/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.flop_kick]
+        path = "transformers/flop_kick/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.frob]
+        path = "transformers/frob/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.pit_file_debt_ceiling]
+        path = "transformers/pit_file/debt_ceiling/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.pit_file_ilk]
+        path = "transformers/pit_file/ilk/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.price_feeds]
+        path = "transformers/price_feeds/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.tend]
+        path = "transformers/tend/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_flux]
+        path = "transformers/vat_flux/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_fold]
+        path = "transformers/vat_fold/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_grab]
+        path = "transformers/vat_grab/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_heal]
+        path = "transformers/vat_heal/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_init]
+        path = "transformers/vat_init/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_move]
+        path = "transformers/vat_move/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_slip]
+        path = "transformers/vat_slip/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_toll]
+        path = "transformers/vat_toll/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vat_tune]
+        path = "transformers/vat_tune/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
+    [exporter.vow_flog]
+        path = "transformers/vow_flog/initializer"
+        type = "eth_event"
+        repository = "github.com/vulcanize/mcd_transformers"
+        migrations = "db/migrations"
 
 [filesystem]
     storageDiffsPath = "INSERT-PATH-TO-STORAGE-DIFFS"

libraries/shared/storage/utils/utils_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestShared(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Shared Suite")
+	RunSpecs(t, "Storage Utils Suite")
 }
 
 var _ = BeforeSuite(func() {

pkg/config/plugin.go

@@ -17,8 +17,6 @@
 package config
 
 import (
-	"errors"
-	"fmt"
 	"path/filepath"
 	"strings"
 
@@ -26,15 +24,19 @@ import (
 )
 
 type Plugin struct {
-	Initializers map[string]string     // Map of import aliases to transformer initializer paths
+	Transformers map[string]Transformer
-	Dependencies map[string]string     // Map of vendor dep names to their repositories
-	Migrations   map[string]string     // Map of vendor dep names to relative path from repository to db migrations
-	Types        map[string]PluginType // Map of import aliases to their transformer initializer type (e.g. eth-event vs eth-storage)
 	FilePath     string
 	FileName     string
 	Save         bool
 }
 
+type Transformer struct {
+	Path           string
+	Type           TransformerType
+	MigrationPath  string
+	RepositoryPath string
+}
+
 func (c *Plugin) GetPluginPaths() (string, string, error) {
 	path, err := helpers.CleanPath(c.FilePath)
 	if err != nil {
@@ -48,55 +50,58 @@ func (c *Plugin) GetPluginPaths() (string, string, error) {
 	return goFile, soFile, nil
 }
 
-func (c *Plugin) GetMigrationsPaths() ([]string, error) {
+// Removes duplicate migration paths before returning them
-	paths := make([]string, 0, len(c.Migrations))
+func (c *Plugin) GetMigrationsPaths() (map[string]bool, error) {
-	for key, relPath := range c.Migrations {
+	paths := make(map[string]bool)
-		repo, ok := c.Dependencies[key]
+	for _, transformer := range c.Transformers {
-		if !ok {
+		repo := transformer.RepositoryPath
-			return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath))
+		mig := transformer.MigrationPath
-		}
+		path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, mig)
-		path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath)
 		cleanPath, err := helpers.CleanPath(path)
 		if err != nil {
 			return nil, err
 		}
-		paths = append(paths, cleanPath)
+		paths[cleanPath] = true
 	}
 
 	return paths, nil
 }
 
-type PluginType int
+// Removes duplicate repo paths before returning them
+func (c *Plugin) GetRepoPaths() map[string]bool {
+	paths := make(map[string]bool)
+	for _, transformer := range c.Transformers {
+		paths[transformer.RepositoryPath] = true
+	}
+
+	return paths
+}
+
+type TransformerType int
 
 const (
-	UnknownTransformerType PluginType = iota + 1
+	UnknownTransformerType TransformerType = iota
 	EthEvent
 	EthStorage
-	IpfsEvent
-	IpfsStorage
 )
 
-func (pt PluginType) String() string {
+func (pt TransformerType) String() string {
 	names := [...]string{
 		"eth_event",
 		"eth_storage",
-		"ipfs_event",
-		"ipfs_storage",
 	}
 
-	if pt > IpfsStorage || pt < EthEvent {
+	if pt > EthStorage || pt < EthEvent {
 		return "Unknown"
 	}
 
 	return names[pt]
 }
 
-func GetPluginType(str string) PluginType {
+func GetTransformerType(str string) TransformerType {
-	types := [...]PluginType{
+	types := [...]TransformerType{
 		EthEvent,
 		EthStorage,
-		IpfsEvent,
-		IpfsStorage,
 	}
 
 	for _, ty := range types {

pkg/omni/full/transformer/transformer_test.go

@@ -324,7 +324,7 @@ var _ = Describe("Transformer", func() {
 			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
 			Expect(res.TokenName).To(Equal(""))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6194636'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
 			Expect(err).To(HaveOccurred())
 		})
 

pkg/omni/light/transformer/transformer_test.go

@@ -331,7 +331,7 @@ var _ = Describe("Transformer", func() {
 			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
 			Expect(res.TokenName).To(Equal(""))
 
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).To(HaveOccurred())
 		})
 

pkg/omni/shared/contract/contract.go

@@ -18,6 +18,7 @@ package contract
 
 import (
 	"errors"
+
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -78,7 +79,7 @@ func (c *Contract) GenerateFilters() error {
 
 	for name, event := range c.Events {
 		c.Filters[name] = filters.LogFilter{
-			Name:      event.Name,
+			Name:      c.Address + "_" + event.Name,
 			FromBlock: c.StartingBlock,
 			ToBlock:   -1,
 			Address:   common.HexToAddress(c.Address).Hex(),

pkg/omni/shared/helpers/test_helpers/mocks/entities.go

@@ -129,7 +129,7 @@ var NewOwnerBlock2 = core.Block{
 }
 
 var ExpectedTransferFilter = filters.LogFilter{
-	Name:      "Transfer",
+	Name:      constants.TusdContractAddress + "_" + "Transfer",
 	Address:   constants.TusdContractAddress,
 	ToBlock:   -1,
 	FromBlock: 6194634,
@@ -137,7 +137,7 @@ var ExpectedTransferFilter = filters.LogFilter{
 }
 
 var ExpectedApprovalFilter = filters.LogFilter{
-	Name:      "Approval",
+	Name:      constants.TusdContractAddress + "_" + "Approval",
 	Address:   constants.TusdContractAddress,
 	ToBlock:   -1,
 	FromBlock: 6194634,

pkg/plugin/builder/builder.go

@@ -37,16 +37,17 @@ type PluginBuilder interface {
 }
 
 type builder struct {
-	GenConfig  config.Plugin
+	GenConfig    config.Plugin
-	tmpVenDirs []string // Keep track of temp vendor directories
+	dependencies []string
-	goFile     string   // Keep track of goFile name
+	tmpVenDirs   []string // Keep track of temp vendor directories
+	goFile       string   // Keep track of goFile name
 }
 
 // Requires populated plugin config
 func NewPluginBuilder(gc config.Plugin) *builder {
 	return &builder{
 		GenConfig:  gc,
-		tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
+		tmpVenDirs: make([]string, 0),
 	}
 }
 
@@ -82,8 +83,10 @@ func (b *builder) setupBuildEnv() error {
 		return err
 	}
 
+	repoPaths := b.GenConfig.GetRepoPaths()
+
 	// Import transformer dependencies so that we can build our plugin
-	for name, importPath := range b.GenConfig.Dependencies {
+	for importPath := range repoPaths {
 		// Use dependency paths in config to form git ssh string
 		// TODO: Change this to https once we are no longer working private transformer repos
 		// Right now since vulcanize/mcd_transformers is a private repo we
@@ -95,7 +98,7 @@ func (b *builder) setupBuildEnv() error {
 		depPath := filepath.Join(vendorPath, importPath)
 		err = exec.Command("git", "clone", importURL, depPath).Run()
 		if err != nil {
-			return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
+			return errors.New(fmt.Sprintf("unable to clone transformer dependency from %s: %s", importPath, err.Error()))
 		}
 		err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
 		if err != nil {

pkg/plugin/generator.go

@@ -18,9 +18,6 @@ package plugin
 
 import (
 	"errors"
-	"fmt"
-	"os"
-
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/builder"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/manager"
@@ -40,14 +37,8 @@ type generator struct {
 
 // Creates a new generator from a plugin and database config
 func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
-	if len(gc.Initializers) < 1 {
+	if len(gc.Transformers) < 1 {
-		return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
+		return nil, errors.New("plugin generator is not configured with any transformers")
-	}
-	if len(gc.Dependencies) < 1 {
-		return nil, errors.New("generator needs to be configured with root repository path(s)")
-	}
-	if len(gc.Migrations) < 1 {
-		fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided for the plugin transformers\r\n")
 	}
 	return &generator{
 		PluginWriter:     writer.NewPluginWriter(gc),

pkg/plugin/generator_test.go

@@ -36,62 +36,76 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers"
 )
 
-var genConfig = config.Plugin{
+var eventConfig = config.Plugin{
-	Initializers: map[string]string{
+	Transformers: map[string]config.Transformer{
-		"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
+		"bite": {
-		"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
+			Path:           "transformers/bite/initializer",
+			Type:           config.EthEvent,
+			MigrationPath:  "db/migrations",
+			RepositoryPath: "github.com/vulcanize/mcd_transformers",
+		},
+		"deal": {
+			Path:           "transformers/deal/initializer",
+			Type:           config.EthEvent,
+			MigrationPath:  "db/migrations",
+			RepositoryPath: "github.com/vulcanize/mcd_transformers",
+		},
 	},
-	Types: map[string]config.PluginType{
+	FileName: "testEventTransformerSet",
-		"bite": config.EthEvent,
+	FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
-		"deal": config.EthEvent,
+	Save:     false,
-	},
-	Dependencies: map[string]string{
-		"mcd_transformers": "github.com/vulcanize/mcd_transformers",
-	},
-	Migrations: map[string]string{"mcd_transformers": "db/migrations"},
-	FileName:   "testEventTransformerSet",
-	FilePath:   "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
-	Save:       false,
 }
 
-var genStorageConfig = config.Plugin{
+var storageConfig = config.Plugin{
-	Initializers: map[string]string{
+	Transformers: map[string]config.Transformer{
-		"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
+		"pit": {
-		"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
+			Path:           "transformers/storage_diffs/maker/pit/initializer",
+			Type:           config.EthStorage,
+			MigrationPath:  "db/migrations",
+			RepositoryPath: "github.com/vulcanize/mcd_transformers",
+		},
+		"vat": {
+			Path:           "transformers/storage_diffs/maker/vat/initializer",
+			Type:           config.EthStorage,
+			MigrationPath:  "db/migrations",
+			RepositoryPath: "github.com/vulcanize/mcd_transformers",
+		},
 	},
-	Types: map[string]config.PluginType{
+	FileName: "testStorageTransformerSet",
-		"pit": config.EthStorage,
+	FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
-		"vat": config.EthStorage,
+	Save:     false,
-	},
-	Dependencies: map[string]string{
-		"mcd_transformers": "github.com/vulcanize/mcd_transformers",
-	},
-	Migrations: map[string]string{"mcd_transformers": "db/migrations"},
-	FileName:   "testStorageTransformerSet",
-	FilePath:   "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
-	Save:       false,
 }
 
 var combinedConfig = config.Plugin{
-	Initializers: map[string]string{
+	Transformers: map[string]config.Transformer{
-		"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
+		"pit": {
-		"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
+			Path:           "transformers/storage_diffs/maker/pit/initializer",
-		"pit":  "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
+			Type:           config.EthStorage,
-		"vat":  "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
+			MigrationPath:  "db/migrations",
+			RepositoryPath: "github.com/vulcanize/mcd_transformers",
+		},
+		"vat": {
+			Path:           "transformers/storage_diffs/maker/vat/initializer",
+			Type:           config.EthStorage,
+			MigrationPath:  "db/migrations",
+			RepositoryPath: "github.com/vulcanize/mcd_transformers",
+		},
+		"bite": {
+			Path:           "transformers/bite/initializer",
+			Type:           config.EthEvent,
+			MigrationPath:  "db/migrations",
+			RepositoryPath: "github.com/vulcanize/mcd_transformers",
+		},
+		"deal": {
+			Path:           "transformers/deal/initializer",
+			Type:           config.EthEvent,
+			MigrationPath:  "db/migrations",
+			RepositoryPath: "github.com/vulcanize/mcd_transformers",
+		},
 	},
-	Types: map[string]config.PluginType{
+	FileName: "testComboTransformerSet",
-		"bite": config.EthEvent,
+	FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
-		"deal": config.EthEvent,
+	Save:     false,
-		"pit":  config.EthStorage,
-		"vat":  config.EthStorage,
-	},
-	Dependencies: map[string]string{
-		"mcd_transformers": "github.com/vulcanize/mcd_transformers",
-	},
-	Migrations: map[string]string{"mcd_transformers": "db/migrations"},
-	FileName:   "testComboTransformerSet",
-	FilePath:   "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
-	Save:       false,
 }
 
 var dbConfig = config.Database{
@@ -117,9 +131,9 @@ var _ = Describe("Generator test", func() {
 
 	Describe("Event Transformers only", func() {
 		BeforeEach(func() {
-			goPath, soPath, err = genConfig.GetPluginPaths()
+			goPath, soPath, err = eventConfig.GetPluginPaths()
 			Expect(err).ToNot(HaveOccurred())
-			g, err = p2.NewGenerator(genConfig, dbConfig)
+			g, err = p2.NewGenerator(eventConfig, dbConfig)
 			Expect(err).ToNot(HaveOccurred())
 			err = g.GenerateExporterPlugin()
 			Expect(err).ToNot(HaveOccurred())
@@ -199,9 +213,9 @@ var _ = Describe("Generator test", func() {
 
 	Describe("Storage Transformers only", func() {
 		BeforeEach(func() {
-			goPath, soPath, err = genStorageConfig.GetPluginPaths()
+			goPath, soPath, err = storageConfig.GetPluginPaths()
 			Expect(err).ToNot(HaveOccurred())
-			g, err = p2.NewGenerator(genStorageConfig, dbConfig)
+			g, err = p2.NewGenerator(storageConfig, dbConfig)
 			Expect(err).ToNot(HaveOccurred())
 			err = g.GenerateExporterPlugin()
 			Expect(err).ToNot(HaveOccurred())

pkg/plugin/helpers/helpers.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package helpers
 
 import (
@@ -32,7 +48,7 @@ func ClearFiles(files ...string) error {
 				return err
 			}
 		} else if os.IsNotExist(err) {
-			// fall through
+			continue
 		} else {
 			return err
 		}

pkg/plugin/manager/manager.go

@@ -89,7 +89,7 @@ func (m *manager) setupMigrationEnv() error {
 	if err != nil {
 		return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir))
 	}
-	err = os.Mkdir(m.tmpMigDir, os.FileMode(0777))
+	err = os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm))
 	if err != nil {
 		return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir))
 	}
@@ -98,9 +98,9 @@ func (m *manager) setupMigrationEnv() error {
 }
 
 // Create copies of db migrations from vendored libs
-func (m *manager) createMigrationCopies(paths []string) error {
+func (m *manager) createMigrationCopies(paths map[string]bool) error {
 	// Iterate through migration paths to find migration directory
-	for _, path := range paths {
+	for path := range paths {
 		dir, err := ioutil.ReadDir(path)
 		if err != nil {
 			return err

pkg/plugin/writer/writer.go

@@ -58,12 +58,15 @@ func (w *writer) WritePlugin() error {
 
 	// Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config
 	f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
-	for alias, relPath := range w.GenConfig.Initializers {
+	for name, transformer := range w.GenConfig.Transformers {
-		f.ImportAlias(w.makePath(alias, relPath), alias)
+		f.ImportAlias(transformer.RepositoryPath+"/"+transformer.Path, name)
 	}
 
 	// Collect initializer code
-	ethEventInitializers, ethStorageInitializers, _, _ := w.sortTransformers()
+	code, err := w.collectTransformers()
+	if err != nil {
+		return err
+	}
 
 	// Create Exporter variable with method to export the set of the imported storage and event transformer initializers
 	f.Type().Id("exporter").String()
@@ -74,10 +77,10 @@ func (w *writer) WritePlugin() error {
 	)).Block(Return(
 		Index().Qual(
 			"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
-			"TransformerInitializer").Values(ethEventInitializers...),
+			"TransformerInitializer").Values(code[config.EthEvent]...),
 		Index().Qual(
 			"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
-			"StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers
+			"StorageTransformerInitializer").Values(code[config.EthStorage]...))) // Exports the collected event and storage transformer initializers
 
 	// Write code to destination file
 	err = f.Save(goFile)
@@ -88,34 +91,21 @@ func (w *writer) WritePlugin() error {
 }
 
 // Collect code for various types of initializers
-func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
+func (w *writer) collectTransformers() (map[config.TransformerType][]Code, error) {
-	importedEthEventInitializers := make([]Code, 0)
+	code := make(map[config.TransformerType][]Code)
-	importerEthStorageInitializers := make([]Code, 0)
+	for _, transformer := range w.GenConfig.Transformers {
-	importedIpfsEventInitializers := make([]Code, 0)
+		path := transformer.RepositoryPath + "/" + transformer.Path
-	importerIpfsStorageInitializers := make([]Code, 0)
+		switch transformer.Type {
-	for name, path := range w.GenConfig.Initializers {
-		switch w.GenConfig.Types[name] {
 		case config.EthEvent:
-			importedEthEventInitializers = append(importedEthEventInitializers, Qual(path, "TransformerInitializer"))
+			code[config.EthEvent] = append(code[config.EthEvent], Qual(path, "TransformerInitializer"))
 		case config.EthStorage:
-			importerEthStorageInitializers = append(importerEthStorageInitializers, Qual(path, "StorageTransformerInitializer"))
+			code[config.EthStorage] = append(code[config.EthStorage], Qual(path, "StorageTransformerInitializer"))
-		case config.IpfsEvent:
+		default:
-			//importedIpfsEventInitializers = append(importedIpfsEventInitializers, Qual(path, "IpfsEventTransformerInitializer"))
+			return nil, errors.New(fmt.Sprintf("invalid transformer type %s", transformer.Type))
-		case config.IpfsStorage:
-			//importerIpfsStorageInitializers = append(importerIpfsStorageInitializers, Qual(path, "IpfsStorageTransformerInitializer"))
 		}
 	}
 
-	return importedEthEventInitializers,
+	return code, nil
-		importerEthStorageInitializers,
-		importedIpfsEventInitializers,
-		importerIpfsStorageInitializers
-}
-
-// Concat relative path with its repo's root path
-func (w *writer) makePath(alias, relPath string) string {
-	pathRoot := w.GenConfig.Dependencies[alias]
-	return pathRoot + "/" + relPath
 }
 
 // Setup the .go, clear old ones if present