diff --git a/Makefile b/Makefile index d3ef32c4..f8658e97 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ lint: test: | $(GINKGO) $(LINT) go vet ./... go fmt ./... - $(GINKGO) -r + $(GINKGO) -r --skipPackage=integration_tests,integration .PHONY: integrationtest integrationtest: | $(GINKGO) $(LINT) diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index eceb9bb4..d543e6d7 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -213,23 +213,45 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) { func prepConfig() { fmt.Println("configuring plugin") - types := viper.GetStringMapString("exporter.types") - genTypes := map[string]config.PluginType{} - for transformerName, transformerType := range types { - genType := config.GetPluginType(transformerType) - if genType == config.UnknownTransformerType { - log.Fatal(errors.New(`unknown transformer type in exporter config -accepted types are "eth_event", "eth_storage", "ipfs_event" and "ipfs_storage"`)) + names := viper.GetStringSlice("exporter.transformerNames") + transformers := make(map[string]config.Transformer) + for _, name := range names { + transformer := viper.GetStringMapString("exporter." + name) + _, ok := transformer["path"] + if !ok { + log.Fatal(fmt.Sprintf("%s transformer config is missing `path` value", name)) + } + _, ok = transformer["repository"] + if !ok { + log.Fatal(fmt.Sprintf("%s transformer config is missing `repository` value", name)) + } + _, ok = transformer["migrations"] + if !ok { + log.Fatal(fmt.Sprintf("%s transformer config is missing `migrations` value", name)) + } + ty, ok := transformer["type"] + if !ok { + log.Fatal(fmt.Sprintf("%s transformer config is missing `type` value", name)) + } + + transformerType := config.GetTransformerType(ty) + if transformerType == config.UnknownTransformerType { + log.Fatal(errors.New(`unknown transformer type in exporter config +accepted types are "eth_event", "eth_storage"`)) + } + + transformers[name] = config.Transformer{ + Path: transformer["path"], + Type: transformerType, + RepositoryPath: transformer["repository"], + MigrationPath: transformer["migrations"], } - genTypes[transformerName] = genType } + genConfig = config.Plugin{ + Transformers: transformers, FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins", FileName: viper.GetString("exporter.name"), Save: viper.GetBool("exporter.save"), - Initializers: viper.GetStringMapString("exporter.transformers"), - Dependencies: viper.GetStringMapString("exporter.repositories"), - Migrations: viper.GetStringMapString("exporter.migrations"), - Types: genTypes, } } diff --git a/cmd/root.go b/cmd/root.go index b7b4d993..100c424b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -95,7 +95,6 @@ func init() { rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata") rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name") rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file") - rootCmd.PersistentFlags().String("exporter-path", "~/go/src/github.com/vulcanize/vulcanizedb/plugins", "file path to transformer exporter plugin") rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin") viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) @@ -107,7 +106,6 @@ func init() { viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath")) viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name")) viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath")) - viper.BindPFlag("exporter.filePath", rootCmd.PersistentFlags().Lookup("exporter-path")) viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name")) } diff --git a/environments/compose.toml b/environments/compose.toml index 2933bb45..48221933 100644 --- a/environments/compose.toml +++ b/environments/compose.toml @@ -14,66 +14,177 @@ [exporter] name = "eventTransformerExporter" save = false - [exporter.transformers] - bite = "transformers/bite/initializer" - cat_chop_lump = "transformers/cat_file/chop_lump/initializer" - cat_flip = "transformers/cat_file/flip/initializer" - cat_pit_vow = "transformers/cat_file/pit_vow/initializer" - deal = "transformers/deal/initializer" - dent = "transformers/dent/initializer" - drip_drip = "transformers/drip_drip/initializer" - drip_file_ilk = "transformers/drip_file/ilk/initializer" - drip_file_repo = "transformers/drip_file/repo/initializer" - drip_file_vow = "transformers/drip_file/vow/initializer" - flap_kick = "transformers/flap_kick/initializer" - flip_kick = "transformers/flip_kick/initializer" - flop_kick = "transformers/flop_kick/initializer" - frob = "transformers/frob/initializer" - pit_file_debt_ceiling = "transformers/pit_file/debt_ceiling/initializer" - pit_file_ilk = "transformers/pit_file/ilk/initializer" - price_feeds = "transformers/price_feeds/initializer" - tend = "transformers/tend/initializer" - vat_flux = "transformers/vat_flux/initializer" - vat_fold = "transformers/vat_fold/initializer" - vat_grab = "transformers/vat_grab/initializer" - vat_heal = "transformers/vat_heal/initializer" - vat_init = "transformers/vat_init/initializer" - vat_move = "transformers/vat_move/initializer" - vat_slip = "transformers/vat_slip/initializer" - vat_toll = "transformers/vat_toll/initializer" - vat_tune = "transformers/vat_tune/initializer" - vow_flog = "transformers/vow_flog/initializer" - [exporter.types] - bite = "eth_event" - cat_chop_lump = "eth_event" - cat_flip = "eth_event" - cat_pit_vow = "eth_event" - deal = "eth_event" - dent = "eth_event" - drip_drip = "eth_event" - drip_file_ilk = "eth_event" - drip_file_repo = "eth_event" - drip_file_vow = "eth_event" - flap_kick = "eth_event" - flip_kick = "eth_event" - flop_kick = "eth_event" - frob = "eth_event" - pit_file_debt_ceiling = "eth_event" - pit_file_ilk = "eth_event" - price_feeds = "eth_event" - tend = "eth_event" - vat_flux = "eth_event" - vat_fold = "eth_event" - vat_grab = "eth_event" - vat_heal = "eth_event" - vat_init = "eth_event" - vat_move = "eth_event" - vat_slip = "eth_event" - vat_toll = "eth_event" - vat_tune = "eth_event" - vow_flog = "eth_event" - [exporter.repositories] - mcd__event_transformers = "github.com/vulcanize/mcd_transformers" + transformerNames = [ + "bite", + "cat_chop_lump", + "cat_flip", + "cat_pit_vow", + "deal", + "dent", + "drip_drip", + "drip_file_ilk", + "drop_file_repo", + "drip_file_vow", + "flap_kick", + "flip_kick", + "flop_kick", + "frob_kick", + "frob", + "pit_file_debt_ceiling", + "pit_file_ilk", + "price_feeds", + "tend", + "vat_flux", + "vat_fold", + "vat_grab", + "vat_heal", + "vat_init", + "vat_move", + "vat_slip", + "vat_toll", + "vat_tune", + "vow_flog" + ] + [exporter.bite] + path = "transformers/bite/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.cat_chop_lump] + path = "transformers/cat_file/chop_lump/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.cat_flip] + path = "transformers/cat_file/flip/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.cat_pit_vow] + path = "transformers/cat_file/pit_vow/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.deal] + path = "transformers/deal/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.dent] + path = "transformers/dent/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.drip_drip] + path = "transformers/drip_drip/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.drip_file_ilk] + path = "transformers/drip_file/ilk/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.drip_file_repo] + path = "transformers/drip_file/repo/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.drip_file_vow] + path = "transformers/drip_file/vow/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.flap_kick] + path = "transformers/flap_kick/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.flip_kick] + path = "transformers/flip_kick/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.flop_kick] + path = "transformers/flop_kick/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.frob] + path = "transformers/frob/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.pit_file_debt_ceiling] + path = "transformers/pit_file/debt_ceiling/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.pit_file_ilk] + path = "transformers/pit_file/ilk/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.price_feeds] + path = "transformers/price_feeds/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.tend] + path = "transformers/tend/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_flux] + path = "transformers/vat_flux/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_fold] + path = "transformers/vat_fold/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_grab] + path = "transformers/vat_grab/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_heal] + path = "transformers/vat_heal/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_init] + path = "transformers/vat_init/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_move] + path = "transformers/vat_move/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_slip] + path = "transformers/vat_slip/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_toll] + path = "transformers/vat_toll/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vat_tune] + path = "transformers/vat_tune/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" + [exporter.vow_flog] + path = "transformers/vow_flog/initializer" + type = "eth_event" + repository = "github.com/vulcanize/mcd_transformers" + migrations = "db/migrations" [filesystem] storageDiffsPath = "INSERT-PATH-TO-STORAGE-DIFFS" diff --git a/libraries/shared/storage/utils/shared_suite_test.go b/libraries/shared/storage/utils/utils_suite_test.go similarity index 96% rename from libraries/shared/storage/utils/shared_suite_test.go rename to libraries/shared/storage/utils/utils_suite_test.go index b20597b7..15f6b76c 100644 --- a/libraries/shared/storage/utils/shared_suite_test.go +++ b/libraries/shared/storage/utils/utils_suite_test.go @@ -27,7 +27,7 @@ import ( func TestShared(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Shared Suite") + RunSpecs(t, "Storage Utils Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/config/plugin.go b/pkg/config/plugin.go index f24a33c1..1b44be61 100644 --- a/pkg/config/plugin.go +++ b/pkg/config/plugin.go @@ -17,8 +17,6 @@ package config import ( - "errors" - "fmt" "path/filepath" "strings" @@ -26,15 +24,19 @@ import ( ) type Plugin struct { - Initializers map[string]string // Map of import aliases to transformer initializer paths - Dependencies map[string]string // Map of vendor dep names to their repositories - Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations - Types map[string]PluginType // Map of import aliases to their transformer initializer type (e.g. eth-event vs eth-storage) + Transformers map[string]Transformer FilePath string FileName string Save bool } +type Transformer struct { + Path string + Type TransformerType + MigrationPath string + RepositoryPath string +} + func (c *Plugin) GetPluginPaths() (string, string, error) { path, err := helpers.CleanPath(c.FilePath) if err != nil { @@ -48,55 +50,58 @@ func (c *Plugin) GetPluginPaths() (string, string, error) { return goFile, soFile, nil } -func (c *Plugin) GetMigrationsPaths() ([]string, error) { - paths := make([]string, 0, len(c.Migrations)) - for key, relPath := range c.Migrations { - repo, ok := c.Dependencies[key] - if !ok { - return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath)) - } - path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath) +// Removes duplicate migration paths before returning them +func (c *Plugin) GetMigrationsPaths() (map[string]bool, error) { + paths := make(map[string]bool) + for _, transformer := range c.Transformers { + repo := transformer.RepositoryPath + mig := transformer.MigrationPath + path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, mig) cleanPath, err := helpers.CleanPath(path) if err != nil { return nil, err } - paths = append(paths, cleanPath) + paths[cleanPath] = true } return paths, nil } -type PluginType int +// Removes duplicate repo paths before returning them +func (c *Plugin) GetRepoPaths() map[string]bool { + paths := make(map[string]bool) + for _, transformer := range c.Transformers { + paths[transformer.RepositoryPath] = true + } + + return paths +} + +type TransformerType int const ( - UnknownTransformerType PluginType = iota + 1 + UnknownTransformerType TransformerType = iota EthEvent EthStorage - IpfsEvent - IpfsStorage ) -func (pt PluginType) String() string { +func (pt TransformerType) String() string { names := [...]string{ "eth_event", "eth_storage", - "ipfs_event", - "ipfs_storage", } - if pt > IpfsStorage || pt < EthEvent { + if pt > EthStorage || pt < EthEvent { return "Unknown" } return names[pt] } -func GetPluginType(str string) PluginType { - types := [...]PluginType{ +func GetTransformerType(str string) TransformerType { + types := [...]TransformerType{ EthEvent, EthStorage, - IpfsEvent, - IpfsStorage, } for _, ty := range types { diff --git a/pkg/omni/full/transformer/transformer_test.go b/pkg/omni/full/transformer/transformer_test.go index 171b1618..323a8e77 100644 --- a/pkg/omni/full/transformer/transformer_test.go +++ b/pkg/omni/full/transformer/transformer_test.go @@ -324,7 +324,7 @@ var _ = Describe("Transformer", func() { Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) Expect(res.TokenName).To(Equal("")) - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6194636'", ensAddr)).StructScan(&res) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) }) diff --git a/pkg/omni/light/transformer/transformer_test.go b/pkg/omni/light/transformer/transformer_test.go index a5c69515..e776cb0b 100644 --- a/pkg/omni/light/transformer/transformer_test.go +++ b/pkg/omni/light/transformer/transformer_test.go @@ -331,7 +331,7 @@ var _ = Describe("Transformer", func() { Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) Expect(res.TokenName).To(Equal("")) - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&res) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) }) diff --git a/pkg/omni/shared/contract/contract.go b/pkg/omni/shared/contract/contract.go index a916e734..70f3c123 100644 --- a/pkg/omni/shared/contract/contract.go +++ b/pkg/omni/shared/contract/contract.go @@ -18,6 +18,7 @@ package contract import ( "errors" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -78,7 +79,7 @@ func (c *Contract) GenerateFilters() error { for name, event := range c.Events { c.Filters[name] = filters.LogFilter{ - Name: event.Name, + Name: c.Address + "_" + event.Name, FromBlock: c.StartingBlock, ToBlock: -1, Address: common.HexToAddress(c.Address).Hex(), diff --git a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go b/pkg/omni/shared/helpers/test_helpers/mocks/entities.go index 4d865382..69e832d7 100644 --- a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/omni/shared/helpers/test_helpers/mocks/entities.go @@ -129,7 +129,7 @@ var NewOwnerBlock2 = core.Block{ } var ExpectedTransferFilter = filters.LogFilter{ - Name: "Transfer", + Name: constants.TusdContractAddress + "_" + "Transfer", Address: constants.TusdContractAddress, ToBlock: -1, FromBlock: 6194634, @@ -137,7 +137,7 @@ var ExpectedTransferFilter = filters.LogFilter{ } var ExpectedApprovalFilter = filters.LogFilter{ - Name: "Approval", + Name: constants.TusdContractAddress + "_" + "Approval", Address: constants.TusdContractAddress, ToBlock: -1, FromBlock: 6194634, diff --git a/pkg/plugin/builder/builder.go b/pkg/plugin/builder/builder.go index fa54c693..f891b2c0 100644 --- a/pkg/plugin/builder/builder.go +++ b/pkg/plugin/builder/builder.go @@ -37,16 +37,17 @@ type PluginBuilder interface { } type builder struct { - GenConfig config.Plugin - tmpVenDirs []string // Keep track of temp vendor directories - goFile string // Keep track of goFile name + GenConfig config.Plugin + dependencies []string + tmpVenDirs []string // Keep track of temp vendor directories + goFile string // Keep track of goFile name } // Requires populated plugin config func NewPluginBuilder(gc config.Plugin) *builder { return &builder{ GenConfig: gc, - tmpVenDirs: make([]string, 0, len(gc.Dependencies)), + tmpVenDirs: make([]string, 0), } } @@ -82,8 +83,10 @@ func (b *builder) setupBuildEnv() error { return err } + repoPaths := b.GenConfig.GetRepoPaths() + // Import transformer dependencies so that we can build our plugin - for name, importPath := range b.GenConfig.Dependencies { + for importPath := range repoPaths { // Use dependency paths in config to form git ssh string // TODO: Change this to https once we are no longer working private transformer repos // Right now since vulcanize/mcd_transformers is a private repo we @@ -95,7 +98,7 @@ func (b *builder) setupBuildEnv() error { depPath := filepath.Join(vendorPath, importPath) err = exec.Command("git", "clone", importURL, depPath).Run() if err != nil { - return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error())) + return errors.New(fmt.Sprintf("unable to clone transformer dependency from %s: %s", importPath, err.Error())) } err := os.RemoveAll(filepath.Join(depPath, "vendor/")) if err != nil { diff --git a/pkg/plugin/generator.go b/pkg/plugin/generator.go index a5882ba6..c5ef1df5 100644 --- a/pkg/plugin/generator.go +++ b/pkg/plugin/generator.go @@ -18,9 +18,6 @@ package plugin import ( "errors" - "fmt" - "os" - "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/plugin/builder" "github.com/vulcanize/vulcanizedb/pkg/plugin/manager" @@ -40,14 +37,8 @@ type generator struct { // Creates a new generator from a plugin and database config func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) { - if len(gc.Initializers) < 1 { - return nil, errors.New("generator needs to be configured with TransformerInitializer import paths") - } - if len(gc.Dependencies) < 1 { - return nil, errors.New("generator needs to be configured with root repository path(s)") - } - if len(gc.Migrations) < 1 { - fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided for the plugin transformers\r\n") + if len(gc.Transformers) < 1 { + return nil, errors.New("plugin generator is not configured with any transformers") } return &generator{ PluginWriter: writer.NewPluginWriter(gc), diff --git a/pkg/plugin/generator_test.go b/pkg/plugin/generator_test.go index bd9f374d..7bb7e74b 100644 --- a/pkg/plugin/generator_test.go +++ b/pkg/plugin/generator_test.go @@ -36,62 +36,76 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers" ) -var genConfig = config.Plugin{ - Initializers: map[string]string{ - "bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer", - "deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer", +var eventConfig = config.Plugin{ + Transformers: map[string]config.Transformer{ + "bite": { + Path: "transformers/bite/initializer", + Type: config.EthEvent, + MigrationPath: "db/migrations", + RepositoryPath: "github.com/vulcanize/mcd_transformers", + }, + "deal": { + Path: "transformers/deal/initializer", + Type: config.EthEvent, + MigrationPath: "db/migrations", + RepositoryPath: "github.com/vulcanize/mcd_transformers", + }, }, - Types: map[string]config.PluginType{ - "bite": config.EthEvent, - "deal": config.EthEvent, - }, - Dependencies: map[string]string{ - "mcd_transformers": "github.com/vulcanize/mcd_transformers", - }, - Migrations: map[string]string{"mcd_transformers": "db/migrations"}, - FileName: "testEventTransformerSet", - FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", - Save: false, + FileName: "testEventTransformerSet", + FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", + Save: false, } -var genStorageConfig = config.Plugin{ - Initializers: map[string]string{ - "pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer", - "vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer", +var storageConfig = config.Plugin{ + Transformers: map[string]config.Transformer{ + "pit": { + Path: "transformers/storage_diffs/maker/pit/initializer", + Type: config.EthStorage, + MigrationPath: "db/migrations", + RepositoryPath: "github.com/vulcanize/mcd_transformers", + }, + "vat": { + Path: "transformers/storage_diffs/maker/vat/initializer", + Type: config.EthStorage, + MigrationPath: "db/migrations", + RepositoryPath: "github.com/vulcanize/mcd_transformers", + }, }, - Types: map[string]config.PluginType{ - "pit": config.EthStorage, - "vat": config.EthStorage, - }, - Dependencies: map[string]string{ - "mcd_transformers": "github.com/vulcanize/mcd_transformers", - }, - Migrations: map[string]string{"mcd_transformers": "db/migrations"}, - FileName: "testStorageTransformerSet", - FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", - Save: false, + FileName: "testStorageTransformerSet", + FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", + Save: false, } var combinedConfig = config.Plugin{ - Initializers: map[string]string{ - "bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer", - "deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer", - "pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer", - "vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer", + Transformers: map[string]config.Transformer{ + "pit": { + Path: "transformers/storage_diffs/maker/pit/initializer", + Type: config.EthStorage, + MigrationPath: "db/migrations", + RepositoryPath: "github.com/vulcanize/mcd_transformers", + }, + "vat": { + Path: "transformers/storage_diffs/maker/vat/initializer", + Type: config.EthStorage, + MigrationPath: "db/migrations", + RepositoryPath: "github.com/vulcanize/mcd_transformers", + }, + "bite": { + Path: "transformers/bite/initializer", + Type: config.EthEvent, + MigrationPath: "db/migrations", + RepositoryPath: "github.com/vulcanize/mcd_transformers", + }, + "deal": { + Path: "transformers/deal/initializer", + Type: config.EthEvent, + MigrationPath: "db/migrations", + RepositoryPath: "github.com/vulcanize/mcd_transformers", + }, }, - Types: map[string]config.PluginType{ - "bite": config.EthEvent, - "deal": config.EthEvent, - "pit": config.EthStorage, - "vat": config.EthStorage, - }, - Dependencies: map[string]string{ - "mcd_transformers": "github.com/vulcanize/mcd_transformers", - }, - Migrations: map[string]string{"mcd_transformers": "db/migrations"}, - FileName: "testComboTransformerSet", - FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", - Save: false, + FileName: "testComboTransformerSet", + FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test", + Save: false, } var dbConfig = config.Database{ @@ -117,9 +131,9 @@ var _ = Describe("Generator test", func() { Describe("Event Transformers only", func() { BeforeEach(func() { - goPath, soPath, err = genConfig.GetPluginPaths() + goPath, soPath, err = eventConfig.GetPluginPaths() Expect(err).ToNot(HaveOccurred()) - g, err = p2.NewGenerator(genConfig, dbConfig) + g, err = p2.NewGenerator(eventConfig, dbConfig) Expect(err).ToNot(HaveOccurred()) err = g.GenerateExporterPlugin() Expect(err).ToNot(HaveOccurred()) @@ -199,9 +213,9 @@ var _ = Describe("Generator test", func() { Describe("Storage Transformers only", func() { BeforeEach(func() { - goPath, soPath, err = genStorageConfig.GetPluginPaths() + goPath, soPath, err = storageConfig.GetPluginPaths() Expect(err).ToNot(HaveOccurred()) - g, err = p2.NewGenerator(genStorageConfig, dbConfig) + g, err = p2.NewGenerator(storageConfig, dbConfig) Expect(err).ToNot(HaveOccurred()) err = g.GenerateExporterPlugin() Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/plugin/helpers/helpers.go b/pkg/plugin/helpers/helpers.go index 391df2c7..2885cbef 100644 --- a/pkg/plugin/helpers/helpers.go +++ b/pkg/plugin/helpers/helpers.go @@ -1,3 +1,19 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package helpers import ( @@ -32,7 +48,7 @@ func ClearFiles(files ...string) error { return err } } else if os.IsNotExist(err) { - // fall through + continue } else { return err } diff --git a/pkg/plugin/manager/manager.go b/pkg/plugin/manager/manager.go index 66b38b84..ad9cfdf5 100644 --- a/pkg/plugin/manager/manager.go +++ b/pkg/plugin/manager/manager.go @@ -89,7 +89,7 @@ func (m *manager) setupMigrationEnv() error { if err != nil { return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir)) } - err = os.Mkdir(m.tmpMigDir, os.FileMode(0777)) + err = os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm)) if err != nil { return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir)) } @@ -98,9 +98,9 @@ func (m *manager) setupMigrationEnv() error { } // Create copies of db migrations from vendored libs -func (m *manager) createMigrationCopies(paths []string) error { +func (m *manager) createMigrationCopies(paths map[string]bool) error { // Iterate through migration paths to find migration directory - for _, path := range paths { + for path := range paths { dir, err := ioutil.ReadDir(path) if err != nil { return err diff --git a/pkg/plugin/writer/writer.go b/pkg/plugin/writer/writer.go index 9d9766ff..7ab4ccd8 100644 --- a/pkg/plugin/writer/writer.go +++ b/pkg/plugin/writer/writer.go @@ -58,12 +58,15 @@ func (w *writer) WritePlugin() error { // Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface") - for alias, relPath := range w.GenConfig.Initializers { - f.ImportAlias(w.makePath(alias, relPath), alias) + for name, transformer := range w.GenConfig.Transformers { + f.ImportAlias(transformer.RepositoryPath+"/"+transformer.Path, name) } // Collect initializer code - ethEventInitializers, ethStorageInitializers, _, _ := w.sortTransformers() + code, err := w.collectTransformers() + if err != nil { + return err + } // Create Exporter variable with method to export the set of the imported storage and event transformer initializers f.Type().Id("exporter").String() @@ -74,10 +77,10 @@ func (w *writer) WritePlugin() error { )).Block(Return( Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "TransformerInitializer").Values(ethEventInitializers...), + "TransformerInitializer").Values(code[config.EthEvent]...), Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers + "StorageTransformerInitializer").Values(code[config.EthStorage]...))) // Exports the collected event and storage transformer initializers // Write code to destination file err = f.Save(goFile) @@ -88,34 +91,21 @@ func (w *writer) WritePlugin() error { } // Collect code for various types of initializers -func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) { - importedEthEventInitializers := make([]Code, 0) - importerEthStorageInitializers := make([]Code, 0) - importedIpfsEventInitializers := make([]Code, 0) - importerIpfsStorageInitializers := make([]Code, 0) - for name, path := range w.GenConfig.Initializers { - switch w.GenConfig.Types[name] { +func (w *writer) collectTransformers() (map[config.TransformerType][]Code, error) { + code := make(map[config.TransformerType][]Code) + for _, transformer := range w.GenConfig.Transformers { + path := transformer.RepositoryPath + "/" + transformer.Path + switch transformer.Type { case config.EthEvent: - importedEthEventInitializers = append(importedEthEventInitializers, Qual(path, "TransformerInitializer")) + code[config.EthEvent] = append(code[config.EthEvent], Qual(path, "TransformerInitializer")) case config.EthStorage: - importerEthStorageInitializers = append(importerEthStorageInitializers, Qual(path, "StorageTransformerInitializer")) - case config.IpfsEvent: - //importedIpfsEventInitializers = append(importedIpfsEventInitializers, Qual(path, "IpfsEventTransformerInitializer")) - case config.IpfsStorage: - //importerIpfsStorageInitializers = append(importerIpfsStorageInitializers, Qual(path, "IpfsStorageTransformerInitializer")) + code[config.EthStorage] = append(code[config.EthStorage], Qual(path, "StorageTransformerInitializer")) + default: + return nil, errors.New(fmt.Sprintf("invalid transformer type %s", transformer.Type)) } } - return importedEthEventInitializers, - importerEthStorageInitializers, - importedIpfsEventInitializers, - importerIpfsStorageInitializers -} - -// Concat relative path with its repo's root path -func (w *writer) makePath(alias, relPath string) string { - pathRoot := w.GenConfig.Dependencies[alias] - return pathRoot + "/" + relPath + return code, nil } // Setup the .go, clear old ones if present