From 9bb2f27a6942dd8b18349708cf69fe27492c1fb7 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Tue, 24 Sep 2019 18:44:04 -0500 Subject: [PATCH] option to write the plugin .go file and run the plugin migrations without building the .so file so that it can be done from the shell instead to afford complete error messages --- cmd/compose.go | 1 + cmd/contractWatcher.go | 2 +- pkg/config/contract.go | 4 +- pkg/config/plugin.go | 1 + pkg/config/plugin_test.go | 8 ++- pkg/plugin/composer/composer.go | 100 ++++++++++++++++++++++++++++++++ pkg/plugin/generator.go | 13 ++++- pkg/plugin/manager/manager.go | 7 ++- 8 files changed, 125 insertions(+), 11 deletions(-) create mode 100644 pkg/plugin/composer/composer.go diff --git a/cmd/compose.go b/cmd/compose.go index f92db41a..00694ce8 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -188,5 +188,6 @@ func prepConfig() { FileName: viper.GetString("exporter.name"), Save: viper.GetBool("exporter.save"), Home: viper.GetString("exporter.home"), + WriteOnly: viper.GetBool("exporter.writeOnly"), } } diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go index 2e2a8814..11238869 100644 --- a/cmd/contractWatcher.go +++ b/cmd/contractWatcher.go @@ -18,13 +18,13 @@ package cmd import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/config" "time" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" + "github.com/vulcanize/vulcanizedb/pkg/config" ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" ht "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer" "github.com/vulcanize/vulcanizedb/utils" diff --git a/pkg/config/contract.go b/pkg/config/contract.go index 50b8366a..baa588a5 100644 --- a/pkg/config/contract.go +++ b/pkg/config/contract.go @@ -17,10 +17,12 @@ package config import ( + "strings" + log "github.com/sirupsen/logrus" "github.com/spf13/viper" + "github.com/vulcanize/vulcanizedb/pkg/geth" - "strings" ) // Config struct for generic contract transformer diff --git a/pkg/config/plugin.go b/pkg/config/plugin.go index fad07cbc..55e94053 100644 --- a/pkg/config/plugin.go +++ b/pkg/config/plugin.go @@ -31,6 +31,7 @@ type Plugin struct { FileName string Save bool Home string + WriteOnly bool } type Transformer struct { diff --git a/pkg/config/plugin_test.go b/pkg/config/plugin_test.go index 1d1616bb..847f8e40 100644 --- a/pkg/config/plugin_test.go +++ b/pkg/config/plugin_test.go @@ -17,11 +17,13 @@ package config_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/config" "os" "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/config" ) var allDifferentPathsConfig = config.Plugin{ diff --git a/pkg/plugin/composer/composer.go b/pkg/plugin/composer/composer.go new file mode 100644 index 00000000..19cef9e3 --- /dev/null +++ b/pkg/plugin/composer/composer.go @@ -0,0 +1,100 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package composer + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" +) + +type composer struct { + GenConfig config.Plugin + dependencies []string + tmpVenDirs []string // Keep track of temp vendor directories + goFile string // Keep track of goFile name +} + +// Requires populated plugin config +func NewPluginComposer(gc config.Plugin) *composer { + return &composer{ + GenConfig: gc, + tmpVenDirs: make([]string, 0), + } +} + +func (b *composer) BuildPlugin() error { + // Get plugin .go and .so file paths + var err error + b.goFile, _, err = b.GenConfig.GetPluginPaths() + if err != nil { + return err + } + + // setup env to build plugin + return b.setupComposeEnv() +} + +// Sets up temporary vendor libs needed for plugin build +// This is to work around a conflict between plugins and vendoring (https://github.com/golang/go/issues/20481) +func (b *composer) setupComposeEnv() error { + // TODO: Less hacky way of handling plugin build deps + vendorPath, err := helpers.CleanPath(filepath.Join("$GOPATH/src", b.GenConfig.Home, "vendor")) + if err != nil { + return err + } + + repoPaths := b.GenConfig.GetRepoPaths() + + // Import transformer dependencies so that we can build our plugin + for importPath := range repoPaths { + dst := filepath.Join(vendorPath, importPath) + src, cleanErr := helpers.CleanPath(filepath.Join("$GOPATH/src", importPath)) + if cleanErr != nil { + return cleanErr + } + + copyErr := helpers.CopyDir(src, dst, "vendor") + if copyErr != nil { + return errors.New(fmt.Sprintf("unable to copy transformer dependency from %s to %s: %v", src, dst, copyErr)) + } + + // Have to clear out the copied over vendor lib or plugin won't build (see issue above) + removeErr := os.RemoveAll(filepath.Join(dst, "vendor")) + if removeErr != nil { + return removeErr + } + // Keep track of this vendor directory to clear later + b.tmpVenDirs = append(b.tmpVenDirs, dst) + } + + return nil +} + +func (c *composer) CleanUp() error { + for _, venDir := range c.tmpVenDirs { + err := os.RemoveAll(venDir) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/plugin/generator.go b/pkg/plugin/generator.go index 7b3310f2..ae9d2f2f 100644 --- a/pkg/plugin/generator.go +++ b/pkg/plugin/generator.go @@ -18,8 +18,10 @@ package plugin import ( "errors" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/plugin/builder" + "github.com/vulcanize/vulcanizedb/pkg/plugin/composer" "github.com/vulcanize/vulcanizedb/pkg/plugin/manager" "github.com/vulcanize/vulcanizedb/pkg/plugin/writer" ) @@ -40,11 +42,16 @@ func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) { if len(gc.Transformers) < 1 { return nil, errors.New("plugin generator is not configured with any transformers") } - return &generator{ + gen := &generator{ PluginWriter: writer.NewPluginWriter(gc), - PluginBuilder: builder.NewPluginBuilder(gc), MigrationManager: manager.NewMigrationManager(gc, dbc), - }, nil + } + if gc.WriteOnly { + gen.PluginBuilder = composer.NewPluginComposer(gc) + } else { + gen.PluginBuilder = builder.NewPluginBuilder(gc) + } + return gen, nil } // Generates plugin for the transformer initializers specified in the generator config diff --git a/pkg/plugin/manager/manager.go b/pkg/plugin/manager/manager.go index 0b526b6e..f627c033 100644 --- a/pkg/plugin/manager/manager.go +++ b/pkg/plugin/manager/manager.go @@ -20,13 +20,14 @@ import ( "database/sql" "errors" "fmt" + "io/ioutil" + "os" + "path/filepath" + "github.com/lib/pq" "github.com/pressly/goose" "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" - "io/ioutil" - "os" - "path/filepath" ) // Interface for managing the db migrations for plugin transformers