diff --git a/README.md b/README.md index 99272e36..6685af83 100644 --- a/README.md +++ b/README.md @@ -273,7 +273,6 @@ The config provides information for composing a set of transformers: [exporter] home = "github.com/vulcanize/vulcanizedb" - clone = false name = "exampleTransformerExporter" save = false transformerNames = [ @@ -304,14 +303,13 @@ The config provides information for composing a set of transformers: migrations = "to/db/migrations" ``` - `home` is the name of the package you are building the plugin for, in most cases this is github.com/vulcanize/vulcanizedb -- `clone` this signifies whether or not to retrieve plugin transformer packages by `git clone`ing them; by default we attempt to work with transformer packages located in -our `$GOPATH` but setting this to `true` overrides that. This needs to be set to `true` for the configs used in tests in order for them to work with Travis. - `name` is the name used for the plugin files (.so and .go) - `save` indicates whether or not the user wants to save the .go file instead of removing it after .so compilation. Sometimes useful for debugging/trouble-shooting purposes. - `transformerNames` is the list of the names of the transformers we are composing together, so we know how to access their submaps in the exporter map - `exporter.`s are the sub-mappings containing config info for the transformers - `repository` is the path for the repository which contains the transformer and its `TransformerInitializer` - - `path` is the relative path from `repository` to the transformer's `TransformerInitializer` directory (initializer package) + - `path` is the relative path from `repository` to the transformer's `TransformerInitializer` directory (initializer package). + - Transformer repositories need to be cloned into the user's $GOPATH (`go get`) - `type` is the type of the transformer; indicating which type of watcher it works with (for now, there are only two options: `eth_event` and `eth_storage`) - `eth_storage` indicates the transformer works with the [storage watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go) that fetches state and storage diffs from an ETH node (instead of, for example, from IPFS) diff --git a/cmd/compose.go b/cmd/compose.go index e51298ac..0a55284f 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -45,7 +45,6 @@ var composeCmd = &cobra.Command{ [exporter] home = "github.com/vulcanize/vulcanizedb" - clone = false name = "exampleTransformerExporter" save = false transformerNames = [ @@ -168,6 +167,5 @@ func prepConfig() { FileName: viper.GetString("exporter.name"), Save: viper.GetBool("exporter.save"), Home: viper.GetString("exporter.home"), - Clone: viper.GetBool("exporter.clone"), } } diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index f57b116c..961d38c9 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -46,7 +46,6 @@ var composeAndExecuteCmd = &cobra.Command{ [exporter] home = "github.com/vulcanize/vulcanizedb" - clone = false name = "exampleTransformerExporter" save = false transformerNames = [ diff --git a/pkg/config/plugin.go b/pkg/config/plugin.go index ef333380..04d41ea9 100644 --- a/pkg/config/plugin.go +++ b/pkg/config/plugin.go @@ -29,7 +29,6 @@ type Plugin struct { FileName string Save bool Home string - Clone bool } type Transformer struct { diff --git a/pkg/plugin/builder/builder.go b/pkg/plugin/builder/builder.go index 993a81ba..c16e50b1 100644 --- a/pkg/plugin/builder/builder.go +++ b/pkg/plugin/builder/builder.go @@ -22,7 +22,6 @@ import ( "os" "os/exec" "path/filepath" - "strings" "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers" @@ -61,15 +60,15 @@ func (b *builder) BuildPlugin() error { } // setup env to build plugin - err = b.setupBuildEnv() - if err != nil { - return err + setupErr := b.setupBuildEnv() + if setupErr != nil { + return setupErr } // Build the .go file into a .so plugin - err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run() - if err != nil { - return errors.New(fmt.Sprintf("unable to build .so file: %s", err.Error())) + execErr := exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run() + if execErr != nil { + return errors.New(fmt.Sprintf("unable to build .so file: %s", execErr.Error())) } return nil } @@ -87,37 +86,24 @@ func (b *builder) setupBuildEnv() error { // Import transformer dependencies so that we can build our plugin for importPath := range repoPaths { - dstPath := filepath.Join(vendorPath, importPath) - // When testing on Travis we need to clone the libs - if b.GenConfig.Clone { - // And if we want to be able to work with a private repo we need to use ssh instead of https - // and upload a permissioned ssh key to travis before deploying tests there - index := strings.Index(importPath, "/") - gitPath := importPath[:index] + ":" + importPath[index+1:] - importURL := "git@" + gitPath + ".git" - err = exec.Command("git", "clone", importURL, dstPath).Run() - if err != nil { - return errors.New(fmt.Sprintf("unable to clone transformer dependency from %s to %s: %s", importPath, dstPath, err.Error())) - } - } else { // If not on Travis we can work with libs at $GOPATH - srcDir, err := helpers.CleanPath(filepath.Join("$GOPATH/src", importPath)) - if err != nil { - return err - } - sp := strings.Split(dstPath, "/") - spj := strings.Join(sp[:len(sp)-1], "/") - err = exec.Command("rsync", "-a", srcDir, spj).Run() - if err != nil { - return errors.New(fmt.Sprintf("unable to copy transformer dependency from %s to %s: %s", srcDir, dstPath, err.Error())) - } + dst := filepath.Join(vendorPath, importPath) + src, cleanErr := helpers.CleanPath(filepath.Join("$GOPATH/src", importPath)) + if cleanErr != nil { + return cleanErr } + + copyErr := helpers.CopyDir(src, dst, "vendor") + if copyErr != nil { + return errors.New(fmt.Sprintf("unable to copy transformer dependency from %s to %s: %v", src, dst, copyErr)) + } + // Have to clear out the copied over vendor lib or plugin won't build (see issue above) - err := os.RemoveAll(filepath.Join(dstPath, "vendor")) - if err != nil { - return err + removeErr := os.RemoveAll(filepath.Join(dst, "vendor")) + if removeErr != nil { + return removeErr } // Keep track of this vendor directory to clear later - b.tmpVenDirs = append(b.tmpVenDirs, dstPath) + b.tmpVenDirs = append(b.tmpVenDirs, dst) } return nil diff --git a/pkg/plugin/helpers/helpers.go b/pkg/plugin/helpers/helpers.go index 2885cbef..0455efdb 100644 --- a/pkg/plugin/helpers/helpers.go +++ b/pkg/plugin/helpers/helpers.go @@ -18,6 +18,7 @@ package helpers import ( "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -62,16 +63,54 @@ func CopyFile(src, dst string) error { if err != nil { return err } - defer in.Close() out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL|os.O_WRONLY, os.FileMode(0666)) // Doesn't overwrite files if err != nil { + in.Close() return err } - defer out.Close() _, err = io.Copy(out, in) + in.Close() + out.Close() + return err +} + +func CopyDir(src string, dst string, excludeRecursiveDir string) error { + var err error + var fds []os.FileInfo + var srcinfo os.FileInfo + + srcinfo, err = os.Stat(src) if err != nil { return err } - return out.Close() + + mkErr := os.MkdirAll(dst, srcinfo.Mode()) + if mkErr != nil { + return mkErr + } + + fds, readErr := ioutil.ReadDir(src) + if err != readErr { + return readErr + } + for _, fd := range fds { + srcfp := filepath.Join(src, fd.Name()) + dstfp := filepath.Join(dst, fd.Name()) + + if fd.IsDir() { + if fd.Name() != excludeRecursiveDir { + dirErr := CopyDir(srcfp, dstfp, "") + if dirErr != nil { + return dirErr + } + } + } else { + fileErr := CopyFile(srcfp, dstfp) + if fileErr != nil { + return fileErr + } + } + } + return nil }