Merge pull request #29 from vulcanize/fix

Fix for Travis issue at https://github.com/vulcanize/mcd_transformers/pull/6

README.md

@@ -273,7 +273,6 @@ The config provides information for composing a set of transformers:
 
 [exporter]
     home     = "github.com/vulcanize/vulcanizedb"
-    clone    = false
     name     = "exampleTransformerExporter"
     save     = false
     transformerNames = [
@@ -304,14 +303,13 @@ The config provides information for composing a set of transformers:
         migrations = "to/db/migrations"
 ```
 - `home` is the name of the package you are building the plugin for, in most cases this is github.com/vulcanize/vulcanizedb
-- `clone` this signifies whether or not to retrieve plugin transformer packages by `git clone`ing them; by default we attempt to work with transformer packages located in
-our `$GOPATH` but setting this to `true` overrides that. This needs to be set to `true` for the configs used in tests in order for them to work with Travis.
 - `name` is the name used for the plugin files (.so and .go)   
 - `save` indicates whether or not the user wants to save the .go file instead of removing it after .so compilation. Sometimes useful for debugging/trouble-shooting purposes.
 - `transformerNames` is the list of the names of the transformers we are composing together, so we know how to access their submaps in the exporter map
 - `exporter.<transformerName>`s are the sub-mappings containing config info for the transformers
     - `repository` is the path for the repository which contains the transformer and its `TransformerInitializer`
-    - `path` is the relative path from `repository` to the transformer's `TransformerInitializer` directory (initializer package)
+    - `path` is the relative path from `repository` to the transformer's `TransformerInitializer` directory (initializer package).
+        - Transformer repositories need to be cloned into the user's $GOPATH (`go get`)
     - `type` is the type of the transformer; indicating which type of watcher it works with (for now, there are only two options: `eth_event` and `eth_storage`)
         - `eth_storage` indicates the transformer works with the [storage watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go)
          that fetches state and storage diffs from an ETH node (instead of, for example, from IPFS)

cmd/compose.go

@@ -45,7 +45,6 @@ var composeCmd = &cobra.Command{
 
 [exporter]
     home     = "github.com/vulcanize/vulcanizedb"
-    clone    = false
     name     = "exampleTransformerExporter"
     save     = false
     transformerNames = [
@@ -168,6 +167,5 @@ func prepConfig() {
 		FileName:     viper.GetString("exporter.name"),
 		Save:         viper.GetBool("exporter.save"),
 		Home:         viper.GetString("exporter.home"),
-		Clone:        viper.GetBool("exporter.clone"),
 	}
 }

cmd/composeAndExecute.go

@@ -46,7 +46,6 @@ var composeAndExecuteCmd = &cobra.Command{
 
 [exporter]
     home     = "github.com/vulcanize/vulcanizedb"
-    clone    = false
     name     = "exampleTransformerExporter"
     save     = false
     transformerNames = [

pkg/config/plugin.go

@@ -29,7 +29,6 @@ type Plugin struct {
 	FileName     string
 	Save         bool
 	Home         string
-	Clone        bool
 }
 
 type Transformer struct {

pkg/plugin/builder/builder.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
@@ -61,15 +60,15 @@ func (b *builder) BuildPlugin() error {
 	}
 
 	// setup env to build plugin
-	err = b.setupBuildEnv()
+	setupErr := b.setupBuildEnv()
-	if err != nil {
+	if setupErr != nil {
-		return err
+		return setupErr
 	}
 
 	// Build the .go file into a .so plugin
-	err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run()
+	execErr := exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run()
-	if err != nil {
+	if execErr != nil {
-		return errors.New(fmt.Sprintf("unable to build .so file: %s", err.Error()))
+		return errors.New(fmt.Sprintf("unable to build .so file: %s", execErr.Error()))
 	}
 	return nil
 }
@@ -87,37 +86,24 @@ func (b *builder) setupBuildEnv() error {
 
 	// Import transformer dependencies so that we can build our plugin
 	for importPath := range repoPaths {
-		dstPath := filepath.Join(vendorPath, importPath)
+		dst := filepath.Join(vendorPath, importPath)
-		// When testing on Travis we need to clone the libs
+		src, cleanErr := helpers.CleanPath(filepath.Join("$GOPATH/src", importPath))
-		if b.GenConfig.Clone {
+		if cleanErr != nil {
-			// And if we want to be able to work with a private repo we need  to use ssh instead of https
+			return cleanErr
-			// and upload a permissioned ssh key to travis before deploying tests there
-			index := strings.Index(importPath, "/")
-			gitPath := importPath[:index] + ":" + importPath[index+1:]
-			importURL := "git@" + gitPath + ".git"
-			err = exec.Command("git", "clone", importURL, dstPath).Run()
-			if err != nil {
-				return errors.New(fmt.Sprintf("unable to clone transformer dependency from %s to %s: %s", importPath, dstPath, err.Error()))
-			}
-		} else { // If not on Travis we can work with libs at $GOPATH
-			srcDir, err := helpers.CleanPath(filepath.Join("$GOPATH/src", importPath))
-			if err != nil {
-				return err
-			}
-			sp := strings.Split(dstPath, "/")
-			spj := strings.Join(sp[:len(sp)-1], "/")
-			err = exec.Command("rsync", "-a", srcDir, spj).Run()
-			if err != nil {
-				return errors.New(fmt.Sprintf("unable to copy transformer dependency from %s to %s: %s", srcDir, dstPath, err.Error()))
-			}
 		}
+
+		copyErr := helpers.CopyDir(src, dst, "vendor")
+		if copyErr != nil {
+			return errors.New(fmt.Sprintf("unable to copy transformer dependency from %s to %s: %v", src, dst, copyErr))
+		}
+
 		// Have to clear out the copied over vendor lib or plugin won't build (see issue above)
-		err := os.RemoveAll(filepath.Join(dstPath, "vendor"))
+		removeErr := os.RemoveAll(filepath.Join(dst, "vendor"))
-		if err != nil {
+		if removeErr != nil {
-			return err
+			return removeErr
 		}
 		// Keep track of this vendor directory to clear later
-		b.tmpVenDirs = append(b.tmpVenDirs, dstPath)
+		b.tmpVenDirs = append(b.tmpVenDirs, dst)
 	}
 
 	return nil

pkg/plugin/helpers/helpers.go

@@ -18,6 +18,7 @@ package helpers
 
 import (
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -62,16 +63,54 @@ func CopyFile(src, dst string) error {
 	if err != nil {
 		return err
 	}
-	defer in.Close()
 	out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL|os.O_WRONLY, os.FileMode(0666)) // Doesn't overwrite files
 	if err != nil {
+		in.Close()
 		return err
 	}
-	defer out.Close()
 
 	_, err = io.Copy(out, in)
+	in.Close()
+	out.Close()
+	return err
+}
+
+func CopyDir(src string, dst string, excludeRecursiveDir string) error {
+	var err error
+	var fds []os.FileInfo
+	var srcinfo os.FileInfo
+
+	srcinfo, err = os.Stat(src)
 	if err != nil {
 		return err
 	}
-	return out.Close()
+
+	mkErr := os.MkdirAll(dst, srcinfo.Mode())
+	if mkErr != nil {
+		return mkErr
+	}
+
+	fds, readErr := ioutil.ReadDir(src)
+	if err != readErr {
+		return readErr
+	}
+	for _, fd := range fds {
+		srcfp := filepath.Join(src, fd.Name())
+		dstfp := filepath.Join(dst, fd.Name())
+
+		if fd.IsDir() {
+			if fd.Name() != excludeRecursiveDir {
+				dirErr := CopyDir(srcfp, dstfp, "")
+				if dirErr != nil {
+					return dirErr
+				}
+			}
+		} else {
+			fileErr := CopyFile(srcfp, dstfp)
+			if fileErr != nil {
+				return fileErr
+			}
+		}
+	}
+	return nil
 }