cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

auto management of dependencies temporarily needed to build plugin and transformer

Browse Source 
migrations; new test showing it is working with external transformers runs
properly when migrations were manually performed but still need
to test automated migration management
This commit is contained in:
Ian Norden
2019-02-24 15:23:35 -06:00
parent 6c2d895023
commit 55fa9b8364
9 changed files with 529 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/autogen/config.go
+45 -3
View File
@@ -16,8 +16,50 @@
package autogen
import (
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/vulcanize/vulcanizedb/utils"
)
type Config struct {
Imports map[string]string // Map of import alias to import path
FilePath string
FileName string
Initializers map[string]string // Map of import aliases to transformer paths
Dependencies map[string]string // Map of vendor dep names to their repositories
Migrations map[string]string // Map of migration names to their paths within the vendored deps
FilePath string
FileName string
}
func (c *Config) GetPluginPaths() (string, string, error) {
path, err := utils.CleanPath(c.FilePath)
if err != nil {
return "", "", err
}
name := strings.Split(c.FileName, ".")[0]
goFile := filepath.Join(path, name+".go")
soFile := filepath.Join(path, name+".so")
return goFile, soFile, nil
}
func (c *Config) GetMigrationsPaths() ([]string, error) {
paths := make([]string, 0, len(c.Migrations))
for key, relPath := range c.Migrations {
repo, ok := c.Dependencies[key]
if !ok {
return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath))
}
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath)
cleanPath, err := utils.CleanPath(path)
if err != nil {
return nil, err
}
paths = append(paths, cleanPath)
}
return paths, nil
}
pkg/autogen/generator.go
+168 -42
View File
@@ -18,65 +18,97 @@ package autogen
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"strconv"
. "github.com/dave/jennifer/jen"
"github.com/mitchellh/go-homedir"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/utils"
)
type Generator interface {
GenerateTransformerPlugin() error
GenerateExporterPlugin() error
}
type generator struct {
*Config
GenConfig *Config
DBConfig config.Database
tmpMigDir string
tmpVenDirs []string
}
func NewGenerator(config Config) *generator {
func NewGenerator(gc Config, dbc config.Database) *generator {
return &generator{
Config: &config,
GenConfig: &gc,
DBConfig: dbc,
}
}
func (g *generator) GenerateTransformerPlugin() error {
if g.Config == nil {
func (g *generator) GenerateExporterPlugin() error {
if g.GenConfig == nil {
return errors.New("generator needs a config file")
}
if g.Config.FilePath == "" {
if g.GenConfig.FilePath == "" {
return errors.New("generator is missing file path")
}
if len(g.Config.Imports) < 1 {
if len(g.GenConfig.Initializers) < 1 {
return errors.New("generator needs to be configured with imports")
}
// Create file path
goFile, soFile, err := GetPaths(*g.Config)
// Get plugin file paths
goFile, soFile, err := g.GenConfig.GetPluginPaths()
if err != nil {
return err
}
// Clear previous .go and .so files if they exist
err = ClearFiles(goFile, soFile)
// Clear .go and .so files of the same name if they exist (overwrite)
err = utils.ClearFiles(goFile, soFile)
if err != nil {
return err
}
// Generate Exporter code
err = g.generateCode(goFile)
if err != nil {
return err
}
// Setup temp vendor lib and migrations directories
err = g.setupTempDirs()
if err != nil {
return err
}
defer g.cleanUp() // Clear these up when we are done building our plugin
// Build the .go file into a .so plugin
err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run()
if err != nil {
return err
}
// Run migrations only after successfully building .so file
return g.runMigrations()
}
// Generates the plugin code
func (g *generator) generateCode(goFile string) error {
// Begin code generation
f := NewFile("main")
f.HeaderComment("This exporter is generated to export the configured transformer initializers")
// Import TransformerInitializers
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, imp := range g.Config.Imports {
for alias, imp := range g.GenConfig.Initializers {
f.ImportAlias(imp, alias)
}
// Collect TransformerInitializer names
importedInitializers := make([]Code, 0, len(g.Config.Imports))
for _, path := range g.Config.Imports {
importedInitializers := make([]Code, 0, len(g.GenConfig.Initializers))
for _, path := range g.GenConfig.Initializers {
importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer"))
}
@@ -90,49 +122,143 @@ func (g *generator) GenerateTransformerPlugin() error {
"TransformerInitializer").Block(
Return(Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(importedInitializers...)))
"TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers
// Write code to destination file
err = f.Save(goFile)
return f.Save(goFile)
}
func (g *generator) runMigrations() error {
// Get paths to db migrations
paths, err := g.GenConfig.GetMigrationsPaths()
if err != nil {
return err
}
if len(paths) < 1 {
return nil
}
// Create temporary copies of migrations to the temporary migrationDir
// These tmps are identical except they have had `1` added in front of their unix_timestamps
// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
// But will still be ran in the same order relative to one another
// TODO: Less hacky way of handing migrations
err = g.createMigrationCopies(paths)
if err != nil {
return err
}
// Build the .go file into a .so plugin
return exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run()
// Run the copied migrations
location := "file://" + g.tmpMigDir
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable up", g.DBConfig.Hostname, g.DBConfig.Port, g.DBConfig.Name)
return exec.Command("migrate", "-source", location, pgStr).Run()
}
func GetPaths(config Config) (string, string, error) {
path, err := homedir.Expand(filepath.Clean(config.FilePath))
// Sets up temporary vendor libs and migration directories
func (g *generator) setupTempDirs() error {
// TODO: Less hacky way of handling plugin build deps
dirPath, err := utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/")
if err != nil {
return "", "", err
return err
}
if strings.Contains(path, "$GOPATH") {
env := os.Getenv("GOPATH")
spl := strings.Split(path, "$GOPATH")[1]
path = filepath.Join(env, spl)
vendorPath := filepath.Join(dirPath, "vendor/")
/*
// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
// Import transformer dependencies so that we build our plugin
for _, importPath := range g.GenConfig.Dependencies {
importURL := "https://" + importPath + ".git"
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return err
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
return err
}
g.tmpVenDirs = append(g.tmpVenDirs, depPath)
}
*/
// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
for _, importPath := range g.GenConfig.Dependencies {
depPath := filepath.Join(vendorPath, importPath)
g.tmpVenDirs = append(g.tmpVenDirs, depPath)
}
name := strings.Split(config.FileName, ".")[0]
goFile := filepath.Join(path, name+".go")
soFile := filepath.Join(path, name+".so")
// Dep ensure to make sure vendor pkgs are in place for building the plugin
err = exec.Command("dep", "ensure").Run()
if err != nil {
return errors.New("failed to vendor transformer packages required to build plugin")
}
return goFile, soFile, nil
// Git checkout our head-state vendor libraries
// This is necessary because we currently need to manual edit our vendored
// go-ethereum abi library to allow for unpacking in empty interfaces and maps
// This can be removed once the PRs against geth merged
err = exec.Command("git", "checkout", dirPath).Run()
if err != nil {
return errors.New("failed to checkout vendored go-ethereum lib")
}
// Initialize temp directory for transformer migrations
g.tmpMigDir, err = utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil {
return err
}
stat, err := os.Stat(g.tmpMigDir)
if err == nil {
if !stat.IsDir() {
return errors.New(fmt.Sprintf("file %s found where directory is expected", stat.Name()))
}
} else if os.IsNotExist(err) {
os.Mkdir(g.tmpMigDir, os.FileMode(0777))
} else {
return err
}
return nil
}
func ClearFiles(files ...string) error {
for _, file := range files {
if _, err := os.Stat(file); err == nil {
err = os.Remove(file)
if err != nil {
return err
}
} else if os.IsNotExist(err) {
// fall through
} else {
func (g *generator) createMigrationCopies(paths []string) error {
for _, path := range paths {
dir, err := ioutil.ReadDir(path)
if err != nil {
return err
}
for _, file := range dir {
if file.IsDir() || len(file.Name()) < 15 { // (10 digit unix time stamp + x + .sql) is bare minimum
continue
}
_, err := strconv.Atoi(file.Name()[:10])
if err != nil {
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix", file.Name())
continue
}
if filepath.Ext(file.Name()) == "sql" {
src := filepath.Join(path, file.Name())
dst := filepath.Join(g.tmpMigDir, "1"+file.Name())
err = utils.CopyFile(src, dst)
if err != nil {
return err
}
}
}
}
return nil
}
func (g *generator) cleanUp() error {
for _, venDir := range g.tmpVenDirs {
err := os.RemoveAll(venDir)
if err != nil {
return err
}
}
return os.RemoveAll(g.tmpMigDir)
}
pkg/autogen/generator_test.go
+146 -67
View File
@@ -27,28 +27,33 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/autogen"
"github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
"github.com/vulcanize/vulcanizedb/utils"
)
var testConfig = autogen.Config{
Imports: map[string]string{
var localConfig = autogen.Config{
Initializers: map[string]string{
"bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite",
"deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal",
},
FileName: "testTransformerSet",
FileName: "localTestTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/",
}
var targetConfig = autogen.Config{
Imports: map[string]string{
"bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite",
"deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal",
var externalConfig = autogen.Config{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal",
},
FileName: "targetTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/target/",
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
FileName: "externalTestTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/",
}
type Exporter interface {
@@ -66,73 +71,147 @@ var _ = Describe("Generator test", func() {
viper.SetConfigName("compose")
viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
BeforeEach(func() {
goPath, soPath, err = autogen.GetPaths(testConfig)
Expect(err).ToNot(HaveOccurred())
g = autogen.NewGenerator(testConfig)
err = g.GenerateTransformerPlugin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
err := autogen.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Describe("Using local config", func() {
BeforeEach(func() {
goPath, soPath, err = localConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
g = autogen.NewGenerator(localConfig, config.Database{})
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
AfterEach(func() {
err := utils.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
bite.BiteModel
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
})
})
Describe("Using external config", func() {
BeforeEach(func() {
goPath, soPath, err = externalConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
type model struct {
bite.BiteModel
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
g = autogen.NewGenerator(externalConfig, config.Database{})
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
AfterEach(func() {
err := utils.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
bite.BiteModel
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
})
})
})