deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

refactoring plugin generating code

Browse Source
This commit is contained in:
Ian Norden 2019-02-02 16:15:09 -06:00
parent decc2a3caf
commit a322372713
16 changed files with 549 additions and 361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
cmd/composeAndExecute.go
View File

@ -27,7 +27,9 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/autogen" "github.com/vulcanize/vulcanizedb/pkg/config"
p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
"github.com/vulcanize/vulcanizedb/utils" "github.com/vulcanize/vulcanizedb/utils"
) )
@ -75,7 +77,7 @@ loaded into and executed over by a generic watcher`,
func composeAndExecute() { func composeAndExecute() {
// generate code to build the plugin according to the config file // generate code to build the plugin according to the config file
autogenConfig = autogen.Config{ genConfig = config.Plugin{
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins", FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
FileName: viper.GetString("exporter.name"), FileName: viper.GetString("exporter.name"),
Save: viper.GetBool("exporter.save"), Save: viper.GetBool("exporter.save"),
@ -85,25 +87,28 @@ func composeAndExecute() {
} }
fmt.Println("generating plugin") fmt.Println("generating plugin")
generator := autogen.NewGenerator(autogenConfig, databaseConfig) generator, err := p2.NewGenerator(genConfig, databaseConfig)
err := generator.GenerateExporterPlugin()
if err != nil { if err != nil {
fmt.Println("generating plugin failed") log.Fatal(err)
}
err = generator.GenerateExporterPlugin()
if err != nil {
fmt.Fprint(os.Stderr, "generating plugin failed")
log.Fatal(err) log.Fatal(err)
} }
// Get the plugin path and load the plugin // Get the plugin path and load the plugin
_, pluginPath, err := autogenConfig.GetPluginPaths() _, pluginPath, err := genConfig.GetPluginPaths()
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
if !autogenConfig.Save { if !genConfig.Save {
defer utils.ClearFiles(pluginPath) defer helpers.ClearFiles(pluginPath)
} }
fmt.Println("opening plugin") fmt.Println("opening plugin")
plug, err := plugin.Open(pluginPath) plug, err := plugin.Open(pluginPath)
if err != nil { if err != nil {
fmt.Println("opening pluggin failed") fmt.Fprint(os.Stderr, "opening pluggin failed")
log.Fatal(err) log.Fatal(err)
} }
@ -111,14 +116,14 @@ func composeAndExecute() {
fmt.Println("loading transformers from plugin") fmt.Println("loading transformers from plugin")
symExporter, err := plug.Lookup("Exporter") symExporter, err := plug.Lookup("Exporter")
if err != nil { if err != nil {
fmt.Println("loading Exporter symbol failed") fmt.Fprint(os.Stderr, "loading Exporter symbol failed")
log.Fatal(err) log.Fatal(err)
} }
// Assert that the symbol is of type Exporter // Assert that the symbol is of type Exporter
exporter, ok := symExporter.(Exporter) exporter, ok := symExporter.(Exporter)
if !ok { if !ok {
fmt.Println("plugged-in symbol not of type Exporter") fmt.Fprint(os.Stderr, "plugged-in symbol not of type Exporter")
os.Exit(1) os.Exit(1)
} }

3
cmd/root.go
View File

@ -28,7 +28,6 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/autogen"
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client" "github.com/vulcanize/vulcanizedb/pkg/geth/client"
@ -39,7 +38,7 @@ import (
var ( var (
cfgFile string cfgFile string
databaseConfig config.Database databaseConfig config.Database
autogenConfig autogen.Config genConfig config.Plugin
ipc string ipc string
levelDbPath string levelDbPath string
startingBlockNumber int64 startingBlockNumber int64

56
environments/compose.toml
View File

@ -15,34 +15,34 @@
name = "exporter" name = "exporter"
save = false save = false
[exporter.transformers] [exporter.transformers]
bite = "github.com/vulcanize/mcd_transformers/transformers/bite" bite = "github.com/vulcanize/mcd_transformers/transformers/bite/initializer"
cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump" cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump/initializer"
cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip" cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip/initializer"
cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow" cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow/initializer"
deal = "github.com/vulcanize/mcd_transformers/transformers/deal" deal = "github.com/vulcanize/mcd_transformers/transformers/deal/initializer"
dent = "github.com/vulcanize/mcd_transformers/transformers/dent" dent = "github.com/vulcanize/mcd_transformers/transformers/dent/initializer"
drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip" drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip/initializer"
drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk" drip_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk/initializer"
drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo" drip_file_repo = "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo/initializer"
drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow" drip_file_vow = "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow/initializer"
flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick" flap_kick = "github.com/vulcanize/mcd_transformers/transformers/flap_kick/initializer"
flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick" flip_kick = "github.com/vulcanize/mcd_transformers/transformers/flip_kick/initializer"
flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick" flop_kick = "github.com/vulcanize/mcd_transformers/transformers/flop_kick/initializer"
frob = "github.com/vulcanize/mcd_transformers/transformers/frob" frob = "github.com/vulcanize/mcd_transformers/transformers/frob/initializer"
pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling" pit_file_debt_ceiling = "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling/initializer"
pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk" pit_file_ilk = "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk/initializer"
price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds" price_feeds = "github.com/vulcanize/mcd_transformers/transformers/price_feeds/initializer"
tend = "github.com/vulcanize/mcd_transformers/transformers/tend" tend = "github.com/vulcanize/mcd_transformers/transformers/tend/initializer"
vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux" vat_flux = "github.com/vulcanize/mcd_transformers/transformers/vat_flux/initializer"
vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold" vat_fold = "github.com/vulcanize/mcd_transformers/transformers/vat_fold/initializer"
vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab" vat_grab = "github.com/vulcanize/mcd_transformers/transformers/vat_grab/initializer"
vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal" vat_heal = "github.com/vulcanize/mcd_transformers/transformers/vat_heal/initializer"
vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init" vat_init = "github.com/vulcanize/mcd_transformers/transformers/vat_init/initializer"
vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move" vat_move = "github.com/vulcanize/mcd_transformers/transformers/vat_move/initializer"
vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip" vat_slip = "github.com/vulcanize/mcd_transformers/transformers/vat_slip/initializer"
vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll" vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll/initializer"
vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune" vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune/initializer"
vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog" vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog/initializer"
[exporter.repositories] [exporter.repositories]
mcd_transformers = "github.com/vulcanize/mcd_transformers" mcd_transformers = "github.com/vulcanize/mcd_transformers"

243
pkg/autogen/generator.go
View File

@ -1,243 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package autogen
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
. "github.com/dave/jennifer/jen"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/utils"
)
type Generator interface {
GenerateExporterPlugin() error
}
type generator struct {
GenConfig *Config
DBConfig config.Database
tmpMigDir string
tmpVenDirs []string
}
func NewGenerator(gc Config, dbc config.Database) *generator {
return &generator{
GenConfig: &gc,
DBConfig: dbc,
}
}
func (g *generator) GenerateExporterPlugin() error {
if len(g.GenConfig.Initializers) < 1 {
return errors.New("generator needs to be configured with TransformerInitializer import paths")
}
if len(g.GenConfig.Dependencies) < 1 {
return errors.New("generator needs to be configured with root repository path(s)")
}
if len(g.GenConfig.Migrations) < 1 {
fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided\r\n")
}
// Get plugin file paths
goFile, soFile, err := g.GenConfig.GetPluginPaths()
if err != nil {
return err
}
// Generate Exporter code
err = g.generateCode(goFile, soFile)
if err != nil {
return err
}
// Setup temp vendor lib and migrations directories
err = g.setupTempDirs()
if err != nil {
return err
}
// Clear tmp files and directories when we exit
defer g.cleanUp(goFile)
// Build the .go file into a .so plugin
err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to build .so file: %s", err.Error()))
}
// Run migrations only after successfully building .so file
return g.runMigrations()
}
// Generates the plugin code
func (g *generator) generateCode(goFile, soFile string) error {
// Clear .go and .so files of the same name if they exist
err := utils.ClearFiles(goFile, soFile)
if err != nil {
return err
}
// Begin code generation
f := NewFile("main")
f.HeaderComment("This exporter is generated to export the configured transformer initializers")
// Import TransformerInitializers specified in config
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, imp := range g.GenConfig.Initializers {
f.ImportAlias(imp, alias)
}
// Collect TransformerInitializer names
importedInitializers := make([]Code, 0, len(g.GenConfig.Initializers))
for _, path := range g.GenConfig.Initializers {
importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer"))
}
// Create Exporter variable with method to export the set of the imported TransformerInitializers
f.Type().Id("exporter").String()
f.Var().Id("Exporter").Id("exporter")
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Block(
Return(Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers
// Write code to destination file
return f.Save(goFile)
}
// Sets up temporary vendor libs and migration directories
func (g *generator) setupTempDirs() error {
// TODO: Less hacky way of handling plugin build deps
dirPath, err := utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/")
if err != nil {
return err
}
vendorPath := filepath.Join(dirPath, "vendor")
// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
// Import transformer dependencies so that we build our plugin
for name, importPath := range g.GenConfig.Dependencies {
index := strings.Index(importPath, "/")
gitPath := importPath[:index] + ":" + importPath[index+1:]
importURL := "git@" + gitPath + ".git"
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
return err
}
g.tmpVenDirs = append(g.tmpVenDirs, depPath)
}
// Initialize temp directory for transformer migrations
g.tmpMigDir, err = utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil {
return err
}
err = os.RemoveAll(g.tmpMigDir)
if err != nil {
return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", g.tmpMigDir))
}
return os.Mkdir(g.tmpMigDir, os.FileMode(0777))
}
func (g *generator) runMigrations() error {
// Get paths to db migrations
paths, err := g.GenConfig.GetMigrationsPaths()
if err != nil {
return err
}
if len(paths) < 1 {
return nil
}
// Create temporary copies of migrations to the temporary migrationDir
// These tmps are identical except they have had `1` added in front of their unix_timestamps
// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
// But will still be ran in the same order relative to one another
// TODO: Less hacky way of handing migrations
err = g.createMigrationCopies(paths)
if err != nil {
return err
}
// Run the copied migrations
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", g.DBConfig.Hostname, g.DBConfig.Port, g.DBConfig.Name)
return exec.Command("migrate", "-path", g.tmpMigDir, "-database", pgStr, "up").Run()
}
func (g *generator) createMigrationCopies(paths []string) error {
for _, path := range paths {
dir, err := ioutil.ReadDir(path)
if err != nil {
return err
}
for _, file := range dir {
if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum
continue
}
_, err := strconv.Atoi(file.Name()[:10])
if err != nil {
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name())
continue
}
src := filepath.Join(path, file.Name())
dst := filepath.Join(g.tmpMigDir, "1"+file.Name())
err = utils.CopyFile(src, dst)
if err != nil {
return err
}
}
}
return nil
}
func (g *generator) cleanUp(goFile string) error {
if !g.GenConfig.Save {
err := utils.ClearFiles(goFile)
if err != nil {
return err
}
}
for _, venDir := range g.tmpVenDirs {
err := os.RemoveAll(venDir)
if err != nil {
return err
}
}
return os.RemoveAll(g.tmpMigDir)
}

14
pkg/autogen/config.go → pkg/config/plugin.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package autogen package config
import ( import (
"errors" "errors"
@ -22,10 +22,10 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/vulcanize/vulcanizedb/utils" "github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
) )
type Config struct { type Plugin struct {
Initializers map[string]string // Map of import aliases to transformer initializer paths Initializers map[string]string // Map of import aliases to transformer initializer paths
Dependencies map[string]string // Map of vendor dep names to their repositories Dependencies map[string]string // Map of vendor dep names to their repositories
Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations
@ -34,8 +34,8 @@ type Config struct {
Save bool Save bool
} }
func (c *Config) GetPluginPaths() (string, string, error) { func (c *Plugin) GetPluginPaths() (string, string, error) {
path, err := utils.CleanPath(c.FilePath) path, err := helpers.CleanPath(c.FilePath)
if err != nil { if err != nil {
return "", "", err return "", "", err
} }
@ -47,7 +47,7 @@ func (c *Config) GetPluginPaths() (string, string, error) {
return goFile, soFile, nil return goFile, soFile, nil
} }
func (c *Config) GetMigrationsPaths() ([]string, error) { func (c *Plugin) GetMigrationsPaths() ([]string, error) {
paths := make([]string, 0, len(c.Migrations)) paths := make([]string, 0, len(c.Migrations))
for key, relPath := range c.Migrations { for key, relPath := range c.Migrations {
repo, ok := c.Dependencies[key] repo, ok := c.Dependencies[key]
@ -55,7 +55,7 @@ func (c *Config) GetMigrationsPaths() ([]string, error) {
return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath)) return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath))
} }
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath) path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath)
cleanPath, err := utils.CleanPath(path) cleanPath, err := helpers.CleanPath(path)
if err != nil { if err != nil {
return nil, err return nil, err
} }

118
pkg/plugin/builder/builder.go Normal file
View File

@ -0,0 +1,118 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package builder
import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
type PluginBuilder interface {
BuildPlugin() error
CleanUp() error
}
type builder struct {
GenConfig config.Plugin
tmpVenDirs []string
goFile string
}
func NewPluginBuilder(gc config.Plugin, dbc config.Database) *builder {
return &builder{
GenConfig: gc,
tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
}
}
func (b *builder) BuildPlugin() error {
// Get plugin .go and .so file paths
var err error
var soFile string
b.goFile, soFile, err = b.GenConfig.GetPluginPaths()
if err != nil {
return err
}
// setup env to build plugin
err = b.setupBuildEnv()
if err != nil {
return err
}
// Build the .go file into a .so plugin
err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to build .so file: %s", err.Error()))
}
return nil
}
// Sets up temporary vendor libs needed for plugin build
func (b *builder) setupBuildEnv() error {
// TODO: Less hacky way of handling plugin build deps
vendorPath, err := helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor")
if err != nil {
return err
}
// Import transformer dependencies so that we can build our plugin
for name, importPath := range b.GenConfig.Dependencies {
index := strings.Index(importPath, "/")
gitPath := importPath[:index] + ":" + importPath[index+1:]
importURL := "git@" + gitPath + ".git"
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
return err
}
b.tmpVenDirs = append(b.tmpVenDirs, depPath)
}
return nil
}
func (b *builder) CleanUp() error {
if !b.GenConfig.Save {
err := helpers.ClearFiles(b.goFile)
if err != nil {
return err
}
}
for _, venDir := range b.tmpVenDirs {
err := os.RemoveAll(venDir)
if err != nil {
return err
}
}
return nil
}

69
pkg/plugin/generator.go Normal file
View File

@ -0,0 +1,69 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package plugin
import (
"errors"
"fmt"
"os"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/builder"
"github.com/vulcanize/vulcanizedb/pkg/plugin/manager"
"github.com/vulcanize/vulcanizedb/pkg/plugin/writer"
)
type Generator interface {
GenerateExporterPlugin() error
}
type generator struct {
writer.PluginWriter
builder.PluginBuilder
manager.MigrationManager
}
func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
if len(gc.Initializers) < 1 {
return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
}
if len(gc.Dependencies) < 1 {
return nil, errors.New("generator needs to be configured with root repository path(s)")
}
if len(gc.Migrations) < 1 {
fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided for the plugin transformers\r\n")
}
return &generator{
PluginWriter: writer.NewPluginWriter(gc),
PluginBuilder: builder.NewPluginBuilder(gc, dbc),
MigrationManager: manager.NewMigrationManager(gc, dbc),
}, nil
}
func (g *generator) GenerateExporterPlugin() error {
err := g.PluginWriter.WritePlugin()
if err != nil {
return err
}
defer g.PluginBuilder.CleanUp()
err = g.PluginBuilder.BuildPlugin()
if err != nil {
return err
}
return g.MigrationManager.RunMigrations()
}

4
pkg/autogen/generator_suite_test.go → pkg/plugin/generator_suite_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package autogen_test package plugin_test
import ( import (
"io/ioutil" "io/ioutil"
@ -27,7 +27,7 @@ import (
func TestRepository(t *testing.T) { func TestRepository(t *testing.T) {
RegisterFailHandler(Fail) RegisterFailHandler(Fail)
RunSpecs(t, "Autogen Suite Test") RunSpecs(t, "Gen Suite Test")
} }
var _ = BeforeSuite(func() { var _ = BeforeSuite(func() {

30
pkg/autogen/generator_test.go → pkg/plugin/generator_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package autogen_test package plugin_test
import ( import (
"plugin" "plugin"
@ -25,34 +25,41 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher" "github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/autogen"
"github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/utils" p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
"github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers"
) )
var genConfig = autogen.Config{ var genConfig = config.Plugin{
Initializers: map[string]string{ Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite", "bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal", "deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
}, },
Dependencies: map[string]string{ Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers", "mcd_transformers": "github.com/vulcanize/mcd_transformers",
}, },
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
FileName: "externalTestTransformerSet", FileName: "externalTestTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test", FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false, Save: false,
} }
var dbConfig = config.Database{
Hostname: "localhost",
Port: 5432,
Name: "vulcanize_private",
}
type Exporter interface { type Exporter interface {
Export() []transformer.TransformerInitializer Export() []transformer.TransformerInitializer
} }
var _ = Describe("Generator test", func() { var _ = Describe("Generator test", func() {
var g autogen.Generator var g p2.Generator
var goPath, soPath string var goPath, soPath string
var err error var err error
var bc core.BlockChain var bc core.BlockChain
@ -65,13 +72,14 @@ var _ = Describe("Generator test", func() {
BeforeEach(func() { BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths() goPath, soPath, err = genConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
g = autogen.NewGenerator(genConfig, config.Database{}) g, err = p2.NewGenerator(genConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin() err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
AfterEach(func() { AfterEach(func() {
err := utils.ClearFiles(goPath, soPath) err := helpers.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })

61
pkg/plugin/helpers/helpers.go Normal file
View File

@ -0,0 +1,61 @@
package helpers
import (
"io"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/mitchellh/go-homedir"
)
func CleanPath(str string) (string, error) {
path, err := homedir.Expand(filepath.Clean(str))
if err != nil {
return "", err
}
if strings.Contains(path, "$GOPATH") {
env := os.Getenv("GOPATH")
spl := strings.Split(path, "$GOPATH")[1]
path = filepath.Join(env, spl)
}
return path, nil
}
func ClearFiles(files ...string) error {
for _, file := range files {
if _, err := os.Stat(file); err == nil {
err = os.Remove(file)
if err != nil {
return err
}
} else if os.IsNotExist(err) {
// fall through
} else {
return err
}
}
return nil
}
func CopyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL|os.O_WRONLY, os.FileMode(0666)) // Doesn't overwrite files
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, in)
if err != nil {
return err
}
return out.Close()
}

134
pkg/plugin/manager/manager.go Normal file
View File

@ -0,0 +1,134 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package manager
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
type MigrationManager interface {
RunMigrations() error
}
type manager struct {
GenConfig config.Plugin
DBConfig config.Database
tmpMigDir string
}
func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
return &manager{
GenConfig: gc,
DBConfig: dbc,
}
}
func (m *manager) RunMigrations() error {
// Get paths to db migrations
paths, err := m.GenConfig.GetMigrationsPaths()
if err != nil {
return err
}
if len(paths) < 1 {
return nil
}
// Init directory for temporary copies
err = m.setupMigrationEnv()
if err != nil {
return err
}
defer m.cleanUp()
// Create temporary copies of migrations to the temporary migrationDir
// These tmps are identical except they have had `1` added in front of their unix_timestamps
// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
// But will still be ran in the same order relative to one another
// TODO: Less hacky way of handing migrations
err = m.createMigrationCopies(paths)
if err != nil {
return err
}
// Run the copied migrations
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
err = exec.Command("migrate", "-path", m.tmpMigDir, "-database", pgStr, "up").Run()
if err != nil {
return errors.New(fmt.Sprintf("db migrations for plugin transformers failed: %s", err.Error()))
}
return nil
}
func (m *manager) setupMigrationEnv() error {
// Initialize temp directory for transformer migrations
var err error
m.tmpMigDir, err = helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil {
return err
}
err = os.RemoveAll(m.tmpMigDir)
if err != nil {
return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir))
}
err = os.Mkdir(m.tmpMigDir, os.FileMode(0777))
if err != nil {
return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir))
}
return nil
}
func (m *manager) createMigrationCopies(paths []string) error {
for _, path := range paths {
dir, err := ioutil.ReadDir(path)
if err != nil {
return err
}
for _, file := range dir {
if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum
continue
}
_, err := strconv.Atoi(file.Name()[:10])
if err != nil {
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name())
continue
}
src := filepath.Join(path, file.Name())
dst := filepath.Join(m.tmpMigDir, "1"+file.Name())
err = helpers.CopyFile(src, dst)
if err != nil {
return err
}
}
}
return nil
}
func (m *manager) cleanUp() error {
return os.RemoveAll(m.tmpMigDir)
}

0
pkg/autogen/test_helpers/database.go → pkg/plugin/test_helpers/database.go
View File

0
pkg/autogen/test_helpers/test/README.md → pkg/plugin/test_helpers/test/README.md
View File

92
pkg/plugin/writer/writer.go Normal file
View File

@ -0,0 +1,92 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package writer
import (
"errors"
"fmt"
. "github.com/dave/jennifer/jen"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
type PluginWriter interface {
WritePlugin() error
}
type writer struct {
GenConfig config.Plugin
}
func NewPluginWriter(gc config.Plugin) *writer {
return &writer{
GenConfig: gc,
}
}
// Generates the plugin code
func (w *writer) WritePlugin() error {
// Setup plugin file paths
goFile, err := w.setupFilePath()
if err != nil {
return err
}
// Begin code generation
f := NewFile("main")
f.HeaderComment("This is a plugin generated to export the configured transformer initializers")
// Import TransformerInitializers specified in config
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, imp := range w.GenConfig.Initializers {
f.ImportAlias(imp, alias)
}
// Collect TransformerInitializer names
importedInitializers := make([]Code, 0, len(w.GenConfig.Initializers))
for _, path := range w.GenConfig.Initializers {
importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer"))
}
// Create Exporter variable with method to export the set of the imported TransformerInitializers
f.Type().Id("exporter").String()
f.Var().Id("Exporter").Id("exporter")
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Block(
Return(Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers
// Write code to destination file
err = f.Save(goFile)
if err != nil {
return errors.New(fmt.Sprintf("failed to save generated .go file: %s\r\n%s", goFile, err.Error()))
}
return nil
}
func (w *writer) setupFilePath() (string, error) {
goFile, soFile, err := w.GenConfig.GetPluginPaths()
if err != nil {
return "", err
}
// Clear .go and .so files of the same name if they exist
return goFile, helpers.ClearFiles(goFile, soFile)
}

2
plugins/README.md
View File

@ -25,7 +25,7 @@ The config file requires, at a minimum, the below fields:
transformer4 = "github.com/different/path/to/transformer1" transformer4 = "github.com/different/path/to/transformer1"
[exporter.repositories] [exporter.repositories]
transformers = "github.com/path/to" transformers = "github.com/path/to"
transformer4 = "github.com/different/path transformer4 = "github.com/different/path"
[exporter.migrations] [exporter.migrations]
transformers = "db/migrations" transformers = "db/migrations"
transformer4 = "to/db/migrations" transformer4 = "to/db/migrations"

57
utils/utils.go
View File

@ -17,15 +17,10 @@
package utils package utils
import ( import (
"io" log "github.com/sirupsen/logrus"
"math/big" "math/big"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"syscall"
"github.com/mitchellh/go-homedir"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
@ -84,53 +79,3 @@ func RequestedBlockNumber(blockNumber *int64) *big.Int {
} }
return _blockNumber return _blockNumber
} }
func CleanPath(str string) (string, error) {
path, err := homedir.Expand(filepath.Clean(str))
if err != nil {
return "", err
}
if strings.Contains(path, "$GOPATH") {
env := os.Getenv("GOPATH")
spl := strings.Split(path, "$GOPATH")[1]
path = filepath.Join(env, spl)
}
return path, nil
}
func ClearFiles(files ...string) error {
for _, file := range files {
if _, err := os.Stat(file); err == nil {
err = os.Remove(file)
if err != nil {
return err
}
} else if os.IsNotExist(err) {
// fall through
} else {
return err
}
}
return nil
}
func CopyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL|os.O_WRONLY, os.FileMode(0666)) // Doesn't overwrite files
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, in)
if err != nil {
return err
}
return out.Close()
}