deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

auto management of dependencies temporarily needed to build plugin and transformer

Browse Source 
migrations; new test showing it is working with external transformers runs
properly when migrations were manually performed but still need
to test automated migration management
This commit is contained in:
Ian Norden 2019-01-30 19:38:56 -06:00
parent 6c2d895023
commit 55fa9b8364
9 changed files with 529 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
cmd/composeAndExecute.go
View File

@ -47,11 +47,19 @@ var composeAndExecuteCmd = &cobra.Command{
ipcPath = "http://kovan0.vulcanize.io:8545"
[exporter]
filePath = "~/go/src/github.com/vulcanize/vulcanizedb/plugins"
fileName = "exporter"
filePath = "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/"
fileName = "exporter"
[exporter.transformers]
transformerImport1 = "github.com/path_to/transformerInitializer1"
transformerImport2 = "github.com/path_to/transformerInitializer2"
transformer1 = "github.com/path/to/transformer1"
transformer2 = "github.com/path/to/transformer2"
transformer3 = "github.com/path/to/transformer3"
transformer4 = "github.com/different/path/to/transformer1"
[exporter.repositories]
transformers = "github.com/path/to"
transformer4 = "github.com/different/path
[exporter.migrations]
transformers = "db/migrations"
transformer4 = "to/db/migrations"
Note: If any of the imported transformer need additional
config variables do not forget to include those as well
@ -66,43 +74,50 @@ loaded into and executed over by a generic watcher`,
}
func composeAndExecute() {
generator := autogen.NewGenerator(autogenConfig)
err := generator.GenerateTransformerPlugin()
// generate code to build the plugin according to the config file
generator := autogen.NewGenerator(autogenConfig, databaseConfig)
err := generator.GenerateExporterPlugin()
if err != nil {
log.Fatal(err)
}
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
blockChain := getBlockChain()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
_, pluginPath, err := autogen.GetPaths(autogenConfig)
// Get the plugin path and load the plugin
_, pluginPath, err := autogenConfig.GetPluginPaths()
if err != nil {
log.Fatal(err)
}
plug, err := plugin.Open(pluginPath)
if err != nil {
log.Fatal(err)
}
// Load the `Exporter` symbol from the plugin
symExporter, err := plug.Lookup("Exporter")
if err != nil {
log.Fatal(err)
}
// Assert that the symbol is of type Exporter
exporter, ok := symExporter.(Exporter)
if !ok {
fmt.Println("plugged-in symbol not of type Exporter")
os.Exit(1)
}
// Use the Exporters export method to load the TransformerInitializer set
initializers := exporter.Export()
// Setup bc and db objects
blockChain := getBlockChain()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
// Create a watcher and load the TransformerInitializer set into it
w := watcher.NewWatcher(&db, blockChain)
w.AddTransformers(initializers)
// Execute over the TransformerInitializer set using the watcher
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
err := w.Execute()
if err != nil {

8
cmd/root.go
View File

@ -77,9 +77,11 @@ func configure(cmd *cobra.Command, args []string) {
Password: viper.GetString("database.password"),
}
autogenConfig = autogen.Config{
FilePath: viper.GetString("exporter.filePath"),
FileName: viper.GetString("exporter.fileName"),
Imports: viper.GetStringMapString("exporter.transformers"),
FilePath: viper.GetString("exporter.filePath"),
FileName: viper.GetString("exporter.fileName"),
Initializers: viper.GetStringMapString("exporter.transformers"),
Dependencies: viper.GetStringMapString("exporter.repositories"),
Migrations: viper.GetStringMapString("exporter.migrations"),
}
viper.Set("database.config", databaseConfig)
}

20
environments/compose.toml
View File

@ -1,24 +1,24 @@
[database]
name = "vulcanize_public"
name = "vulcanize_public"
hostname = "localhost"
user = "vulcanize"
user = "vulcanize"
password = "vulcanize"
port = 5432
port = 5432
[client]
ipcPath = "http://kovan0.vulcanize.io:8545"
ipcPath = "http://kovan0.vulcanize.io:8545"
[datadog]
name = "maker_vdb_staging"
name = "maker_vdb_staging"
[exporter]
filePath = "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/"
fileName = "exporter"
[exporter.transformers]
bite = "github.com/vulcanize/mcd_transformers/transformers/bite"
cat_chop_lump = "github.com/vulcanize/maker_transformers/cat/chop_lump"
cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat/flip"
cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat/pit_vow"
cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump"
cat_flip = "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip"
cat_pit_vow = "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow"
deal = "github.com/vulcanize/mcd_transformers/transformers/deal"
dent = "github.com/vulcanize/mcd_transformers/transformers/dent"
drip_drip = "github.com/vulcanize/mcd_transformers/transformers/drip_drip"
@ -43,6 +43,10 @@
vat_toll = "github.com/vulcanize/mcd_transformers/transformers/vat_toll"
vat_tune = "github.com/vulcanize/mcd_transformers/transformers/vat_tune"
vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog"
[exporter.repositories]
mcd_transformers = "github.com/vulcanize/mcd_transformers"
[exporter.migrations]
mcd_transformers = "db/migrations"
[contract]
[contract.address]

48
pkg/autogen/config.go
View File

@ -16,8 +16,50 @@
package autogen
import (
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/vulcanize/vulcanizedb/utils"
)
type Config struct {
Imports map[string]string // Map of import alias to import path
FilePath string
FileName string
Initializers map[string]string // Map of import aliases to transformer paths
Dependencies map[string]string // Map of vendor dep names to their repositories
Migrations map[string]string // Map of migration names to their paths within the vendored deps
FilePath string
FileName string
}
func (c *Config) GetPluginPaths() (string, string, error) {
path, err := utils.CleanPath(c.FilePath)
if err != nil {
return "", "", err
}
name := strings.Split(c.FileName, ".")[0]
goFile := filepath.Join(path, name+".go")
soFile := filepath.Join(path, name+".so")
return goFile, soFile, nil
}
func (c *Config) GetMigrationsPaths() ([]string, error) {
paths := make([]string, 0, len(c.Migrations))
for key, relPath := range c.Migrations {
repo, ok := c.Dependencies[key]
if !ok {
return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath))
}
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath)
cleanPath, err := utils.CleanPath(path)
if err != nil {
return nil, err
}
paths = append(paths, cleanPath)
}
return paths, nil
}

210
pkg/autogen/generator.go
View File

@ -18,65 +18,97 @@ package autogen
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"strconv"
. "github.com/dave/jennifer/jen"
"github.com/mitchellh/go-homedir"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/utils"
)
type Generator interface {
GenerateTransformerPlugin() error
GenerateExporterPlugin() error
}
type generator struct {
*Config
GenConfig *Config
DBConfig config.Database
tmpMigDir string
tmpVenDirs []string
}
func NewGenerator(config Config) *generator {
func NewGenerator(gc Config, dbc config.Database) *generator {
return &generator{
Config: &config,
GenConfig: &gc,
DBConfig: dbc,
}
}
func (g *generator) GenerateTransformerPlugin() error {
if g.Config == nil {
func (g *generator) GenerateExporterPlugin() error {
if g.GenConfig == nil {
return errors.New("generator needs a config file")
}
if g.Config.FilePath == "" {
if g.GenConfig.FilePath == "" {
return errors.New("generator is missing file path")
}
if len(g.Config.Imports) < 1 {
if len(g.GenConfig.Initializers) < 1 {
return errors.New("generator needs to be configured with imports")
}
// Create file path
goFile, soFile, err := GetPaths(*g.Config)
// Get plugin file paths
goFile, soFile, err := g.GenConfig.GetPluginPaths()
if err != nil {
return err
}
// Clear previous .go and .so files if they exist
err = ClearFiles(goFile, soFile)
// Clear .go and .so files of the same name if they exist (overwrite)
err = utils.ClearFiles(goFile, soFile)
if err != nil {
return err
}
// Generate Exporter code
err = g.generateCode(goFile)
if err != nil {
return err
}
// Setup temp vendor lib and migrations directories
err = g.setupTempDirs()
if err != nil {
return err
}
defer g.cleanUp() // Clear these up when we are done building our plugin
// Build the .go file into a .so plugin
err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run()
if err != nil {
return err
}
// Run migrations only after successfully building .so file
return g.runMigrations()
}
// Generates the plugin code
func (g *generator) generateCode(goFile string) error {
// Begin code generation
f := NewFile("main")
f.HeaderComment("This exporter is generated to export the configured transformer initializers")
// Import TransformerInitializers
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, imp := range g.Config.Imports {
for alias, imp := range g.GenConfig.Initializers {
f.ImportAlias(imp, alias)
}
// Collect TransformerInitializer names
importedInitializers := make([]Code, 0, len(g.Config.Imports))
for _, path := range g.Config.Imports {
importedInitializers := make([]Code, 0, len(g.GenConfig.Initializers))
for _, path := range g.GenConfig.Initializers {
importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer"))
}
@ -90,49 +122,143 @@ func (g *generator) GenerateTransformerPlugin() error {
"TransformerInitializer").Block(
Return(Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(importedInitializers...)))
"TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers
// Write code to destination file
err = f.Save(goFile)
return f.Save(goFile)
}
func (g *generator) runMigrations() error {
// Get paths to db migrations
paths, err := g.GenConfig.GetMigrationsPaths()
if err != nil {
return err
}
if len(paths) < 1 {
return nil
}
// Create temporary copies of migrations to the temporary migrationDir
// These tmps are identical except they have had `1` added in front of their unix_timestamps
// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
// But will still be ran in the same order relative to one another
// TODO: Less hacky way of handing migrations
err = g.createMigrationCopies(paths)
if err != nil {
return err
}
// Build the .go file into a .so plugin
return exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run()
// Run the copied migrations
location := "file://" + g.tmpMigDir
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable up", g.DBConfig.Hostname, g.DBConfig.Port, g.DBConfig.Name)
return exec.Command("migrate", "-source", location, pgStr).Run()
}
func GetPaths(config Config) (string, string, error) {
path, err := homedir.Expand(filepath.Clean(config.FilePath))
// Sets up temporary vendor libs and migration directories
func (g *generator) setupTempDirs() error {
// TODO: Less hacky way of handling plugin build deps
dirPath, err := utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/")
if err != nil {
return "", "", err
return err
}
if strings.Contains(path, "$GOPATH") {
env := os.Getenv("GOPATH")
spl := strings.Split(path, "$GOPATH")[1]
path = filepath.Join(env, spl)
vendorPath := filepath.Join(dirPath, "vendor/")
/*
// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
// Import transformer dependencies so that we build our plugin
for _, importPath := range g.GenConfig.Dependencies {
importURL := "https://" + importPath + ".git"
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return err
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
return err
}
g.tmpVenDirs = append(g.tmpVenDirs, depPath)
}
*/
// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
for _, importPath := range g.GenConfig.Dependencies {
depPath := filepath.Join(vendorPath, importPath)
g.tmpVenDirs = append(g.tmpVenDirs, depPath)
}
name := strings.Split(config.FileName, ".")[0]
goFile := filepath.Join(path, name+".go")
soFile := filepath.Join(path, name+".so")
// Dep ensure to make sure vendor pkgs are in place for building the plugin
err = exec.Command("dep", "ensure").Run()
if err != nil {
return errors.New("failed to vendor transformer packages required to build plugin")
}
return goFile, soFile, nil
// Git checkout our head-state vendor libraries
// This is necessary because we currently need to manual edit our vendored
// go-ethereum abi library to allow for unpacking in empty interfaces and maps
// This can be removed once the PRs against geth merged
err = exec.Command("git", "checkout", dirPath).Run()
if err != nil {
return errors.New("failed to checkout vendored go-ethereum lib")
}
// Initialize temp directory for transformer migrations
g.tmpMigDir, err = utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil {
return err
}
stat, err := os.Stat(g.tmpMigDir)
if err == nil {
if !stat.IsDir() {
return errors.New(fmt.Sprintf("file %s found where directory is expected", stat.Name()))
}
} else if os.IsNotExist(err) {
os.Mkdir(g.tmpMigDir, os.FileMode(0777))
} else {
return err
}
return nil
}
func ClearFiles(files ...string) error {
for _, file := range files {
if _, err := os.Stat(file); err == nil {
err = os.Remove(file)
if err != nil {
return err
}
} else if os.IsNotExist(err) {
// fall through
} else {
func (g *generator) createMigrationCopies(paths []string) error {
for _, path := range paths {
dir, err := ioutil.ReadDir(path)
if err != nil {
return err
}
for _, file := range dir {
if file.IsDir() || len(file.Name()) < 15 { // (10 digit unix time stamp + x + .sql) is bare minimum
continue
}
_, err := strconv.Atoi(file.Name()[:10])
if err != nil {
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix", file.Name())
continue
}
if filepath.Ext(file.Name()) == "sql" {
src := filepath.Join(path, file.Name())
dst := filepath.Join(g.tmpMigDir, "1"+file.Name())
err = utils.CopyFile(src, dst)
if err != nil {
return err
}
}
}
}
return nil
}
func (g *generator) cleanUp() error {
for _, venDir := range g.tmpVenDirs {
err := os.RemoveAll(venDir)
if err != nil {
return err
}
}
return os.RemoveAll(g.tmpMigDir)
}

213
pkg/autogen/generator_test.go
View File

@ -27,28 +27,33 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/autogen"
"github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
"github.com/vulcanize/vulcanizedb/utils"
)
var testConfig = autogen.Config{
Imports: map[string]string{
var localConfig = autogen.Config{
Initializers: map[string]string{
"bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite",
"deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal",
},
FileName: "testTransformerSet",
FileName: "localTestTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/",
}
var targetConfig = autogen.Config{
Imports: map[string]string{
"bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite",
"deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal",
var externalConfig = autogen.Config{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal",
},
FileName: "targetTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/target/",
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
FileName: "externalTestTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/",
}
type Exporter interface {
@ -66,73 +71,147 @@ var _ = Describe("Generator test", func() {
viper.SetConfigName("compose")
viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
BeforeEach(func() {
goPath, soPath, err = autogen.GetPaths(testConfig)
Expect(err).ToNot(HaveOccurred())
g = autogen.NewGenerator(testConfig)
err = g.GenerateTransformerPlugin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
err := autogen.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Describe("Using local config", func() {
BeforeEach(func() {
goPath, soPath, err = localConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
g = autogen.NewGenerator(localConfig, config.Database{})
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
AfterEach(func() {
err := utils.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
bite.BiteModel
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
})
})
Describe("Using external config", func() {
BeforeEach(func() {
goPath, soPath, err = externalConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
type model struct {
bite.BiteModel
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
g = autogen.NewGenerator(externalConfig, config.Database{})
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
AfterEach(func() {
err := utils.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
bite.BiteModel
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
})
})
})

34
plugins/README.md
View File

@ -23,36 +23,54 @@ The config file requires, at a minimum, the below fields:
transformer1 = "github.com/path/to/transformer1"
transformer2 = "github.com/path/to/transformer2"
transformer3 = "github.com/path/to/transformer3"
transformer4 = "github.com/different/path/to/transformer1"
[exporter.repositories]
transformers = "github.com/path/to"
transformer4 = "github.com/different/path
[exporter.migrations]
transformers = "db/migrations"
transformer4 = "to/db/migrations"
```
In the above, the exporter.transformers are mappings of import aliases to their import paths
- `exporter.transformers` are mappings of import aliases to paths to `TransformerInitializer`s
- Import aliases can be arbitrarily named but note that `interface1` is a reserved alias needed for the generic TransformerInitializer type
- `exporter.repositores` are the paths to the repositories which contain the transformers
- `exporter.migrations` are the relative paths to the db migrations found within the `exporter.repositores`
- Migrations need to be located in the repos in `exporter.repositores`
- Keys should match the keys for the corresponding repo
If the individual transformers require additional configuration variables be sure to include them in the .toml file
The general structure of a plugin .go file, and what we would see with the above config is below
Note that `shared_transformer` is a reserved alias needed for the generic TransformerInitializer type:
The general structure of a plugin .go file, and what we would see with the above config is shown below
```go
package main
import (
shared_transformer "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
interface1 "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
transformer1 "github.com/path/to/transformer1"
transformer2 "github.com/path/to/transformer2"
transformer3 "github.com/path/to/transformer3"
transformer4 "github.com/different/path/to/transformer1"
)
type exporter string
var Exporter exporter
func (e exporter) Export() []shared_transformer.TransformerInitializer {
return []shared_transformer.TransformerInitializer{
func (e exporter) Export() []interface1.TransformerInitializer {
return []interface1.TransformerInitializer{
transformer1.TransformerInitializer,
transformer2.TransformerInitializer,
transformer3.TransformerInitializer,
transformer4.TransformerInitializer,
}
}
```
As such, to plug in an external transformer all we need to do is create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19)
As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them
As such, to plug in an external transformer we need to create a [package](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/pkg/autogen/test_helpers/bite/initializer.go) that exports a variable `TransformerInitializer` that is of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/transformer.go#L19)
As long as the imported transformers abide by the required interfaces, we can execute over any arbitrary set of them
Note: currently the transformers must also operate using the watcher's [execution mode](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/watcher.go#L80)
For each transformer we will also need to create db migrations to run against vulcanizeDB so that we can store the transformed data
The migrations needed for a specific transformer need to be included in the same repository as the transformers that require them, and their relative paths in that repo specified in the config as discussed above

43
plugins/example_maker_exporter Normal file
View File

@ -0,0 +1,43 @@
// This should be the output from running composeAndExecute with compose.toml
package main
import (
cat_chop_lump "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump"
bite "github.com/vulcanize/mcd_transformers/transformers/bite"
cat_flip "github.com/vulcanize/mcd_transformers/transformers/cat_file/flip"
cat_pit_vow "github.com/vulcanize/mcd_transformers/transformers/cat_file/pit_vow"
deal "github.com/vulcanize/mcd_transformers/transformers/deal"
dent "github.com/vulcanize/mcd_transformers/transformers/dent"
drip_drip "github.com/vulcanize/mcd_transformers/transformers/drip_drip"
drip_file_ilk "github.com/vulcanize/mcd_transformers/transformers/drip_file/ilk"
drip_file_repo "github.com/vulcanize/mcd_transformers/transformers/drip_file/repo"
drip_file_vow "github.com/vulcanize/mcd_transformers/transformers/drip_file/vow"
flap_kick "github.com/vulcanize/mcd_transformers/transformers/flap_kick"
flip_kick "github.com/vulcanize/mcd_transformers/transformers/flip_kick"
flop_kick "github.com/vulcanize/mcd_transformers/transformers/flop_kick"
frob "github.com/vulcanize/mcd_transformers/transformers/frob"
pit_file_debt_ceiling "github.com/vulcanize/mcd_transformers/transformers/pit_file/debt_ceiling"
pit_file_ilk "github.com/vulcanize/mcd_transformers/transformers/pit_file/ilk"
price_feeds "github.com/vulcanize/mcd_transformers/transformers/price_feeds"
tend "github.com/vulcanize/mcd_transformers/transformers/tend"
vat_flux "github.com/vulcanize/mcd_transformers/transformers/vat_flux"
vat_fold "github.com/vulcanize/mcd_transformers/transformers/vat_fold"
vat_grab "github.com/vulcanize/mcd_transformers/transformers/vat_grab"
vat_heal "github.com/vulcanize/mcd_transformers/transformers/vat_heal"
vat_init "github.com/vulcanize/mcd_transformers/transformers/vat_init"
vat_move "github.com/vulcanize/mcd_transformers/transformers/vat_move"
vat_slip "github.com/vulcanize/mcd_transformers/transformers/vat_slip"
vat_toll "github.com/vulcanize/mcd_transformers/transformers/vat_toll"
vat_tune "github.com/vulcanize/mcd_transformers/transformers/vat_tune"
vow_flog "github.com/vulcanize/mcd_transformers/transformers/vow_flog"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
)
type exporter string
var Exporter exporter
func (e exporter) Export() []transformer.TransformerInitializer {
return []transformer.TransformerInitializer{deal.TransformerInitializer, cat_chop_lump.TransformerInitializer, vat_slip.TransformerInitializer, bite.TransformerInitializer, vat_heal.TransformerInitializer, vat_toll.TransformerInitializer, price_feeds.TransformerInitializer, vat_init.TransformerInitializer, cat_pit_vow.TransformerInitializer, drip_drip.TransformerInitializer, vat_grab.TransformerInitializer, tend.TransformerInitializer, pit_file_ilk.TransformerInitializer, vat_fold.TransformerInitializer, vat_tune.TransformerInitializer, dent.TransformerInitializer, vow_flog.TransformerInitializer, flip_kick.TransformerInitializer, vat_flux.TransformerInitializer, frob.TransformerInitializer, flap_kick.TransformerInitializer, drip_file_repo.TransformerInitializer, flop_kick.TransformerInitializer, vat_move.TransformerInitializer, cat_flip.TransformerInitializer, drip_file_ilk.TransformerInitializer, drip_file_vow.TransformerInitializer, pit_file_debt_ceiling.TransformerInitializer}
}

55
utils/utils.go
View File

@ -17,10 +17,14 @@
package utils
import (
"io"
"math/big"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/mitchellh/go-homedir"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/config"
@ -80,3 +84,54 @@ func RequestedBlockNumber(blockNumber *int64) *big.Int {
}
return _blockNumber
}
func CleanPath(str string) (string, error) {
path, err := homedir.Expand(filepath.Clean(str))
if err != nil {
return "", err
}
if strings.Contains(path, "$GOPATH") {
env := os.Getenv("GOPATH")
spl := strings.Split(path, "$GOPATH")[1]
path = filepath.Join(env, spl)
}
return path, nil
}
func ClearFiles(files ...string) error {
for _, file := range files {
if _, err := os.Stat(file); err == nil {
err = os.Remove(file)
if err != nil {
return err
}
} else if os.IsNotExist(err) {
// fall through
} else {
return err
}
}
return nil
}
func CopyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL, os.FileMode(0666)) // Doesn't overwrite files
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, in)
if err != nil {
return err
}
return out.Close()
}