cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

goose changes; update plugin readme; use rel paths for transformers in config; more commenting in pkg/plugin; fix PluginWriter to work with different transformer types; reopen generator tests

Browse Source
This commit is contained in:
Ian Norden
2019-02-24 21:38:48 -06:00
parent 90e67d6da9
commit 2868cf2f73
10 changed files with 212 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/plugin/builder/builder.go
+18 -5
View File
@@ -28,6 +28,9 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
// Interface for compile Go code written by the
// PluginWriter into a shared object (.so file)
// which can be used loaded as a plugin
type PluginBuilder interface {
BuildPlugin() error
CleanUp() error
@@ -35,11 +38,12 @@ type PluginBuilder interface {
type builder struct {
GenConfig config.Plugin
tmpVenDirs []string
goFile string
tmpVenDirs []string // Keep track of temp vendor directories
goFile string // Keep track of goFile name
}
func NewPluginBuilder(gc config.Plugin, dbc config.Database) *builder {
// Requires populated plugin config
func NewPluginBuilder(gc config.Plugin) *builder {
return &builder{
GenConfig: gc,
tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
@@ -70,6 +74,7 @@ func (b *builder) BuildPlugin() error {
}
// Sets up temporary vendor libs needed for plugin build
// This is to work around a conflict between plugins and vendoring (https://github.com/golang/go/issues/20481)
func (b *builder) setupBuildEnv() error {
// TODO: Less hacky way of handling plugin build deps
vendorPath, err := helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor")
@@ -79,6 +84,11 @@ func (b *builder) setupBuildEnv() error {
// Import transformer dependencies so that we can build our plugin
for name, importPath := range b.GenConfig.Dependencies {
// Use dependency paths in config to form git ssh string
// TODO: Change this to https once we are no longer working private transformer repos
// Right now since vulcanize/mcd_transformers is a private repo we
// are using ssh and uploading ssh key to travis for testing
// This is slower and more involved than using https urls
index := strings.Index(importPath, "/")
gitPath := importPath[:index] + ":" + importPath[index+1:]
importURL := "git@" + gitPath + ".git"
@@ -87,18 +97,21 @@ func (b *builder) setupBuildEnv() error {
if err != nil {
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
return err
}
// Keep track of this vendor directory to clear later
b.tmpVenDirs = append(b.tmpVenDirs, depPath)
}
return nil
}
// Used to clear all of the tmp vendor libs used to build the plugin
// Also clears the go file if saving it has not been specified in the config
// Do not call until after the MigrationManager has performed its operations
// as it needs to pull the db migrations from the tmpVenDirs
func (b *builder) CleanUp() error {
if !b.GenConfig.Save {
err := helpers.ClearFiles(b.goFile)
pkg/plugin/generator.go
+9 -1
View File
@@ -27,6 +27,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/plugin/writer"
)
// Generator is the top-level interface for creating transformer plugins
type Generator interface {
GenerateExporterPlugin() error
}
@@ -37,6 +38,7 @@ type generator struct {
manager.MigrationManager
}
// Creates a new generator from a plugin and database config
func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
if len(gc.Initializers) < 1 {
return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
@@ -49,21 +51,27 @@ func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
}
return &generator{
PluginWriter: writer.NewPluginWriter(gc),
PluginBuilder: builder.NewPluginBuilder(gc, dbc),
PluginBuilder: builder.NewPluginBuilder(gc),
MigrationManager: manager.NewMigrationManager(gc, dbc),
}, nil
}
// Generates plugin for the transformer initializers specified in the generator config
// Writes plugin code => Sets up build environment => Builds .so file => Performs db migrations for the plugin transformers => Clean up
func (g *generator) GenerateExporterPlugin() error {
// Use plugin writer interface to write the plugin code
err := g.PluginWriter.WritePlugin()
if err != nil {
return err
}
// Clean up temporary files and directories when we are done
defer g.PluginBuilder.CleanUp()
// Use plugin builder interface to setup build environment and compile .go file into a .so file
err = g.PluginBuilder.BuildPlugin()
if err != nil {
return err
}
// Perform db migrations for the transformers
return g.MigrationManager.RunMigrations()
}
pkg/plugin/generator_suite_test.go
+1 -1
View File
@@ -27,7 +27,7 @@ import (
func TestRepository(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Gen Suite Test")
RunSpecs(t, "Plugin Suite Test")
}
var _ = BeforeSuite(func() {
pkg/plugin/generator_test.go
+36 -31
View File
@@ -16,7 +16,6 @@
package plugin_test
/*
import (
"plugin"
@@ -24,6 +23,7 @@ import (
. "github.com/onsi/gomega"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/config"
@@ -48,26 +48,28 @@ var genConfig = config.Plugin{
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
FileName: "testEventTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testEventTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var genStorageConfig = config.Plugin{
Initializers: map[string]string{
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
},
Types: map[string]config.PluginType{
"pit": config.EthStorage,
"vat": config.EthStorage,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var combinedConfig = config.Plugin{
@@ -75,19 +77,21 @@ var combinedConfig = config.Plugin{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
},
Types: map[string]config.PluginType{
"bite": config.EthEvent,
"deal": config.EthEvent,
"pit": config.EthStorage,
"vat": config.EthStorage,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testComboTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var dbConfig = config.Database{
@@ -156,9 +160,9 @@ var _ = Describe("Generator test", func() {
Expect(ok).To(Equal(true))
initializers, _ := exporter.Export()
w := watcher.NewWatcher(db, bc)
w := watcher.NewEventWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
err = w.Execute(constants.HeaderMissing)
Expect(err).ToNot(HaveOccurred())
type model struct {
@@ -180,8 +184,8 @@ var _ = Describe("Generator test", func() {
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(returned.Urn).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
@@ -195,7 +199,7 @@ var _ = Describe("Generator test", func() {
Describe("Storage Transformers only", func() {
BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths()
goPath, soPath, err = genStorageConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(genStorageConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
@@ -216,12 +220,12 @@ var _ = Describe("Generator test", func() {
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
event, initializers := exporter.Export()
Expect(len(initializers)).To(Equal(1))
Expect(len(initializers)).To(Equal(2))
Expect(len(event)).To(Equal(0))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of StorageTransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
db, _ = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
plug, err := plugin.Open(soPath)
@@ -235,15 +239,16 @@ var _ = Describe("Generator test", func() {
tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")}
w := watcher.NewStorageWatcher(tailer, db)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
// This blocks right now, need to make test file to read from
//err = w.Execute()
//Expect(err).ToNot(HaveOccurred())
})
})
})
Describe("Event and Storage Transformers in same instance", func() {
BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths()
goPath, soPath, err = combinedConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(combinedConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
@@ -265,7 +270,7 @@ var _ = Describe("Generator test", func() {
Expect(ok).To(Equal(true))
eventInitializers, storageInitializers := exporter.Export()
Expect(len(eventInitializers)).To(Equal(2))
Expect(len(storageInitializers)).To(Equal(1))
Expect(len(storageInitializers)).To(Equal(2))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers and StorageTransformerInitializers that we can execute over", func() {
@@ -286,9 +291,9 @@ var _ = Describe("Generator test", func() {
Expect(ok).To(Equal(true))
eventInitializers, storageInitializers := exporter.Export()
ew := watcher.NewWatcher(db, bc)
ew := watcher.NewEventWatcher(db, bc)
ew.AddTransformers(eventInitializers)
err = ew.Execute()
err = ew.Execute(constants.HeaderMissing)
Expect(err).ToNot(HaveOccurred())
type model struct {
@@ -310,8 +315,8 @@ var _ = Describe("Generator test", func() {
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(returned.Urn).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
@@ -323,10 +328,10 @@ var _ = Describe("Generator test", func() {
tailer := fs.FileTailer{Path: viper.GetString("filesystem.storageDiffsPath")}
sw := watcher.NewStorageWatcher(tailer, db)
sw.AddTransformers(storageInitializers)
err = sw.Execute()
Expect(err).ToNot(HaveOccurred())
// This blocks right now, need to make test file to read from
//err = w.Execute()
//Expect(err).ToNot(HaveOccurred())
})
})
})
})
*/
pkg/plugin/manager/manager.go
+18 -24
View File
@@ -19,16 +19,15 @@ package manager
import (
"errors"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
// Interface for managing the db migrations for plugin transformers
type MigrationManager interface {
RunMigrations() error
}
@@ -39,6 +38,7 @@ type manager struct {
tmpMigDir string
}
// Manager requires both filled in generator and database configs
func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
return &manager{
GenConfig: gc,
@@ -47,7 +47,7 @@ func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
}
func (m *manager) RunMigrations() error {
// Get paths to db migrations
// Get paths to db migrations from the plugin config
paths, err := m.GenConfig.GetMigrationsPaths()
if err != nil {
return err
@@ -55,27 +55,22 @@ func (m *manager) RunMigrations() error {
if len(paths) < 1 {
return nil
}
// Init directory for temporary copies
// Init directory for temporary copies of migrations
err = m.setupMigrationEnv()
if err != nil {
return err
}
defer m.cleanUp()
// Create temporary copies of migrations to the temporary migrationDir
// These tmps are identical except they have had `1` added in front of their unix_timestamps
// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
// But will still be ran in the same order relative to one another
// TODO: Less hacky way of handing migrations
// Creates copies of migrations for all the plugin's transformers in a tmp dir
err = m.createMigrationCopies(paths)
if err != nil {
return err
}
// Run the copied migrations
// Run the copied migrations with goose
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
err = exec.Command("migrate", "-path", m.tmpMigDir, "-database", pgStr, "up").Run()
cmd := exec.Command("goose", "postgres", pgStr, "up")
cmd.Dir = m.tmpMigDir
err = cmd.Run()
if err != nil {
return errors.New(fmt.Sprintf("db migrations for plugin transformers failed: %s", err.Error()))
}
@@ -83,8 +78,8 @@ func (m *manager) RunMigrations() error {
return nil
}
// Setup a temporary directory to hold transformer db migrations
func (m *manager) setupMigrationEnv() error {
// Initialize temp directory for transformer migrations
var err error
m.tmpMigDir, err = helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil {
@@ -102,23 +97,22 @@ func (m *manager) setupMigrationEnv() error {
return nil
}
// Create copies of db migrations from vendored libs
func (m *manager) createMigrationCopies(paths []string) error {
// Iterate through migration paths to find migration directory
for _, path := range paths {
dir, err := ioutil.ReadDir(path)
if err != nil {
return err
}
// For each file in the directory check if it is a migration
for _, file := range dir {
if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum
continue
}
_, err := strconv.Atoi(file.Name()[:10])
if err != nil {
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name())
if file.IsDir() || filepath.Ext(file.Name()) != ".sql" {
continue
}
src := filepath.Join(path, file.Name())
dst := filepath.Join(m.tmpMigDir, "1"+file.Name())
dst := filepath.Join(m.tmpMigDir, file.Name())
// and if it is make a copy of it to our tmp migration directory
err = helpers.CopyFile(src, dst)
if err != nil {
return err
pkg/plugin/test_helpers/database.go
+16
View File
@@ -1,3 +1,19 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package test_helpers
import (
pkg/plugin/writer/writer.go
+24 -14
View File
@@ -26,6 +26,9 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
// Interface for writing a .go file for a simple
// plugin that exports the set of transformer
// initializers specified in the config
type PluginWriter interface {
WritePlugin() error
}
@@ -34,13 +37,14 @@ type writer struct {
GenConfig config.Plugin
}
// Requires populated plugin config
func NewPluginWriter(gc config.Plugin) *writer {
return &writer{
GenConfig: gc,
}
}
// Generates the plugin code
// Generates the plugin code according to config specification
func (w *writer) WritePlugin() error {
// Setup plugin file paths
goFile, err := w.setupFilePath()
@@ -52,10 +56,10 @@ func (w *writer) WritePlugin() error {
f := NewFile("main")
f.HeaderComment("This is a plugin generated to export the configured transformer initializers")
// Import TransformerInitializers specified in config
// Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, imp := range w.GenConfig.Initializers {
f.ImportAlias(imp, alias)
for alias, relPath := range w.GenConfig.Initializers {
f.ImportAlias(w.makePath(alias, relPath), alias)
}
// Collect initializer code
@@ -64,17 +68,16 @@ func (w *writer) WritePlugin() error {
// Create Exporter variable with method to export the set of the imported storage and event transformer initializers
f.Type().Id("exporter").String()
f.Var().Id("Exporter").Id("exporter")
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Block(
Return(Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(ethEventInitializers...)),
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Parens(List(
Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "TransformerInitializer"),
Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "StorageTransformerInitializer"),
)).Block(Return(
Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Values(ethStorageInitializers...)) // Exports the collected initializers
"TransformerInitializer").Values(ethEventInitializers...),
Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers
// Write code to destination file
err = f.Save(goFile)
@@ -84,8 +87,8 @@ func (w *writer) WritePlugin() error {
return nil
}
// Collect code for various types of initializers
func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
// Collect code for various initializers
importedEthEventInitializers := make([]Code, 0)
importerEthStorageInitializers := make([]Code, 0)
importedIpfsEventInitializers := make([]Code, 0)
@@ -109,6 +112,13 @@ func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
importerIpfsStorageInitializers
}
// Concat relative path with its repo's root path
func (w *writer) makePath(alias, relPath string) string {
pathRoot := w.GenConfig.Dependencies[alias]
return pathRoot + "/" + relPath
}
// Setup the .go, clear old ones if present
func (w *writer) setupFilePath() (string, error) {
goFile, soFile, err := w.GenConfig.GetPluginPaths()
if err != nil {