refactoring plugin generating code

This commit is contained in:
Ian Norden
2019-02-24 21:38:47 -06:00
parent decc2a3caf
commit a322372713
16 changed files with 549 additions and 361 deletions
+118
View File
@@ -0,0 +1,118 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package builder
import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
type PluginBuilder interface {
BuildPlugin() error
CleanUp() error
}
type builder struct {
GenConfig config.Plugin
tmpVenDirs []string
goFile string
}
func NewPluginBuilder(gc config.Plugin, dbc config.Database) *builder {
return &builder{
GenConfig: gc,
tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
}
}
func (b *builder) BuildPlugin() error {
// Get plugin .go and .so file paths
var err error
var soFile string
b.goFile, soFile, err = b.GenConfig.GetPluginPaths()
if err != nil {
return err
}
// setup env to build plugin
err = b.setupBuildEnv()
if err != nil {
return err
}
// Build the .go file into a .so plugin
err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, b.goFile).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to build .so file: %s", err.Error()))
}
return nil
}
// Sets up temporary vendor libs needed for plugin build
func (b *builder) setupBuildEnv() error {
// TODO: Less hacky way of handling plugin build deps
vendorPath, err := helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor")
if err != nil {
return err
}
// Import transformer dependencies so that we can build our plugin
for name, importPath := range b.GenConfig.Dependencies {
index := strings.Index(importPath, "/")
gitPath := importPath[:index] + ":" + importPath[index+1:]
importURL := "git@" + gitPath + ".git"
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
return err
}
b.tmpVenDirs = append(b.tmpVenDirs, depPath)
}
return nil
}
func (b *builder) CleanUp() error {
if !b.GenConfig.Save {
err := helpers.ClearFiles(b.goFile)
if err != nil {
return err
}
}
for _, venDir := range b.tmpVenDirs {
err := os.RemoveAll(venDir)
if err != nil {
return err
}
}
return nil
}
+69
View File
@@ -0,0 +1,69 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package plugin
import (
"errors"
"fmt"
"os"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/builder"
"github.com/vulcanize/vulcanizedb/pkg/plugin/manager"
"github.com/vulcanize/vulcanizedb/pkg/plugin/writer"
)
type Generator interface {
GenerateExporterPlugin() error
}
type generator struct {
writer.PluginWriter
builder.PluginBuilder
manager.MigrationManager
}
func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
if len(gc.Initializers) < 1 {
return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
}
if len(gc.Dependencies) < 1 {
return nil, errors.New("generator needs to be configured with root repository path(s)")
}
if len(gc.Migrations) < 1 {
fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided for the plugin transformers\r\n")
}
return &generator{
PluginWriter: writer.NewPluginWriter(gc),
PluginBuilder: builder.NewPluginBuilder(gc, dbc),
MigrationManager: manager.NewMigrationManager(gc, dbc),
}, nil
}
func (g *generator) GenerateExporterPlugin() error {
err := g.PluginWriter.WritePlugin()
if err != nil {
return err
}
defer g.PluginBuilder.CleanUp()
err = g.PluginBuilder.BuildPlugin()
if err != nil {
return err
}
return g.MigrationManager.RunMigrations()
}
+35
View File
@@ -0,0 +1,35 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package plugin_test
import (
"io/ioutil"
"log"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestRepository(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Gen Suite Test")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})
+151
View File
@@ -0,0 +1,151 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package plugin_test
import (
"plugin"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
"github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers"
)
var genConfig = config.Plugin{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
//Migrations: map[string]string{"mcd_transformers" : "db/migrations"},
FileName: "externalTestTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var dbConfig = config.Database{
Hostname: "localhost",
Port: 5432,
Name: "vulcanize_private",
}
type Exporter interface {
Export() []transformer.TransformerInitializer
}
var _ = Describe("Generator test", func() {
var g p2.Generator
var goPath, soPath string
var err error
var bc core.BlockChain
var db *postgres.DB
var hr repositories.HeaderRepository
var headerID int64
viper.SetConfigName("compose")
viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(genConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
err := helpers.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
Ilk string
Urn string
Ink string
Art string
IArt string
Tab string
NFlip string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
})
})
+61
View File
@@ -0,0 +1,61 @@
package helpers
import (
"io"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/mitchellh/go-homedir"
)
func CleanPath(str string) (string, error) {
path, err := homedir.Expand(filepath.Clean(str))
if err != nil {
return "", err
}
if strings.Contains(path, "$GOPATH") {
env := os.Getenv("GOPATH")
spl := strings.Split(path, "$GOPATH")[1]
path = filepath.Join(env, spl)
}
return path, nil
}
func ClearFiles(files ...string) error {
for _, file := range files {
if _, err := os.Stat(file); err == nil {
err = os.Remove(file)
if err != nil {
return err
}
} else if os.IsNotExist(err) {
// fall through
} else {
return err
}
}
return nil
}
func CopyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, syscall.O_CREAT|syscall.O_EXCL|os.O_WRONLY, os.FileMode(0666)) // Doesn't overwrite files
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, in)
if err != nil {
return err
}
return out.Close()
}
+134
View File
@@ -0,0 +1,134 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package manager
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
type MigrationManager interface {
RunMigrations() error
}
type manager struct {
GenConfig config.Plugin
DBConfig config.Database
tmpMigDir string
}
func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
return &manager{
GenConfig: gc,
DBConfig: dbc,
}
}
func (m *manager) RunMigrations() error {
// Get paths to db migrations
paths, err := m.GenConfig.GetMigrationsPaths()
if err != nil {
return err
}
if len(paths) < 1 {
return nil
}
// Init directory for temporary copies
err = m.setupMigrationEnv()
if err != nil {
return err
}
defer m.cleanUp()
// Create temporary copies of migrations to the temporary migrationDir
// These tmps are identical except they have had `1` added in front of their unix_timestamps
// As such, they will be ran on top of all core migrations (at least, for the next ~317 years)
// But will still be ran in the same order relative to one another
// TODO: Less hacky way of handing migrations
err = m.createMigrationCopies(paths)
if err != nil {
return err
}
// Run the copied migrations
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
err = exec.Command("migrate", "-path", m.tmpMigDir, "-database", pgStr, "up").Run()
if err != nil {
return errors.New(fmt.Sprintf("db migrations for plugin transformers failed: %s", err.Error()))
}
return nil
}
func (m *manager) setupMigrationEnv() error {
// Initialize temp directory for transformer migrations
var err error
m.tmpMigDir, err = helpers.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil {
return err
}
err = os.RemoveAll(m.tmpMigDir)
if err != nil {
return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir))
}
err = os.Mkdir(m.tmpMigDir, os.FileMode(0777))
if err != nil {
return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir))
}
return nil
}
func (m *manager) createMigrationCopies(paths []string) error {
for _, path := range paths {
dir, err := ioutil.ReadDir(path)
if err != nil {
return err
}
for _, file := range dir {
if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum
continue
}
_, err := strconv.Atoi(file.Name()[:10])
if err != nil {
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name())
continue
}
src := filepath.Join(path, file.Name())
dst := filepath.Join(m.tmpMigDir, "1"+file.Name())
err = helpers.CopyFile(src, dst)
if err != nil {
return err
}
}
}
return nil
}
func (m *manager) cleanUp() error {
return os.RemoveAll(m.tmpMigDir)
}
+65
View File
@@ -0,0 +1,65 @@
package test_helpers
import (
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
)
func SetupDBandBC() (*postgres.DB, core.BlockChain) {
infuraIPC := "http://kovan0.vulcanize.io:8545"
rawRpcClient, err := rpc.Dial(infuraIPC)
Expect(err).NotTo(HaveOccurred())
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",
Name: "vulcanize_private",
Port: 5432,
}, blockChain.Node())
Expect(err).NotTo(HaveOccurred())
return db, blockChain
}
func TearDown(db *postgres.DB) {
tx, err := db.Begin()
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM headers`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM logs`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM log_filters`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM transactions`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM receipts`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM checked_headers`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM maker.bite`)
Expect(err).NotTo(HaveOccurred())
err = tx.Commit()
Expect(err).NotTo(HaveOccurred())
}
+3
View File
@@ -0,0 +1,3 @@
### Test
This empty directory is for holding the temporary .so and .go files generated during the generator_tests
+92
View File
@@ -0,0 +1,92 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package writer
import (
"errors"
"fmt"
. "github.com/dave/jennifer/jen"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
type PluginWriter interface {
WritePlugin() error
}
type writer struct {
GenConfig config.Plugin
}
func NewPluginWriter(gc config.Plugin) *writer {
return &writer{
GenConfig: gc,
}
}
// Generates the plugin code
func (w *writer) WritePlugin() error {
// Setup plugin file paths
goFile, err := w.setupFilePath()
if err != nil {
return err
}
// Begin code generation
f := NewFile("main")
f.HeaderComment("This is a plugin generated to export the configured transformer initializers")
// Import TransformerInitializers specified in config
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, imp := range w.GenConfig.Initializers {
f.ImportAlias(imp, alias)
}
// Collect TransformerInitializer names
importedInitializers := make([]Code, 0, len(w.GenConfig.Initializers))
for _, path := range w.GenConfig.Initializers {
importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer"))
}
// Create Exporter variable with method to export the set of the imported TransformerInitializers
f.Type().Id("exporter").String()
f.Var().Id("Exporter").Id("exporter")
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Block(
Return(Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(importedInitializers...))) // Exports the collected TransformerInitializers
// Write code to destination file
err = f.Save(goFile)
if err != nil {
return errors.New(fmt.Sprintf("failed to save generated .go file: %s\r\n%s", goFile, err.Error()))
}
return nil
}
func (w *writer) setupFilePath() (string, error) {
goFile, soFile, err := w.GenConfig.GetPluginPaths()
if err != nil {
return "", err
}
// Clear .go and .so files of the same name if they exist
return goFile, helpers.ClearFiles(goFile, soFile)
}