addressing review comments; still need to reorg migrations and add helper tests
This commit is contained in:
parent
2868cf2f73
commit
b449193b16
2
Makefile
2
Makefile
@ -50,7 +50,7 @@ lint:
|
||||
test: | $(GINKGO) $(LINT)
|
||||
go vet ./...
|
||||
go fmt ./...
|
||||
$(GINKGO) -r
|
||||
$(GINKGO) -r --skipPackage=integration_tests,integration
|
||||
|
||||
.PHONY: integrationtest
|
||||
integrationtest: | $(GINKGO) $(LINT)
|
||||
|
@ -213,23 +213,45 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
|
||||
|
||||
func prepConfig() {
|
||||
fmt.Println("configuring plugin")
|
||||
types := viper.GetStringMapString("exporter.types")
|
||||
genTypes := map[string]config.PluginType{}
|
||||
for transformerName, transformerType := range types {
|
||||
genType := config.GetPluginType(transformerType)
|
||||
if genType == config.UnknownTransformerType {
|
||||
log.Fatal(errors.New(`unknown transformer type in exporter config
|
||||
accepted types are "eth_event", "eth_storage", "ipfs_event" and "ipfs_storage"`))
|
||||
names := viper.GetStringSlice("exporter.transformerNames")
|
||||
transformers := make(map[string]config.Transformer)
|
||||
for _, name := range names {
|
||||
transformer := viper.GetStringMapString("exporter." + name)
|
||||
_, ok := transformer["path"]
|
||||
if !ok {
|
||||
log.Fatal(fmt.Sprintf("%s transformer config is missing `path` value", name))
|
||||
}
|
||||
_, ok = transformer["repository"]
|
||||
if !ok {
|
||||
log.Fatal(fmt.Sprintf("%s transformer config is missing `repository` value", name))
|
||||
}
|
||||
_, ok = transformer["migrations"]
|
||||
if !ok {
|
||||
log.Fatal(fmt.Sprintf("%s transformer config is missing `migrations` value", name))
|
||||
}
|
||||
ty, ok := transformer["type"]
|
||||
if !ok {
|
||||
log.Fatal(fmt.Sprintf("%s transformer config is missing `type` value", name))
|
||||
}
|
||||
|
||||
transformerType := config.GetTransformerType(ty)
|
||||
if transformerType == config.UnknownTransformerType {
|
||||
log.Fatal(errors.New(`unknown transformer type in exporter config
|
||||
accepted types are "eth_event", "eth_storage"`))
|
||||
}
|
||||
|
||||
transformers[name] = config.Transformer{
|
||||
Path: transformer["path"],
|
||||
Type: transformerType,
|
||||
RepositoryPath: transformer["repository"],
|
||||
MigrationPath: transformer["migrations"],
|
||||
}
|
||||
genTypes[transformerName] = genType
|
||||
}
|
||||
|
||||
genConfig = config.Plugin{
|
||||
Transformers: transformers,
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
|
||||
FileName: viper.GetString("exporter.name"),
|
||||
Save: viper.GetBool("exporter.save"),
|
||||
Initializers: viper.GetStringMapString("exporter.transformers"),
|
||||
Dependencies: viper.GetStringMapString("exporter.repositories"),
|
||||
Migrations: viper.GetStringMapString("exporter.migrations"),
|
||||
Types: genTypes,
|
||||
}
|
||||
}
|
||||
|
@ -95,7 +95,6 @@ func init() {
|
||||
rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
|
||||
rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
|
||||
rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
|
||||
rootCmd.PersistentFlags().String("exporter-path", "~/go/src/github.com/vulcanize/vulcanizedb/plugins", "file path to transformer exporter plugin")
|
||||
rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
|
||||
|
||||
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
|
||||
@ -107,7 +106,6 @@ func init() {
|
||||
viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
|
||||
viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
|
||||
viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
|
||||
viper.BindPFlag("exporter.filePath", rootCmd.PersistentFlags().Lookup("exporter-path"))
|
||||
viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
|
||||
}
|
||||
|
||||
|
@ -14,66 +14,177 @@
|
||||
[exporter]
|
||||
name = "eventTransformerExporter"
|
||||
save = false
|
||||
[exporter.transformers]
|
||||
bite = "transformers/bite/initializer"
|
||||
cat_chop_lump = "transformers/cat_file/chop_lump/initializer"
|
||||
cat_flip = "transformers/cat_file/flip/initializer"
|
||||
cat_pit_vow = "transformers/cat_file/pit_vow/initializer"
|
||||
deal = "transformers/deal/initializer"
|
||||
dent = "transformers/dent/initializer"
|
||||
drip_drip = "transformers/drip_drip/initializer"
|
||||
drip_file_ilk = "transformers/drip_file/ilk/initializer"
|
||||
drip_file_repo = "transformers/drip_file/repo/initializer"
|
||||
drip_file_vow = "transformers/drip_file/vow/initializer"
|
||||
flap_kick = "transformers/flap_kick/initializer"
|
||||
flip_kick = "transformers/flip_kick/initializer"
|
||||
flop_kick = "transformers/flop_kick/initializer"
|
||||
frob = "transformers/frob/initializer"
|
||||
pit_file_debt_ceiling = "transformers/pit_file/debt_ceiling/initializer"
|
||||
pit_file_ilk = "transformers/pit_file/ilk/initializer"
|
||||
price_feeds = "transformers/price_feeds/initializer"
|
||||
tend = "transformers/tend/initializer"
|
||||
vat_flux = "transformers/vat_flux/initializer"
|
||||
vat_fold = "transformers/vat_fold/initializer"
|
||||
vat_grab = "transformers/vat_grab/initializer"
|
||||
vat_heal = "transformers/vat_heal/initializer"
|
||||
vat_init = "transformers/vat_init/initializer"
|
||||
vat_move = "transformers/vat_move/initializer"
|
||||
vat_slip = "transformers/vat_slip/initializer"
|
||||
vat_toll = "transformers/vat_toll/initializer"
|
||||
vat_tune = "transformers/vat_tune/initializer"
|
||||
vow_flog = "transformers/vow_flog/initializer"
|
||||
[exporter.types]
|
||||
bite = "eth_event"
|
||||
cat_chop_lump = "eth_event"
|
||||
cat_flip = "eth_event"
|
||||
cat_pit_vow = "eth_event"
|
||||
deal = "eth_event"
|
||||
dent = "eth_event"
|
||||
drip_drip = "eth_event"
|
||||
drip_file_ilk = "eth_event"
|
||||
drip_file_repo = "eth_event"
|
||||
drip_file_vow = "eth_event"
|
||||
flap_kick = "eth_event"
|
||||
flip_kick = "eth_event"
|
||||
flop_kick = "eth_event"
|
||||
frob = "eth_event"
|
||||
pit_file_debt_ceiling = "eth_event"
|
||||
pit_file_ilk = "eth_event"
|
||||
price_feeds = "eth_event"
|
||||
tend = "eth_event"
|
||||
vat_flux = "eth_event"
|
||||
vat_fold = "eth_event"
|
||||
vat_grab = "eth_event"
|
||||
vat_heal = "eth_event"
|
||||
vat_init = "eth_event"
|
||||
vat_move = "eth_event"
|
||||
vat_slip = "eth_event"
|
||||
vat_toll = "eth_event"
|
||||
vat_tune = "eth_event"
|
||||
vow_flog = "eth_event"
|
||||
[exporter.repositories]
|
||||
mcd__event_transformers = "github.com/vulcanize/mcd_transformers"
|
||||
transformerNames = [
|
||||
"bite",
|
||||
"cat_chop_lump",
|
||||
"cat_flip",
|
||||
"cat_pit_vow",
|
||||
"deal",
|
||||
"dent",
|
||||
"drip_drip",
|
||||
"drip_file_ilk",
|
||||
"drop_file_repo",
|
||||
"drip_file_vow",
|
||||
"flap_kick",
|
||||
"flip_kick",
|
||||
"flop_kick",
|
||||
"frob_kick",
|
||||
"frob",
|
||||
"pit_file_debt_ceiling",
|
||||
"pit_file_ilk",
|
||||
"price_feeds",
|
||||
"tend",
|
||||
"vat_flux",
|
||||
"vat_fold",
|
||||
"vat_grab",
|
||||
"vat_heal",
|
||||
"vat_init",
|
||||
"vat_move",
|
||||
"vat_slip",
|
||||
"vat_toll",
|
||||
"vat_tune",
|
||||
"vow_flog"
|
||||
]
|
||||
[exporter.bite]
|
||||
path = "transformers/bite/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.cat_chop_lump]
|
||||
path = "transformers/cat_file/chop_lump/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.cat_flip]
|
||||
path = "transformers/cat_file/flip/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.cat_pit_vow]
|
||||
path = "transformers/cat_file/pit_vow/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.deal]
|
||||
path = "transformers/deal/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.dent]
|
||||
path = "transformers/dent/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.drip_drip]
|
||||
path = "transformers/drip_drip/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.drip_file_ilk]
|
||||
path = "transformers/drip_file/ilk/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.drip_file_repo]
|
||||
path = "transformers/drip_file/repo/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.drip_file_vow]
|
||||
path = "transformers/drip_file/vow/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.flap_kick]
|
||||
path = "transformers/flap_kick/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.flip_kick]
|
||||
path = "transformers/flip_kick/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.flop_kick]
|
||||
path = "transformers/flop_kick/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.frob]
|
||||
path = "transformers/frob/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.pit_file_debt_ceiling]
|
||||
path = "transformers/pit_file/debt_ceiling/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.pit_file_ilk]
|
||||
path = "transformers/pit_file/ilk/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.price_feeds]
|
||||
path = "transformers/price_feeds/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.tend]
|
||||
path = "transformers/tend/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_flux]
|
||||
path = "transformers/vat_flux/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_fold]
|
||||
path = "transformers/vat_fold/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_grab]
|
||||
path = "transformers/vat_grab/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_heal]
|
||||
path = "transformers/vat_heal/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_init]
|
||||
path = "transformers/vat_init/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_move]
|
||||
path = "transformers/vat_move/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_slip]
|
||||
path = "transformers/vat_slip/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_toll]
|
||||
path = "transformers/vat_toll/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vat_tune]
|
||||
path = "transformers/vat_tune/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
[exporter.vow_flog]
|
||||
path = "transformers/vow_flog/initializer"
|
||||
type = "eth_event"
|
||||
repository = "github.com/vulcanize/mcd_transformers"
|
||||
migrations = "db/migrations"
|
||||
|
||||
[filesystem]
|
||||
storageDiffsPath = "INSERT-PATH-TO-STORAGE-DIFFS"
|
||||
|
@ -27,7 +27,7 @@ import (
|
||||
|
||||
func TestShared(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Shared Suite")
|
||||
RunSpecs(t, "Storage Utils Suite")
|
||||
}
|
||||
|
||||
var _ = BeforeSuite(func() {
|
@ -17,8 +17,6 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
@ -26,15 +24,19 @@ import (
|
||||
)
|
||||
|
||||
type Plugin struct {
|
||||
Initializers map[string]string // Map of import aliases to transformer initializer paths
|
||||
Dependencies map[string]string // Map of vendor dep names to their repositories
|
||||
Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations
|
||||
Types map[string]PluginType // Map of import aliases to their transformer initializer type (e.g. eth-event vs eth-storage)
|
||||
Transformers map[string]Transformer
|
||||
FilePath string
|
||||
FileName string
|
||||
Save bool
|
||||
}
|
||||
|
||||
type Transformer struct {
|
||||
Path string
|
||||
Type TransformerType
|
||||
MigrationPath string
|
||||
RepositoryPath string
|
||||
}
|
||||
|
||||
func (c *Plugin) GetPluginPaths() (string, string, error) {
|
||||
path, err := helpers.CleanPath(c.FilePath)
|
||||
if err != nil {
|
||||
@ -48,55 +50,58 @@ func (c *Plugin) GetPluginPaths() (string, string, error) {
|
||||
return goFile, soFile, nil
|
||||
}
|
||||
|
||||
func (c *Plugin) GetMigrationsPaths() ([]string, error) {
|
||||
paths := make([]string, 0, len(c.Migrations))
|
||||
for key, relPath := range c.Migrations {
|
||||
repo, ok := c.Dependencies[key]
|
||||
if !ok {
|
||||
return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath))
|
||||
}
|
||||
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath)
|
||||
// Removes duplicate migration paths before returning them
|
||||
func (c *Plugin) GetMigrationsPaths() (map[string]bool, error) {
|
||||
paths := make(map[string]bool)
|
||||
for _, transformer := range c.Transformers {
|
||||
repo := transformer.RepositoryPath
|
||||
mig := transformer.MigrationPath
|
||||
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, mig)
|
||||
cleanPath, err := helpers.CleanPath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths = append(paths, cleanPath)
|
||||
paths[cleanPath] = true
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
type PluginType int
|
||||
// Removes duplicate repo paths before returning them
|
||||
func (c *Plugin) GetRepoPaths() map[string]bool {
|
||||
paths := make(map[string]bool)
|
||||
for _, transformer := range c.Transformers {
|
||||
paths[transformer.RepositoryPath] = true
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
type TransformerType int
|
||||
|
||||
const (
|
||||
UnknownTransformerType PluginType = iota + 1
|
||||
UnknownTransformerType TransformerType = iota
|
||||
EthEvent
|
||||
EthStorage
|
||||
IpfsEvent
|
||||
IpfsStorage
|
||||
)
|
||||
|
||||
func (pt PluginType) String() string {
|
||||
func (pt TransformerType) String() string {
|
||||
names := [...]string{
|
||||
"eth_event",
|
||||
"eth_storage",
|
||||
"ipfs_event",
|
||||
"ipfs_storage",
|
||||
}
|
||||
|
||||
if pt > IpfsStorage || pt < EthEvent {
|
||||
if pt > EthStorage || pt < EthEvent {
|
||||
return "Unknown"
|
||||
}
|
||||
|
||||
return names[pt]
|
||||
}
|
||||
|
||||
func GetPluginType(str string) PluginType {
|
||||
types := [...]PluginType{
|
||||
func GetTransformerType(str string) TransformerType {
|
||||
types := [...]TransformerType{
|
||||
EthEvent,
|
||||
EthStorage,
|
||||
IpfsEvent,
|
||||
IpfsStorage,
|
||||
}
|
||||
|
||||
for _, ty := range types {
|
||||
|
@ -324,7 +324,7 @@ var _ = Describe("Transformer", func() {
|
||||
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
|
||||
Expect(res.TokenName).To(Equal(""))
|
||||
|
||||
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6194636'", ensAddr)).StructScan(&res)
|
||||
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
|
@ -331,7 +331,7 @@ var _ = Describe("Transformer", func() {
|
||||
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
|
||||
Expect(res.TokenName).To(Equal(""))
|
||||
|
||||
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&res)
|
||||
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
|
@ -18,6 +18,7 @@ package contract
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/ethereum/go-ethereum/accounts/abi"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
@ -78,7 +79,7 @@ func (c *Contract) GenerateFilters() error {
|
||||
|
||||
for name, event := range c.Events {
|
||||
c.Filters[name] = filters.LogFilter{
|
||||
Name: event.Name,
|
||||
Name: c.Address + "_" + event.Name,
|
||||
FromBlock: c.StartingBlock,
|
||||
ToBlock: -1,
|
||||
Address: common.HexToAddress(c.Address).Hex(),
|
||||
|
@ -129,7 +129,7 @@ var NewOwnerBlock2 = core.Block{
|
||||
}
|
||||
|
||||
var ExpectedTransferFilter = filters.LogFilter{
|
||||
Name: "Transfer",
|
||||
Name: constants.TusdContractAddress + "_" + "Transfer",
|
||||
Address: constants.TusdContractAddress,
|
||||
ToBlock: -1,
|
||||
FromBlock: 6194634,
|
||||
@ -137,7 +137,7 @@ var ExpectedTransferFilter = filters.LogFilter{
|
||||
}
|
||||
|
||||
var ExpectedApprovalFilter = filters.LogFilter{
|
||||
Name: "Approval",
|
||||
Name: constants.TusdContractAddress + "_" + "Approval",
|
||||
Address: constants.TusdContractAddress,
|
||||
ToBlock: -1,
|
||||
FromBlock: 6194634,
|
||||
|
@ -37,16 +37,17 @@ type PluginBuilder interface {
|
||||
}
|
||||
|
||||
type builder struct {
|
||||
GenConfig config.Plugin
|
||||
tmpVenDirs []string // Keep track of temp vendor directories
|
||||
goFile string // Keep track of goFile name
|
||||
GenConfig config.Plugin
|
||||
dependencies []string
|
||||
tmpVenDirs []string // Keep track of temp vendor directories
|
||||
goFile string // Keep track of goFile name
|
||||
}
|
||||
|
||||
// Requires populated plugin config
|
||||
func NewPluginBuilder(gc config.Plugin) *builder {
|
||||
return &builder{
|
||||
GenConfig: gc,
|
||||
tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
|
||||
tmpVenDirs: make([]string, 0),
|
||||
}
|
||||
}
|
||||
|
||||
@ -82,8 +83,10 @@ func (b *builder) setupBuildEnv() error {
|
||||
return err
|
||||
}
|
||||
|
||||
repoPaths := b.GenConfig.GetRepoPaths()
|
||||
|
||||
// Import transformer dependencies so that we can build our plugin
|
||||
for name, importPath := range b.GenConfig.Dependencies {
|
||||
for importPath := range repoPaths {
|
||||
// Use dependency paths in config to form git ssh string
|
||||
// TODO: Change this to https once we are no longer working private transformer repos
|
||||
// Right now since vulcanize/mcd_transformers is a private repo we
|
||||
@ -95,7 +98,7 @@ func (b *builder) setupBuildEnv() error {
|
||||
depPath := filepath.Join(vendorPath, importPath)
|
||||
err = exec.Command("git", "clone", importURL, depPath).Run()
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
|
||||
return errors.New(fmt.Sprintf("unable to clone transformer dependency from %s: %s", importPath, err.Error()))
|
||||
}
|
||||
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
|
||||
if err != nil {
|
||||
|
@ -18,9 +18,6 @@ package plugin
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/config"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/builder"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/manager"
|
||||
@ -40,14 +37,8 @@ type generator struct {
|
||||
|
||||
// Creates a new generator from a plugin and database config
|
||||
func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
|
||||
if len(gc.Initializers) < 1 {
|
||||
return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
|
||||
}
|
||||
if len(gc.Dependencies) < 1 {
|
||||
return nil, errors.New("generator needs to be configured with root repository path(s)")
|
||||
}
|
||||
if len(gc.Migrations) < 1 {
|
||||
fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided for the plugin transformers\r\n")
|
||||
if len(gc.Transformers) < 1 {
|
||||
return nil, errors.New("plugin generator is not configured with any transformers")
|
||||
}
|
||||
return &generator{
|
||||
PluginWriter: writer.NewPluginWriter(gc),
|
||||
|
@ -36,62 +36,76 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers"
|
||||
)
|
||||
|
||||
var genConfig = config.Plugin{
|
||||
Initializers: map[string]string{
|
||||
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
|
||||
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
|
||||
var eventConfig = config.Plugin{
|
||||
Transformers: map[string]config.Transformer{
|
||||
"bite": {
|
||||
Path: "transformers/bite/initializer",
|
||||
Type: config.EthEvent,
|
||||
MigrationPath: "db/migrations",
|
||||
RepositoryPath: "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
"deal": {
|
||||
Path: "transformers/deal/initializer",
|
||||
Type: config.EthEvent,
|
||||
MigrationPath: "db/migrations",
|
||||
RepositoryPath: "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
},
|
||||
Types: map[string]config.PluginType{
|
||||
"bite": config.EthEvent,
|
||||
"deal": config.EthEvent,
|
||||
},
|
||||
Dependencies: map[string]string{
|
||||
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
|
||||
FileName: "testEventTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
FileName: "testEventTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
}
|
||||
|
||||
var genStorageConfig = config.Plugin{
|
||||
Initializers: map[string]string{
|
||||
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
|
||||
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
|
||||
var storageConfig = config.Plugin{
|
||||
Transformers: map[string]config.Transformer{
|
||||
"pit": {
|
||||
Path: "transformers/storage_diffs/maker/pit/initializer",
|
||||
Type: config.EthStorage,
|
||||
MigrationPath: "db/migrations",
|
||||
RepositoryPath: "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
"vat": {
|
||||
Path: "transformers/storage_diffs/maker/vat/initializer",
|
||||
Type: config.EthStorage,
|
||||
MigrationPath: "db/migrations",
|
||||
RepositoryPath: "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
},
|
||||
Types: map[string]config.PluginType{
|
||||
"pit": config.EthStorage,
|
||||
"vat": config.EthStorage,
|
||||
},
|
||||
Dependencies: map[string]string{
|
||||
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
|
||||
FileName: "testStorageTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
FileName: "testStorageTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
}
|
||||
|
||||
var combinedConfig = config.Plugin{
|
||||
Initializers: map[string]string{
|
||||
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
|
||||
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
|
||||
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
|
||||
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
|
||||
Transformers: map[string]config.Transformer{
|
||||
"pit": {
|
||||
Path: "transformers/storage_diffs/maker/pit/initializer",
|
||||
Type: config.EthStorage,
|
||||
MigrationPath: "db/migrations",
|
||||
RepositoryPath: "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
"vat": {
|
||||
Path: "transformers/storage_diffs/maker/vat/initializer",
|
||||
Type: config.EthStorage,
|
||||
MigrationPath: "db/migrations",
|
||||
RepositoryPath: "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
"bite": {
|
||||
Path: "transformers/bite/initializer",
|
||||
Type: config.EthEvent,
|
||||
MigrationPath: "db/migrations",
|
||||
RepositoryPath: "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
"deal": {
|
||||
Path: "transformers/deal/initializer",
|
||||
Type: config.EthEvent,
|
||||
MigrationPath: "db/migrations",
|
||||
RepositoryPath: "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
},
|
||||
Types: map[string]config.PluginType{
|
||||
"bite": config.EthEvent,
|
||||
"deal": config.EthEvent,
|
||||
"pit": config.EthStorage,
|
||||
"vat": config.EthStorage,
|
||||
},
|
||||
Dependencies: map[string]string{
|
||||
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
|
||||
},
|
||||
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
|
||||
FileName: "testComboTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
FileName: "testComboTransformerSet",
|
||||
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
|
||||
Save: false,
|
||||
}
|
||||
|
||||
var dbConfig = config.Database{
|
||||
@ -117,9 +131,9 @@ var _ = Describe("Generator test", func() {
|
||||
|
||||
Describe("Event Transformers only", func() {
|
||||
BeforeEach(func() {
|
||||
goPath, soPath, err = genConfig.GetPluginPaths()
|
||||
goPath, soPath, err = eventConfig.GetPluginPaths()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
g, err = p2.NewGenerator(genConfig, dbConfig)
|
||||
g, err = p2.NewGenerator(eventConfig, dbConfig)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = g.GenerateExporterPlugin()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@ -199,9 +213,9 @@ var _ = Describe("Generator test", func() {
|
||||
|
||||
Describe("Storage Transformers only", func() {
|
||||
BeforeEach(func() {
|
||||
goPath, soPath, err = genStorageConfig.GetPluginPaths()
|
||||
goPath, soPath, err = storageConfig.GetPluginPaths()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
g, err = p2.NewGenerator(genStorageConfig, dbConfig)
|
||||
g, err = p2.NewGenerator(storageConfig, dbConfig)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = g.GenerateExporterPlugin()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
@ -1,3 +1,19 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2018 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package helpers
|
||||
|
||||
import (
|
||||
@ -32,7 +48,7 @@ func ClearFiles(files ...string) error {
|
||||
return err
|
||||
}
|
||||
} else if os.IsNotExist(err) {
|
||||
// fall through
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
|
@ -89,7 +89,7 @@ func (m *manager) setupMigrationEnv() error {
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir))
|
||||
}
|
||||
err = os.Mkdir(m.tmpMigDir, os.FileMode(0777))
|
||||
err = os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm))
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir))
|
||||
}
|
||||
@ -98,9 +98,9 @@ func (m *manager) setupMigrationEnv() error {
|
||||
}
|
||||
|
||||
// Create copies of db migrations from vendored libs
|
||||
func (m *manager) createMigrationCopies(paths []string) error {
|
||||
func (m *manager) createMigrationCopies(paths map[string]bool) error {
|
||||
// Iterate through migration paths to find migration directory
|
||||
for _, path := range paths {
|
||||
for path := range paths {
|
||||
dir, err := ioutil.ReadDir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -58,12 +58,15 @@ func (w *writer) WritePlugin() error {
|
||||
|
||||
// Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config
|
||||
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
|
||||
for alias, relPath := range w.GenConfig.Initializers {
|
||||
f.ImportAlias(w.makePath(alias, relPath), alias)
|
||||
for name, transformer := range w.GenConfig.Transformers {
|
||||
f.ImportAlias(transformer.RepositoryPath+"/"+transformer.Path, name)
|
||||
}
|
||||
|
||||
// Collect initializer code
|
||||
ethEventInitializers, ethStorageInitializers, _, _ := w.sortTransformers()
|
||||
code, err := w.collectTransformers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create Exporter variable with method to export the set of the imported storage and event transformer initializers
|
||||
f.Type().Id("exporter").String()
|
||||
@ -74,10 +77,10 @@ func (w *writer) WritePlugin() error {
|
||||
)).Block(Return(
|
||||
Index().Qual(
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
|
||||
"TransformerInitializer").Values(ethEventInitializers...),
|
||||
"TransformerInitializer").Values(code[config.EthEvent]...),
|
||||
Index().Qual(
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
|
||||
"StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers
|
||||
"StorageTransformerInitializer").Values(code[config.EthStorage]...))) // Exports the collected event and storage transformer initializers
|
||||
|
||||
// Write code to destination file
|
||||
err = f.Save(goFile)
|
||||
@ -88,34 +91,21 @@ func (w *writer) WritePlugin() error {
|
||||
}
|
||||
|
||||
// Collect code for various types of initializers
|
||||
func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
|
||||
importedEthEventInitializers := make([]Code, 0)
|
||||
importerEthStorageInitializers := make([]Code, 0)
|
||||
importedIpfsEventInitializers := make([]Code, 0)
|
||||
importerIpfsStorageInitializers := make([]Code, 0)
|
||||
for name, path := range w.GenConfig.Initializers {
|
||||
switch w.GenConfig.Types[name] {
|
||||
func (w *writer) collectTransformers() (map[config.TransformerType][]Code, error) {
|
||||
code := make(map[config.TransformerType][]Code)
|
||||
for _, transformer := range w.GenConfig.Transformers {
|
||||
path := transformer.RepositoryPath + "/" + transformer.Path
|
||||
switch transformer.Type {
|
||||
case config.EthEvent:
|
||||
importedEthEventInitializers = append(importedEthEventInitializers, Qual(path, "TransformerInitializer"))
|
||||
code[config.EthEvent] = append(code[config.EthEvent], Qual(path, "TransformerInitializer"))
|
||||
case config.EthStorage:
|
||||
importerEthStorageInitializers = append(importerEthStorageInitializers, Qual(path, "StorageTransformerInitializer"))
|
||||
case config.IpfsEvent:
|
||||
//importedIpfsEventInitializers = append(importedIpfsEventInitializers, Qual(path, "IpfsEventTransformerInitializer"))
|
||||
case config.IpfsStorage:
|
||||
//importerIpfsStorageInitializers = append(importerIpfsStorageInitializers, Qual(path, "IpfsStorageTransformerInitializer"))
|
||||
code[config.EthStorage] = append(code[config.EthStorage], Qual(path, "StorageTransformerInitializer"))
|
||||
default:
|
||||
return nil, errors.New(fmt.Sprintf("invalid transformer type %s", transformer.Type))
|
||||
}
|
||||
}
|
||||
|
||||
return importedEthEventInitializers,
|
||||
importerEthStorageInitializers,
|
||||
importedIpfsEventInitializers,
|
||||
importerIpfsStorageInitializers
|
||||
}
|
||||
|
||||
// Concat relative path with its repo's root path
|
||||
func (w *writer) makePath(alias, relPath string) string {
|
||||
pathRoot := w.GenConfig.Dependencies[alias]
|
||||
return pathRoot + "/" + relPath
|
||||
return code, nil
|
||||
}
|
||||
|
||||
// Setup the .go, clear old ones if present
|
||||
|
Loading…
Reference in New Issue
Block a user