cerc-io/ipld-eth-server
17
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

addressing review comments; still need to reorg migrations and add helper tests

Browse Source
This commit is contained in:
Ian Norden 2019-02-13 13:04:07 -06:00
parent 2868cf2f73
commit b449193b16
16 changed files with 360 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -50,7 +50,7 @@ lint:
test: | $(GINKGO) $(LINT)
go vet ./...
go fmt ./...
$(GINKGO) -r
$(GINKGO) -r --skipPackage=integration_tests,integration
.PHONY: integrationtest
integrationtest: | $(GINKGO) $(LINT)

46
cmd/composeAndExecute.go
View File

@ -213,23 +213,45 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
func prepConfig() {
fmt.Println("configuring plugin")
types := viper.GetStringMapString("exporter.types")
genTypes := map[string]config.PluginType{}
for transformerName, transformerType := range types {
genType := config.GetPluginType(transformerType)
if genType == config.UnknownTransformerType {
log.Fatal(errors.New(`unknown transformer type in exporter config
accepted types are "eth_event", "eth_storage", "ipfs_event" and "ipfs_storage"`))
names := viper.GetStringSlice("exporter.transformerNames")
transformers := make(map[string]config.Transformer)
for _, name := range names {
transformer := viper.GetStringMapString("exporter." + name)
_, ok := transformer["path"]
if !ok {
log.Fatal(fmt.Sprintf("%s transformer config is missing `path` value", name))
}
_, ok = transformer["repository"]
if !ok {
log.Fatal(fmt.Sprintf("%s transformer config is missing `repository` value", name))
}
_, ok = transformer["migrations"]
if !ok {
log.Fatal(fmt.Sprintf("%s transformer config is missing `migrations` value", name))
}
ty, ok := transformer["type"]
if !ok {
log.Fatal(fmt.Sprintf("%s transformer config is missing `type` value", name))
}
transformerType := config.GetTransformerType(ty)
if transformerType == config.UnknownTransformerType {
log.Fatal(errors.New(`unknown transformer type in exporter config
accepted types are "eth_event", "eth_storage"`))
}
transformers[name] = config.Transformer{
Path: transformer["path"],
Type: transformerType,
RepositoryPath: transformer["repository"],
MigrationPath: transformer["migrations"],
}
genTypes[transformerName] = genType
}
genConfig = config.Plugin{
Transformers: transformers,
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
FileName: viper.GetString("exporter.name"),
Save: viper.GetBool("exporter.save"),
Initializers: viper.GetStringMapString("exporter.transformers"),
Dependencies: viper.GetStringMapString("exporter.repositories"),
Migrations: viper.GetStringMapString("exporter.migrations"),
Types: genTypes,
}
}

2
cmd/root.go
View File

@ -95,7 +95,6 @@ func init() {
rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
rootCmd.PersistentFlags().String("exporter-path", "~/go/src/github.com/vulcanize/vulcanizedb/plugins", "file path to transformer exporter plugin")
rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
@ -107,7 +106,6 @@ func init() {
viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
viper.BindPFlag("exporter.filePath", rootCmd.PersistentFlags().Lookup("exporter-path"))
viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
}

231
environments/compose.toml
View File

@ -14,66 +14,177 @@
[exporter]
name = "eventTransformerExporter"
save = false
[exporter.transformers]
bite = "transformers/bite/initializer"
cat_chop_lump = "transformers/cat_file/chop_lump/initializer"
cat_flip = "transformers/cat_file/flip/initializer"
cat_pit_vow = "transformers/cat_file/pit_vow/initializer"
deal = "transformers/deal/initializer"
dent = "transformers/dent/initializer"
drip_drip = "transformers/drip_drip/initializer"
drip_file_ilk = "transformers/drip_file/ilk/initializer"
drip_file_repo = "transformers/drip_file/repo/initializer"
drip_file_vow = "transformers/drip_file/vow/initializer"
flap_kick = "transformers/flap_kick/initializer"
flip_kick = "transformers/flip_kick/initializer"
flop_kick = "transformers/flop_kick/initializer"
frob = "transformers/frob/initializer"
pit_file_debt_ceiling = "transformers/pit_file/debt_ceiling/initializer"
pit_file_ilk = "transformers/pit_file/ilk/initializer"
price_feeds = "transformers/price_feeds/initializer"
tend = "transformers/tend/initializer"
vat_flux = "transformers/vat_flux/initializer"
vat_fold = "transformers/vat_fold/initializer"
vat_grab = "transformers/vat_grab/initializer"
vat_heal = "transformers/vat_heal/initializer"
vat_init = "transformers/vat_init/initializer"
vat_move = "transformers/vat_move/initializer"
vat_slip = "transformers/vat_slip/initializer"
vat_toll = "transformers/vat_toll/initializer"
vat_tune = "transformers/vat_tune/initializer"
vow_flog = "transformers/vow_flog/initializer"
[exporter.types]
bite = "eth_event"
cat_chop_lump = "eth_event"
cat_flip = "eth_event"
cat_pit_vow = "eth_event"
deal = "eth_event"
dent = "eth_event"
drip_drip = "eth_event"
drip_file_ilk = "eth_event"
drip_file_repo = "eth_event"
drip_file_vow = "eth_event"
flap_kick = "eth_event"
flip_kick = "eth_event"
flop_kick = "eth_event"
frob = "eth_event"
pit_file_debt_ceiling = "eth_event"
pit_file_ilk = "eth_event"
price_feeds = "eth_event"
tend = "eth_event"
vat_flux = "eth_event"
vat_fold = "eth_event"
vat_grab = "eth_event"
vat_heal = "eth_event"
vat_init = "eth_event"
vat_move = "eth_event"
vat_slip = "eth_event"
vat_toll = "eth_event"
vat_tune = "eth_event"
vow_flog = "eth_event"
[exporter.repositories]
mcd__event_transformers = "github.com/vulcanize/mcd_transformers"
transformerNames = [
"bite",
"cat_chop_lump",
"cat_flip",
"cat_pit_vow",
"deal",
"dent",
"drip_drip",
"drip_file_ilk",
"drop_file_repo",
"drip_file_vow",
"flap_kick",
"flip_kick",
"flop_kick",
"frob_kick",
"frob",
"pit_file_debt_ceiling",
"pit_file_ilk",
"price_feeds",
"tend",
"vat_flux",
"vat_fold",
"vat_grab",
"vat_heal",
"vat_init",
"vat_move",
"vat_slip",
"vat_toll",
"vat_tune",
"vow_flog"
]
[exporter.bite]
path = "transformers/bite/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.cat_chop_lump]
path = "transformers/cat_file/chop_lump/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.cat_flip]
path = "transformers/cat_file/flip/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.cat_pit_vow]
path = "transformers/cat_file/pit_vow/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.deal]
path = "transformers/deal/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.dent]
path = "transformers/dent/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.drip_drip]
path = "transformers/drip_drip/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.drip_file_ilk]
path = "transformers/drip_file/ilk/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.drip_file_repo]
path = "transformers/drip_file/repo/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.drip_file_vow]
path = "transformers/drip_file/vow/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.flap_kick]
path = "transformers/flap_kick/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.flip_kick]
path = "transformers/flip_kick/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.flop_kick]
path = "transformers/flop_kick/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.frob]
path = "transformers/frob/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.pit_file_debt_ceiling]
path = "transformers/pit_file/debt_ceiling/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.pit_file_ilk]
path = "transformers/pit_file/ilk/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.price_feeds]
path = "transformers/price_feeds/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.tend]
path = "transformers/tend/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_flux]
path = "transformers/vat_flux/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_fold]
path = "transformers/vat_fold/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_grab]
path = "transformers/vat_grab/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_heal]
path = "transformers/vat_heal/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_init]
path = "transformers/vat_init/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_move]
path = "transformers/vat_move/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_slip]
path = "transformers/vat_slip/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_toll]
path = "transformers/vat_toll/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vat_tune]
path = "transformers/vat_tune/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[exporter.vow_flog]
path = "transformers/vow_flog/initializer"
type = "eth_event"
repository = "github.com/vulcanize/mcd_transformers"
migrations = "db/migrations"
[filesystem]
storageDiffsPath = "INSERT-PATH-TO-STORAGE-DIFFS"

2
libraries/shared/storage/utils/shared_suite_test.go → libraries/shared/storage/utils/utils_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestShared(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Shared Suite")
RunSpecs(t, "Storage Utils Suite")
}
var _ = BeforeSuite(func() {

59
pkg/config/plugin.go
View File

@ -17,8 +17,6 @@
package config
import (
"errors"
"fmt"
"path/filepath"
"strings"
@ -26,15 +24,19 @@ import (
)
type Plugin struct {
Initializers map[string]string // Map of import aliases to transformer initializer paths
Dependencies map[string]string // Map of vendor dep names to their repositories
Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations
Types map[string]PluginType // Map of import aliases to their transformer initializer type (e.g. eth-event vs eth-storage)
Transformers map[string]Transformer
FilePath string
FileName string
Save bool
}
type Transformer struct {
Path string
Type TransformerType
MigrationPath string
RepositoryPath string
}
func (c *Plugin) GetPluginPaths() (string, string, error) {
path, err := helpers.CleanPath(c.FilePath)
if err != nil {
@ -48,55 +50,58 @@ func (c *Plugin) GetPluginPaths() (string, string, error) {
return goFile, soFile, nil
}
func (c *Plugin) GetMigrationsPaths() ([]string, error) {
paths := make([]string, 0, len(c.Migrations))
for key, relPath := range c.Migrations {
repo, ok := c.Dependencies[key]
if !ok {
return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath))
}
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath)
// Removes duplicate migration paths before returning them
func (c *Plugin) GetMigrationsPaths() (map[string]bool, error) {
paths := make(map[string]bool)
for _, transformer := range c.Transformers {
repo := transformer.RepositoryPath
mig := transformer.MigrationPath
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, mig)
cleanPath, err := helpers.CleanPath(path)
if err != nil {
return nil, err
}
paths = append(paths, cleanPath)
paths[cleanPath] = true
}
return paths, nil
}
type PluginType int
// Removes duplicate repo paths before returning them
func (c *Plugin) GetRepoPaths() map[string]bool {
paths := make(map[string]bool)
for _, transformer := range c.Transformers {
paths[transformer.RepositoryPath] = true
}
return paths
}
type TransformerType int
const (
UnknownTransformerType PluginType = iota + 1
UnknownTransformerType TransformerType = iota
EthEvent
EthStorage
IpfsEvent
IpfsStorage
)
func (pt PluginType) String() string {
func (pt TransformerType) String() string {
names := [...]string{
"eth_event",
"eth_storage",
"ipfs_event",
"ipfs_storage",
}
if pt > IpfsStorage || pt < EthEvent {
if pt > EthStorage || pt < EthEvent {
return "Unknown"
}
return names[pt]
}
func GetPluginType(str string) PluginType {
types := [...]PluginType{
func GetTransformerType(str string) TransformerType {
types := [...]TransformerType{
EthEvent,
EthStorage,
IpfsEvent,
IpfsStorage,
}
for _, ty := range types {

2
pkg/omni/full/transformer/transformer_test.go
View File

@ -324,7 +324,7 @@ var _ = Describe("Transformer", func() {
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6194636'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})

2
pkg/omni/light/transformer/transformer_test.go
View File

@ -331,7 +331,7 @@ var _ = Describe("Transformer", func() {
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})

3
pkg/omni/shared/contract/contract.go
View File

@ -18,6 +18,7 @@ package contract
import (
"errors"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -78,7 +79,7 @@ func (c *Contract) GenerateFilters() error {
for name, event := range c.Events {
c.Filters[name] = filters.LogFilter{
Name: event.Name,
Name: c.Address + "_" + event.Name,
FromBlock: c.StartingBlock,
ToBlock: -1,
Address: common.HexToAddress(c.Address).Hex(),

4
pkg/omni/shared/helpers/test_helpers/mocks/entities.go
View File

@ -129,7 +129,7 @@ var NewOwnerBlock2 = core.Block{
}
var ExpectedTransferFilter = filters.LogFilter{
Name: "Transfer",
Name: constants.TusdContractAddress + "_" + "Transfer",
Address: constants.TusdContractAddress,
ToBlock: -1,
FromBlock: 6194634,
@ -137,7 +137,7 @@ var ExpectedTransferFilter = filters.LogFilter{
}
var ExpectedApprovalFilter = filters.LogFilter{
Name: "Approval",
Name: constants.TusdContractAddress + "_" + "Approval",
Address: constants.TusdContractAddress,
ToBlock: -1,
FromBlock: 6194634,

15
pkg/plugin/builder/builder.go
View File

@ -37,16 +37,17 @@ type PluginBuilder interface {
}
type builder struct {
GenConfig config.Plugin
tmpVenDirs []string // Keep track of temp vendor directories
goFile string // Keep track of goFile name
GenConfig config.Plugin
dependencies []string
tmpVenDirs []string // Keep track of temp vendor directories
goFile string // Keep track of goFile name
}
// Requires populated plugin config
func NewPluginBuilder(gc config.Plugin) *builder {
return &builder{
GenConfig: gc,
tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
tmpVenDirs: make([]string, 0),
}
}
@ -82,8 +83,10 @@ func (b *builder) setupBuildEnv() error {
return err
}
repoPaths := b.GenConfig.GetRepoPaths()
// Import transformer dependencies so that we can build our plugin
for name, importPath := range b.GenConfig.Dependencies {
for importPath := range repoPaths {
// Use dependency paths in config to form git ssh string
// TODO: Change this to https once we are no longer working private transformer repos
// Right now since vulcanize/mcd_transformers is a private repo we
@ -95,7 +98,7 @@ func (b *builder) setupBuildEnv() error {
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
return errors.New(fmt.Sprintf("unable to clone transformer dependency from %s: %s", importPath, err.Error()))
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {

13
pkg/plugin/generator.go
View File

@ -18,9 +18,6 @@ package plugin
import (
"errors"
"fmt"
"os"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/builder"
"github.com/vulcanize/vulcanizedb/pkg/plugin/manager"
@ -40,14 +37,8 @@ type generator struct {
// Creates a new generator from a plugin and database config
func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
if len(gc.Initializers) < 1 {
return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
}
if len(gc.Dependencies) < 1 {
return nil, errors.New("generator needs to be configured with root repository path(s)")
}
if len(gc.Migrations) < 1 {
fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided for the plugin transformers\r\n")
if len(gc.Transformers) < 1 {
return nil, errors.New("plugin generator is not configured with any transformers")
}
return &generator{
PluginWriter: writer.NewPluginWriter(gc),

118
pkg/plugin/generator_test.go
View File

@ -36,62 +36,76 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers"
)
var genConfig = config.Plugin{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
var eventConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"bite": {
Path: "transformers/bite/initializer",
Type: config.EthEvent,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"deal": {
Path: "transformers/deal/initializer",
Type: config.EthEvent,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
},
Types: map[string]config.PluginType{
"bite": config.EthEvent,
"deal": config.EthEvent,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testEventTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
FileName: "testEventTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var genStorageConfig = config.Plugin{
Initializers: map[string]string{
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
var storageConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"pit": {
Path: "transformers/storage_diffs/maker/pit/initializer",
Type: config.EthStorage,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"vat": {
Path: "transformers/storage_diffs/maker/vat/initializer",
Type: config.EthStorage,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
},
Types: map[string]config.PluginType{
"pit": config.EthStorage,
"vat": config.EthStorage,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var combinedConfig = config.Plugin{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
Transformers: map[string]config.Transformer{
"pit": {
Path: "transformers/storage_diffs/maker/pit/initializer",
Type: config.EthStorage,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"vat": {
Path: "transformers/storage_diffs/maker/vat/initializer",
Type: config.EthStorage,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"bite": {
Path: "transformers/bite/initializer",
Type: config.EthEvent,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"deal": {
Path: "transformers/deal/initializer",
Type: config.EthEvent,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
},
Types: map[string]config.PluginType{
"bite": config.EthEvent,
"deal": config.EthEvent,
"pit": config.EthStorage,
"vat": config.EthStorage,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testComboTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
FileName: "testComboTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var dbConfig = config.Database{
@ -117,9 +131,9 @@ var _ = Describe("Generator test", func() {
Describe("Event Transformers only", func() {
BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths()
goPath, soPath, err = eventConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(genConfig, dbConfig)
g, err = p2.NewGenerator(eventConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
@ -199,9 +213,9 @@ var _ = Describe("Generator test", func() {
Describe("Storage Transformers only", func() {
BeforeEach(func() {
goPath, soPath, err = genStorageConfig.GetPluginPaths()
goPath, soPath, err = storageConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(genStorageConfig, dbConfig)
g, err = p2.NewGenerator(storageConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())

18
pkg/plugin/helpers/helpers.go
View File

@ -1,3 +1,19 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package helpers
import (
@ -32,7 +48,7 @@ func ClearFiles(files ...string) error {
return err
}
} else if os.IsNotExist(err) {
// fall through
continue
} else {
return err
}

6
pkg/plugin/manager/manager.go
View File

@ -89,7 +89,7 @@ func (m *manager) setupMigrationEnv() error {
if err != nil {
return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir))
}
err = os.Mkdir(m.tmpMigDir, os.FileMode(0777))
err = os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm))
if err != nil {
return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir))
}
@ -98,9 +98,9 @@ func (m *manager) setupMigrationEnv() error {
}
// Create copies of db migrations from vendored libs
func (m *manager) createMigrationCopies(paths []string) error {
func (m *manager) createMigrationCopies(paths map[string]bool) error {
// Iterate through migration paths to find migration directory
for _, path := range paths {
for path := range paths {
dir, err := ioutil.ReadDir(path)
if err != nil {
return err

46
pkg/plugin/writer/writer.go
View File

@ -58,12 +58,15 @@ func (w *writer) WritePlugin() error {
// Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, relPath := range w.GenConfig.Initializers {
f.ImportAlias(w.makePath(alias, relPath), alias)
for name, transformer := range w.GenConfig.Transformers {
f.ImportAlias(transformer.RepositoryPath+"/"+transformer.Path, name)
}
// Collect initializer code
ethEventInitializers, ethStorageInitializers, _, _ := w.sortTransformers()
code, err := w.collectTransformers()
if err != nil {
return err
}
// Create Exporter variable with method to export the set of the imported storage and event transformer initializers
f.Type().Id("exporter").String()
@ -74,10 +77,10 @@ func (w *writer) WritePlugin() error {
)).Block(Return(
Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(ethEventInitializers...),
"TransformerInitializer").Values(code[config.EthEvent]...),
Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers
"StorageTransformerInitializer").Values(code[config.EthStorage]...))) // Exports the collected event and storage transformer initializers
// Write code to destination file
err = f.Save(goFile)
@ -88,34 +91,21 @@ func (w *writer) WritePlugin() error {
}
// Collect code for various types of initializers
func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
importedEthEventInitializers := make([]Code, 0)
importerEthStorageInitializers := make([]Code, 0)
importedIpfsEventInitializers := make([]Code, 0)
importerIpfsStorageInitializers := make([]Code, 0)
for name, path := range w.GenConfig.Initializers {
switch w.GenConfig.Types[name] {
func (w *writer) collectTransformers() (map[config.TransformerType][]Code, error) {
code := make(map[config.TransformerType][]Code)
for _, transformer := range w.GenConfig.Transformers {
path := transformer.RepositoryPath + "/" + transformer.Path
switch transformer.Type {
case config.EthEvent:
importedEthEventInitializers = append(importedEthEventInitializers, Qual(path, "TransformerInitializer"))
code[config.EthEvent] = append(code[config.EthEvent], Qual(path, "TransformerInitializer"))
case config.EthStorage:
importerEthStorageInitializers = append(importerEthStorageInitializers, Qual(path, "StorageTransformerInitializer"))
case config.IpfsEvent:
//importedIpfsEventInitializers = append(importedIpfsEventInitializers, Qual(path, "IpfsEventTransformerInitializer"))
case config.IpfsStorage:
//importerIpfsStorageInitializers = append(importerIpfsStorageInitializers, Qual(path, "IpfsStorageTransformerInitializer"))
code[config.EthStorage] = append(code[config.EthStorage], Qual(path, "StorageTransformerInitializer"))
default:
return nil, errors.New(fmt.Sprintf("invalid transformer type %s", transformer.Type))
}
}
return importedEthEventInitializers,
importerEthStorageInitializers,
importedIpfsEventInitializers,
importerIpfsStorageInitializers
}
// Concat relative path with its repo's root path
func (w *writer) makePath(alias, relPath string) string {
pathRoot := w.GenConfig.Dependencies[alias]
return pathRoot + "/" + relPath
return code, nil
}
// Setup the .go, clear old ones if present