cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

addressing review comments; still need to reorg migrations and add helper tests

Browse Source
This commit is contained in:
Ian Norden
2019-02-24 21:38:48 -06:00
parent 2868cf2f73
commit b449193b16
16 changed files with 359 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/config/plugin.go
+32 -27
View File
@@ -17,8 +17,6 @@
package config
import (
"errors"
"fmt"
"path/filepath"
"strings"
@@ -26,15 +24,19 @@ import (
)
type Plugin struct {
Initializers map[string]string // Map of import aliases to transformer initializer paths
Dependencies map[string]string // Map of vendor dep names to their repositories
Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations
Types map[string]PluginType // Map of import aliases to their transformer initializer type (e.g. eth-event vs eth-storage)
Transformers map[string]Transformer
FilePath string
FileName string
Save bool
}
type Transformer struct {
Path string
Type TransformerType
MigrationPath string
RepositoryPath string
}
func (c *Plugin) GetPluginPaths() (string, string, error) {
path, err := helpers.CleanPath(c.FilePath)
if err != nil {
@@ -48,55 +50,58 @@ func (c *Plugin) GetPluginPaths() (string, string, error) {
return goFile, soFile, nil
}
func (c *Plugin) GetMigrationsPaths() ([]string, error) {
paths := make([]string, 0, len(c.Migrations))
for key, relPath := range c.Migrations {
repo, ok := c.Dependencies[key]
if !ok {
return nil, errors.New(fmt.Sprintf("migration %s with path %s missing repository", key, relPath))
}
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, relPath)
// Removes duplicate migration paths before returning them
func (c *Plugin) GetMigrationsPaths() (map[string]bool, error) {
paths := make(map[string]bool)
for _, transformer := range c.Transformers {
repo := transformer.RepositoryPath
mig := transformer.MigrationPath
path := filepath.Join("$GOPATH/src/github.com/vulcanize/vulcanizedb/vendor", repo, mig)
cleanPath, err := helpers.CleanPath(path)
if err != nil {
return nil, err
}
paths = append(paths, cleanPath)
paths[cleanPath] = true
}
return paths, nil
}
type PluginType int
// Removes duplicate repo paths before returning them
func (c *Plugin) GetRepoPaths() map[string]bool {
paths := make(map[string]bool)
for _, transformer := range c.Transformers {
paths[transformer.RepositoryPath] = true
}
return paths
}
type TransformerType int
const (
UnknownTransformerType PluginType = iota + 1
UnknownTransformerType TransformerType = iota
EthEvent
EthStorage
IpfsEvent
IpfsStorage
)
func (pt PluginType) String() string {
func (pt TransformerType) String() string {
names := [...]string{
"eth_event",
"eth_storage",
"ipfs_event",
"ipfs_storage",
}
if pt > IpfsStorage || pt < EthEvent {
if pt > EthStorage || pt < EthEvent {
return "Unknown"
}
return names[pt]
}
func GetPluginType(str string) PluginType {
types := [...]PluginType{
func GetTransformerType(str string) TransformerType {
types := [...]TransformerType{
EthEvent,
EthStorage,
IpfsEvent,
IpfsStorage,
}
for _, ty := range types {
pkg/omni/full/transformer/transformer_test.go
+1 -1
View File
@@ -324,7 +324,7 @@ var _ = Describe("Transformer", func() {
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6194636'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
pkg/omni/light/transformer/transformer_test.go
+1 -1
View File
@@ -331,7 +331,7 @@ var _ = Describe("Transformer", func() {
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
pkg/omni/shared/contract/contract.go
+2 -1
View File
@@ -18,6 +18,7 @@ package contract
import (
"errors"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@@ -78,7 +79,7 @@ func (c *Contract) GenerateFilters() error {
for name, event := range c.Events {
c.Filters[name] = filters.LogFilter{
Name: event.Name,
Name: c.Address + "_" + event.Name,
FromBlock: c.StartingBlock,
ToBlock: -1,
Address: common.HexToAddress(c.Address).Hex(),
pkg/omni/shared/helpers/test_helpers/mocks/entities.go
+2 -2
View File
@@ -129,7 +129,7 @@ var NewOwnerBlock2 = core.Block{
}
var ExpectedTransferFilter = filters.LogFilter{
Name: "Transfer",
Name: constants.TusdContractAddress + "_" + "Transfer",
Address: constants.TusdContractAddress,
ToBlock: -1,
FromBlock: 6194634,
@@ -137,7 +137,7 @@ var ExpectedTransferFilter = filters.LogFilter{
}
var ExpectedApprovalFilter = filters.LogFilter{
Name: "Approval",
Name: constants.TusdContractAddress + "_" + "Approval",
Address: constants.TusdContractAddress,
ToBlock: -1,
FromBlock: 6194634,
pkg/plugin/builder/builder.go
+9 -6
View File
@@ -37,16 +37,17 @@ type PluginBuilder interface {
}
type builder struct {
GenConfig config.Plugin
tmpVenDirs []string // Keep track of temp vendor directories
goFile string // Keep track of goFile name
GenConfig config.Plugin
dependencies []string
tmpVenDirs []string // Keep track of temp vendor directories
goFile string // Keep track of goFile name
}
// Requires populated plugin config
func NewPluginBuilder(gc config.Plugin) *builder {
return &builder{
GenConfig: gc,
tmpVenDirs: make([]string, 0, len(gc.Dependencies)),
tmpVenDirs: make([]string, 0),
}
}
@@ -82,8 +83,10 @@ func (b *builder) setupBuildEnv() error {
return err
}
repoPaths := b.GenConfig.GetRepoPaths()
// Import transformer dependencies so that we can build our plugin
for name, importPath := range b.GenConfig.Dependencies {
for importPath := range repoPaths {
// Use dependency paths in config to form git ssh string
// TODO: Change this to https once we are no longer working private transformer repos
// Right now since vulcanize/mcd_transformers is a private repo we
@@ -95,7 +98,7 @@ func (b *builder) setupBuildEnv() error {
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
return errors.New(fmt.Sprintf("unable to clone transformer dependency from %s: %s", importPath, err.Error()))
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
pkg/plugin/generator.go
+2 -11
View File
@@ -18,9 +18,6 @@ package plugin
import (
"errors"
"fmt"
"os"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/plugin/builder"
"github.com/vulcanize/vulcanizedb/pkg/plugin/manager"
@@ -40,14 +37,8 @@ type generator struct {
// Creates a new generator from a plugin and database config
func NewGenerator(gc config.Plugin, dbc config.Database) (*generator, error) {
if len(gc.Initializers) < 1 {
return nil, errors.New("generator needs to be configured with TransformerInitializer import paths")
}
if len(gc.Dependencies) < 1 {
return nil, errors.New("generator needs to be configured with root repository path(s)")
}
if len(gc.Migrations) < 1 {
fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided for the plugin transformers\r\n")
if len(gc.Transformers) < 1 {
return nil, errors.New("plugin generator is not configured with any transformers")
}
return &generator{
PluginWriter: writer.NewPluginWriter(gc),
pkg/plugin/generator_test.go
+66 -52
View File
@@ -36,62 +36,76 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers"
)
var genConfig = config.Plugin{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
var eventConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"bite": {
Path: "transformers/bite/initializer",
Type: config.EthEvent,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"deal": {
Path: "transformers/deal/initializer",
Type: config.EthEvent,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
},
Types: map[string]config.PluginType{
"bite": config.EthEvent,
"deal": config.EthEvent,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testEventTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
FileName: "testEventTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var genStorageConfig = config.Plugin{
Initializers: map[string]string{
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
var storageConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"pit": {
Path: "transformers/storage_diffs/maker/pit/initializer",
Type: config.EthStorage,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"vat": {
Path: "transformers/storage_diffs/maker/vat/initializer",
Type: config.EthStorage,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
},
Types: map[string]config.PluginType{
"pit": config.EthStorage,
"vat": config.EthStorage,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
FileName: "testStorageTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var combinedConfig = config.Plugin{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite/initializer",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal/initializer",
"pit": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/pit/initializer",
"vat": "github.com/vulcanize/mcd_transformers/transformers/storage_diffs/maker/vat/initializer",
Transformers: map[string]config.Transformer{
"pit": {
Path: "transformers/storage_diffs/maker/pit/initializer",
Type: config.EthStorage,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"vat": {
Path: "transformers/storage_diffs/maker/vat/initializer",
Type: config.EthStorage,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"bite": {
Path: "transformers/bite/initializer",
Type: config.EthEvent,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
"deal": {
Path: "transformers/deal/initializer",
Type: config.EthEvent,
MigrationPath: "db/migrations",
RepositoryPath: "github.com/vulcanize/mcd_transformers",
},
},
Types: map[string]config.PluginType{
"bite": config.EthEvent,
"deal": config.EthEvent,
"pit": config.EthStorage,
"vat": config.EthStorage,
},
Dependencies: map[string]string{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
Migrations: map[string]string{"mcd_transformers": "db/migrations"},
FileName: "testComboTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
FileName: "testComboTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/plugin/test_helpers/test",
Save: false,
}
var dbConfig = config.Database{
@@ -117,9 +131,9 @@ var _ = Describe("Generator test", func() {
Describe("Event Transformers only", func() {
BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths()
goPath, soPath, err = eventConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(genConfig, dbConfig)
g, err = p2.NewGenerator(eventConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
@@ -199,9 +213,9 @@ var _ = Describe("Generator test", func() {
Describe("Storage Transformers only", func() {
BeforeEach(func() {
goPath, soPath, err = genStorageConfig.GetPluginPaths()
goPath, soPath, err = storageConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g, err = p2.NewGenerator(genStorageConfig, dbConfig)
g, err = p2.NewGenerator(storageConfig, dbConfig)
Expect(err).ToNot(HaveOccurred())
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
pkg/plugin/helpers/helpers.go
+17 -1
View File
@@ -1,3 +1,19 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package helpers
import (
@@ -32,7 +48,7 @@ func ClearFiles(files ...string) error {
return err
}
} else if os.IsNotExist(err) {
// fall through
continue
} else {
return err
}
pkg/plugin/manager/manager.go
+3 -3
View File
@@ -89,7 +89,7 @@ func (m *manager) setupMigrationEnv() error {
if err != nil {
return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir))
}
err = os.Mkdir(m.tmpMigDir, os.FileMode(0777))
err = os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm))
if err != nil {
return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir))
}
@@ -98,9 +98,9 @@ func (m *manager) setupMigrationEnv() error {
}
// Create copies of db migrations from vendored libs
func (m *manager) createMigrationCopies(paths []string) error {
func (m *manager) createMigrationCopies(paths map[string]bool) error {
// Iterate through migration paths to find migration directory
for _, path := range paths {
for path := range paths {
dir, err := ioutil.ReadDir(path)
if err != nil {
return err
pkg/plugin/writer/writer.go
+18 -28
View File
@@ -58,12 +58,15 @@ func (w *writer) WritePlugin() error {
// Import pkgs for generic TransformerInitializer interface and specific TransformerInitializers specified in config
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, relPath := range w.GenConfig.Initializers {
f.ImportAlias(w.makePath(alias, relPath), alias)
for name, transformer := range w.GenConfig.Transformers {
f.ImportAlias(transformer.RepositoryPath+"/"+transformer.Path, name)
}
// Collect initializer code
ethEventInitializers, ethStorageInitializers, _, _ := w.sortTransformers()
code, err := w.collectTransformers()
if err != nil {
return err
}
// Create Exporter variable with method to export the set of the imported storage and event transformer initializers
f.Type().Id("exporter").String()
@@ -74,10 +77,10 @@ func (w *writer) WritePlugin() error {
)).Block(Return(
Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Values(ethEventInitializers...),
"TransformerInitializer").Values(code[config.EthEvent]...),
Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"StorageTransformerInitializer").Values(ethStorageInitializers...))) // Exports the collected initializers
"StorageTransformerInitializer").Values(code[config.EthStorage]...))) // Exports the collected event and storage transformer initializers
// Write code to destination file
err = f.Save(goFile)
@@ -88,34 +91,21 @@ func (w *writer) WritePlugin() error {
}
// Collect code for various types of initializers
func (w *writer) sortTransformers() ([]Code, []Code, []Code, []Code) {
importedEthEventInitializers := make([]Code, 0)
importerEthStorageInitializers := make([]Code, 0)
importedIpfsEventInitializers := make([]Code, 0)
importerIpfsStorageInitializers := make([]Code, 0)
for name, path := range w.GenConfig.Initializers {
switch w.GenConfig.Types[name] {
func (w *writer) collectTransformers() (map[config.TransformerType][]Code, error) {
code := make(map[config.TransformerType][]Code)
for _, transformer := range w.GenConfig.Transformers {
path := transformer.RepositoryPath + "/" + transformer.Path
switch transformer.Type {
case config.EthEvent:
importedEthEventInitializers = append(importedEthEventInitializers, Qual(path, "TransformerInitializer"))
code[config.EthEvent] = append(code[config.EthEvent], Qual(path, "TransformerInitializer"))
case config.EthStorage:
importerEthStorageInitializers = append(importerEthStorageInitializers, Qual(path, "StorageTransformerInitializer"))
case config.IpfsEvent:
//importedIpfsEventInitializers = append(importedIpfsEventInitializers, Qual(path, "IpfsEventTransformerInitializer"))
case config.IpfsStorage:
//importerIpfsStorageInitializers = append(importerIpfsStorageInitializers, Qual(path, "IpfsStorageTransformerInitializer"))
code[config.EthStorage] = append(code[config.EthStorage], Qual(path, "StorageTransformerInitializer"))
default:
return nil, errors.New(fmt.Sprintf("invalid transformer type %s", transformer.Type))
}
}
return importedEthEventInitializers,
importerEthStorageInitializers,
importedIpfsEventInitializers,
importerIpfsStorageInitializers
}
// Concat relative path with its repo's root path
func (w *writer) makePath(alias, relPath string) string {
pathRoot := w.GenConfig.Dependencies[alias]
return pathRoot + "/" + relPath
return code, nil
}
// Setup the .go, clear old ones if present