cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

trimming down to ipfs watchers

Browse Source
This commit is contained in:
Ian Norden
2020-05-29 22:02:47 -05:00
parent 78e9fbd248
commit fe083c12f9
422 changed files with 1310 additions and 28341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/config/config_test.go
+1 -1
View File
@@ -38,7 +38,7 @@ var _ = Describe("Loading the config", func() {
It("reads the private config using the environment", func() {
viper.SetConfigName("config")
viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/ipfs-chain-watcher/environments/")
Expect(viper.Get("client.ipcpath")).To(BeNil())
testConfig := viper.New()
pkg/config/contract.go
-218
View File
@@ -1,218 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"strings"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/eth"
)
// Config struct for generic contract transformer
type ContractConfig struct {
// Name for the transformer
Name string
// Ethereum network name; default "" is mainnet
Network string
// List of contract addresses (map to ensure no duplicates)
Addresses map[string]bool
// Map of contract address to abi
// If an address has no associated abi the parser will attempt to fetch one from etherscan
Abis map[string]string
// Map of contract address to slice of events
// Used to set which addresses to watch
// If any events are listed in the slice only those will be watched
// Otherwise all events in the contract ABI are watched
Events map[string][]string
// Map of contract address to slice of methods
// If any methods are listed in the slice only those will be polled
// Otherwise no methods will be polled
Methods map[string][]string
// Map of contract address to slice of event arguments to filter for
// If arguments are provided then only events which emit those arguments are watched
// Otherwise arguments are not filtered on events
EventArgs map[string][]string
// Map of contract address to slice of method arguments to limit polling to
// If arguments are provided then only those arguments are allowed as arguments in method polling
// Otherwise any argument of the right type seen emitted from events at that contract will be used in method polling
MethodArgs map[string][]string
// Map of contract address to their starting block
StartingBlocks map[string]int64
// Map of contract address to whether or not to pipe method polling results forward into subsequent method calls
Piping map[string]bool
}
func (contractConfig *ContractConfig) PrepConfig() {
addrs := viper.GetStringSlice("contract.addresses")
contractConfig.Network = viper.GetString("contract.network")
contractConfig.Addresses = make(map[string]bool, len(addrs))
contractConfig.Abis = make(map[string]string, len(addrs))
contractConfig.Methods = make(map[string][]string, len(addrs))
contractConfig.Events = make(map[string][]string, len(addrs))
contractConfig.MethodArgs = make(map[string][]string, len(addrs))
contractConfig.EventArgs = make(map[string][]string, len(addrs))
contractConfig.StartingBlocks = make(map[string]int64, len(addrs))
contractConfig.Piping = make(map[string]bool, len(addrs))
// De-dupe addresses
for _, addr := range addrs {
contractConfig.Addresses[strings.ToLower(addr)] = true
}
// Iterate over addresses to pull out config info for each contract
for _, addr := range addrs {
transformer := viper.GetStringMap("contract." + addr)
// Get and check abi
var abi string
abiInterface, abiOK := transformer["abi"]
if !abiOK {
log.Warnf("contract %s not configured with an ABI, will attempt to fetch it from Etherscan\r\n", addr)
} else {
abi, abiOK = abiInterface.(string)
if !abiOK {
log.Fatal(addr, "transformer `abi` not of type []string")
}
}
if abi != "" {
if _, abiErr := eth.ParseAbi(abi); abiErr != nil {
log.Fatal(addr, "transformer `abi` not valid JSON")
}
}
contractConfig.Abis[strings.ToLower(addr)] = abi
// Get and check events
events := make([]string, 0)
eventsInterface, eventsOK := transformer["events"]
if !eventsOK {
log.Warnf("contract %s not configured with a list of events to watch, will watch all events\r\n", addr)
events = []string{}
} else {
eventsI, eventsOK := eventsInterface.([]interface{})
if !eventsOK {
log.Fatal(addr, "transformer `events` not of type []string\r\n")
}
for _, strI := range eventsI {
str, strOK := strI.(string)
if !strOK {
log.Fatal(addr, "transformer `events` not of type []string\r\n")
}
events = append(events, str)
}
}
contractConfig.Events[strings.ToLower(addr)] = events
// Get and check methods
methods := make([]string, 0)
methodsInterface, methodsOK := transformer["methods"]
if !methodsOK {
log.Warnf("contract %s not configured with a list of methods to poll, will not poll any methods\r\n", addr)
methods = []string{}
} else {
methodsI, methodsOK := methodsInterface.([]interface{})
if !methodsOK {
log.Fatal(addr, "transformer `methods` not of type []string\r\n")
}
for _, strI := range methodsI {
str, strOK := strI.(string)
if !strOK {
log.Fatal(addr, "transformer `methods` not of type []string\r\n")
}
methods = append(methods, str)
}
}
contractConfig.Methods[strings.ToLower(addr)] = methods
// Get and check eventArgs
eventArgs := make([]string, 0)
eventArgsInterface, eventArgsOK := transformer["eventArgs"]
if !eventArgsOK {
log.Warnf("contract %s not configured with a list of event arguments to filter for, will not filter events for specific emitted values\r\n", addr)
eventArgs = []string{}
} else {
eventArgsI, eventArgsOK := eventArgsInterface.([]interface{})
if !eventArgsOK {
log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n")
}
for _, strI := range eventArgsI {
str, strOK := strI.(string)
if !strOK {
log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n")
}
eventArgs = append(eventArgs, str)
}
}
contractConfig.EventArgs[strings.ToLower(addr)] = eventArgs
// Get and check methodArgs
methodArgs := make([]string, 0)
methodArgsInterface, methodArgsOK := transformer["methodArgs"]
if !methodArgsOK {
log.Warnf("contract %s not configured with a list of method argument values to poll with, will poll methods with all available arguments\r\n", addr)
methodArgs = []string{}
} else {
methodArgsI, methodArgsOK := methodArgsInterface.([]interface{})
if !methodArgsOK {
log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n")
}
for _, strI := range methodArgsI {
str, strOK := strI.(string)
if !strOK {
log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n")
}
methodArgs = append(methodArgs, str)
}
}
contractConfig.MethodArgs[strings.ToLower(addr)] = methodArgs
// Get and check startingBlock
startInterface, startOK := transformer["startingblock"]
if !startOK {
log.Fatal(addr, "transformer config is missing `startingBlock` value\r\n")
}
start, startOK := startInterface.(int64)
if !startOK {
log.Fatal(addr, "transformer `startingBlock` not of type int\r\n")
}
contractConfig.StartingBlocks[strings.ToLower(addr)] = start
// Get pipping
var piping bool
_, pipeOK := transformer["piping"]
if !pipeOK {
log.Warnf("contract %s does not have its `piping` set, by default piping is turned off\r\n", addr)
piping = false
} else {
pipingInterface := transformer["piping"]
piping, pipeOK = pipingInterface.(bool)
if !pipeOK {
log.Fatal(addr, "transformer `piping` not of type bool\r\n")
}
}
contractConfig.Piping[strings.ToLower(addr)] = piping
}
}
pkg/config/plugin.go
-167
View File
@@ -1,167 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
)
type Plugin struct {
Transformers map[string]Transformer
FilePath string
FileName string
Save bool
Home string
}
type Transformer struct {
Path string
Type TransformerType
MigrationPath string
MigrationRank uint64
RepositoryPath string
}
func (pluginConfig *Plugin) GetPluginPaths() (string, string, error) {
path, err := helpers.CleanPath(pluginConfig.FilePath)
if err != nil {
return "", "", err
}
name := strings.Split(pluginConfig.FileName, ".")[0]
goFile := filepath.Join(path, name+".go")
soFile := filepath.Join(path, name+".so")
return goFile, soFile, nil
}
// Removes duplicate migration paths and returns them in ranked order
func (pluginConfig *Plugin) GetMigrationsPaths() ([]string, error) {
paths := make(map[uint64]string)
highestRank := -1
for name, transformer := range pluginConfig.Transformers {
repo := transformer.RepositoryPath
mig := transformer.MigrationPath
path := filepath.Join("$GOPATH/src", pluginConfig.Home, "vendor", repo, mig)
cleanPath, err := helpers.CleanPath(path)
if err != nil {
return nil, err
}
// If there is a different path with the same rank then we have a conflict
_, ok := paths[transformer.MigrationRank]
if ok {
conflictingPath := paths[transformer.MigrationRank]
if conflictingPath != cleanPath {
return nil, fmt.Errorf("transformer %s has the same migration rank (%d) as another transformer", name, transformer.MigrationRank)
}
}
paths[transformer.MigrationRank] = cleanPath
if int(transformer.MigrationRank) >= highestRank {
highestRank = int(transformer.MigrationRank)
}
}
// Check for gaps and duplicates
if len(paths) != (highestRank + 1) {
return []string{}, errors.New("number of distinct ranks does not match number of distinct migration paths")
}
if anyDupes(paths) {
return []string{}, errors.New("duplicate paths with different ranks present")
}
sortedPaths := make([]string, len(paths))
for rank, path := range paths {
sortedPaths[rank] = path
}
return sortedPaths, nil
}
// Removes duplicate repo paths before returning them
func (pluginConfig *Plugin) GetRepoPaths() map[string]bool {
paths := make(map[string]bool)
for _, transformer := range pluginConfig.Transformers {
paths[transformer.RepositoryPath] = true
}
return paths
}
type TransformerType int
const (
UnknownTransformerType TransformerType = iota
EthEvent
EthStorage
EthContract
)
func (transformerType TransformerType) String() string {
names := [...]string{
"Unknown",
"eth_event",
"eth_storage",
"eth_contract",
}
if transformerType > EthContract || transformerType < EthEvent {
return "Unknown"
}
return names[transformerType]
}
func GetTransformerType(str string) TransformerType {
types := [...]TransformerType{
EthEvent,
EthStorage,
EthContract,
}
for _, ty := range types {
if ty.String() == str {
return ty
}
}
return UnknownTransformerType
}
func anyDupes(list map[uint64]string) bool {
seen := make([]string, 0, len(list))
for _, str := range list {
dupe := inList(str, seen)
if dupe {
return true
}
seen = append(seen, str)
}
return false
}
func inList(str string, list []string) bool {
for _, element := range list {
if str == element {
return true
}
}
return false
}
pkg/config/plugin_test.go
-242
View File
@@ -1,242 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config_test
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/config"
)
var allDifferentPathsConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"transformer1": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path1",
MigrationRank: 0,
RepositoryPath: "test/repo/path",
},
"transformer2": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path2",
MigrationRank: 2,
RepositoryPath: "test/repo/path",
},
"transformer3": {
Path: "test/init/path2",
Type: config.EthEvent,
MigrationPath: "test/migration/path3",
MigrationRank: 1,
RepositoryPath: "test/repo/path",
},
},
}
var overlappingPathsConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"transformer1": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path1",
MigrationRank: 0,
RepositoryPath: "test/repo/path",
},
"transformer2": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path1",
MigrationRank: 0,
RepositoryPath: "test/repo/path",
},
"transformer3": {
Path: "test/init/path2",
Type: config.EthEvent,
MigrationPath: "test/migration/path3",
MigrationRank: 1,
RepositoryPath: "test/repo/path",
},
},
}
var conflictErrorConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"transformer1": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path1",
MigrationRank: 0,
RepositoryPath: "test/repo/path",
},
"transformer2": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path2",
MigrationRank: 0,
RepositoryPath: "test/repo/path",
},
"transformer3": {
Path: "test/init/path2",
Type: config.EthEvent,
MigrationPath: "test/migration/path3",
MigrationRank: 1,
RepositoryPath: "test/repo/path",
},
},
}
var gapErrorConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"transformer1": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path1",
MigrationRank: 0,
RepositoryPath: "test/repo/path",
},
"transformer2": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path2",
MigrationRank: 3,
RepositoryPath: "test/repo/path",
},
"transformer3": {
Path: "test/init/path2",
Type: config.EthEvent,
MigrationPath: "test/migration/path3",
MigrationRank: 1,
RepositoryPath: "test/repo/path",
},
},
}
var missingRankErrorConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"transformer1": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path1",
MigrationRank: 0,
RepositoryPath: "test/repo/path",
},
"transformer2": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path2",
RepositoryPath: "test/repo/path",
},
"transformer3": {
Path: "test/init/path2",
Type: config.EthEvent,
MigrationPath: "test/migration/path3",
MigrationRank: 1,
RepositoryPath: "test/repo/path",
},
},
}
var duplicateErrorConfig = config.Plugin{
Transformers: map[string]config.Transformer{
"transformer1": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path1",
MigrationRank: 0,
RepositoryPath: "test/repo/path",
},
"transformer2": {
Path: "test/init/path",
Type: config.EthEvent,
MigrationPath: "test/migration/path1",
RepositoryPath: "test/repo/path",
MigrationRank: 2,
},
"transformer3": {
Path: "test/init/path2",
Type: config.EthEvent,
MigrationPath: "test/migration/path3",
MigrationRank: 1,
RepositoryPath: "test/repo/path",
},
},
}
var _ = Describe("GetMigrationsPaths", func() {
It("Sorts migration paths by rank", func() {
plugin := allDifferentPathsConfig
migrationPaths, err := plugin.GetMigrationsPaths()
Expect(err).ToNot(HaveOccurred())
Expect(len(migrationPaths)).To(Equal(3))
env := os.Getenv("GOPATH")
path1 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path1")
path2 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path3")
path3 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path2")
expectedMigrationPaths := []string{path1, path2, path3}
Expect(migrationPaths).To(Equal(expectedMigrationPaths))
})
It("Expects identical migration paths to have the same rank", func() {
plugin := overlappingPathsConfig
migrationPaths, err := plugin.GetMigrationsPaths()
Expect(err).ToNot(HaveOccurred())
Expect(len(migrationPaths)).To(Equal(2))
env := os.Getenv("GOPATH")
path1 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path1")
path2 := filepath.Join(env, "src/vendor/test/repo/path/test/migration/path3")
expectedMigrationPaths := []string{path1, path2}
Expect(migrationPaths).To(Equal(expectedMigrationPaths))
})
It("Fails if two different migration paths have the same rank", func() {
plugin := conflictErrorConfig
migrationPaths, err := plugin.GetMigrationsPaths()
Expect(err).To(HaveOccurred())
Expect(len(migrationPaths)).To(Equal(0))
Expect(err.Error()).To(ContainSubstring("has the same migration rank"))
})
It("Fails if there is a gap in the ranks of the migration paths", func() {
plugin := gapErrorConfig
migrationPaths, err := plugin.GetMigrationsPaths()
Expect(err).To(HaveOccurred())
Expect(len(migrationPaths)).To(Equal(0))
Expect(err.Error()).To(ContainSubstring("number of distinct ranks does not match number of distinct migration paths"))
})
It("Fails if a transformer is missing its rank", func() {
plugin := missingRankErrorConfig
migrationPaths, err := plugin.GetMigrationsPaths()
Expect(err).To(HaveOccurred())
Expect(len(migrationPaths)).To(Equal(0))
Expect(err.Error()).To(ContainSubstring("has the same migration rank"))
})
It("Fails if the same migration path has more than one rank", func() {
plugin := duplicateErrorConfig
migrationPaths, err := plugin.GetMigrationsPaths()
Expect(err).To(HaveOccurred())
Expect(len(migrationPaths)).To(Equal(0))
Expect(err.Error()).To(ContainSubstring("duplicate paths with different ranks present"))
})
})