cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

compose and execute as separate commands; update README and add git checkout vendor/github.com/ethereum/go-ethereum/accounts/abi to make build

Browse Source
This commit is contained in:
Ian Norden
2019-02-28 18:15:16 -06:00
parent 2c5ddd03ba
commit 83f7daf37d
10 changed files with 377 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/compose.go
+173
View File
@@ -0,0 +1,173 @@
// Copyright © 2019 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"errors"
"fmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/config"
p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
)
// composeCmd represents the compose command
var composeCmd = &cobra.Command{
Use: "compose",
Short: "Composes transformer initializer plugin",
Long: `This command needs a config .toml file of form:
[database]
name = "vulcanize_public"
hostname = "localhost"
user = "vulcanize"
password = "vulcanize"
port = 5432
[client]
ipcPath = "http://kovan0.vulcanize.io:8545"
[exporter]
home = "github.com/vulcanize/vulcanizedb"
clone = false
name = "exampleTransformerExporter"
save = false
transformerNames = [
"transformer1",
"transformer2",
"transformer3",
"transformer4",
]
[exporter.transformer1]
path = "path/to/transformer1"
type = "eth_event"
repository = "github.com/account/repo"
migrations = "db/migrations"
[exporter.transformer2]
path = "path/to/transformer2"
type = "eth_event"
repository = "github.com/account/repo"
migrations = "db/migrations"
[exporter.transformer3]
path = "path/to/transformer3"
type = "eth_event"
repository = "github.com/account/repo"
migrations = "db/migrations"
[exporter.transformer4]
path = "path/to/transformer4"
type = "eth_storage"
repository = "github.com/account2/repo2"
migrations = "to/db/migrations"
Note: If any of the plugin transformer need additional
configuration variables include them in the .toml file as well
This information is used to write and build a go plugin with a transformer
set composed from the transformer imports specified in the config file
This plugin is loaded and the set of transformer initializers is exported
from it and loaded into and executed over by the appropriate watcher.
The type of watcher that the transformer works with is specified using the
type variable for each transformer in the config. Currently there are watchers
of event data from an eth node (eth_event) and storage data from an eth node
(eth_storage).
Transformers of different types can be ran together in the same command using a
single config file or in separate command instances using different config files
Specify config location when executing the command:
./vulcanizedb compose --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) {
compose()
},
}
func compose() {
// Build plugin generator config
prepConfig()
// Generate code to build the plugin according to the config file
log.Info("generating plugin")
generator, err := p2.NewGenerator(genConfig, databaseConfig)
if err != nil {
log.Fatal(err)
}
err = generator.GenerateExporterPlugin()
if err != nil {
log.Debug("generating plugin failed")
log.Fatal(err)
}
// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
_, pluginPath, err := genConfig.GetPluginPaths()
if err != nil {
log.Fatal(err)
}
fmt.Printf("Composed plugin %s", pluginPath)
log.Info("plugin .so file output to", pluginPath)
}
func init() {
rootCmd.AddCommand(composeCmd)
}
func prepConfig() {
log.Info("configuring plugin")
names := viper.GetStringSlice("exporter.transformerNames")
transformers := make(map[string]config.Transformer)
for _, name := range names {
transformer := viper.GetStringMapString("exporter." + name)
p, ok := transformer["path"]
if !ok || p == "" {
log.Fatal(fmt.Sprintf("%s transformer config is missing `path` value", name))
}
r, ok := transformer["repository"]
if !ok || r == "" {
log.Fatal(fmt.Sprintf("%s transformer config is missing `repository` value", name))
}
m, ok := transformer["migrations"]
if !ok || m == "" {
log.Fatal(fmt.Sprintf("%s transformer config is missing `migrations` value", name))
}
t, ok := transformer["type"]
if !ok {
log.Fatal(fmt.Sprintf("%s transformer config is missing `type` value", name))
}
transformerType := config.GetTransformerType(t)
if transformerType == config.UnknownTransformerType {
log.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
}
transformers[name] = config.Transformer{
Path: p,
Type: transformerType,
RepositoryPath: r,
MigrationPath: m,
}
}
genConfig = config.Plugin{
Transformers: transformers,
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
FileName: viper.GetString("exporter.name"),
Save: viper.GetBool("exporter.save"),
Home: viper.GetString("exporter.home"),
Clone: viper.GetBool("exporter.clone"),
}
}
cmd/composeAndExecute.go
+8 -100
View File
@@ -16,28 +16,19 @@
package cmd
import (
"errors"
"fmt"
"os"
"plugin"
syn "sync"
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/fs"
p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
"github.com/vulcanize/vulcanizedb/utils"
"os"
"plugin"
syn "sync"
)
// executePluginCmd represents the execute command
// composeAndExecuteCmd represents the composeAndExecute command
var composeAndExecuteCmd = &cobra.Command{
Use: "composeAndExecute",
Short: "Composes, loads, and executes transformer initializer plugin",
@@ -86,8 +77,8 @@ var composeAndExecuteCmd = &cobra.Command{
migrations = "to/db/migrations"
Note: If any of the imported transformer need additional
config variables do not forget to include those as well
Note: If any of the plugin transformer need additional
configuration variables include them in the .toml file as well
This information is used to write and build a go plugin with a transformer
set composed from the transformer imports specified in the config file
@@ -133,10 +124,10 @@ func composeAndExecute() {
if !genConfig.Save {
defer helpers.ClearFiles(pluginPath)
}
log.Info("opening plugin")
log.Info("linking plugin", pluginPath)
plug, err := plugin.Open(pluginPath)
if err != nil {
log.Debug("opening pluggin failed")
log.Debug("linking plugin failed")
log.Fatal(err)
}
@@ -182,90 +173,7 @@ func composeAndExecute() {
wg.Wait()
}
type Exporter interface {
Export() ([]transformer.TransformerInitializer, []transformer.StorageTransformerInitializer)
}
func init() {
rootCmd.AddCommand(composeAndExecuteCmd)
composeAndExecuteCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
}
func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the TransformerInitializer set using the watcher
log.Info("executing event transformers")
var recheck constants.TransformerExecution
if recheckHeadersArg {
recheck = constants.HeaderRecheck
} else {
recheck = constants.HeaderMissing
}
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
err := w.Execute(recheck)
if err != nil {
// TODO Handle watcher errors in composeAndExecute
}
}
}
func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the TransformerInitializer set using the watcher
log.Info("executing storage transformers")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
err := w.Execute()
if err != nil {
// TODO Handle watcher errors in composeAndExecute
}
}
}
func prepConfig() {
log.Info("configuring plugin")
names := viper.GetStringSlice("exporter.transformerNames")
transformers := make(map[string]config.Transformer)
for _, name := range names {
transformer := viper.GetStringMapString("exporter." + name)
p, ok := transformer["path"]
if !ok || p == "" {
log.Fatal(fmt.Sprintf("%s transformer config is missing `path` value", name))
}
r, ok := transformer["repository"]
if !ok || r == "" {
log.Fatal(fmt.Sprintf("%s transformer config is missing `repository` value", name))
}
m, ok := transformer["migrations"]
if !ok || m == "" {
log.Fatal(fmt.Sprintf("%s transformer config is missing `migrations` value", name))
}
t, ok := transformer["type"]
if !ok {
log.Fatal(fmt.Sprintf("%s transformer config is missing `type` value", name))
}
transformerType := config.GetTransformerType(t)
if transformerType == config.UnknownTransformerType {
log.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
}
transformers[name] = config.Transformer{
Path: p,
Type: transformerType,
RepositoryPath: r,
MigrationPath: m,
}
}
genConfig = config.Plugin{
Transformers: transformers,
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
FileName: viper.GetString("exporter.name"),
Save: viper.GetBool("exporter.save"),
Home: viper.GetString("exporter.home"),
Clone: viper.GetBool("exporter.clone"),
}
}
cmd/execute.go
+170
View File
@@ -0,0 +1,170 @@
// Copyright © 2019 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"os"
"plugin"
syn "sync"
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/fs"
"github.com/vulcanize/vulcanizedb/utils"
)
// executeCmd represents the execute command
var executeCmd = &cobra.Command{
Use: "execute",
Short: "executes a precomposed transformer initializer plugin",
Long: `This command needs a config .toml file of form:
[database]
name = "vulcanize_public"
hostname = "localhost"
user = "vulcanize"
password = "vulcanize"
port = 5432
[client]
ipcPath = "http://kovan0.vulcanize.io:8545"
[exporter]
name = "exampleTransformerExporter"
Note: If any of the plugin transformer need additional
configuration variables include them in the .toml file as well
The exporter.name is the name (without extension) of the plugin to be loaded.
The plugin file needs to be located in the /plugins directory and this command assumes
the db migrations remain from when the plugin was composed. Additionally, the plugin
must have been composed by the same version of vulcanizedb or else it will not be compatible.
Specify config location when executing the command:
./vulcanizedb execute --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) {
execute()
},
}
func execute() {
// Build plugin generator config
prepConfig()
// Get the plugin path and load the plugin
_, pluginPath, err := genConfig.GetPluginPaths()
if err != nil {
log.Fatal(err)
}
fmt.Printf("Executing plugin %s", pluginPath)
log.Info("linking plugin", pluginPath)
plug, err := plugin.Open(pluginPath)
if err != nil {
log.Debug("linking plugin failed")
log.Fatal(err)
}
// Load the `Exporter` symbol from the plugin
log.Info("loading transformers from plugin")
symExporter, err := plug.Lookup("Exporter")
if err != nil {
log.Debug("loading Exporter symbol failed")
log.Fatal(err)
}
// Assert that the symbol is of type Exporter
exporter, ok := symExporter.(Exporter)
if !ok {
log.Debug("plugged-in symbol not of type Exporter")
os.Exit(1)
}
// Use the Exporters export method to load the TransformerInitializer and StorageTransformerInitializer sets
ethEventInitializers, ethStorageInitializers := exporter.Export()
// Setup bc and db objects
blockChain := getBlockChain()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
// Execute over transformer sets returned by the exporter
// Use WaitGroup to wait on both goroutines
var wg syn.WaitGroup
if len(ethEventInitializers) > 0 {
ew := watcher.NewEventWatcher(&db, blockChain)
ew.AddTransformers(ethEventInitializers)
wg.Add(1)
go watchEthEvents(&ew, &wg)
}
if len(ethStorageInitializers) > 0 {
tailer := fs.FileTailer{Path: storageDiffsPath}
sw := watcher.NewStorageWatcher(tailer, &db)
sw.AddTransformers(ethStorageInitializers)
wg.Add(1)
go watchEthStorage(&sw, &wg)
}
wg.Wait()
}
func init() {
rootCmd.AddCommand(executeCmd)
executeCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
}
type Exporter interface {
Export() ([]transformer.TransformerInitializer, []transformer.StorageTransformerInitializer)
}
func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the TransformerInitializer set using the watcher
log.Info("executing event transformers")
var recheck constants.TransformerExecution
if recheckHeadersArg {
recheck = constants.HeaderRecheck
} else {
recheck = constants.HeaderMissing
}
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
err := w.Execute(recheck)
if err != nil {
// TODO Handle watcher errors in execute
}
}
}
func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the TransformerInitializer set using the watcher
log.Info("executing storage transformers")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
err := w.Execute()
if err != nil {
// TODO Handle watcher errors in execute
}
}
}