composeAndExecute command that loads and executes over arbitrary transformer set exported from a go plugin generated according to config file; test for plugin generation, loading, and execution; work on plugins README

2019-02-24 15:23:35 -06:00
parent fa4abe4e5c
commit 6c2d895023
116 changed files with 10554 additions and 454 deletions
@@ -1,62 +0,0 @@
-// VulcanizeDB
-// Copyright © 2018 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/vulcanize/vulcanizedb/libraries/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
-)
-
-// backfillMakerLogsCmd represents the backfillMakerLogs command
-var backfillMakerLogsCmd = &cobra.Command{
-	Use:   "backfillMakerLogs",
-	Short: "Backfill Maker event logs",
-	Long: `Backfills Maker event logs based on previously populated block Header records.
-This currently includes logs related to Multi-collateral Dai (frob), Auctions (flip-kick),
-and Price Feeds (ETH/USD, MKR/USD, and REP/USD - LogValue).
-
-vulcanizedb backfillMakerLogs --config environments/local.toml
-
-This command expects a light sync to have been run, and the presence of header records in the Vulcanize database.`,
-	Run: func(cmd *cobra.Command, args []string) {
-		backfillMakerLogs()
-	},
-}
-
-func backfillMakerLogs() {
-	blockChain := getBlockChain()
-	db, err := postgres.NewDB(databaseConfig, blockChain.Node())
-	if err != nil {
-		log.Fatal("Failed to initialize database.")
-	}
-
-	watcher := shared.NewEventWatcher(db, blockChain)
-
-	watcher.AddTransformers(transformers.TransformerInitializers())
-	err = watcher.Execute(constants.HeaderMissing)
-	if err != nil {
-		// TODO Handle watcher error in backfillMakerLogs
-	}
-}
-
-func init() {
-	rootCmd.AddCommand(backfillMakerLogsCmd)
-}
@@ -0,0 +1,121 @@
+// Copyright © 2019 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"plugin"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
+	"github.com/vulcanize/vulcanizedb/pkg/autogen"
+	"github.com/vulcanize/vulcanizedb/utils"
+)
+
+// executePluginCmd represents the execute command
+var composeAndExecuteCmd = &cobra.Command{
+	Use:   "composeAndExecute",
+	Short: "Composes, loads, and executes transformer initializer plugin",
+	Long: `This command needs a config .toml file of form:
+
+[database]
+    name = "vulcanize_public"
+    hostname = "localhost"
+    user = "vulcanize"
+    password = "vulcanize"
+    port = 5432
+
+[client]
+    ipcPath = "http://kovan0.vulcanize.io:8545"
+
+[exporter]
+    filePath = "~/go/src/github.com/vulcanize/vulcanizedb/plugins"
+	fileName = "exporter"
+    [exporter.transformers]
+		transformerImport1 = "github.com/path_to/transformerInitializer1"
+		transformerImport2 = "github.com/path_to/transformerInitializer2"
+
+Note: If any of the imported transformer need additional
+config variables do not forget to include those as well
+
+This information is used to write and build a .so with an arbitrary transformer 
+set composed from the transformer imports specified in the config file
+This .so is loaded as a plugin and the set of transformer initializers is 
+loaded into and executed over by a generic watcher`,
+	Run: func(cmd *cobra.Command, args []string) {
+		composeAndExecute()
+	},
+}
+
+func composeAndExecute() {
+	generator := autogen.NewGenerator(autogenConfig)
+	err := generator.GenerateTransformerPlugin()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	ticker := time.NewTicker(pollingInterval)
+	defer ticker.Stop()
+
+	blockChain := getBlockChain()
+	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
+
+	_, pluginPath, err := autogen.GetPaths(autogenConfig)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	plug, err := plugin.Open(pluginPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	symExporter, err := plug.Lookup("Exporter")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	exporter, ok := symExporter.(Exporter)
+	if !ok {
+		fmt.Println("plugged-in symbol not of type Exporter")
+		os.Exit(1)
+	}
+
+	initializers := exporter.Export()
+	w := watcher.NewWatcher(&db, blockChain)
+	w.AddTransformers(initializers)
+
+	for range ticker.C {
+		err := w.Execute()
+		if err != nil {
+			// TODO Handle watcher errors in composeAndExecute
+		}
+	}
+}
+
+type Exporter interface {
+	Export() []transformer.TransformerInitializer
+}
+
+func init() {
+	rootCmd.AddCommand(composeAndExecuteCmd)
+	composeAndExecuteCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start transformer execution from")
+}
@@ -1,134 +0,0 @@
-// VulcanizeDB
-// Copyright © 2018 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers"
-	shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
-)
-
-// continuousLogSyncCmd represents the continuousLogSync command
-var continuousLogSyncCmd = &cobra.Command{
-	Use:   "continuousLogSync",
-	Short: "Continuously sync logs at the head of the chain",
-	Long: fmt.Sprintf(`Continously syncs logs based on the configured transformers.
-
-vulcanizedb continousLogSync --config environments/local.toml
-	
-Available transformers for (optional) selection with --transformers:
-%v
-
-This command expects a light sync to have been run, and the presence of header records in the Vulcanize database.`,
-		constants.AllTransformerLabels()),
-	Run: func(cmd *cobra.Command, args []string) {
-		syncMakerLogs()
-	},
-}
-
-var transformerNames []string
-var recheckHeadersArg bool
-
-func syncMakerLogs() {
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-
-	blockChain := getBlockChain()
-	db, err := postgres.NewDB(databaseConfig, blockChain.Node())
-	if err != nil {
-		log.Fatal("Failed to initialize database.")
-	}
-
-	initializers := getTransformerInitializers(transformerNames)
-
-	watcher := shared.NewEventWatcher(db, blockChain)
-	watcher.AddTransformers(initializers)
-
-	for range ticker.C {
-		if recheckHeadersArg {
-			err = watcher.Execute(constants.HeaderRecheck)
-		} else {
-			err = watcher.Execute(constants.HeaderMissing)
-		}
-		if err != nil {
-			// TODO Handle watcher errors in ContinuousLogSync
-		}
-	}
-}
-
-func getTransformerInitializers(transformerNames []string) []shared2.TransformerInitializer {
-	var initializers []shared2.TransformerInitializer
-
-	if transformerNames[0] == "all" {
-		initializers = transformers.TransformerInitializers()
-	} else {
-		initializerMap := buildTransformerInitializerMap()
-		for _, transformerName := range transformerNames {
-			initializers = append(initializers, initializerMap[transformerName])
-		}
-	}
-	return initializers
-}
-
-func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer {
-	initializerMap := make(map[string]shared2.TransformerInitializer)
-
-	initializerMap[constants.BiteLabel] = transformers.GetBiteTransformer().NewTransformer
-	initializerMap[constants.CatFileChopLumpLabel] = transformers.GetCatFileChopLumpTransformer().NewLogNoteTransformer
-	initializerMap[constants.CatFileFlipLabel] = transformers.GetCatFileFlipTransformer().NewLogNoteTransformer
-	initializerMap[constants.CatFilePitVowLabel] = transformers.GetCatFilePitVowTransformer().NewLogNoteTransformer
-	initializerMap[constants.DealLabel] = transformers.GetDealTransformer().NewLogNoteTransformer
-	initializerMap[constants.DentLabel] = transformers.GetDentTransformer().NewLogNoteTransformer
-	initializerMap[constants.DripDripLabel] = transformers.GetDripDripTransformer().NewLogNoteTransformer
-	initializerMap[constants.DripFileIlkLabel] = transformers.GetDripFileIlkTransformer().NewLogNoteTransformer
-	initializerMap[constants.DripFileRepoLabel] = transformers.GetDripFileRepoTransformer().NewLogNoteTransformer
-	initializerMap[constants.DripFileVowLabel] = transformers.GetDripFileVowTransformer().NewLogNoteTransformer
-	initializerMap[constants.FlapKickLabel] = transformers.GetFlapKickTransformer().NewTransformer
-	initializerMap[constants.FlipKickLabel] = transformers.GetFlipKickTransformer().NewTransformer
-	initializerMap[constants.FlopKickLabel] = transformers.GetFlopKickTransformer().NewTransformer
-	initializerMap[constants.FrobLabel] = transformers.GetFrobTransformer().NewTransformer
-	initializerMap[constants.PitFileDebtCeilingLabel] = transformers.GetPitFileDebtCeilingTransformer().NewLogNoteTransformer
-	initializerMap[constants.PitFileIlkLabel] = transformers.GetPitFileIlkTransformer().NewLogNoteTransformer
-	initializerMap[constants.PriceFeedLabel] = transformers.GetPriceFeedTransformer().NewLogNoteTransformer
-	initializerMap[constants.TendLabel] = transformers.GetTendTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatFluxLabel] = transformers.GetVatFluxTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatFoldLabel] = transformers.GetVatFoldTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatGrabLabel] = transformers.GetVatGrabTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatHealLabel] = transformers.GetVatHealTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatInitLabel] = transformers.GetVatInitTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatMoveLabel] = transformers.GetVatMoveTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatSlipLabel] = transformers.GetVatSlipTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatTollLabel] = transformers.GetVatTollTransformer().NewLogNoteTransformer
-	initializerMap[constants.VatTuneLabel] = transformers.GetVatTuneTransformer().NewLogNoteTransformer
-	initializerMap[constants.VowFlogLabel] = transformers.GetFlogTransformer().NewLogNoteTransformer
-
-	return initializerMap
-}
-
-func init() {
-	rootCmd.AddCommand(continuousLogSyncCmd)
-	continuousLogSyncCmd.Flags().StringSliceVar(&transformerNames, "transformers", []string{"all"}, "transformer names to be run during this command")
-	continuousLogSyncCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
-}
@@ -28,6 +28,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

+	"github.com/vulcanize/vulcanizedb/pkg/autogen"
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
 	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
@@ -38,6 +39,7 @@ import (
 var (
 	cfgFile             string
 	databaseConfig      config.Database
+	autogenConfig       autogen.Config
 	ipc                 string
 	levelDbPath         string
 	startingBlockNumber int64
@@ -53,7 +55,7 @@ const (

 var rootCmd = &cobra.Command{
 	Use:              "vulcanizedb",
-	PersistentPreRun: database,
+	PersistentPreRun: configure,
 }

 func Execute() {
@@ -63,7 +65,7 @@ func Execute() {
 	}
 }

-func database(cmd *cobra.Command, args []string) {
+func configure(cmd *cobra.Command, args []string) {
 	ipc = viper.GetString("client.ipcpath")
 	levelDbPath = viper.GetString("client.leveldbpath")
 	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
@@ -74,6 +76,11 @@ func database(cmd *cobra.Command, args []string) {
 		User:     viper.GetString("database.user"),
 		Password: viper.GetString("database.password"),
 	}
+	autogenConfig = autogen.Config{
+		FilePath: viper.GetString("exporter.filePath"),
+		FileName: viper.GetString("exporter.fileName"),
+		Imports:  viper.GetStringMapString("exporter.transformers"),
+	}
 	viper.Set("database.config", databaseConfig)
 }

@@ -93,6 +100,8 @@ func init() {
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
 	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
+	rootCmd.PersistentFlags().String("exporter-path", "~/go/src/github.com/vulcanize/vulcanizedb/plugins", "file path to transformer exporter plugin")
+	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")

 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@@ -103,6 +112,8 @@ func init() {
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
 	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
+	viper.BindPFlag("exporter.filePath", rootCmd.PersistentFlags().Lookup("exporter-path"))
+	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
 }

 func initConfig() {