trimming down to ipfs watchers

cmd/compose.go

@@ -1,193 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"errors"
-	"fmt"
-	"strconv"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
-)
-
-// composeCmd represents the compose command
-var composeCmd = &cobra.Command{
-	Use:   "compose",
-	Short: "Composes transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "exampleTransformerExporter"
-    save     = false
-    transformerNames = [
-        "transformer1",
-        "transformer2",
-        "transformer3",
-        "transformer4",
-    ]
-    [exporter.transformer1]
-        path = "path/to/transformer1"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer2]
-        path = "path/to/transformer2"
-        type = "eth_contract"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer3]
-        path = "path/to/transformer3"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer4]
-        path = "path/to/transformer4"
-        type = "eth_storage"
-        repository = "github.com/account2/repo2"
-        migrations = "to/db/migrations"
-        rank = "1"
-
-
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-
-This information is used to write and build a go plugin with a transformer 
-set composed from the transformer imports specified in the config file
-This plugin is loaded and the set of transformer initializers is exported
-from it and loaded into and executed over by the appropriate watcher. 
-
-The type of watcher that the transformer works with is specified using the 
-type variable for each transformer in the config. Currently there are watchers 
-of event data from an eth node (eth_event) and storage data from an eth node 
-(eth_storage), and a more generic interface for accepting contract_watcher pkg
-based transformers which can perform both event watching and public method 
-polling (eth_contract).
-
-Transformers of different types can be ran together in the same command using a 
-single config file or in separate command instances using different config files
-
-Specify config location when executing the command:
-./vulcanizedb compose --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		compose()
-	},
-}
-
-func compose() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Generate code to build the plugin according to the config file
-	logWithCommand.Info("generating plugin")
-	generator, err := p2.NewGenerator(genConfig, databaseConfig)
-	if err != nil {
-		logWithCommand.Debug("initializing plugin generator failed")
-		logWithCommand.Fatal(err)
-	}
-	err = generator.GenerateExporterPlugin()
-	if err != nil {
-		logWithCommand.Debug("generating plugin failed")
-		logWithCommand.Fatal(err)
-	}
-	// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		logWithCommand.Debug("getting plugin path failed")
-		logWithCommand.Fatal(err)
-	}
-	fmt.Printf("Composed plugin %s", pluginPath)
-	logWithCommand.Info("plugin .so file output to ", pluginPath)
-}
-
-func init() {
-	rootCmd.AddCommand(composeCmd)
-}
-
-func prepConfig() {
-	logWithCommand.Info("configuring plugin")
-	names := viper.GetStringSlice("exporter.transformerNames")
-	transformers := make(map[string]config.Transformer)
-	for _, name := range names {
-		logWithCommand.Debug("Configuring " + name + " transformer")
-		transformer := viper.GetStringMapString("exporter." + name)
-		p, pOK := transformer["path"]
-		if !pOK || p == "" {
-			logWithCommand.Fatal(name, " transformer config is missing `path` value")
-		}
-		r, rOK := transformer["repository"]
-		if !rOK || r == "" {
-			logWithCommand.Fatal(name, " transformer config is missing `repository` value")
-		}
-		m, mOK := transformer["migrations"]
-		if !mOK || m == "" {
-			logWithCommand.Fatal(name, " transformer config is missing `migrations` value")
-		}
-		mr, mrOK := transformer["rank"]
-		if !mrOK || mr == "" {
-			logWithCommand.Fatal(name, " transformer config is missing `rank` value")
-		}
-		rank, err := strconv.ParseUint(mr, 10, 64)
-		if err != nil {
-			logWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
-		}
-		t, tOK := transformer["type"]
-		if !tOK {
-			logWithCommand.Fatal(name, " transformer config is missing `type` value")
-		}
-		transformerType := config.GetTransformerType(t)
-		if transformerType == config.UnknownTransformerType {
-			logWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
-		}
-
-		transformers[name] = config.Transformer{
-			Path:           p,
-			Type:           transformerType,
-			RepositoryPath: r,
-			MigrationPath:  m,
-			MigrationRank:  rank,
-		}
-	}
-
-	genConfig = config.Plugin{
-		Transformers: transformers,
-		FilePath:     "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
-		FileName:     viper.GetString("exporter.name"),
-		Save:         viper.GetBool("exporter.save"),
-		Home:         viper.GetString("exporter.home"),
-	}
-}

cmd/composeAndExecute.go

@@ -1,217 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"os"
-	"plugin"
-	syn "sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
-	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// composeAndExecuteCmd represents the composeAndExecute command
-var composeAndExecuteCmd = &cobra.Command{
-	Use:   "composeAndExecute",
-	Short: "Composes, loads, and executes transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-[exporter]
-    home     = "github.com/vulcanize/vulcanizedb"
-    name     = "exampleTransformerExporter"
-    save     = false
-    transformerNames = [
-        "transformer1",
-        "transformer2",
-        "transformer3",
-        "transformer4",
-    ]
-    [exporter.transformer1]
-        path = "path/to/transformer1"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer2]
-        path = "path/to/transformer2"
-        type = "eth_contract"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "2"
-    [exporter.transformer3]
-        path = "path/to/transformer3"
-        type = "eth_event"
-        repository = "github.com/account/repo"
-        migrations = "db/migrations"
-        rank = "0"
-    [exporter.transformer4]
-        path = "path/to/transformer4"
-        type = "eth_storage"
-        repository = "github.com/account2/repo2"
-        migrations = "to/db/migrations"
-        rank = "1"
-
-
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-
-This information is used to write and build a go plugin with a transformer 
-set composed from the transformer imports specified in the config file
-This plugin is loaded and the set of transformer initializers is exported
-from it and loaded into and executed over by the appropriate watcher. 
-
-The type of watcher that the transformer works with is specified using the 
-type variable for each transformer in the config. Currently there are watchers 
-of event data from an eth node (eth_event) and storage data from an eth node 
-(eth_storage), and a more generic interface for accepting contract_watcher pkg
-based transformers which can perform both event watching and public method 
-polling (eth_contract).
-
-Transformers of different types can be ran together in the same command using a 
-single config file or in separate command instances using different config files
-
-Specify config location when executing the command:
-./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		composeAndExecute()
-	},
-}
-
-func composeAndExecute() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Generate code to build the plugin according to the config file
-	logWithCommand.Info("generating plugin")
-	generator, err := p2.NewGenerator(genConfig, databaseConfig)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	err = generator.GenerateExporterPlugin()
-	if err != nil {
-		logWithCommand.Debug("generating plugin failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Get the plugin path and load the plugin
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	if !genConfig.Save {
-		defer helpers.ClearFiles(pluginPath)
-	}
-	logWithCommand.Info("linking plugin ", pluginPath)
-	plug, err := plugin.Open(pluginPath)
-	if err != nil {
-		logWithCommand.Debug("linking plugin failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Load the `Exporter` symbol from the plugin
-	logWithCommand.Info("loading transformers from plugin")
-	symExporter, err := plug.Lookup("Exporter")
-	if err != nil {
-		logWithCommand.Debug("loading Exporter symbol failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Assert that the symbol is of type Exporter
-	exporter, ok := symExporter.(Exporter)
-	if !ok {
-		logWithCommand.Debug("plugged-in symbol not of type Exporter")
-		os.Exit(1)
-	}
-
-	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
-	ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export()
-
-	// Setup bc and db objects
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	// Execute over transformer sets returned by the exporter
-	// Use WaitGroup to wait on both goroutines
-	var wg syn.WaitGroup
-	if len(ethEventInitializers) > 0 {
-		ew := watcher.NewEventWatcher(&db, blockChain)
-		err := ew.AddTransformers(ethEventInitializers)
-		if err != nil {
-			logWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
-		}
-		wg.Add(1)
-		go watchEthEvents(&ew, &wg)
-	}
-
-	if len(ethStorageInitializers) > 0 {
-		switch storageDiffsSource {
-		case "geth":
-			log.Debug("fetching storage diffs from geth pub sub")
-			rpcClient, _ := getClients()
-			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
-			storageFetcher := fetcher.NewGethRPCStorageFetcher(stateDiffStreamer)
-			sw := watcher.NewStorageWatcher(storageFetcher, &db)
-			sw.AddTransformers(ethStorageInitializers)
-			wg.Add(1)
-			go watchEthStorage(sw, &wg)
-		default:
-			log.Debug("fetching storage diffs from csv")
-			tailer := fs.FileTailer{Path: storageDiffsPath}
-			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-			sw := watcher.NewStorageWatcher(storageFetcher, &db)
-			sw.AddTransformers(ethStorageInitializers)
-			wg.Add(1)
-			go watchEthStorage(sw, &wg)
-		}
-	}
-
-	if len(ethContractInitializers) > 0 {
-		gw := watcher.NewContractWatcher(&db, blockChain)
-		gw.AddTransformers(ethContractInitializers)
-		wg.Add(1)
-		go watchEthContract(&gw, &wg)
-	}
-	wg.Wait()
-}
-
-func init() {
-	rootCmd.AddCommand(composeAndExecuteCmd)
-	composeAndExecuteCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
-}

cmd/contractWatcher.go

@@ -1,118 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	ht "github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/transformer"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// contractWatcherCmd represents the contractWatcher command
-var contractWatcherCmd = &cobra.Command{
-	Use:   "contractWatcher",
-	Short: "Watches events at the provided contract address using fully synced vDB",
-	Long: `Uses input contract address and event filters to watch events
-
-Expects an ethereum node to be running
-Expects an archival node synced into vulcanizeDB
-Requires a .toml config file:
-
-  [database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-
-  [client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-
-  [contract]
-    network  = ""
-    addresses  = [
-        "contractAddress1",
-        "contractAddress2"
-    ]
-    [contract.contractAddress1]
-        abi    = 'ABI for contract 1'
-        startingBlock = 982463
-    [contract.contractAddress2]
-        abi    = 'ABI for contract 2'
-        events = [
-            "event1",
-            "event2"
-        ]
-		eventArgs = [
-			"arg1",
-			"arg2"
-		]
-        methods = [
-            "method1",
-			"method2"
-        ]
-		methodArgs = [
-			"arg1",
-			"arg2"
-		]
-        startingBlock = 4448566
-        piping = true
-`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		contractWatcher()
-	},
-}
-
-var (
-	mode string
-)
-
-func contractWatcher() {
-	ticker := time.NewTicker(5 * time.Second)
-	defer ticker.Stop()
-
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	var t st.ContractTransformer
-	con := config.ContractConfig{}
-	con.PrepConfig()
-	t = ht.NewTransformer(con, blockChain, &db)
-
-	err := t.Init()
-	if err != nil {
-		logWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
-	}
-
-	for range ticker.C {
-		err = t.Execute()
-		if err != nil {
-			logWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
-		}
-	}
-}
-
-func init() {
-	rootCmd.AddCommand(contractWatcherCmd)
-}

cmd/execute.go

@@ -1,207 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"plugin"
-	syn "sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// executeCmd represents the execute command
-var executeCmd = &cobra.Command{
-	Use:   "execute",
-	Short: "executes a precomposed transformer initializer plugin",
-	Long: `This command needs a config .toml file of form:
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    user     = "vulcanize"
-    password = "vulcanize"
-    port     = 5432
-[client]
-    ipcPath  = "/Users/user/Library/Ethereum/geth.ipc"
-[exporter]
-    name     = "exampleTransformerExporter"
-Note: If any of the plugin transformer need additional
-configuration variables include them in the .toml file as well
-The exporter.name is the name (without extension) of the plugin to be loaded.
-The plugin file needs to be located in the /plugins directory and this command assumes 
-the db migrations remain from when the plugin was composed. Additionally, the plugin 
-must have been composed by the same version of vulcanizedb or else it will not be compatible.
-Specify config location when executing the command:
-./vulcanizedb execute --config=./environments/config_name.toml`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		execute()
-	},
-}
-
-func execute() {
-	// Build plugin generator config
-	prepConfig()
-
-	// Get the plugin path and load the plugin
-	_, pluginPath, err := genConfig.GetPluginPaths()
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
-	fmt.Printf("Executing plugin %s", pluginPath)
-	logWithCommand.Info("linking plugin ", pluginPath)
-	plug, err := plugin.Open(pluginPath)
-	if err != nil {
-		logWithCommand.Warn("linking plugin failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Load the `Exporter` symbol from the plugin
-	logWithCommand.Info("loading transformers from plugin")
-	symExporter, err := plug.Lookup("Exporter")
-	if err != nil {
-		logWithCommand.Warn("loading Exporter symbol failed")
-		logWithCommand.Fatal(err)
-	}
-
-	// Assert that the symbol is of type Exporter
-	exporter, ok := symExporter.(Exporter)
-	if !ok {
-		logWithCommand.Fatal("plugged-in symbol not of type Exporter")
-	}
-
-	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
-	ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export()
-
-	// Setup bc and db objects
-	blockChain := getBlockChain()
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	// Execute over transformer sets returned by the exporter
-	// Use WaitGroup to wait on both goroutines
-	var wg syn.WaitGroup
-	if len(ethEventInitializers) > 0 {
-		ew := watcher.NewEventWatcher(&db, blockChain)
-		err = ew.AddTransformers(ethEventInitializers)
-		if err != nil {
-			logWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
-		}
-		wg.Add(1)
-		go watchEthEvents(&ew, &wg)
-	}
-
-	if len(ethStorageInitializers) > 0 {
-		switch storageDiffsSource {
-		case "geth":
-			log.Debug("fetching storage diffs from geth pub sub")
-			wsClient := getWSClient()
-			stateDiffStreamer := streamer.NewStateDiffStreamer(wsClient)
-			storageFetcher := fetcher.NewGethRPCStorageFetcher(stateDiffStreamer)
-			sw := watcher.NewStorageWatcher(storageFetcher, &db)
-			sw.AddTransformers(ethStorageInitializers)
-			wg.Add(1)
-			go watchEthStorage(sw, &wg)
-		default:
-			log.Debug("fetching storage diffs from csv")
-			tailer := fs.FileTailer{Path: storageDiffsPath}
-			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-			sw := watcher.NewStorageWatcher(storageFetcher, &db)
-			sw.AddTransformers(ethStorageInitializers)
-			wg.Add(1)
-			go watchEthStorage(sw, &wg)
-		}
-	}
-
-	if len(ethContractInitializers) > 0 {
-		gw := watcher.NewContractWatcher(&db, blockChain)
-		gw.AddTransformers(ethContractInitializers)
-		wg.Add(1)
-		go watchEthContract(&gw, &wg)
-	}
-	wg.Wait()
-}
-
-func init() {
-	rootCmd.AddCommand(executeCmd)
-	executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
-}
-
-type Exporter interface {
-	Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer, []transformer.ContractTransformerInitializer)
-}
-
-func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the EventTransformerInitializer set using the watcher
-	logWithCommand.Info("executing event transformers")
-	var recheck constants.TransformerExecution
-	if recheckHeadersArg {
-		recheck = constants.HeaderRecheck
-	} else {
-		recheck = constants.HeaderUnchecked
-	}
-	err := w.Execute(recheck)
-	if err != nil {
-		logWithCommand.Fatalf("error executing event watcher: %s", err.Error())
-	}
-}
-
-func watchEthStorage(w watcher.IStorageWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the StorageTransformerInitializer set using the storage watcher
-	logWithCommand.Info("executing storage transformers")
-	on := viper.GetBool("storageBackFill.on")
-	if on {
-		backFillStorage(w)
-	}
-	w.Execute(queueRecheckInterval, on)
-}
-
-func backFillStorage(w watcher.IStorageWatcher) {
-	rpcClient, _ := getClients()
-	// find min deployment block
-	minDeploymentBlock := constants.GetMinDeploymentBlock()
-	stateDiffFetcher := fetcher.NewStateDiffFetcher(rpcClient)
-	backFiller := storage.NewStorageBackFiller(stateDiffFetcher, storage.DefaultMaxBatchSize)
-	go w.BackFill(minDeploymentBlock, backFiller)
-}
-
-func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
-	defer wg.Done()
-	// Execute over the ContractTransformerInitializer set using the contract watcher
-	logWithCommand.Info("executing contract_watcher transformers")
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-	for range ticker.C {
-		w.Execute()
-	}
-}

cmd/headerSync.go

@@ -1,114 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
-	"github.com/vulcanize/vulcanizedb/utils"
-)
-
-// headerSyncCmd represents the headerSync command
-var headerSyncCmd = &cobra.Command{
-	Use:   "headerSync",
-	Short: "Syncs VulcanizeDB with local ethereum node's block headers",
-	Long: `Syncs VulcanizeDB with local ethereum node. Populates
-Postgres with block headers.
-
-./vulcanizedb headerSync --starting-block-number 0 --config public.toml
-
-Expects ethereum node to be running and requires a .toml config:
-
-  [database]
-  name = "vulcanize_public"
-  hostname = "localhost"
-  port = 5432
-
-  [client]
-  ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
-`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		headerSync()
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(headerSyncCmd)
-	headerSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
-}
-
-func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
-	populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
-	if err != nil {
-		// TODO Lots of possible errors in the call stack above. If errors occur, we still put
-		// 0 in the channel, triggering another round
-		logWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
-	}
-	missingBlocksPopulated <- populated
-}
-
-func headerSync() {
-	ticker := time.NewTicker(pollingInterval)
-	defer ticker.Stop()
-	blockChain := getBlockChain()
-	validateArgs(blockChain)
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-
-	headerRepository := repositories.NewHeaderRepository(&db)
-	validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow)
-	missingBlocksPopulated := make(chan int)
-	go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)
-
-	for {
-		select {
-		case <-ticker.C:
-			window, err := validator.ValidateHeaders()
-			if err != nil {
-				logWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
-			}
-			logWithCommand.Debug(window.GetString())
-		case n := <-missingBlocksPopulated:
-			if n == 0 {
-				time.Sleep(3 * time.Second)
-			}
-			go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)
-		}
-	}
-}
-
-func validateArgs(blockChain *eth.BlockChain) {
-	lastBlock, err := blockChain.LastBlock()
-	if err != nil {
-		logWithCommand.Error("validateArgs: Error getting last block: ", err)
-	}
-	if lastBlock.Int64() == 0 {
-		logWithCommand.Fatal("geth initial: state sync not finished")
-	}
-	if startingBlockNumber > lastBlock.Int64() {
-		logWithCommand.Fatal("starting block number > current block number")
-	}
-}

cmd/resync.go

@@ -19,11 +19,11 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/resync"
-	v "github.com/vulcanize/vulcanizedb/version"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/resync"
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
 )
 
 // resyncCmd represents the resync command

cmd/root.go

@@ -17,38 +17,24 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/config"
 )
 
 var (
-	cfgFile              string
-	databaseConfig       config.Database
-	genConfig            config.Plugin
-	ipc                  string
-	queueRecheckInterval time.Duration
-	startingBlockNumber  int64
-	storageDiffsPath     string
-	recheckHeadersArg    bool
-	subCommand           string
-	logWithCommand       log.Entry
-	storageDiffsSource   string
+	cfgFile        string
+	databaseConfig config.Database
+	ipc            string
+	subCommand     string
+	logWithCommand log.Entry
 )
 
 const (
@@ -91,8 +77,6 @@ func initFuncs(cmd *cobra.Command, args []string) {
 
 func setViperConfigs() {
 	ipc = viper.GetString("client.ipcpath")
-	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
-	storageDiffsSource = viper.GetString("storageDiffs.source")
 	databaseConfig = config.Database{
 		Name:     viper.GetString("database.name"),
 		Hostname: viper.GetString("database.hostname"),
@@ -130,10 +114,6 @@ func init() {
 	rootCmd.PersistentFlags().String("database-user", "", "database user")
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
-	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
-	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
-	rootCmd.PersistentFlags().String("storageDiffs-source", "csv", "where to get the state diffs: csv or geth")
-	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
 
 	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
@@ -143,10 +123,6 @@ func init() {
 	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
-	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
-	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
-	viper.BindPFlag("storageDiffs.source", rootCmd.PersistentFlags().Lookup("storageDiffs-source"))
-	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }
 
@@ -162,35 +138,3 @@ func initConfig() {
 		log.Warn("No config file passed with --config flag")
 	}
 }
-
-func getBlockChain() *eth.BlockChain {
-	rpcClient, ethClient := getClients()
-	vdbEthClient := client.NewEthClient(ethClient)
-	vdbNode := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRPCTransactionConverter(ethClient)
-	return eth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
-}
-
-func getClients() (client.RPCClient, *ethclient.Client) {
-	rawRPCClient, err := rpc.Dial(ipc)
-
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	rpcClient := client.NewRPCClient(rawRPCClient, ipc)
-	ethClient := ethclient.NewClient(rawRPCClient)
-
-	return rpcClient, ethClient
-}
-
-func getWSClient() core.RPCClient {
-	wsRPCpath := viper.GetString("client.wsPath")
-	if wsRPCpath == "" {
-		logWithCommand.Fatal(errors.New("getWSClient() was called but no ws rpc path is provided"))
-	}
-	wsRPCClient, dialErr := rpc.Dial(wsRPCpath)
-	if dialErr != nil {
-		logWithCommand.Fatal(dialErr)
-	}
-	return client.NewRPCClient(wsRPCClient, wsRPCpath)
-}

cmd/streamEthSubscribe.go

@@ -28,11 +28,11 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/client"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/core"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/eth"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/streamer"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/watcher"
 )
 
 // streamEthSubscriptionCmd represents the streamEthSubscription command
@@ -64,7 +64,7 @@ func streamEthSubscription() {
 	str := streamer.NewSuperNodeStreamer(rpcClient)
 
 	// Buffered channel for reading subscription payloads
-	payloadChan := make(chan super_node.SubscriptionPayload, 20000)
+	payloadChan := make(chan watcher.SubscriptionPayload, 20000)
 
 	// Subscribe to the super node service with the given config/filter parameters
 	rlpParams, err := rlp.EncodeToBytes(ethSubConfig)

cmd/version.go

@@ -19,20 +19,20 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	v "github.com/vulcanize/vulcanizedb/version"
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
 )
 
 // versionCmd represents the version command
 var versionCmd = &cobra.Command{
 	Use:   "version",
-	Short: "Prints the version of vulcanizeDB",
-	Long: `Use this command to fetch the version of vulcanizeDB
+	Short: "Prints the version of ipfs-chain-watcher",
+	Long: `Use this command to fetch the version of ipfs-chain-watcher
 
-Usage: ./vulcanizedb version`,
+Usage: ./ipfs-chain-watcher version`,
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *log.WithField("SubCommand", subCommand)
-		logWithCommand.Infof("VulcanizeDB version: %s", v.VersionWithMeta)
+		logWithCommand.Infof("ipfs-chain-watcher version: %s", v.VersionWithMeta)
 	},
 }
 

cmd/watch.go

@@ -25,17 +25,17 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
-	v "github.com/vulcanize/vulcanizedb/version"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/ipfs"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/shared"
+	"github.com/vulcanize/ipfs-chain-watcher/pkg/watcher"
+	v "github.com/vulcanize/ipfs-chain-watcher/version"
 )
 
 // superNodeCmd represents the superNode command
 var superNodeCmd = &cobra.Command{
 	Use:   "superNode",
-	Short: "VulcanizeDB SuperNode",
-	Long: `This command configures a VulcanizeDB SuperNode.
+	Short: "sync chain data into PG-IPFS",
+	Long: `This command configures a VulcanizeDB ipfs-chain-watcher.
 
 The Sync process streams all chain data from the appropriate chain, processes this data into IPLD objects
 and publishes them to IPFS. It then indexes the CIDs against useful data fields/metadata in Postgres. 
@@ -56,7 +56,7 @@ and fill in gaps in the data
 func superNode() {
 	logWithCommand.Infof("running vdb version: %s", v.VersionWithMeta)
 	logWithCommand.Debug("loading super node configuration variables")
-	superNodeConfig, err := super_node.NewSuperNodeConfig()
+	superNodeConfig, err := watcher.NewSuperNodeConfig()
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -68,14 +68,14 @@ func superNode() {
 	}
 	wg := &sync.WaitGroup{}
 	logWithCommand.Debug("initializing new super node service")
-	superNode, err := super_node.NewSuperNode(superNodeConfig)
+	superNode, err := watcher.NewSuperNode(superNodeConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
 	var forwardPayloadChan chan shared.ConvertedData
 	if superNodeConfig.Serve {
 		logWithCommand.Info("starting up super node servers")
-		forwardPayloadChan = make(chan shared.ConvertedData, super_node.PayloadChanBufferSize)
+		forwardPayloadChan = make(chan shared.ConvertedData, watcher.PayloadChanBufferSize)
 		superNode.Serve(wg, forwardPayloadChan)
 		if err := startServers(superNode, superNodeConfig); err != nil {
 			logWithCommand.Fatal(err)
@@ -87,10 +87,10 @@ func superNode() {
 			logWithCommand.Fatal(err)
 		}
 	}
-	var backFiller super_node.BackFillInterface
+	var backFiller watcher.BackFillInterface
 	if superNodeConfig.BackFill {
 		logWithCommand.Debug("initializing new super node backfill service")
-		backFiller, err = super_node.NewBackFillService(superNodeConfig, forwardPayloadChan)
+		backFiller, err = watcher.NewBackFillService(superNodeConfig, forwardPayloadChan)
 		if err != nil {
 			logWithCommand.Fatal(err)
 		}
@@ -107,7 +107,7 @@ func superNode() {
 	wg.Wait()
 }
 
-func startServers(superNode super_node.SuperNode, settings *super_node.Config) error {
+func startServers(superNode watcher.SuperNode, settings *watcher.Config) error {
 	logWithCommand.Debug("starting up IPC server")
 	_, _, err := rpc.StartIPCEndpoint(settings.IPCEndpoint, superNode.APIs())
 	if err != nil {