(VDB-298) Consume Pit contract storage diffs

- Continuously parse storage diffs CSV data to read Pit contract state
- Convert ilks in database to raw bytes32 value for use in generating
  storage keys dynamically
- Persist storage diffs with block number and hash for validation

cmd/backfillMakerLogs.go

@@ -47,7 +47,7 @@ func backfillMakerLogs() {
 		log.Fatal("Failed to initialize database.")
 	}
 
-	watcher := shared.NewWatcher(db, blockChain)
+	watcher := shared.NewEventWatcher(db, blockChain)
 
 	watcher.AddTransformers(transformers.TransformerInitializers())
 	err = watcher.Execute()

cmd/continuousLogSync.go

@@ -62,7 +62,7 @@ func syncMakerLogs() {
 
 	initializers := getTransformerInitializers(transformerNames)
 
-	watcher := shared.NewWatcher(db, blockChain)
+	watcher := shared.NewEventWatcher(db, blockChain)
 	watcher.AddTransformers(initializers)
 
 	for range ticker.C {

cmd/parseStorageDiffs.go

@@ -0,0 +1,66 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/vulcanize/vulcanizedb/libraries/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
+	"log"
+)
+
+// parseStorageDiffsCmd represents the parseStorageDiffs command
+var parseStorageDiffsCmd = &cobra.Command{
+	Use:   "parseStorageDiffs",
+	Short: "Continuously ingest storage diffs from a CSV file",
+	Long: `Read storage diffs out of a CSV file that is constantly receiving
+new rows from an Ethereum node. For example:
+
+./vulcanizedb parseStorageDiffs --config=environments/staging.toml
+
+Note that the path to your storage diffs must be configured in your toml
+file under storageDiffsPath.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		parseStorageDiffs()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(parseStorageDiffsCmd)
+}
+
+func parseStorageDiffs() {
+	blockChain := getBlockChain()
+	db, err := postgres.NewDB(databaseConfig, blockChain.Node())
+	if err != nil {
+		log.Fatal("Failed to initialize database: ", err)
+	}
+
+	tailer := fs.FileTailer{Path: storageDiffsPath}
+
+	// TODO: configure transformers
+	watcher := shared.NewStorageWatcher(tailer, db)
+	watcher.AddTransformers([]storage.TransformerInitializer{transformers.GetPitStorageTransformer().NewTransformer})
+
+	err = watcher.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

cmd/root.go

@@ -40,6 +40,7 @@ var (
 	ipc                 string
 	levelDbPath         string
 	startingBlockNumber int64
+	storageDiffsPath    string
 	syncAll             bool
 	endingBlockNumber   int64
 )
@@ -64,6 +65,7 @@ func Execute() {
 func database(cmd *cobra.Command, args []string) {
 	ipc = viper.GetString("client.ipcpath")
 	levelDbPath = viper.GetString("client.leveldbpath")
+	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
 	databaseConfig = config.Database{
 		Name:     viper.GetString("database.name"),
 		Hostname: viper.GetString("database.hostname"),
@@ -86,6 +88,7 @@ func init() {
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
+	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
 
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@@ -95,6 +98,7 @@ func init() {
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
+	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
 }
 
 func initConfig() {