distinguish between differential state/storage nodes and eventual ones

cmd/root.go

@@ -20,42 +20,31 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/config"
 )
 
 var (
 	cfgFile        string
-	databaseConfig config.Database
-	ipc            string
 	subCommand     string
 	logWithCommand log.Entry
 )
 
-const (
-	pollingInterval  = 7 * time.Second
-	validationWindow = 15
-)
-
 var rootCmd = &cobra.Command{
-	Use:              "vulcanizedb",
+	Use:              "ipfs-blockchain-watcher",
 	PersistentPreRun: initFuncs,
 }
 
 func Execute() {
-	log.Info("----- Starting vDB -----")
+	log.Info("----- Starting IPFS blockchain watcher -----")
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
 	}
 }
 
 func initFuncs(cmd *cobra.Command, args []string) {
-	setViperConfigs()
 	logfile := viper.GetString("logfile")
 	if logfile != "" {
 		file, err := os.OpenFile(logfile,
@@ -75,18 +64,6 @@ func initFuncs(cmd *cobra.Command, args []string) {
 	}
 }
 
-func setViperConfigs() {
-	ipc = viper.GetString("client.ipcpath")
-	databaseConfig = config.Database{
-		Name:     viper.GetString("database.name"),
-		Hostname: viper.GetString("database.hostname"),
-		Port:     viper.GetInt("database.port"),
-		User:     viper.GetString("database.user"),
-		Password: viper.GetString("database.password"),
-	}
-	viper.Set("database.config", databaseConfig)
-}
-
 func logLevel() error {
 	lvl, err := log.ParseLevel(viper.GetString("log.level"))
 	if err != nil {
@@ -102,7 +79,6 @@ func logLevel() error {
 
 func init() {
 	cobra.OnInitialize(initConfig)
-	// When searching for env variables, replace dots in config keys with underscores
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv()
 
@@ -122,7 +98,6 @@ func init() {
 	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
 	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
-	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }
 

db/migrations/00025_eth_add_diff_row.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+ALTER TABLE eth.state_cids
+ADD COLUMN diff BOOLEAN NOT NULL DEFAULT FALSE;
+
+ALTER TABLE eth.storage_cids
+ADD COLUMN diff BOOLEAN NOT NULL DEFAULT FALSE;
+
+-- +goose Down
+ALTER TABLE  eth.state_cids
+DROP COLUMN diff;
+
+ALTER TABLE  eth.storage_cids
+DROP COLUMN diff;

db/schema.sql

@@ -413,7 +413,8 @@ CREATE TABLE eth.state_cids (
     state_leaf_key character varying(66),
     cid text NOT NULL,
     state_path bytea,
-    node_type integer
+    node_type integer,
+    diff boolean DEFAULT false NOT NULL
 );
 
 
@@ -447,7 +448,8 @@ CREATE TABLE eth.storage_cids (
     storage_leaf_key character varying(66),
     cid text NOT NULL,
     storage_path bytea,
-    node_type integer
+    node_type integer,
+    diff boolean DEFAULT false NOT NULL
 );
 
 

pkg/config/config_test.go

@@ -29,17 +29,12 @@ var vulcanizeConfig = []byte(`
 name = "dbname"
 hostname = "localhost"
 port = 5432
-
-[client]
-ipcPath = "IPCPATH/geth.ipc"
 `)
 
 var _ = Describe("Loading the config", func() {
-
 	It("reads the private config using the environment", func() {
 		viper.SetConfigName("config")
 		viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/ipfs-blockchain-watcher/environments/")
-		Expect(viper.Get("client.ipcpath")).To(BeNil())
 
 		testConfig := viper.New()
 		testConfig.SetConfigType("toml")
@@ -48,7 +43,6 @@ var _ = Describe("Loading the config", func() {
 		Expect(testConfig.Get("database.hostname")).To(Equal("localhost"))
 		Expect(testConfig.Get("database.name")).To(Equal("dbname"))
 		Expect(testConfig.Get("database.port")).To(Equal(int64(5432)))
-		Expect(testConfig.Get("client.ipcpath")).To(Equal("IPCPATH/geth.ipc"))
 	})
 
 })

pkg/eth/indexer.go

@@ -149,10 +149,10 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
 		if stateCID.StateKey != nullHash.String() {
 			stateKey = stateCID.StateKey
 		}
-		err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
+		err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6)
-									ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5)
+									ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6)
 									RETURNING id`,
-			headerID, stateKey, stateCID.CID, stateCID.Path, stateCID.NodeType).Scan(&stateID)
+			headerID, stateKey, stateCID.CID, stateCID.Path, stateCID.NodeType, true).Scan(&stateID)
 		if err != nil {
 			return err
 		}
@@ -180,10 +180,10 @@ func (in *CIDIndexer) indexStateCID(tx *sqlx.Tx, stateNode StateNodeModel, heade
 	if stateNode.StateKey != nullHash.String() {
 		stateKey = stateNode.StateKey
 	}
-	err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
+	err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6)
-									ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5)
+									ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6)
 									RETURNING id`,
-		headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType).Scan(&stateID)
+		headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true).Scan(&stateID)
 	return stateID, err
 }
 
@@ -199,8 +199,8 @@ func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel,
 	if storageCID.StorageKey != nullHash.String() {
 		storageKey = storageCID.StorageKey
 	}
-	_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5) 
+	_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6) 
-							  ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type) = ($2, $3, $5)`,
+							  ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6)`,
-		stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType)
+		stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType, true)
 	return err
 }

pkg/eth/models.go

@@ -80,6 +80,7 @@ type StateNodeModel struct {
 	StateKey string `db:"state_leaf_key"`
 	NodeType int    `db:"node_type"`
 	CID      string `db:"cid"`
+	Diff     bool   `db:"diff"`
 }
 
 // StorageNodeModel is the db model for eth.storage_cids
@@ -90,6 +91,7 @@ type StorageNodeModel struct {
 	StorageKey string `db:"storage_leaf_key"`
 	NodeType   int    `db:"node_type"`
 	CID        string `db:"cid"`
+	Diff       bool   `db:"diff"`
 }
 
 // StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key
@@ -101,6 +103,7 @@ type StorageNodeWithStateKeyModel struct {
 	StorageKey string `db:"storage_leaf_key"`
 	NodeType   int    `db:"node_type"`
 	CID        string `db:"cid"`
+	Diff       bool   `db:"diff"`
 }
 
 // StateAccountModel is a db model for an eth state account (decoded value of state leaf node)