From 0ab55ef9d81e2e5b3302f4d158c7fadff13ec361 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 1 Jul 2020 13:44:04 -0500 Subject: [PATCH] distinguish between differential state/storage nodes and eventual ones --- cmd/root.go | 29 ++---------------------- db/migrations/00025_eth_add_diff_row.sql | 13 +++++++++++ db/schema.sql | 6 +++-- pkg/config/config_test.go | 6 ----- pkg/eth/indexer.go | 18 +++++++-------- pkg/eth/models.go | 3 +++ 6 files changed, 31 insertions(+), 44 deletions(-) create mode 100644 db/migrations/00025_eth_add_diff_row.sql diff --git a/cmd/root.go b/cmd/root.go index 36a09eb3..942ab5f8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,42 +20,31 @@ import ( "fmt" "os" "strings" - "time" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - - "github.com/vulcanize/ipfs-blockchain-watcher/pkg/config" ) var ( cfgFile string - databaseConfig config.Database - ipc string subCommand string logWithCommand log.Entry ) -const ( - pollingInterval = 7 * time.Second - validationWindow = 15 -) - var rootCmd = &cobra.Command{ - Use: "vulcanizedb", + Use: "ipfs-blockchain-watcher", PersistentPreRun: initFuncs, } func Execute() { - log.Info("----- Starting vDB -----") + log.Info("----- Starting IPFS blockchain watcher -----") if err := rootCmd.Execute(); err != nil { log.Fatal(err) } } func initFuncs(cmd *cobra.Command, args []string) { - setViperConfigs() logfile := viper.GetString("logfile") if logfile != "" { file, err := os.OpenFile(logfile, @@ -75,18 +64,6 @@ func initFuncs(cmd *cobra.Command, args []string) { } } -func setViperConfigs() { - ipc = viper.GetString("client.ipcpath") - databaseConfig = config.Database{ - Name: viper.GetString("database.name"), - Hostname: viper.GetString("database.hostname"), - Port: viper.GetInt("database.port"), - User: viper.GetString("database.user"), - Password: viper.GetString("database.password"), - } - viper.Set("database.config", databaseConfig) -} - func logLevel() error { lvl, err := log.ParseLevel(viper.GetString("log.level")) if err != nil { @@ -102,7 +79,6 @@ func logLevel() error { func init() { cobra.OnInitialize(initConfig) - // When searching for env variables, replace dots in config keys with underscores viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.AutomaticEnv() @@ -122,7 +98,6 @@ func init() { viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname")) viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user")) viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password")) - viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath")) viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) } diff --git a/db/migrations/00025_eth_add_diff_row.sql b/db/migrations/00025_eth_add_diff_row.sql new file mode 100644 index 00000000..fa2c5640 --- /dev/null +++ b/db/migrations/00025_eth_add_diff_row.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE eth.state_cids +ADD COLUMN diff BOOLEAN NOT NULL DEFAULT FALSE; + +ALTER TABLE eth.storage_cids +ADD COLUMN diff BOOLEAN NOT NULL DEFAULT FALSE; + +-- +goose Down +ALTER TABLE eth.state_cids +DROP COLUMN diff; + +ALTER TABLE eth.storage_cids +DROP COLUMN diff; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index a13e13a1..3520e20b 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -413,7 +413,8 @@ CREATE TABLE eth.state_cids ( state_leaf_key character varying(66), cid text NOT NULL, state_path bytea, - node_type integer + node_type integer, + diff boolean DEFAULT false NOT NULL ); @@ -447,7 +448,8 @@ CREATE TABLE eth.storage_cids ( storage_leaf_key character varying(66), cid text NOT NULL, storage_path bytea, - node_type integer + node_type integer, + diff boolean DEFAULT false NOT NULL ); diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index c9e368d0..9fccc554 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -29,17 +29,12 @@ var vulcanizeConfig = []byte(` name = "dbname" hostname = "localhost" port = 5432 - -[client] -ipcPath = "IPCPATH/geth.ipc" `) var _ = Describe("Loading the config", func() { - It("reads the private config using the environment", func() { viper.SetConfigName("config") viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/ipfs-blockchain-watcher/environments/") - Expect(viper.Get("client.ipcpath")).To(BeNil()) testConfig := viper.New() testConfig.SetConfigType("toml") @@ -48,7 +43,6 @@ var _ = Describe("Loading the config", func() { Expect(testConfig.Get("database.hostname")).To(Equal("localhost")) Expect(testConfig.Get("database.name")).To(Equal("dbname")) Expect(testConfig.Get("database.port")).To(Equal(int64(5432))) - Expect(testConfig.Get("client.ipcpath")).To(Equal("IPCPATH/geth.ipc")) }) }) diff --git a/pkg/eth/indexer.go b/pkg/eth/indexer.go index 299a7573..48f57373 100644 --- a/pkg/eth/indexer.go +++ b/pkg/eth/indexer.go @@ -149,10 +149,10 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, if stateCID.StateKey != nullHash.String() { stateKey = stateCID.StateKey } - err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5) + err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6) RETURNING id`, - headerID, stateKey, stateCID.CID, stateCID.Path, stateCID.NodeType).Scan(&stateID) + headerID, stateKey, stateCID.CID, stateCID.Path, stateCID.NodeType, true).Scan(&stateID) if err != nil { return err } @@ -180,10 +180,10 @@ func (in *CIDIndexer) indexStateCID(tx *sqlx.Tx, stateNode StateNodeModel, heade if stateNode.StateKey != nullHash.String() { stateKey = stateNode.StateKey } - err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5) + err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6) RETURNING id`, - headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType).Scan(&stateID) + headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true).Scan(&stateID) return stateID, err } @@ -199,8 +199,8 @@ func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, if storageCID.StorageKey != nullHash.String() { storageKey = storageCID.StorageKey } - _, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type) = ($2, $3, $5)`, - stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType) + _, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6)`, + stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType, true) return err } diff --git a/pkg/eth/models.go b/pkg/eth/models.go index 5f73bfe1..e9b860ec 100644 --- a/pkg/eth/models.go +++ b/pkg/eth/models.go @@ -80,6 +80,7 @@ type StateNodeModel struct { StateKey string `db:"state_leaf_key"` NodeType int `db:"node_type"` CID string `db:"cid"` + Diff bool `db:"diff"` } // StorageNodeModel is the db model for eth.storage_cids @@ -90,6 +91,7 @@ type StorageNodeModel struct { StorageKey string `db:"storage_leaf_key"` NodeType int `db:"node_type"` CID string `db:"cid"` + Diff bool `db:"diff"` } // StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key @@ -101,6 +103,7 @@ type StorageNodeWithStateKeyModel struct { StorageKey string `db:"storage_leaf_key"` NodeType int `db:"node_type"` CID string `db:"cid"` + Diff bool `db:"diff"` } // StateAccountModel is a db model for an eth state account (decoded value of state leaf node)