distinguish between differential state/storage nodes and eventual ones
This commit is contained in:
Ian Norden
parent
e2bcc06f8a
commit
0ab55ef9d8
29
cmd/root.go
29
cmd/root.go
@ -20,42 +20,31 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/config"
|
||||
)
|
||||
|
||||
var (
|
||||
cfgFile string
|
||||
databaseConfig config.Database
|
||||
ipc string
|
||||
subCommand string
|
||||
logWithCommand log.Entry
|
||||
)
|
||||
|
||||
const (
|
||||
pollingInterval = 7 * time.Second
|
||||
validationWindow = 15
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "vulcanizedb",
|
||||
Use: "ipfs-blockchain-watcher",
|
||||
PersistentPreRun: initFuncs,
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
log.Info("----- Starting vDB -----")
|
||||
log.Info("----- Starting IPFS blockchain watcher -----")
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func initFuncs(cmd *cobra.Command, args []string) {
|
||||
setViperConfigs()
|
||||
logfile := viper.GetString("logfile")
|
||||
if logfile != "" {
|
||||
file, err := os.OpenFile(logfile,
|
||||
@ -75,18 +64,6 @@ func initFuncs(cmd *cobra.Command, args []string) {
|
||||
}
|
||||
}
|
||||
|
||||
func setViperConfigs() {
|
||||
ipc = viper.GetString("client.ipcpath")
|
||||
databaseConfig = config.Database{
|
||||
Name: viper.GetString("database.name"),
|
||||
Hostname: viper.GetString("database.hostname"),
|
||||
Port: viper.GetInt("database.port"),
|
||||
User: viper.GetString("database.user"),
|
||||
Password: viper.GetString("database.password"),
|
||||
}
|
||||
viper.Set("database.config", databaseConfig)
|
||||
}
|
||||
|
||||
func logLevel() error {
|
||||
lvl, err := log.ParseLevel(viper.GetString("log.level"))
|
||||
if err != nil {
|
||||
@ -102,7 +79,6 @@ func logLevel() error {
|
||||
|
||||
func init() {
|
||||
cobra.OnInitialize(initConfig)
|
||||
// When searching for env variables, replace dots in config keys with underscores
|
||||
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
||||
viper.AutomaticEnv()
|
||||
|
||||
@ -122,7 +98,6 @@ func init() {
|
||||
viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
|
||||
viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
|
||||
viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
|
||||
viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
|
||||
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
|
||||
}
|
||||
|
||||
|
||||
13
db/migrations/00025_eth_add_diff_row.sql
Normal file
13
db/migrations/00025_eth_add_diff_row.sql
Normal file
@ -0,0 +1,13 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE eth.state_cids
|
||||
ADD COLUMN diff BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
ADD COLUMN diff BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE eth.state_cids
|
||||
DROP COLUMN diff;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
DROP COLUMN diff;
|
||||
@ -413,7 +413,8 @@ CREATE TABLE eth.state_cids (
|
||||
state_leaf_key character varying(66),
|
||||
cid text NOT NULL,
|
||||
state_path bytea,
|
||||
node_type integer
|
||||
node_type integer,
|
||||
diff boolean DEFAULT false NOT NULL
|
||||
);
|
||||
|
||||
|
||||
@ -447,7 +448,8 @@ CREATE TABLE eth.storage_cids (
|
||||
storage_leaf_key character varying(66),
|
||||
cid text NOT NULL,
|
||||
storage_path bytea,
|
||||
node_type integer
|
||||
node_type integer,
|
||||
diff boolean DEFAULT false NOT NULL
|
||||
);
|
||||
|
||||
|
||||
|
||||
@ -29,17 +29,12 @@ var vulcanizeConfig = []byte(`
|
||||
name = "dbname"
|
||||
hostname = "localhost"
|
||||
port = 5432
|
||||
|
||||
[client]
|
||||
ipcPath = "IPCPATH/geth.ipc"
|
||||
`)
|
||||
|
||||
var _ = Describe("Loading the config", func() {
|
||||
|
||||
It("reads the private config using the environment", func() {
|
||||
viper.SetConfigName("config")
|
||||
viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/ipfs-blockchain-watcher/environments/")
|
||||
Expect(viper.Get("client.ipcpath")).To(BeNil())
|
||||
|
||||
testConfig := viper.New()
|
||||
testConfig.SetConfigType("toml")
|
||||
@ -48,7 +43,6 @@ var _ = Describe("Loading the config", func() {
|
||||
Expect(testConfig.Get("database.hostname")).To(Equal("localhost"))
|
||||
Expect(testConfig.Get("database.name")).To(Equal("dbname"))
|
||||
Expect(testConfig.Get("database.port")).To(Equal(int64(5432)))
|
||||
Expect(testConfig.Get("client.ipcpath")).To(Equal("IPCPATH/geth.ipc"))
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
@ -149,10 +149,10 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
|
||||
if stateCID.StateKey != nullHash.String() {
|
||||
stateKey = stateCID.StateKey
|
||||
}
|
||||
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5)
|
||||
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6)
|
||||
RETURNING id`,
|
||||
headerID, stateKey, stateCID.CID, stateCID.Path, stateCID.NodeType).Scan(&stateID)
|
||||
headerID, stateKey, stateCID.CID, stateCID.Path, stateCID.NodeType, true).Scan(&stateID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -180,10 +180,10 @@ func (in *CIDIndexer) indexStateCID(tx *sqlx.Tx, stateNode StateNodeModel, heade
|
||||
if stateNode.StateKey != nullHash.String() {
|
||||
stateKey = stateNode.StateKey
|
||||
}
|
||||
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5)
|
||||
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6)
|
||||
RETURNING id`,
|
||||
headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType).Scan(&stateID)
|
||||
headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true).Scan(&stateID)
|
||||
return stateID, err
|
||||
}
|
||||
|
||||
@ -199,8 +199,8 @@ func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel,
|
||||
if storageCID.StorageKey != nullHash.String() {
|
||||
storageKey = storageCID.StorageKey
|
||||
}
|
||||
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type) = ($2, $3, $5)`,
|
||||
stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType)
|
||||
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff) VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff) = ($2, $3, $5, $6)`,
|
||||
stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType, true)
|
||||
return err
|
||||
}
|
||||
|
||||
@ -80,6 +80,7 @@ type StateNodeModel struct {
|
||||
StateKey string `db:"state_leaf_key"`
|
||||
NodeType int `db:"node_type"`
|
||||
CID string `db:"cid"`
|
||||
Diff bool `db:"diff"`
|
||||
}
|
||||
|
||||
// StorageNodeModel is the db model for eth.storage_cids
|
||||
@ -90,6 +91,7 @@ type StorageNodeModel struct {
|
||||
StorageKey string `db:"storage_leaf_key"`
|
||||
NodeType int `db:"node_type"`
|
||||
CID string `db:"cid"`
|
||||
Diff bool `db:"diff"`
|
||||
}
|
||||
|
||||
// StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key
|
||||
@ -101,6 +103,7 @@ type StorageNodeWithStateKeyModel struct {
|
||||
StorageKey string `db:"storage_leaf_key"`
|
||||
NodeType int `db:"node_type"`
|
||||
CID string `db:"cid"`
|
||||
Diff bool `db:"diff"`
|
||||
}
|
||||
|
||||
// StateAccountModel is a db model for an eth state account (decoded value of state leaf node)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user