From 57bdcca43c7432e657cc9997ad2c89858185d912 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 20 Mar 2020 13:15:50 -0500 Subject: [PATCH] combine resync and supernode configs; load config params from env variables if available; allow config params to be passed in as cli flags --- cmd/resync.go | 51 +++++++++++- cmd/superNode.go | 64 ++++++++++++++- environments/resyncBTC.toml | 24 ------ environments/resyncETH.toml | 18 ----- environments/superNodeBTC.toml | 60 ++++++++------ environments/superNodeETH.toml | 48 ++++++----- pkg/config/database.go | 28 ++++++- pkg/super_node/config.go | 124 ++++++++++++++--------------- pkg/super_node/resync/config.go | 81 ++++++++----------- pkg/super_node/shared/env.go | 96 ++++++++++++++++++++++ pkg/super_node/shared/functions.go | 1 - 11 files changed, 388 insertions(+), 207 deletions(-) delete mode 100644 environments/resyncBTC.toml delete mode 100644 environments/resyncETH.toml create mode 100644 pkg/super_node/shared/env.go diff --git a/cmd/resync.go b/cmd/resync.go index fd48cfca..560bbb0a 100644 --- a/cmd/resync.go +++ b/cmd/resync.go @@ -18,6 +18,7 @@ package cmd import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/vulcanize/vulcanizedb/pkg/ipfs" "github.com/vulcanize/vulcanizedb/pkg/super_node/resync" @@ -35,10 +36,6 @@ var resyncCmd = &cobra.Command{ }, } -func init() { - rootCmd.AddCommand(resyncCmd) -} - func rsyncCmdCommand() { rConfig, err := resync.NewReSyncConfig() if err != nil { @@ -56,3 +53,49 @@ func rsyncCmdCommand() { } logWithCommand.Infof("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String()) } + +func init() { + rootCmd.AddCommand(resyncCmd) + + // flags + resyncCmd.PersistentFlags().String("ipfs-path", "", "ipfs repository path") + + resyncCmd.PersistentFlags().String("resync-chain", "", "which chain to support, options are currently Ethereum or Bitcoin.") + resyncCmd.PersistentFlags().String("resync-type", "", "which type of data to resync") + resyncCmd.PersistentFlags().Int("resync-start", 0, "block height to start resync") + resyncCmd.PersistentFlags().Int("resync-stop", 0, "block height to stop resync") + resyncCmd.PersistentFlags().Int("resync-batch-size", 0, "data fetching batch size") + resyncCmd.PersistentFlags().Int("resync-batch-number", 0, "how many goroutines to fetch data concurrently") + resyncCmd.PersistentFlags().Bool("resync-clear-old", false, "if true, clear out old data of the provided type within the resync range before resyncing") + + resyncCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node") + resyncCmd.PersistentFlags().String("btc-password", "", "password for btc node") + resyncCmd.PersistentFlags().String("btc-username", "", "username for btc node") + resyncCmd.PersistentFlags().String("btc-node-id", "", "btc node id") + resyncCmd.PersistentFlags().String("btc-client-name", "", "btc client name") + resyncCmd.PersistentFlags().String("btc-genesis-block", "", "btc genesis block hash") + resyncCmd.PersistentFlags().String("btc-network-id", "", "btc network id") + + resyncCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node") + + // and their bindings + viper.BindPFlag("ipfs.path", resyncCmd.PersistentFlags().Lookup("ipfs-path")) + + viper.BindPFlag("resync.chain", resyncCmd.PersistentFlags().Lookup("resync-chain")) + viper.BindPFlag("resync.type", resyncCmd.PersistentFlags().Lookup("resync-type")) + viper.BindPFlag("resync.start", resyncCmd.PersistentFlags().Lookup("resync-start")) + viper.BindPFlag("resync.stop", resyncCmd.PersistentFlags().Lookup("resync-stop")) + viper.BindPFlag("resync.batchSize", resyncCmd.PersistentFlags().Lookup("resync-batch-size")) + viper.BindPFlag("resync.batchNumber", resyncCmd.PersistentFlags().Lookup("resync-batch-number")) + viper.BindPFlag("resync.clearOldCache", resyncCmd.PersistentFlags().Lookup("resync-clear-old")) + + viper.BindPFlag("bitcoin.httpPath", resyncCmd.PersistentFlags().Lookup("btc-http-path")) + viper.BindPFlag("bitcoin.pass", resyncCmd.PersistentFlags().Lookup("btc-password")) + viper.BindPFlag("bitcoin.user", resyncCmd.PersistentFlags().Lookup("btc-username")) + viper.BindPFlag("bitcoin.nodeID", resyncCmd.PersistentFlags().Lookup("btc-node-id")) + viper.BindPFlag("bitcoin.clientName", resyncCmd.PersistentFlags().Lookup("btc-client-name")) + viper.BindPFlag("bitcoin.genesisBlock", resyncCmd.PersistentFlags().Lookup("btc-genesis-block")) + viper.BindPFlag("bitcoin.networkID", resyncCmd.PersistentFlags().Lookup("btc-network-id")) + + viper.BindPFlag("ethereum.httpPath", resyncCmd.PersistentFlags().Lookup("eth-http-path")) +} diff --git a/cmd/superNode.go b/cmd/superNode.go index 93f1cbc9..b1239fd2 100644 --- a/cmd/superNode.go +++ b/cmd/superNode.go @@ -18,6 +18,8 @@ package cmd import ( "sync" + "github.com/spf13/viper" + "github.com/ethereum/go-ethereum/rpc" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -49,10 +51,6 @@ and fill in gaps in the data }, } -func init() { - rootCmd.AddCommand(superNodeCmd) -} - func superNode() { superNodeConfig, err := super_node.NewSuperNodeConfig() if err != nil { @@ -101,3 +99,61 @@ func startServers(superNode super_node.SuperNode, settings *super_node.Config) e _, _, err = rpc.StartHTTPEndpoint(settings.HTTPEndpoint, superNode.APIs(), []string{settings.Chain.API()}, nil, nil, rpc.HTTPTimeouts{}) return err } + +func init() { + rootCmd.AddCommand(superNodeCmd) + + // flags + superNodeCmd.PersistentFlags().String("ipfs-path", "", "ipfs repository path") + + superNodeCmd.PersistentFlags().String("supernode-chain", "", "which chain to support, options are currently Ethereum or Bitcoin.") + superNodeCmd.PersistentFlags().Bool("supernode-server", false, "turn vdb server on or off") + superNodeCmd.PersistentFlags().String("supernode-ws-path", "", "vdb server ws path") + superNodeCmd.PersistentFlags().String("supernode-http-path", "", "vdb server http path") + superNodeCmd.PersistentFlags().String("supernode-ipc-path", "", "vdb server ipc path") + superNodeCmd.PersistentFlags().Bool("supernode-sync", false, "turn vdb sync on or off") + superNodeCmd.PersistentFlags().Int("supernode-workers", 0, "how many worker goroutines to publish and index data") + superNodeCmd.PersistentFlags().Bool("supernode-back-fill", false, "turn vdb backfill on or off") + superNodeCmd.PersistentFlags().Int("supernode-frequency", 0, "how often (in seconds) the backfill process checks for gaps") + superNodeCmd.PersistentFlags().Int("supernode-batch-size", 0, "data fetching batch size") + superNodeCmd.PersistentFlags().Int("supernode-batch-number", 0, "how many goroutines to fetch data concurrently") + + superNodeCmd.PersistentFlags().String("btc-ws-path", "", "ws url for bitcoin node") + superNodeCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node") + superNodeCmd.PersistentFlags().String("btc-password", "", "password for btc node") + superNodeCmd.PersistentFlags().String("btc-username", "", "username for btc node") + superNodeCmd.PersistentFlags().String("btc-node-id", "", "btc node id") + superNodeCmd.PersistentFlags().String("btc-client-name", "", "btc client name") + superNodeCmd.PersistentFlags().String("btc-genesis-block", "", "btc genesis block hash") + superNodeCmd.PersistentFlags().String("btc-network-id", "", "btc network id") + + superNodeCmd.PersistentFlags().String("eth-ws-path", "", "ws url for ethereum node") + superNodeCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node") + + // and their bindings + viper.BindPFlag("ipfs.path", superNodeCmd.PersistentFlags().Lookup("ipfs-path")) + + viper.BindPFlag("superNode.chain", superNodeCmd.PersistentFlags().Lookup("supernode-chain")) + viper.BindPFlag("superNode.server", superNodeCmd.PersistentFlags().Lookup("supernode-server")) + viper.BindPFlag("superNode.wsPath", superNodeCmd.PersistentFlags().Lookup("supernode-ws-path")) + viper.BindPFlag("superNode.httpPath", superNodeCmd.PersistentFlags().Lookup("supernode-http-path")) + viper.BindPFlag("superNode.ipcPath", superNodeCmd.PersistentFlags().Lookup("supernode-ipc-path")) + viper.BindPFlag("superNode.sync", superNodeCmd.PersistentFlags().Lookup("supernode-sync")) + viper.BindPFlag("superNode.workers", superNodeCmd.PersistentFlags().Lookup("supernode-workers")) + viper.BindPFlag("superNode.backFill", superNodeCmd.PersistentFlags().Lookup("supernode-back-fill")) + viper.BindPFlag("superNode.frequency", superNodeCmd.PersistentFlags().Lookup("supernode-frequency")) + viper.BindPFlag("superNode.batchSize", superNodeCmd.PersistentFlags().Lookup("supernode-batch-size")) + viper.BindPFlag("superNode.batchNumber", superNodeCmd.PersistentFlags().Lookup("supernode-batch-number")) + + viper.BindPFlag("bitcoin.wsPath", superNodeCmd.PersistentFlags().Lookup("btc-ws-path")) + viper.BindPFlag("bitcoin.httpPath", superNodeCmd.PersistentFlags().Lookup("btc-http-path")) + viper.BindPFlag("bitcoin.pass", superNodeCmd.PersistentFlags().Lookup("btc-password")) + viper.BindPFlag("bitcoin.user", superNodeCmd.PersistentFlags().Lookup("btc-username")) + viper.BindPFlag("bitcoin.nodeID", superNodeCmd.PersistentFlags().Lookup("btc-node-id")) + viper.BindPFlag("bitcoin.clientName", superNodeCmd.PersistentFlags().Lookup("btc-client-name")) + viper.BindPFlag("bitcoin.genesisBlock", superNodeCmd.PersistentFlags().Lookup("btc-genesis-block")) + viper.BindPFlag("bitcoin.networkID", superNodeCmd.PersistentFlags().Lookup("btc-network-id")) + + viper.BindPFlag("ethereum.wsPath", superNodeCmd.PersistentFlags().Lookup("eth-ws-path")) + viper.BindPFlag("ethereum.httpPath", superNodeCmd.PersistentFlags().Lookup("eth-http-path")) +} diff --git a/environments/resyncBTC.toml b/environments/resyncBTC.toml deleted file mode 100644 index b35bd61e..00000000 --- a/environments/resyncBTC.toml +++ /dev/null @@ -1,24 +0,0 @@ -[database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - user = "vdbm" - -[resync] - chain = "bitcoin" - type = "full" - clearOldCache = true - ipfsPath = "/root/.ipfs" - batchSize = 1 - batchNumber = 50 - start = 0 - stop = 0 - -[bitcoin] - httpPath = "127.0.0.1:8332" - pass = "password" - user = "username" - nodeID = "ocd0" - clientName = "Omnicore" - genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" - networkID = "0xD9B4BEF9" \ No newline at end of file diff --git a/environments/resyncETH.toml b/environments/resyncETH.toml deleted file mode 100644 index 98253e67..00000000 --- a/environments/resyncETH.toml +++ /dev/null @@ -1,18 +0,0 @@ -[database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - user = "vdbm" - -[resync] - chain = "ethereum" - type = "state" - clearOldCache = true - ipfsPath = "/root/.ipfs" - batchSize = 5 - batchNumber = 50 - start = 0 - stop = 0 - -[ethereum] - httpPath = "127.0.0.1:8545" \ No newline at end of file diff --git a/environments/superNodeBTC.toml b/environments/superNodeBTC.toml index 52b0d633..42582702 100644 --- a/environments/superNodeBTC.toml +++ b/environments/superNodeBTC.toml @@ -1,29 +1,41 @@ [database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - user = "vdbm" + name = "vulcanize_public" # $DATABASE_NAME + hostname = "localhost" # &DATABASE_HOSTNAME + port = 5432 # $DATABASE_PORT + user = "vdbm" # $DATABASE_USER + password = "" # $DATABASE_PASSWORD + +[ipfs] + path = "/root/.ipfs" # $IPFS_PATH + +[resync] + chain = "bitcoin" # $RESYNC_CHAIN + type = "full" # $RESYNC_TYPE + start = 0 # $RESYNC_START + stop = 0 # $RESYNC_STOP + batchSize = 1 # $RESYNC_BATCH_SIZE + batchNumber = 50 # $RESYNC_BATCH_NUMBER + clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE [superNode] - chain = "bitcoin" - ipfsPath = "/root/.ipfs" - server = true - ipcPath = "/root/.vulcanize/btc/vulcanize.ipc" - wsPath = "127.0.0.1:8082" - httpPath = "127.0.0.1:8083" - sync = true - workers = 1 - backFill = true - frequency = 45 - batchSize = 1 - batchNumber = 50 + chain = "bitcoin" # $SUPERNODE_CHAIN + server = true # $SUPERNODE_SERVER + ipcPath = "/root/.vulcanize/btc/vulcanize.ipc" # $SUPERNODE_IPC_PATH + wsPath = "127.0.0.1:8082" # $SUPERNODE_WS_PATH + httpPath = "127.0.0.1:8083" # $SUPERNODE_HTTP_PATH + sync = true # $SUPERNODE_SYNC + workers = 1 # $SUPERNODE_WORKERS + backFill = true # $SUPERNODE_BACKFILL + frequency = 45 # $SUPERNODE_FREQUENCY + batchSize = 1 # $SUPERNODE_BATCH_SIZE + batchNumber = 50 # $SUPERNODE_BATCH_NUMBER [bitcoin] - wsPath = "127.0.0.1:8332" - httpPath = "127.0.0.1:8332" - pass = "password" - user = "username" - nodeID = "ocd0" - clientName = "Omnicore" - genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" - networkID = "0xD9B4BEF9" \ No newline at end of file + wsPath = "127.0.0.1:8332" # $BTC_WS_PATH + httpPath = "127.0.0.1:8332" # $BTC_HTTP_PATH + pass = "password" # $BTC_NODE_PASSWORD + user = "username" # $BTC_NODE_USER + nodeID = "ocd0" # $BTC_NODE_ID + clientName = "Omnicore" # $BTC_CLIENT_NAME + genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" # $BTC_GENESIS_BLOCK + networkID = "0xD9B4BEF9" # $BTC_NETWORK_ID \ No newline at end of file diff --git a/environments/superNodeETH.toml b/environments/superNodeETH.toml index cc2b7274..074da362 100644 --- a/environments/superNodeETH.toml +++ b/environments/superNodeETH.toml @@ -1,23 +1,35 @@ [database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - user = "vdbm" + name = "vulcanize_public" # $DATABASE_NAME + hostname = "localhost" # &DATABASE_HOSTNAME + port = 5432 # $DATABASE_PORT + user = "vdbm" # $DATABASE_USER + password = "" # $DATABASE_PASSWORD + +[ipfs] + path = "/root/.ipfs" # $IPFS_PATH + +[resync] + chain = "ethereum" # $RESYNC_CHAIN + type = "state" # $RESYNC_TYPE + start = 0 # $RESYNC_START + stop = 0 # $RESYNC_STOP + batchSize = 5 # $RESYNC_BATCH_SIZE + batchNumber = 50 # $RESYNC_BATCH_NUMBER + clearOldCache = true # $RESYNC_CLEAR_OLD_CACHE [superNode] - chain = "ethereum" - ipfsPath = "/root/.ipfs" - server = true - ipcPath = "/root/.vulcanize/eth/vulcanize.ipc" - wsPath = "127.0.0.1:8081" - httpPath = "127.0.0.1:8082" - sync = true - workers = 1 - backFill = true - frequency = 15 - batchSize = 5 - batchNumber = 50 + chain = "ethereum" # $SUPERNODE_CHAIN + server = true # $SUPERNODE_SERVER + ipcPath = "/root/.vulcanize/eth/vulcanize.ipc" # $SUPERNODE_IPC_PATH + wsPath = "127.0.0.1:8081" # $SUPERNODE_WS_PATH + httpPath = "127.0.0.1:8082" # $SUPERNODE_HTTP_PATH + sync = true # $SUPERNODE_SYNC + workers = 1 # $SUPERNODE_WORKERS + backFill = true # $SUPERNODE_BACKFILL + frequency = 15 # $SUPERNODE_FREQUENCY + batchSize = 5 # $SUPERNODE_BATCH_SIZE + batchNumber = 50 # $SUPERNODE_BATCH_NUMBER [ethereum] - wsPath = "127.0.0.1:8546" - httpPath = "127.0.0.1:8545" \ No newline at end of file + wsPath = "127.0.0.1:8546" # $ETH_WS_PATH + httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH \ No newline at end of file diff --git a/pkg/config/database.go b/pkg/config/database.go index 4e08ca41..57b16faa 100644 --- a/pkg/config/database.go +++ b/pkg/config/database.go @@ -16,7 +16,20 @@ package config -import "fmt" +import ( + "fmt" + + "github.com/spf13/viper" +) + +// Env variables +const ( + DATABASE_NAME = "DATABASE_NAME" + DATABASE_HOSTNAME = "DATABASE_HOSTNAME" + DATABASE_PORT = "DATABASE_PORT" + DATABASE_USER = "DATABASE_USER" + DATABASE_PASSWORD = "DATABASE_PASSWORD" +) type Database struct { Hostname string @@ -37,3 +50,16 @@ func DbConnectionString(dbConfig Database) string { } return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", dbConfig.Hostname, dbConfig.Port, dbConfig.Name) } + +func (d *Database) Init() { + viper.BindEnv("database.name", DATABASE_NAME) + viper.BindEnv("database.hostname", DATABASE_HOSTNAME) + viper.BindEnv("database.port", DATABASE_PORT) + viper.BindEnv("database.user", DATABASE_USER) + viper.BindEnv("database.password", DATABASE_PASSWORD) + d.Name = viper.GetString("database.name") + d.Hostname = viper.GetString("database.hostname") + d.Port = viper.GetInt("database.port") + d.User = viper.GetString("database.user") + d.Password = viper.GetString("database.password") +} diff --git a/pkg/super_node/config.go b/pkg/super_node/config.go index dfa29543..e8afe345 100644 --- a/pkg/super_node/config.go +++ b/pkg/super_node/config.go @@ -22,19 +22,30 @@ import ( "path/filepath" "time" - "github.com/btcsuite/btcd/rpcclient" - "github.com/ethereum/go-ethereum/rpc" "github.com/spf13/viper" "github.com/vulcanize/vulcanizedb/pkg/config" - "github.com/vulcanize/vulcanizedb/pkg/eth/client" "github.com/vulcanize/vulcanizedb/pkg/eth/core" - "github.com/vulcanize/vulcanizedb/pkg/eth/node" "github.com/vulcanize/vulcanizedb/pkg/postgres" "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" "github.com/vulcanize/vulcanizedb/utils" ) +// Env variables +const ( + SUPERNODE_CHAIN = "SUPERNODE_CHAIN" + SUPERNODE_SYNC = "SUPERNODE_SYNC" + SUPERNODE_WORKERS = "SUPERNODE_WORKERS" + SUPERNODE_SERVER = "SUPERNODE_SERVER" + SUPERNODE_WS_PATH = "SUPERNODE_WS_PATH" + SUPERNODE_IPC_PATH = "SUPERNODE_IPC_PATH" + SUPERNODE_HTTP_PATH = "SUPERNODE_HTTP_PATH" + SUPERNODE_BACKFILL = "SUPERNODE_BACKFILL" + SUPERNODE_FREQUENCY = "SUPERNODE_FREQUENCY" + SUPERNODE_BATCH_SIZE = "SUPERNODE_BATCH_SIZE" + SUPERNODE_BATCH_NUMBER = "SUPERNODE_BATCH_NUMBER" +) + // Config struct type Config struct { // Ubiquitous fields @@ -67,21 +78,27 @@ func NewSuperNodeConfig() (*Config, error) { c := new(Config) var err error + viper.BindEnv("superNode.chain", SUPERNODE_CHAIN) + viper.BindEnv("superNode.sync", SUPERNODE_SYNC) + viper.BindEnv("superNode.workers", SUPERNODE_WORKERS) + viper.BindEnv("ethereum.wsPath", shared.ETH_WS_PATH) + viper.BindEnv("bitcoin.wsPath", shared.BTC_WS_PATH) + viper.BindEnv("superNode.server", SUPERNODE_SERVER) + viper.BindEnv("superNode.wsPath", SUPERNODE_WS_PATH) + viper.BindEnv("superNode.ipcPath", SUPERNODE_IPC_PATH) + viper.BindEnv("superNode.httpPath", SUPERNODE_HTTP_PATH) + viper.BindEnv("superNode.backFill", SUPERNODE_BACKFILL) + chain := viper.GetString("superNode.chain") c.Chain, err = shared.NewChainType(chain) if err != nil { return nil, err } - ipfsPath := viper.GetString("superNode.ipfsPath") - if ipfsPath == "" { - home, err := os.UserHomeDir() - if err != nil { - return nil, err - } - ipfsPath = filepath.Join(home, ".ipfs") + c.IPFSPath, err = shared.GetIPFSPath() + if err != nil { + return nil, err } - c.IPFSPath = ipfsPath c.Sync = viper.GetBool("superNode.sync") if c.Sync { @@ -92,25 +109,14 @@ func NewSuperNodeConfig() (*Config, error) { c.Workers = workers switch c.Chain { case shared.Ethereum: - c.NodeInfo, c.WSClient, err = getEthNodeAndClient(fmt.Sprintf("ws://%s", viper.GetString("ethereum.wsPath"))) + ethWS := viper.GetString("ethereum.wsPath") + c.NodeInfo, c.WSClient, err = shared.GetEthNodeAndClient(fmt.Sprintf("ws://%s", ethWS)) if err != nil { return nil, err } case shared.Bitcoin: - c.NodeInfo = core.Node{ - ID: viper.GetString("bitcoin.nodeID"), - ClientName: viper.GetString("bitcoin.clientName"), - GenesisBlock: viper.GetString("bitcoin.genesisBlock"), - NetworkID: viper.GetString("bitcoin.networkID"), - } - // For bitcoin we load in node info from the config because there is no RPC endpoint to retrieve this from the node - c.WSClient = &rpcclient.ConnConfig{ - Host: viper.GetString("bitcoin.wsPath"), - HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode - DisableTLS: true, // Bitcoin core does not provide TLS by default - Pass: viper.GetString("bitcoin.pass"), - User: viper.GetString("bitcoin.user"), - } + btcWS := viper.GetString("bitcoin.wsPath") + c.NodeInfo, c.WSClient = shared.GetBtcNodeAndClient(btcWS) } } @@ -137,47 +143,43 @@ func NewSuperNodeConfig() (*Config, error) { c.HTTPEndpoint = httpPath } - c.DBConfig = config.Database{ - Name: viper.GetString("database.name"), - Hostname: viper.GetString("database.hostname"), - Port: viper.GetInt("database.port"), - User: viper.GetString("database.user"), - Password: viper.GetString("database.password"), - } - - db := utils.LoadPostgres(c.DBConfig, c.NodeInfo) - c.DB = &db - c.Quit = make(chan bool) - if viper.GetBool("superNode.backFill") { - if err := c.BackFillFields(chain); err != nil { + c.BackFill = viper.GetBool("superNode.backFill") + if c.BackFill { + if err := c.BackFillFields(); err != nil { return nil, err } } + c.DBConfig.Init() + db := utils.LoadPostgres(c.DBConfig, c.NodeInfo) + c.DB = &db + c.Quit = make(chan bool) + return c, nil } // BackFillFields is used to fill in the BackFill fields of the config -func (sn *Config) BackFillFields(chain string) error { - sn.BackFill = true - var httpClient interface{} +func (c *Config) BackFillFields() error { var err error - switch sn.Chain { + + viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH) + viper.BindEnv("bitcoin.httpPath", shared.BTC_HTTP_PATH) + viper.BindEnv("superNode.frequency", SUPERNODE_FREQUENCY) + viper.BindEnv("superNode.batchSize", SUPERNODE_BATCH_SIZE) + viper.BindEnv("superNode.batchNumber", SUPERNODE_BATCH_NUMBER) + + switch c.Chain { case shared.Ethereum: - _, httpClient, err = getEthNodeAndClient(fmt.Sprintf("http://%s", viper.GetString("ethereum.httpPath"))) + ethHTTP := viper.GetString("ethereum.httpPath") + c.NodeInfo, c.HTTPClient, err = shared.GetEthNodeAndClient(fmt.Sprintf("http://%s", ethHTTP)) if err != nil { return err } case shared.Bitcoin: - httpClient = &rpcclient.ConnConfig{ - Host: viper.GetString("bitcoin.httpPath"), - HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode - DisableTLS: true, // Bitcoin core does not provide TLS by default - Pass: viper.GetString("bitcoin.pass"), - User: viper.GetString("bitcoin.user"), - } + btcHTTP := viper.GetString("bitcoin.httpPath") + c.NodeInfo, c.HTTPClient = shared.GetBtcNodeAndClient(btcHTTP) } - sn.HTTPClient = httpClient + freq := viper.GetInt("superNode.frequency") var frequency time.Duration if freq <= 0 { @@ -185,18 +187,8 @@ func (sn *Config) BackFillFields(chain string) error { } else { frequency = time.Second * time.Duration(freq) } - sn.Frequency = frequency - sn.BatchSize = uint64(viper.GetInt64("superNode.batchSize")) - sn.BatchNumber = uint64(viper.GetInt64("superNode.batchNumber")) + c.Frequency = frequency + c.BatchSize = uint64(viper.GetInt64("superNode.batchSize")) + c.BatchNumber = uint64(viper.GetInt64("superNode.batchNumber")) return nil } - -func getEthNodeAndClient(path string) (core.Node, interface{}, error) { - rawRPCClient, err := rpc.Dial(path) - if err != nil { - return core.Node{}, nil, err - } - rpcClient := client.NewRPCClient(rawRPCClient, path) - vdbNode := node.MakeNode(rpcClient) - return vdbNode, rpcClient, nil -} diff --git a/pkg/super_node/resync/config.go b/pkg/super_node/resync/config.go index ab8b1cdc..04c5b118 100644 --- a/pkg/super_node/resync/config.go +++ b/pkg/super_node/resync/config.go @@ -18,22 +18,27 @@ package resync import ( "fmt" - "os" - "path/filepath" - "github.com/btcsuite/btcd/rpcclient" - "github.com/ethereum/go-ethereum/rpc" "github.com/spf13/viper" "github.com/vulcanize/vulcanizedb/pkg/config" - "github.com/vulcanize/vulcanizedb/pkg/eth/client" "github.com/vulcanize/vulcanizedb/pkg/eth/core" - "github.com/vulcanize/vulcanizedb/pkg/eth/node" "github.com/vulcanize/vulcanizedb/pkg/postgres" "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" "github.com/vulcanize/vulcanizedb/utils" ) +// Env variables +const ( + RESYNC_CHAIN = "RESYNC_CHAIN" + RESYNC_START = "RESYNC_START" + RESYNC_STOP = "RESYNC_STOP" + RESYNC_BATCH_SIZE = "RESYNC_BATCH_SIZE" + RESYNC_BATCH_NUMBER = "RESYNC_BATCH_NUMBER" + RESYNC_CLEAR_OLD_CACHE = "RESYNC_CLEAR_OLD_CACHE" + RESYNC_TYPE = "RESYNC_TYPE" +) + // Config holds the parameters needed to perform a resync type Config struct { Chain shared.ChainType // The type of resync to perform @@ -58,26 +63,26 @@ type Config struct { func NewReSyncConfig() (*Config, error) { c := new(Config) var err error + + viper.BindEnv("resync.start", RESYNC_START) + viper.BindEnv("resync.stop", RESYNC_STOP) + viper.BindEnv("resync.clearOldCache", RESYNC_CLEAR_OLD_CACHE) + viper.BindEnv("resync.type", RESYNC_TYPE) + viper.BindEnv("resync.chain", RESYNC_CHAIN) + viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH) + viper.BindEnv("bitcoin.httpPath", shared.BTC_HTTP_PATH) + viper.BindEnv("resync.batchSize", RESYNC_BATCH_SIZE) + viper.BindEnv("resync.batchNumber", RESYNC_BATCH_NUMBER) + start := uint64(viper.GetInt64("resync.start")) stop := uint64(viper.GetInt64("resync.stop")) c.Ranges = [][2]uint64{{start, stop}} - ipfsPath := viper.GetString("resync.ipfsPath") - if ipfsPath == "" { - home, err := os.UserHomeDir() - if err != nil { - return nil, err - } - ipfsPath = filepath.Join(home, ".ipfs") - } - c.IPFSPath = ipfsPath - c.DBConfig = config.Database{ - Name: viper.GetString("database.name"), - Hostname: viper.GetString("database.hostname"), - Port: viper.GetInt("database.port"), - User: viper.GetString("database.user"), - Password: viper.GetString("database.password"), - } c.ClearOldCache = viper.GetBool("resync.clearOldCache") + + c.IPFSPath, err = shared.GetIPFSPath() + if err != nil { + return nil, err + } resyncType := viper.GetString("resync.type") c.ResyncType, err = shared.GenerateResyncTypeFromString(resyncType) if err != nil { @@ -97,40 +102,22 @@ func NewReSyncConfig() (*Config, error) { switch c.Chain { case shared.Ethereum: - c.NodeInfo, c.HTTPClient, err = getEthNodeAndClient(fmt.Sprintf("http://%s", viper.GetString("ethereum.httpPath"))) + ethHTTP := viper.GetString("ethereum.httpPath") + c.NodeInfo, c.HTTPClient, err = shared.GetEthNodeAndClient(fmt.Sprintf("http://%s", ethHTTP)) if err != nil { return nil, err } case shared.Bitcoin: - c.NodeInfo = core.Node{ - ID: viper.GetString("bitcoin.nodeID"), - ClientName: viper.GetString("bitcoin.clientName"), - GenesisBlock: viper.GetString("bitcoin.genesisBlock"), - NetworkID: viper.GetString("bitcoin.networkID"), - } - // For bitcoin we load in node info from the config because there is no RPC endpoint to retrieve this from the node - c.HTTPClient = &rpcclient.ConnConfig{ - Host: viper.GetString("bitcoin.httpPath"), - HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode - DisableTLS: true, // Bitcoin core does not provide TLS by default - Pass: viper.GetString("bitcoin.pass"), - User: viper.GetString("bitcoin.user"), - } + btcHTTP := viper.GetString("bitcoin.httpPath") + c.NodeInfo, c.HTTPClient = shared.GetBtcNodeAndClient(btcHTTP) } + + c.DBConfig.Init() db := utils.LoadPostgres(c.DBConfig, c.NodeInfo) c.DB = &db + c.Quit = make(chan bool) c.BatchSize = uint64(viper.GetInt64("resync.batchSize")) c.BatchNumber = uint64(viper.GetInt64("resync.batchNumber")) return c, nil } - -func getEthNodeAndClient(path string) (core.Node, interface{}, error) { - rawRPCClient, err := rpc.Dial(path) - if err != nil { - return core.Node{}, nil, err - } - rpcClient := client.NewRPCClient(rawRPCClient, path) - vdbNode := node.MakeNode(rpcClient) - return vdbNode, rpcClient, nil -} diff --git a/pkg/super_node/shared/env.go b/pkg/super_node/shared/env.go new file mode 100644 index 00000000..fbd66dd4 --- /dev/null +++ b/pkg/super_node/shared/env.go @@ -0,0 +1,96 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "os" + "path/filepath" + + "github.com/ethereum/go-ethereum/rpc" + + "github.com/btcsuite/btcd/rpcclient" + "github.com/spf13/viper" + "github.com/vulcanize/vulcanizedb/pkg/eth/client" + "github.com/vulcanize/vulcanizedb/pkg/eth/core" + "github.com/vulcanize/vulcanizedb/pkg/eth/node" +) + +// Env variables +const ( + IPFS_PATH = "IPFS_PATH" + + ETH_WS_PATH = "ETH_WS_PATH" + ETH_HTTP_PATH = "ETH_HTTP_PATH" + + BTC_WS_PATH = "BTC_WS_PATH" + BTC_HTTP_PATH = "BTC_HTTP_PATH" + BTC_NODE_PASSWORD = "BTC_NODE_PASSWORD" + BTC_NODE_USER = "BTC_NODE_USER" + BTC_NODE_ID = "BTC_NODE_ID" + BTC_CLIENT_NAME = "BTC_CLIENT_NAME" + BTC_GENESIS_BLOCK = "BTC_GENESIS_BLOCK" + BTC_NETWORK_ID = "BTC_NETWORK_ID" +) + +// GetEthNodeAndClient returns eth node info and client from path url +func GetEthNodeAndClient(path string) (core.Node, core.RPCClient, error) { + rawRPCClient, err := rpc.Dial(path) + if err != nil { + return core.Node{}, nil, err + } + rpcClient := client.NewRPCClient(rawRPCClient, path) + vdbNode := node.MakeNode(rpcClient) + return vdbNode, rpcClient, nil +} + +// GetIPFSPath returns the ipfs path from the config or env variable +func GetIPFSPath() (string, error) { + viper.BindEnv("ipfs.path", IPFS_PATH) + ipfsPath := viper.GetString("ipfs.path") + if ipfsPath == "" { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + ipfsPath = filepath.Join(home, ".ipfs") + } + return ipfsPath, nil +} + +// GetBtcNodeAndClient returns btc node info from path url +func GetBtcNodeAndClient(path string) (core.Node, *rpcclient.ConnConfig) { + viper.BindEnv("bitcoin.nodeID", BTC_NODE_ID) + viper.BindEnv("bitcoin.clientName", BTC_CLIENT_NAME) + viper.BindEnv("bitcoin.genesisBlock", BTC_GENESIS_BLOCK) + viper.BindEnv("bitcoin.networkID", BTC_NETWORK_ID) + viper.BindEnv("bitcoin.pass", BTC_NODE_PASSWORD) + viper.BindEnv("bitcoin.user", BTC_NODE_USER) + + // For bitcoin we load in node info from the config because there is no RPC endpoint to retrieve this from the node + return core.Node{ + ID: viper.GetString("bitcoin.nodeID"), + ClientName: viper.GetString("bitcoin.clientName"), + GenesisBlock: viper.GetString("bitcoin.genesisBlock"), + NetworkID: viper.GetString("bitcoin.networkID"), + }, &rpcclient.ConnConfig{ + Host: path, + HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode + DisableTLS: true, // Bitcoin core does not provide TLS by default + Pass: viper.GetString("bitcoin.pass"), + User: viper.GetString("bitcoin.user"), + } +} diff --git a/pkg/super_node/shared/functions.go b/pkg/super_node/shared/functions.go index ecb6ce3f..efb1cd64 100644 --- a/pkg/super_node/shared/functions.go +++ b/pkg/super_node/shared/functions.go @@ -20,7 +20,6 @@ import ( "bytes" "github.com/ethereum/go-ethereum/common" - "github.com/vulcanize/vulcanizedb/pkg/ipfs" )