cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

combine resync and supernode configs; load config params from env variables if available; allow config params to be passed in as cli flags

Browse Source
This commit is contained in:
Ian Norden 2020-03-20 13:15:50 -05:00
parent 1d4b37aca9
commit 57bdcca43c
11 changed files with 388 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
cmd/resync.go
View File

@ -18,6 +18,7 @@ package cmd
import ( import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/ipfs" "github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/super_node/resync" "github.com/vulcanize/vulcanizedb/pkg/super_node/resync"
@ -35,10 +36,6 @@ var resyncCmd = &cobra.Command{
}, },
} }
func init() {
rootCmd.AddCommand(resyncCmd)
}
func rsyncCmdCommand() { func rsyncCmdCommand() {
rConfig, err := resync.NewReSyncConfig() rConfig, err := resync.NewReSyncConfig()
if err != nil { if err != nil {
@ -56,3 +53,49 @@ func rsyncCmdCommand() {
} }
logWithCommand.Infof("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String()) logWithCommand.Infof("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String())
} }
func init() {
rootCmd.AddCommand(resyncCmd)
// flags
resyncCmd.PersistentFlags().String("ipfs-path", "", "ipfs repository path")
resyncCmd.PersistentFlags().String("resync-chain", "", "which chain to support, options are currently Ethereum or Bitcoin.")
resyncCmd.PersistentFlags().String("resync-type", "", "which type of data to resync")
resyncCmd.PersistentFlags().Int("resync-start", 0, "block height to start resync")
resyncCmd.PersistentFlags().Int("resync-stop", 0, "block height to stop resync")
resyncCmd.PersistentFlags().Int("resync-batch-size", 0, "data fetching batch size")
resyncCmd.PersistentFlags().Int("resync-batch-number", 0, "how many goroutines to fetch data concurrently")
resyncCmd.PersistentFlags().Bool("resync-clear-old", false, "if true, clear out old data of the provided type within the resync range before resyncing")
resyncCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node")
resyncCmd.PersistentFlags().String("btc-password", "", "password for btc node")
resyncCmd.PersistentFlags().String("btc-username", "", "username for btc node")
resyncCmd.PersistentFlags().String("btc-node-id", "", "btc node id")
resyncCmd.PersistentFlags().String("btc-client-name", "", "btc client name")
resyncCmd.PersistentFlags().String("btc-genesis-block", "", "btc genesis block hash")
resyncCmd.PersistentFlags().String("btc-network-id", "", "btc network id")
resyncCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
// and their bindings
viper.BindPFlag("ipfs.path", resyncCmd.PersistentFlags().Lookup("ipfs-path"))
viper.BindPFlag("resync.chain", resyncCmd.PersistentFlags().Lookup("resync-chain"))
viper.BindPFlag("resync.type", resyncCmd.PersistentFlags().Lookup("resync-type"))
viper.BindPFlag("resync.start", resyncCmd.PersistentFlags().Lookup("resync-start"))
viper.BindPFlag("resync.stop", resyncCmd.PersistentFlags().Lookup("resync-stop"))
viper.BindPFlag("resync.batchSize", resyncCmd.PersistentFlags().Lookup("resync-batch-size"))
viper.BindPFlag("resync.batchNumber", resyncCmd.PersistentFlags().Lookup("resync-batch-number"))
viper.BindPFlag("resync.clearOldCache", resyncCmd.PersistentFlags().Lookup("resync-clear-old"))
viper.BindPFlag("bitcoin.httpPath", resyncCmd.PersistentFlags().Lookup("btc-http-path"))
viper.BindPFlag("bitcoin.pass", resyncCmd.PersistentFlags().Lookup("btc-password"))
viper.BindPFlag("bitcoin.user", resyncCmd.PersistentFlags().Lookup("btc-username"))
viper.BindPFlag("bitcoin.nodeID", resyncCmd.PersistentFlags().Lookup("btc-node-id"))
viper.BindPFlag("bitcoin.clientName", resyncCmd.PersistentFlags().Lookup("btc-client-name"))
viper.BindPFlag("bitcoin.genesisBlock", resyncCmd.PersistentFlags().Lookup("btc-genesis-block"))
viper.BindPFlag("bitcoin.networkID", resyncCmd.PersistentFlags().Lookup("btc-network-id"))
viper.BindPFlag("ethereum.httpPath", resyncCmd.PersistentFlags().Lookup("eth-http-path"))
}

64
cmd/superNode.go
View File

@ -18,6 +18,8 @@ package cmd
import ( import (
"sync" "sync"
"github.com/spf13/viper"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -49,10 +51,6 @@ and fill in gaps in the data
}, },
} }
func init() {
rootCmd.AddCommand(superNodeCmd)
}
func superNode() { func superNode() {
superNodeConfig, err := super_node.NewSuperNodeConfig() superNodeConfig, err := super_node.NewSuperNodeConfig()
if err != nil { if err != nil {
@ -101,3 +99,61 @@ func startServers(superNode super_node.SuperNode, settings *super_node.Config) e
_, _, err = rpc.StartHTTPEndpoint(settings.HTTPEndpoint, superNode.APIs(), []string{settings.Chain.API()}, nil, nil, rpc.HTTPTimeouts{}) _, _, err = rpc.StartHTTPEndpoint(settings.HTTPEndpoint, superNode.APIs(), []string{settings.Chain.API()}, nil, nil, rpc.HTTPTimeouts{})
return err return err
} }
func init() {
rootCmd.AddCommand(superNodeCmd)
// flags
superNodeCmd.PersistentFlags().String("ipfs-path", "", "ipfs repository path")
superNodeCmd.PersistentFlags().String("supernode-chain", "", "which chain to support, options are currently Ethereum or Bitcoin.")
superNodeCmd.PersistentFlags().Bool("supernode-server", false, "turn vdb server on or off")
superNodeCmd.PersistentFlags().String("supernode-ws-path", "", "vdb server ws path")
superNodeCmd.PersistentFlags().String("supernode-http-path", "", "vdb server http path")
superNodeCmd.PersistentFlags().String("supernode-ipc-path", "", "vdb server ipc path")
superNodeCmd.PersistentFlags().Bool("supernode-sync", false, "turn vdb sync on or off")
superNodeCmd.PersistentFlags().Int("supernode-workers", 0, "how many worker goroutines to publish and index data")
superNodeCmd.PersistentFlags().Bool("supernode-back-fill", false, "turn vdb backfill on or off")
superNodeCmd.PersistentFlags().Int("supernode-frequency", 0, "how often (in seconds) the backfill process checks for gaps")
superNodeCmd.PersistentFlags().Int("supernode-batch-size", 0, "data fetching batch size")
superNodeCmd.PersistentFlags().Int("supernode-batch-number", 0, "how many goroutines to fetch data concurrently")
superNodeCmd.PersistentFlags().String("btc-ws-path", "", "ws url for bitcoin node")
superNodeCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node")
superNodeCmd.PersistentFlags().String("btc-password", "", "password for btc node")
superNodeCmd.PersistentFlags().String("btc-username", "", "username for btc node")
superNodeCmd.PersistentFlags().String("btc-node-id", "", "btc node id")
superNodeCmd.PersistentFlags().String("btc-client-name", "", "btc client name")
superNodeCmd.PersistentFlags().String("btc-genesis-block", "", "btc genesis block hash")
superNodeCmd.PersistentFlags().String("btc-network-id", "", "btc network id")
superNodeCmd.PersistentFlags().String("eth-ws-path", "", "ws url for ethereum node")
superNodeCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
// and their bindings
viper.BindPFlag("ipfs.path", superNodeCmd.PersistentFlags().Lookup("ipfs-path"))
viper.BindPFlag("superNode.chain", superNodeCmd.PersistentFlags().Lookup("supernode-chain"))
viper.BindPFlag("superNode.server", superNodeCmd.PersistentFlags().Lookup("supernode-server"))
viper.BindPFlag("superNode.wsPath", superNodeCmd.PersistentFlags().Lookup("supernode-ws-path"))
viper.BindPFlag("superNode.httpPath", superNodeCmd.PersistentFlags().Lookup("supernode-http-path"))
viper.BindPFlag("superNode.ipcPath", superNodeCmd.PersistentFlags().Lookup("supernode-ipc-path"))
viper.BindPFlag("superNode.sync", superNodeCmd.PersistentFlags().Lookup("supernode-sync"))
viper.BindPFlag("superNode.workers", superNodeCmd.PersistentFlags().Lookup("supernode-workers"))
viper.BindPFlag("superNode.backFill", superNodeCmd.PersistentFlags().Lookup("supernode-back-fill"))
viper.BindPFlag("superNode.frequency", superNodeCmd.PersistentFlags().Lookup("supernode-frequency"))
viper.BindPFlag("superNode.batchSize", superNodeCmd.PersistentFlags().Lookup("supernode-batch-size"))
viper.BindPFlag("superNode.batchNumber", superNodeCmd.PersistentFlags().Lookup("supernode-batch-number"))
viper.BindPFlag("bitcoin.wsPath", superNodeCmd.PersistentFlags().Lookup("btc-ws-path"))
viper.BindPFlag("bitcoin.httpPath", superNodeCmd.PersistentFlags().Lookup("btc-http-path"))
viper.BindPFlag("bitcoin.pass", superNodeCmd.PersistentFlags().Lookup("btc-password"))
viper.BindPFlag("bitcoin.user", superNodeCmd.PersistentFlags().Lookup("btc-username"))
viper.BindPFlag("bitcoin.nodeID", superNodeCmd.PersistentFlags().Lookup("btc-node-id"))
viper.BindPFlag("bitcoin.clientName", superNodeCmd.PersistentFlags().Lookup("btc-client-name"))
viper.BindPFlag("bitcoin.genesisBlock", superNodeCmd.PersistentFlags().Lookup("btc-genesis-block"))
viper.BindPFlag("bitcoin.networkID", superNodeCmd.PersistentFlags().Lookup("btc-network-id"))
viper.BindPFlag("ethereum.wsPath", superNodeCmd.PersistentFlags().Lookup("eth-ws-path"))
viper.BindPFlag("ethereum.httpPath", superNodeCmd.PersistentFlags().Lookup("eth-http-path"))
}

24
environments/resyncBTC.toml
View File

@ -1,24 +0,0 @@
[database]
name = "vulcanize_public"
hostname = "localhost"
port = 5432
user = "vdbm"
[resync]
chain = "bitcoin"
type = "full"
clearOldCache = true
ipfsPath = "/root/.ipfs"
batchSize = 1
batchNumber = 50
start = 0
stop = 0
[bitcoin]
httpPath = "127.0.0.1:8332"
pass = "password"
user = "username"
nodeID = "ocd0"
clientName = "Omnicore"
genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"
networkID = "0xD9B4BEF9"

18
environments/resyncETH.toml
View File

@ -1,18 +0,0 @@
[database]
name = "vulcanize_public"
hostname = "localhost"
port = 5432
user = "vdbm"
[resync]
chain = "ethereum"
type = "state"
clearOldCache = true
ipfsPath = "/root/.ipfs"
batchSize = 5
batchNumber = 50
start = 0
stop = 0
[ethereum]
httpPath = "127.0.0.1:8545"

60
environments/superNodeBTC.toml
View File

@ -1,29 +1,41 @@
[database] [database]
name = "vulcanize_public" name = "vulcanize_public" # $DATABASE_NAME
hostname = "localhost" hostname = "localhost" # &DATABASE_HOSTNAME
port = 5432 port = 5432 # $DATABASE_PORT
user = "vdbm" user = "vdbm" # $DATABASE_USER
password = "" # $DATABASE_PASSWORD
[ipfs]
path = "/root/.ipfs" # $IPFS_PATH
[resync]
chain = "bitcoin" # $RESYNC_CHAIN
type = "full" # $RESYNC_TYPE
start = 0 # $RESYNC_START
stop = 0 # $RESYNC_STOP
batchSize = 1 # $RESYNC_BATCH_SIZE
batchNumber = 50 # $RESYNC_BATCH_NUMBER
clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE
[superNode] [superNode]
chain = "bitcoin" chain = "bitcoin" # $SUPERNODE_CHAIN
ipfsPath = "/root/.ipfs" server = true # $SUPERNODE_SERVER
server = true ipcPath = "/root/.vulcanize/btc/vulcanize.ipc" # $SUPERNODE_IPC_PATH
ipcPath = "/root/.vulcanize/btc/vulcanize.ipc" wsPath = "127.0.0.1:8082" # $SUPERNODE_WS_PATH
wsPath = "127.0.0.1:8082" httpPath = "127.0.0.1:8083" # $SUPERNODE_HTTP_PATH
httpPath = "127.0.0.1:8083" sync = true # $SUPERNODE_SYNC
sync = true workers = 1 # $SUPERNODE_WORKERS
workers = 1 backFill = true # $SUPERNODE_BACKFILL
backFill = true frequency = 45 # $SUPERNODE_FREQUENCY
frequency = 45 batchSize = 1 # $SUPERNODE_BATCH_SIZE
batchSize = 1 batchNumber = 50 # $SUPERNODE_BATCH_NUMBER
batchNumber = 50
[bitcoin] [bitcoin]
wsPath = "127.0.0.1:8332" wsPath = "127.0.0.1:8332" # $BTC_WS_PATH
httpPath = "127.0.0.1:8332" httpPath = "127.0.0.1:8332" # $BTC_HTTP_PATH
pass = "password" pass = "password" # $BTC_NODE_PASSWORD
user = "username" user = "username" # $BTC_NODE_USER
nodeID = "ocd0" nodeID = "ocd0" # $BTC_NODE_ID
clientName = "Omnicore" clientName = "Omnicore" # $BTC_CLIENT_NAME
genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" genesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" # $BTC_GENESIS_BLOCK
networkID = "0xD9B4BEF9" networkID = "0xD9B4BEF9" # $BTC_NETWORK_ID

48
environments/superNodeETH.toml
View File

@ -1,23 +1,35 @@
[database] [database]
name = "vulcanize_public" name = "vulcanize_public" # $DATABASE_NAME
hostname = "localhost" hostname = "localhost" # &DATABASE_HOSTNAME
port = 5432 port = 5432 # $DATABASE_PORT
user = "vdbm" user = "vdbm" # $DATABASE_USER
password = "" # $DATABASE_PASSWORD
[ipfs]
path = "/root/.ipfs" # $IPFS_PATH
[resync]
chain = "ethereum" # $RESYNC_CHAIN
type = "state" # $RESYNC_TYPE
start = 0 # $RESYNC_START
stop = 0 # $RESYNC_STOP
batchSize = 5 # $RESYNC_BATCH_SIZE
batchNumber = 50 # $RESYNC_BATCH_NUMBER
clearOldCache = true # $RESYNC_CLEAR_OLD_CACHE
[superNode] [superNode]
chain = "ethereum" chain = "ethereum" # $SUPERNODE_CHAIN
ipfsPath = "/root/.ipfs" server = true # $SUPERNODE_SERVER
server = true ipcPath = "/root/.vulcanize/eth/vulcanize.ipc" # $SUPERNODE_IPC_PATH
ipcPath = "/root/.vulcanize/eth/vulcanize.ipc" wsPath = "127.0.0.1:8081" # $SUPERNODE_WS_PATH
wsPath = "127.0.0.1:8081" httpPath = "127.0.0.1:8082" # $SUPERNODE_HTTP_PATH
httpPath = "127.0.0.1:8082" sync = true # $SUPERNODE_SYNC
sync = true workers = 1 # $SUPERNODE_WORKERS
workers = 1 backFill = true # $SUPERNODE_BACKFILL
backFill = true frequency = 15 # $SUPERNODE_FREQUENCY
frequency = 15 batchSize = 5 # $SUPERNODE_BATCH_SIZE
batchSize = 5 batchNumber = 50 # $SUPERNODE_BATCH_NUMBER
batchNumber = 50
[ethereum] [ethereum]
wsPath = "127.0.0.1:8546" wsPath = "127.0.0.1:8546" # $ETH_WS_PATH
httpPath = "127.0.0.1:8545" httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH

28
pkg/config/database.go
View File

@ -16,7 +16,20 @@
package config package config
import "fmt" import (
"fmt"
"github.com/spf13/viper"
)
// Env variables
const (
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
)
type Database struct { type Database struct {
Hostname string Hostname string
@ -37,3 +50,16 @@ func DbConnectionString(dbConfig Database) string {
} }
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", dbConfig.Hostname, dbConfig.Port, dbConfig.Name) return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", dbConfig.Hostname, dbConfig.Port, dbConfig.Name)
} }
func (d *Database) Init() {
viper.BindEnv("database.name", DATABASE_NAME)
viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
viper.BindEnv("database.port", DATABASE_PORT)
viper.BindEnv("database.user", DATABASE_USER)
viper.BindEnv("database.password", DATABASE_PASSWORD)
d.Name = viper.GetString("database.name")
d.Hostname = viper.GetString("database.hostname")
d.Port = viper.GetInt("database.port")
d.User = viper.GetString("database.user")
d.Password = viper.GetString("database.password")
}

120
pkg/super_node/config.go
View File

@ -22,19 +22,30 @@ import (
"path/filepath" "path/filepath"
"time" "time"
"github.com/btcsuite/btcd/rpcclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/eth/client"
"github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/node"
"github.com/vulcanize/vulcanizedb/pkg/postgres" "github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared" "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/vulcanize/vulcanizedb/utils" "github.com/vulcanize/vulcanizedb/utils"
) )
// Env variables
const (
SUPERNODE_CHAIN = "SUPERNODE_CHAIN"
SUPERNODE_SYNC = "SUPERNODE_SYNC"
SUPERNODE_WORKERS = "SUPERNODE_WORKERS"
SUPERNODE_SERVER = "SUPERNODE_SERVER"
SUPERNODE_WS_PATH = "SUPERNODE_WS_PATH"
SUPERNODE_IPC_PATH = "SUPERNODE_IPC_PATH"
SUPERNODE_HTTP_PATH = "SUPERNODE_HTTP_PATH"
SUPERNODE_BACKFILL = "SUPERNODE_BACKFILL"
SUPERNODE_FREQUENCY = "SUPERNODE_FREQUENCY"
SUPERNODE_BATCH_SIZE = "SUPERNODE_BATCH_SIZE"
SUPERNODE_BATCH_NUMBER = "SUPERNODE_BATCH_NUMBER"
)
// Config struct // Config struct
type Config struct { type Config struct {
// Ubiquitous fields // Ubiquitous fields
@ -67,21 +78,27 @@ func NewSuperNodeConfig() (*Config, error) {
c := new(Config) c := new(Config)
var err error var err error
viper.BindEnv("superNode.chain", SUPERNODE_CHAIN)
viper.BindEnv("superNode.sync", SUPERNODE_SYNC)
viper.BindEnv("superNode.workers", SUPERNODE_WORKERS)
viper.BindEnv("ethereum.wsPath", shared.ETH_WS_PATH)
viper.BindEnv("bitcoin.wsPath", shared.BTC_WS_PATH)
viper.BindEnv("superNode.server", SUPERNODE_SERVER)
viper.BindEnv("superNode.wsPath", SUPERNODE_WS_PATH)
viper.BindEnv("superNode.ipcPath", SUPERNODE_IPC_PATH)
viper.BindEnv("superNode.httpPath", SUPERNODE_HTTP_PATH)
viper.BindEnv("superNode.backFill", SUPERNODE_BACKFILL)
chain := viper.GetString("superNode.chain") chain := viper.GetString("superNode.chain")
c.Chain, err = shared.NewChainType(chain) c.Chain, err = shared.NewChainType(chain)
if err != nil { if err != nil {
return nil, err return nil, err
} }
ipfsPath := viper.GetString("superNode.ipfsPath") c.IPFSPath, err = shared.GetIPFSPath()
if ipfsPath == "" {
home, err := os.UserHomeDir()
if err != nil { if err != nil {
return nil, err return nil, err
} }
ipfsPath = filepath.Join(home, ".ipfs")
}
c.IPFSPath = ipfsPath
c.Sync = viper.GetBool("superNode.sync") c.Sync = viper.GetBool("superNode.sync")
if c.Sync { if c.Sync {
@ -92,25 +109,14 @@ func NewSuperNodeConfig() (*Config, error) {
c.Workers = workers c.Workers = workers
switch c.Chain { switch c.Chain {
case shared.Ethereum: case shared.Ethereum:
c.NodeInfo, c.WSClient, err = getEthNodeAndClient(fmt.Sprintf("ws://%s", viper.GetString("ethereum.wsPath"))) ethWS := viper.GetString("ethereum.wsPath")
c.NodeInfo, c.WSClient, err = shared.GetEthNodeAndClient(fmt.Sprintf("ws://%s", ethWS))
if err != nil { if err != nil {
return nil, err return nil, err
} }
case shared.Bitcoin: case shared.Bitcoin:
c.NodeInfo = core.Node{ btcWS := viper.GetString("bitcoin.wsPath")
ID: viper.GetString("bitcoin.nodeID"), c.NodeInfo, c.WSClient = shared.GetBtcNodeAndClient(btcWS)
ClientName: viper.GetString("bitcoin.clientName"),
GenesisBlock: viper.GetString("bitcoin.genesisBlock"),
NetworkID: viper.GetString("bitcoin.networkID"),
}
// For bitcoin we load in node info from the config because there is no RPC endpoint to retrieve this from the node
c.WSClient = &rpcclient.ConnConfig{
Host: viper.GetString("bitcoin.wsPath"),
HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode
DisableTLS: true, // Bitcoin core does not provide TLS by default
Pass: viper.GetString("bitcoin.pass"),
User: viper.GetString("bitcoin.user"),
}
} }
} }
@ -137,47 +143,43 @@ func NewSuperNodeConfig() (*Config, error) {
c.HTTPEndpoint = httpPath c.HTTPEndpoint = httpPath
} }
c.DBConfig = config.Database{ c.BackFill = viper.GetBool("superNode.backFill")
Name: viper.GetString("database.name"), if c.BackFill {
Hostname: viper.GetString("database.hostname"), if err := c.BackFillFields(); err != nil {
Port: viper.GetInt("database.port"),
User: viper.GetString("database.user"),
Password: viper.GetString("database.password"),
}
db := utils.LoadPostgres(c.DBConfig, c.NodeInfo)
c.DB = &db
c.Quit = make(chan bool)
if viper.GetBool("superNode.backFill") {
if err := c.BackFillFields(chain); err != nil {
return nil, err return nil, err
} }
} }
c.DBConfig.Init()
db := utils.LoadPostgres(c.DBConfig, c.NodeInfo)
c.DB = &db
c.Quit = make(chan bool)
return c, nil return c, nil
} }
// BackFillFields is used to fill in the BackFill fields of the config // BackFillFields is used to fill in the BackFill fields of the config
func (sn *Config) BackFillFields(chain string) error { func (c *Config) BackFillFields() error {
sn.BackFill = true
var httpClient interface{}
var err error var err error
switch sn.Chain {
viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
viper.BindEnv("bitcoin.httpPath", shared.BTC_HTTP_PATH)
viper.BindEnv("superNode.frequency", SUPERNODE_FREQUENCY)
viper.BindEnv("superNode.batchSize", SUPERNODE_BATCH_SIZE)
viper.BindEnv("superNode.batchNumber", SUPERNODE_BATCH_NUMBER)
switch c.Chain {
case shared.Ethereum: case shared.Ethereum:
_, httpClient, err = getEthNodeAndClient(fmt.Sprintf("http://%s", viper.GetString("ethereum.httpPath"))) ethHTTP := viper.GetString("ethereum.httpPath")
c.NodeInfo, c.HTTPClient, err = shared.GetEthNodeAndClient(fmt.Sprintf("http://%s", ethHTTP))
if err != nil { if err != nil {
return err return err
} }
case shared.Bitcoin: case shared.Bitcoin:
httpClient = &rpcclient.ConnConfig{ btcHTTP := viper.GetString("bitcoin.httpPath")
Host: viper.GetString("bitcoin.httpPath"), c.NodeInfo, c.HTTPClient = shared.GetBtcNodeAndClient(btcHTTP)
HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode
DisableTLS: true, // Bitcoin core does not provide TLS by default
Pass: viper.GetString("bitcoin.pass"),
User: viper.GetString("bitcoin.user"),
} }
}
sn.HTTPClient = httpClient
freq := viper.GetInt("superNode.frequency") freq := viper.GetInt("superNode.frequency")
var frequency time.Duration var frequency time.Duration
if freq <= 0 { if freq <= 0 {
@ -185,18 +187,8 @@ func (sn *Config) BackFillFields(chain string) error {
} else { } else {
frequency = time.Second * time.Duration(freq) frequency = time.Second * time.Duration(freq)
} }
sn.Frequency = frequency c.Frequency = frequency
sn.BatchSize = uint64(viper.GetInt64("superNode.batchSize")) c.BatchSize = uint64(viper.GetInt64("superNode.batchSize"))
sn.BatchNumber = uint64(viper.GetInt64("superNode.batchNumber")) c.BatchNumber = uint64(viper.GetInt64("superNode.batchNumber"))
return nil return nil
} }
func getEthNodeAndClient(path string) (core.Node, interface{}, error) {
rawRPCClient, err := rpc.Dial(path)
if err != nil {
return core.Node{}, nil, err
}
rpcClient := client.NewRPCClient(rawRPCClient, path)
vdbNode := node.MakeNode(rpcClient)
return vdbNode, rpcClient, nil
}

77
pkg/super_node/resync/config.go
View File

@ -18,22 +18,27 @@ package resync
import ( import (
"fmt" "fmt"
"os"
"path/filepath"
"github.com/btcsuite/btcd/rpcclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/eth/client"
"github.com/vulcanize/vulcanizedb/pkg/eth/core" "github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/node"
"github.com/vulcanize/vulcanizedb/pkg/postgres" "github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared" "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/vulcanize/vulcanizedb/utils" "github.com/vulcanize/vulcanizedb/utils"
) )
// Env variables
const (
RESYNC_CHAIN = "RESYNC_CHAIN"
RESYNC_START = "RESYNC_START"
RESYNC_STOP = "RESYNC_STOP"
RESYNC_BATCH_SIZE = "RESYNC_BATCH_SIZE"
RESYNC_BATCH_NUMBER = "RESYNC_BATCH_NUMBER"
RESYNC_CLEAR_OLD_CACHE = "RESYNC_CLEAR_OLD_CACHE"
RESYNC_TYPE = "RESYNC_TYPE"
)
// Config holds the parameters needed to perform a resync // Config holds the parameters needed to perform a resync
type Config struct { type Config struct {
Chain shared.ChainType // The type of resync to perform Chain shared.ChainType // The type of resync to perform
@ -58,26 +63,26 @@ type Config struct {
func NewReSyncConfig() (*Config, error) { func NewReSyncConfig() (*Config, error) {
c := new(Config) c := new(Config)
var err error var err error
viper.BindEnv("resync.start", RESYNC_START)
viper.BindEnv("resync.stop", RESYNC_STOP)
viper.BindEnv("resync.clearOldCache", RESYNC_CLEAR_OLD_CACHE)
viper.BindEnv("resync.type", RESYNC_TYPE)
viper.BindEnv("resync.chain", RESYNC_CHAIN)
viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
viper.BindEnv("bitcoin.httpPath", shared.BTC_HTTP_PATH)
viper.BindEnv("resync.batchSize", RESYNC_BATCH_SIZE)
viper.BindEnv("resync.batchNumber", RESYNC_BATCH_NUMBER)
start := uint64(viper.GetInt64("resync.start")) start := uint64(viper.GetInt64("resync.start"))
stop := uint64(viper.GetInt64("resync.stop")) stop := uint64(viper.GetInt64("resync.stop"))
c.Ranges = [][2]uint64{{start, stop}} c.Ranges = [][2]uint64{{start, stop}}
ipfsPath := viper.GetString("resync.ipfsPath") c.ClearOldCache = viper.GetBool("resync.clearOldCache")
if ipfsPath == "" {
home, err := os.UserHomeDir() c.IPFSPath, err = shared.GetIPFSPath()
if err != nil { if err != nil {
return nil, err return nil, err
} }
ipfsPath = filepath.Join(home, ".ipfs")
}
c.IPFSPath = ipfsPath
c.DBConfig = config.Database{
Name: viper.GetString("database.name"),
Hostname: viper.GetString("database.hostname"),
Port: viper.GetInt("database.port"),
User: viper.GetString("database.user"),
Password: viper.GetString("database.password"),
}
c.ClearOldCache = viper.GetBool("resync.clearOldCache")
resyncType := viper.GetString("resync.type") resyncType := viper.GetString("resync.type")
c.ResyncType, err = shared.GenerateResyncTypeFromString(resyncType) c.ResyncType, err = shared.GenerateResyncTypeFromString(resyncType)
if err != nil { if err != nil {
@ -97,40 +102,22 @@ func NewReSyncConfig() (*Config, error) {
switch c.Chain { switch c.Chain {
case shared.Ethereum: case shared.Ethereum:
c.NodeInfo, c.HTTPClient, err = getEthNodeAndClient(fmt.Sprintf("http://%s", viper.GetString("ethereum.httpPath"))) ethHTTP := viper.GetString("ethereum.httpPath")
c.NodeInfo, c.HTTPClient, err = shared.GetEthNodeAndClient(fmt.Sprintf("http://%s", ethHTTP))
if err != nil { if err != nil {
return nil, err return nil, err
} }
case shared.Bitcoin: case shared.Bitcoin:
c.NodeInfo = core.Node{ btcHTTP := viper.GetString("bitcoin.httpPath")
ID: viper.GetString("bitcoin.nodeID"), c.NodeInfo, c.HTTPClient = shared.GetBtcNodeAndClient(btcHTTP)
ClientName: viper.GetString("bitcoin.clientName"),
GenesisBlock: viper.GetString("bitcoin.genesisBlock"),
NetworkID: viper.GetString("bitcoin.networkID"),
}
// For bitcoin we load in node info from the config because there is no RPC endpoint to retrieve this from the node
c.HTTPClient = &rpcclient.ConnConfig{
Host: viper.GetString("bitcoin.httpPath"),
HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode
DisableTLS: true, // Bitcoin core does not provide TLS by default
Pass: viper.GetString("bitcoin.pass"),
User: viper.GetString("bitcoin.user"),
}
} }
c.DBConfig.Init()
db := utils.LoadPostgres(c.DBConfig, c.NodeInfo) db := utils.LoadPostgres(c.DBConfig, c.NodeInfo)
c.DB = &db c.DB = &db
c.Quit = make(chan bool) c.Quit = make(chan bool)
c.BatchSize = uint64(viper.GetInt64("resync.batchSize")) c.BatchSize = uint64(viper.GetInt64("resync.batchSize"))
c.BatchNumber = uint64(viper.GetInt64("resync.batchNumber")) c.BatchNumber = uint64(viper.GetInt64("resync.batchNumber"))
return c, nil return c, nil
} }
func getEthNodeAndClient(path string) (core.Node, interface{}, error) {
rawRPCClient, err := rpc.Dial(path)
if err != nil {
return core.Node{}, nil, err
}
rpcClient := client.NewRPCClient(rawRPCClient, path)
vdbNode := node.MakeNode(rpcClient)
return vdbNode, rpcClient, nil
}

96
pkg/super_node/shared/env.go Normal file
View File

@ -0,0 +1,96 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import (
"os"
"path/filepath"
"github.com/ethereum/go-ethereum/rpc"
"github.com/btcsuite/btcd/rpcclient"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/eth/client"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/node"
)
// Env variables
const (
IPFS_PATH = "IPFS_PATH"
ETH_WS_PATH = "ETH_WS_PATH"
ETH_HTTP_PATH = "ETH_HTTP_PATH"
BTC_WS_PATH = "BTC_WS_PATH"
BTC_HTTP_PATH = "BTC_HTTP_PATH"
BTC_NODE_PASSWORD = "BTC_NODE_PASSWORD"
BTC_NODE_USER = "BTC_NODE_USER"
BTC_NODE_ID = "BTC_NODE_ID"
BTC_CLIENT_NAME = "BTC_CLIENT_NAME"
BTC_GENESIS_BLOCK = "BTC_GENESIS_BLOCK"
BTC_NETWORK_ID = "BTC_NETWORK_ID"
)
// GetEthNodeAndClient returns eth node info and client from path url
func GetEthNodeAndClient(path string) (core.Node, core.RPCClient, error) {
rawRPCClient, err := rpc.Dial(path)
if err != nil {
return core.Node{}, nil, err
}
rpcClient := client.NewRPCClient(rawRPCClient, path)
vdbNode := node.MakeNode(rpcClient)
return vdbNode, rpcClient, nil
}
// GetIPFSPath returns the ipfs path from the config or env variable
func GetIPFSPath() (string, error) {
viper.BindEnv("ipfs.path", IPFS_PATH)
ipfsPath := viper.GetString("ipfs.path")
if ipfsPath == "" {
home, err := os.UserHomeDir()
if err != nil {
return "", err
}
ipfsPath = filepath.Join(home, ".ipfs")
}
return ipfsPath, nil
}
// GetBtcNodeAndClient returns btc node info from path url
func GetBtcNodeAndClient(path string) (core.Node, *rpcclient.ConnConfig) {
viper.BindEnv("bitcoin.nodeID", BTC_NODE_ID)
viper.BindEnv("bitcoin.clientName", BTC_CLIENT_NAME)
viper.BindEnv("bitcoin.genesisBlock", BTC_GENESIS_BLOCK)
viper.BindEnv("bitcoin.networkID", BTC_NETWORK_ID)
viper.BindEnv("bitcoin.pass", BTC_NODE_PASSWORD)
viper.BindEnv("bitcoin.user", BTC_NODE_USER)
// For bitcoin we load in node info from the config because there is no RPC endpoint to retrieve this from the node
return core.Node{
ID: viper.GetString("bitcoin.nodeID"),
ClientName: viper.GetString("bitcoin.clientName"),
GenesisBlock: viper.GetString("bitcoin.genesisBlock"),
NetworkID: viper.GetString("bitcoin.networkID"),
}, &rpcclient.ConnConfig{
Host: path,
HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode
DisableTLS: true, // Bitcoin core does not provide TLS by default
Pass: viper.GetString("bitcoin.pass"),
User: viper.GetString("bitcoin.user"),
}
}

1
pkg/super_node/shared/functions.go
View File

@ -20,7 +20,6 @@ import (
"bytes" "bytes"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/ipfs" "github.com/vulcanize/vulcanizedb/pkg/ipfs"
) )