From e2ccd3ffdb1771162baed1dd18d4bb2dd03cf930 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 May 2020 18:00:30 -0500 Subject: [PATCH] update configs --- environments/superNodeBTC.toml | 14 +++++---- environments/superNodeETH.toml | 12 ++++--- pkg/config/config.go | 22 ------------- pkg/super_node/config.go | 57 ++++++++++++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 36 deletions(-) delete mode 100644 pkg/config/config.go diff --git a/environments/superNodeBTC.toml b/environments/superNodeBTC.toml index f84ff97c..73178b98 100644 --- a/environments/superNodeBTC.toml +++ b/environments/superNodeBTC.toml @@ -5,16 +5,18 @@ user = "vdbm" # $DATABASE_USER password = "" # $DATABASE_PASSWORD -[ipfs] - path = "~/.ipfs" # $IPFS_PATH + [database.sync] + maxIdle = 1 + [database.backFill] + maxIdle = 5 [resync] chain = "bitcoin" # $RESYNC_CHAIN type = "full" # $RESYNC_TYPE start = 0 # $RESYNC_START stop = 0 # $RESYNC_STOP - batchSize = 1 # $RESYNC_BATCH_SIZE - batchNumber = 50 # $RESYNC_BATCH_NUMBER + batchSize = 5 # $RESYNC_BATCH_SIZE + batchNumber = 5 # $RESYNC_BATCH_NUMBER clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE resetValidation = true # $RESYNC_RESET_VALIDATION @@ -28,8 +30,8 @@ workers = 1 # $SUPERNODE_WORKERS backFill = true # $SUPERNODE_BACKFILL frequency = 45 # $SUPERNODE_FREQUENCY - batchSize = 1 # $SUPERNODE_BATCH_SIZE - batchNumber = 50 # $SUPERNODE_BATCH_NUMBER + batchSize = 5 # $SUPERNODE_BATCH_SIZE + batchNumber = 5 # $SUPERNODE_BATCH_NUMBER validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL [bitcoin] diff --git a/environments/superNodeETH.toml b/environments/superNodeETH.toml index 837c6afa..997ba92e 100644 --- a/environments/superNodeETH.toml +++ b/environments/superNodeETH.toml @@ -5,8 +5,10 @@ user = "vdbm" # $DATABASE_USER password = "" # $DATABASE_PASSWORD -[ipfs] - path = "~/.ipfs" # $IPFS_PATH + [database.sync] + maxIdle = 1 + [database.backFill] + maxIdle = 5 [resync] chain = "ethereum" # $RESYNC_CHAIN @@ -14,9 +16,9 @@ start = 0 # $RESYNC_START stop = 0 # $RESYNC_STOP batchSize = 5 # $RESYNC_BATCH_SIZE - batchNumber = 50 # $RESYNC_BATCH_NUMBER + batchNumber = 5 # $RESYNC_BATCH_NUMBER timeout = 300 # $HTTP_TIMEOUT - clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE + clearOldCache = true # $RESYNC_CLEAR_OLD_CACHE resetValidation = true # $RESYNC_RESET_VALIDATION [superNode] @@ -30,7 +32,7 @@ backFill = true # $SUPERNODE_BACKFILL frequency = 15 # $SUPERNODE_FREQUENCY batchSize = 5 # $SUPERNODE_BATCH_SIZE - batchNumber = 50 # $SUPERNODE_BATCH_NUMBER + batchNumber = 5 # $SUPERNODE_BATCH_NUMBER timeout = 300 # $HTTP_TIMEOUT validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL diff --git a/pkg/config/config.go b/pkg/config/config.go deleted file mode 100644 index 87588f9d..00000000 --- a/pkg/config/config.go +++ /dev/null @@ -1,22 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package config - -type Config struct { - Database Database - Client Client -} diff --git a/pkg/super_node/config.go b/pkg/super_node/config.go index 84582a80..ba1547b7 100644 --- a/pkg/super_node/config.go +++ b/pkg/super_node/config.go @@ -45,6 +45,18 @@ const ( SUPERNODE_BATCH_SIZE = "SUPERNODE_BATCH_SIZE" SUPERNODE_BATCH_NUMBER = "SUPERNODE_BATCH_NUMBER" SUPERNODE_VALIDATION_LEVEL = "SUPERNODE_VALIDATION_LEVEL" + + SYNC_MAX_IDLE_CONNECTIONS = "SYNC_MAX_IDLE_CONNECTIONS" + SYNC_MAX_OPEN_CONNECTIONS = "SYNC_MAX_OPEN_CONNECTIONS" + SYNC_MAX_CONN_LIFETIME = "SYNC_MAX_CONN_LIFETIME" + + BACKFILL_MAX_IDLE_CONNECTIONS = "BACKFILL_MAX_IDLE_CONNECTIONS" + BACKFILL_MAX_OPEN_CONNECTIONS = "BACKFILL_MAX_OPEN_CONNECTIONS" + BACKFILL_MAX_CONN_LIFETIME = "BACKFILL_MAX_CONN_LIFETIME" + + SERVER_MAX_IDLE_CONNECTIONS = "SERVER_MAX_IDLE_CONNECTIONS" + SERVER_MAX_OPEN_CONNECTIONS = "SERVER_MAX_OPEN_CONNECTIONS" + SERVER_MAX_CONN_LIFETIME = "SERVER_MAX_CONN_LIFETIME" ) // Config struct @@ -132,7 +144,8 @@ func NewSuperNodeConfig() (*Config, error) { btcWS := viper.GetString("bitcoin.wsPath") c.NodeInfo, c.WSClient = shared.GetBtcNodeAndClient(btcWS) } - syncDB := utils.LoadPostgres(c.DBConfig, c.NodeInfo) + syncDBConn := overrideDBConnConfig(c.DBConfig, Sync) + syncDB := utils.LoadPostgres(syncDBConn, c.NodeInfo) c.SyncDBConn = &syncDB } @@ -157,7 +170,8 @@ func NewSuperNodeConfig() (*Config, error) { httpPath = "127.0.0.1:8081" } c.HTTPEndpoint = httpPath - serveDB := utils.LoadPostgres(c.DBConfig, c.NodeInfo) + serveDBConn := overrideDBConnConfig(c.DBConfig, Serve) + serveDB := utils.LoadPostgres(serveDBConn, c.NodeInfo) c.ServeDBConn = &serveDB } @@ -215,7 +229,44 @@ func (c *Config) BackFillFields() error { c.BatchNumber = uint64(viper.GetInt64("superNode.batchNumber")) c.ValidationLevel = viper.GetInt("superNode.validationLevel") - backFillDB := utils.LoadPostgres(c.DBConfig, c.NodeInfo) + backFillDBConn := overrideDBConnConfig(c.DBConfig, BackFill) + backFillDB := utils.LoadPostgres(backFillDBConn, c.NodeInfo) c.BackFillDBConn = &backFillDB return nil } + +type mode string + +var ( + Sync mode = "sync" + BackFill mode = "backFill" + Serve mode = "serve" +) + +func overrideDBConnConfig(con config.Database, m mode) config.Database { + switch m { + case Sync: + viper.BindEnv("database.sync.maxIdle", SYNC_MAX_IDLE_CONNECTIONS) + viper.BindEnv("database.sync.maxOpen", SYNC_MAX_OPEN_CONNECTIONS) + viper.BindEnv("database.sync.maxLifetime", SYNC_MAX_CONN_LIFETIME) + con.MaxIdle = viper.GetInt("database.sync.maxIdle") + con.MaxOpen = viper.GetInt("database.sync.maxOpen") + con.MaxLifetime = viper.GetInt("database.sync.maxLifetime") + case BackFill: + viper.BindEnv("database.backFill.maxIdle", BACKFILL_MAX_IDLE_CONNECTIONS) + viper.BindEnv("database.backFill.maxOpen", BACKFILL_MAX_OPEN_CONNECTIONS) + viper.BindEnv("database.backFill.maxLifetime", BACKFILL_MAX_CONN_LIFETIME) + con.MaxIdle = viper.GetInt("database.backFill.maxIdle") + con.MaxOpen = viper.GetInt("database.backFill.maxOpen") + con.MaxLifetime = viper.GetInt("database.backFill.maxLifetime") + case Serve: + viper.BindEnv("database.server.maxIdle", SERVER_MAX_IDLE_CONNECTIONS) + viper.BindEnv("database.server.maxOpen", SERVER_MAX_OPEN_CONNECTIONS) + viper.BindEnv("database.server.maxLifetime", SERVER_MAX_CONN_LIFETIME) + con.MaxIdle = viper.GetInt("database.server.maxIdle") + con.MaxOpen = viper.GetInt("database.server.maxOpen") + con.MaxLifetime = viper.GetInt("database.server.maxLifetime") + default: + } + return con +}