update configs

environments/superNodeBTC.toml

@@ -5,16 +5,18 @@
     user     = "vdbm" # $DATABASE_USER
     password = "" # $DATABASE_PASSWORD
 
-[ipfs]
+    [database.sync]
-    path = "~/.ipfs" # $IPFS_PATH
+        maxIdle = 1
+    [database.backFill]
+        maxIdle = 5
 
 [resync]
     chain = "bitcoin" # $RESYNC_CHAIN
     type = "full" # $RESYNC_TYPE
     start = 0 # $RESYNC_START
     stop = 0 # $RESYNC_STOP
-    batchSize = 1 # $RESYNC_BATCH_SIZE
+    batchSize = 5 # $RESYNC_BATCH_SIZE
-    batchNumber = 50 # $RESYNC_BATCH_NUMBER
+    batchNumber = 5 # $RESYNC_BATCH_NUMBER
     clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE
     resetValidation = true # $RESYNC_RESET_VALIDATION
 
@@ -28,8 +30,8 @@
     workers = 1 # $SUPERNODE_WORKERS
     backFill = true # $SUPERNODE_BACKFILL
     frequency = 45 # $SUPERNODE_FREQUENCY
-    batchSize = 1 # $SUPERNODE_BATCH_SIZE
+    batchSize = 5 # $SUPERNODE_BATCH_SIZE
-    batchNumber = 50 # $SUPERNODE_BATCH_NUMBER
+    batchNumber = 5 # $SUPERNODE_BATCH_NUMBER
     validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL
 
 [bitcoin]

environments/superNodeETH.toml

@@ -5,8 +5,10 @@
     user     = "vdbm" # $DATABASE_USER
     password = "" # $DATABASE_PASSWORD
 
-[ipfs]
+    [database.sync]
-    path = "~/.ipfs" # $IPFS_PATH
+        maxIdle = 1
+    [database.backFill]
+        maxIdle = 5
 
 [resync]
     chain = "ethereum" # $RESYNC_CHAIN
@@ -14,9 +16,9 @@
     start = 0 # $RESYNC_START
     stop = 0 # $RESYNC_STOP
     batchSize = 5 # $RESYNC_BATCH_SIZE
-    batchNumber = 50 # $RESYNC_BATCH_NUMBER
+    batchNumber = 5 # $RESYNC_BATCH_NUMBER
     timeout = 300 # $HTTP_TIMEOUT
-    clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE
+    clearOldCache = true # $RESYNC_CLEAR_OLD_CACHE
     resetValidation = true # $RESYNC_RESET_VALIDATION
 
 [superNode]
@@ -30,7 +32,7 @@
     backFill = true # $SUPERNODE_BACKFILL
     frequency = 15 # $SUPERNODE_FREQUENCY
     batchSize = 5 # $SUPERNODE_BATCH_SIZE
-    batchNumber = 50 # $SUPERNODE_BATCH_NUMBER
+    batchNumber = 5 # $SUPERNODE_BATCH_NUMBER
     timeout = 300 # $HTTP_TIMEOUT
     validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL
 

pkg/config/config.go

@@ -1,22 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package config
-
-type Config struct {
-	Database Database
-	Client   Client
-}

pkg/super_node/config.go

@@ -45,6 +45,18 @@ const (
 	SUPERNODE_BATCH_SIZE       = "SUPERNODE_BATCH_SIZE"
 	SUPERNODE_BATCH_NUMBER     = "SUPERNODE_BATCH_NUMBER"
 	SUPERNODE_VALIDATION_LEVEL = "SUPERNODE_VALIDATION_LEVEL"
+
+	SYNC_MAX_IDLE_CONNECTIONS = "SYNC_MAX_IDLE_CONNECTIONS"
+	SYNC_MAX_OPEN_CONNECTIONS = "SYNC_MAX_OPEN_CONNECTIONS"
+	SYNC_MAX_CONN_LIFETIME    = "SYNC_MAX_CONN_LIFETIME"
+
+	BACKFILL_MAX_IDLE_CONNECTIONS = "BACKFILL_MAX_IDLE_CONNECTIONS"
+	BACKFILL_MAX_OPEN_CONNECTIONS = "BACKFILL_MAX_OPEN_CONNECTIONS"
+	BACKFILL_MAX_CONN_LIFETIME    = "BACKFILL_MAX_CONN_LIFETIME"
+
+	SERVER_MAX_IDLE_CONNECTIONS = "SERVER_MAX_IDLE_CONNECTIONS"
+	SERVER_MAX_OPEN_CONNECTIONS = "SERVER_MAX_OPEN_CONNECTIONS"
+	SERVER_MAX_CONN_LIFETIME    = "SERVER_MAX_CONN_LIFETIME"
 )
 
 // Config struct
@@ -132,7 +144,8 @@ func NewSuperNodeConfig() (*Config, error) {
 			btcWS := viper.GetString("bitcoin.wsPath")
 			c.NodeInfo, c.WSClient = shared.GetBtcNodeAndClient(btcWS)
 		}
-		syncDB := utils.LoadPostgres(c.DBConfig, c.NodeInfo)
+		syncDBConn := overrideDBConnConfig(c.DBConfig, Sync)
+		syncDB := utils.LoadPostgres(syncDBConn, c.NodeInfo)
 		c.SyncDBConn = &syncDB
 	}
 
@@ -157,7 +170,8 @@ func NewSuperNodeConfig() (*Config, error) {
 			httpPath = "127.0.0.1:8081"
 		}
 		c.HTTPEndpoint = httpPath
-		serveDB := utils.LoadPostgres(c.DBConfig, c.NodeInfo)
+		serveDBConn := overrideDBConnConfig(c.DBConfig, Serve)
+		serveDB := utils.LoadPostgres(serveDBConn, c.NodeInfo)
 		c.ServeDBConn = &serveDB
 	}
 
@@ -215,7 +229,44 @@ func (c *Config) BackFillFields() error {
 	c.BatchNumber = uint64(viper.GetInt64("superNode.batchNumber"))
 	c.ValidationLevel = viper.GetInt("superNode.validationLevel")
 
-	backFillDB := utils.LoadPostgres(c.DBConfig, c.NodeInfo)
+	backFillDBConn := overrideDBConnConfig(c.DBConfig, BackFill)
+	backFillDB := utils.LoadPostgres(backFillDBConn, c.NodeInfo)
 	c.BackFillDBConn = &backFillDB
 	return nil
 }
+
+type mode string
+
+var (
+	Sync     mode = "sync"
+	BackFill mode = "backFill"
+	Serve    mode = "serve"
+)
+
+func overrideDBConnConfig(con config.Database, m mode) config.Database {
+	switch m {
+	case Sync:
+		viper.BindEnv("database.sync.maxIdle", SYNC_MAX_IDLE_CONNECTIONS)
+		viper.BindEnv("database.sync.maxOpen", SYNC_MAX_OPEN_CONNECTIONS)
+		viper.BindEnv("database.sync.maxLifetime", SYNC_MAX_CONN_LIFETIME)
+		con.MaxIdle = viper.GetInt("database.sync.maxIdle")
+		con.MaxOpen = viper.GetInt("database.sync.maxOpen")
+		con.MaxLifetime = viper.GetInt("database.sync.maxLifetime")
+	case BackFill:
+		viper.BindEnv("database.backFill.maxIdle", BACKFILL_MAX_IDLE_CONNECTIONS)
+		viper.BindEnv("database.backFill.maxOpen", BACKFILL_MAX_OPEN_CONNECTIONS)
+		viper.BindEnv("database.backFill.maxLifetime", BACKFILL_MAX_CONN_LIFETIME)
+		con.MaxIdle = viper.GetInt("database.backFill.maxIdle")
+		con.MaxOpen = viper.GetInt("database.backFill.maxOpen")
+		con.MaxLifetime = viper.GetInt("database.backFill.maxLifetime")
+	case Serve:
+		viper.BindEnv("database.server.maxIdle", SERVER_MAX_IDLE_CONNECTIONS)
+		viper.BindEnv("database.server.maxOpen", SERVER_MAX_OPEN_CONNECTIONS)
+		viper.BindEnv("database.server.maxLifetime", SERVER_MAX_CONN_LIFETIME)
+		con.MaxIdle = viper.GetInt("database.server.maxIdle")
+		con.MaxOpen = viper.GetInt("database.server.maxOpen")
+		con.MaxLifetime = viper.GetInt("database.server.maxLifetime")
+	default:
+	}
+	return con
+}