Add DB Connection and Logging

* Utilize LogRus
* Create a DB connection using PGX.
* Create an internal boot package for starting the application.

.gitignore

@@ -1,2 +1,3 @@
 
 ipld-ethcl-indexer
+ipld-ethcl-indexer.log

cmd/head.go

@@ -7,7 +7,10 @@ package cmd
 import (
 	"fmt"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/spf13/cobra"
+	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
 )
 
 // headCmd represents the head command
@@ -17,9 +20,17 @@ var headCmd = &cobra.Command{
 	Long:  `Capture only the blocks and state at head.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("head called")
+		startHeadTracking()
 	},
 }
 
+func startHeadTracking() {
+	_, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver)
+	if err != nil {
+		log.Fatal("Unable to Start application with error: ", err)
+	}
+}
+
 func init() {
 	captureCmd.AddCommand(headCmd)
 

cmd/root.go

@@ -8,18 +8,22 @@ import (
 	"fmt"
 	"os"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 var (
-	cfgFile    string
-	dbUserName string
-	dbPassword string
-	dbAddress  string
-	dbPort     uint16
-	lhAddress  string
-	lhPort     uint16
+	cfgFile        string
+	dbUsername     string
+	dbPassword     string
+	dbName         string
+	dbAddress      string
+	dbDriver       string
+	dbPort         int
+	lhAddress      string
+	lhPort         uint16
+	logWithCommand log.Entry
 )
 
 // rootCmd represents the base command when called without any subcommands
@@ -28,6 +32,7 @@ var rootCmd = &cobra.Command{
 	Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.",
 	Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain.
 It can either do this will keeping track of head, or backfilling historic data.`,
+	PersistentPreRun: initFuncs,
 	// Uncomment the following line if your bare application
 	// has an action associated with it:
 	// Run: func(cmd *cobra.Command, args []string) {},
@@ -42,6 +47,43 @@ func Execute() {
 	}
 }
 
+// Prerun for Cobra
+func initFuncs(cmd *cobra.Command, args []string) {
+	viper.BindEnv("log.file", "LOGRUS_FILE")
+	logfile := viper.GetString("log.file")
+	if logfile != "" {
+		file, err := os.OpenFile(logfile,
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err == nil {
+			log.Infof("Directing output to %s", logfile)
+			log.SetOutput(file)
+		} else {
+			log.SetOutput(os.Stdout)
+			log.Info("Failed to log to file, using default stdout")
+		}
+	} else {
+		log.SetOutput(os.Stdout)
+	}
+	if err := logLevel(); err != nil {
+		log.Fatal("Could not set log level: ", err)
+	}
+}
+
+// Set the log level for the application
+func logLevel() error {
+	viper.BindEnv("log.level", "LOGRUS_LEVEL")
+	lvl, err := log.ParseLevel(viper.GetString("log.level"))
+	if err != nil {
+		return err
+	}
+	log.SetLevel(lvl)
+	if lvl > log.InfoLevel {
+		log.SetReportCaller(true)
+	}
+	log.Info("Log level set to ", lvl.String())
+	return nil
+}
+
 func init() {
 	cobra.OnInitialize(initConfig)
 
@@ -51,18 +93,24 @@ func init() {
 
 	// Optional Flags
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)")
+	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
+	rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging")
 
 	// Required Flags
 
 	//// DB Specific
-	rootCmd.PersistentFlags().StringVarP(&dbUserName, "db.username", "u", "", "Database username (required)")
-	rootCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "p", "", "Database Password (required)")
-	rootCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "a", "", "Port to connect to DB(required)")
-	rootCmd.PersistentFlags().Uint16VarP(&dbPort, "db.port", "o", 0, "Port to connect to DB(required)")
+	rootCmd.PersistentFlags().StringVarP(&dbUsername, "db.username", "", "", "Database username (required)")
+	rootCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "", "", "Database Password (required)")
+	rootCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "", "", "Port to connect to DB(required)")
+	rootCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)")
+	rootCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)")
+	rootCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)")
 	rootCmd.MarkPersistentFlagRequired("db.username")
 	rootCmd.MarkPersistentFlagRequired("db.password")
 	rootCmd.MarkPersistentFlagRequired("db.address")
 	rootCmd.MarkPersistentFlagRequired("db.port")
+	rootCmd.MarkPersistentFlagRequired("db.name")
+	rootCmd.MarkPersistentFlagRequired("db.driver")
 
 	//// Lighthouse Specific
 	rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)")
@@ -71,11 +119,17 @@ func init() {
 	rootCmd.MarkPersistentFlagRequired("lh.port")
 
 	// Bind Flags with Viper
+	// Optional
+	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
+
 	//// DB Flags
 	viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username"))
 	viper.BindPFlag("db.password", rootCmd.PersistentFlags().Lookup("db.password"))
 	viper.BindPFlag("db.address", rootCmd.PersistentFlags().Lookup("db.address"))
 	viper.BindPFlag("db.port", rootCmd.PersistentFlags().Lookup("db.port"))
+	viper.BindPFlag("db.name", rootCmd.PersistentFlags().Lookup("db.name"))
+	viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver"))
 
 	// LH specific
 	viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address"))

internal/boot/boot.go

@@ -0,0 +1,23 @@
+package boot
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
+)
+
+func setUpLightHouse() {
+
+}
+
+// This function will perform some boot operations.
+// 1. Setup a logger
+// 2. Connect to the database.
+// 3. Connect to to the lighthouse client.
+func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) {
+	log.Debug("Setting up DB connection")
+	DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName)
+	if err != nil {
+		return nil, err
+	}
+	return DB, nil
+}

internal/boot/setup_database.go

@@ -0,0 +1,29 @@
+// This file will allow users to setup a new DB based on the user provided inputs.
+
+package boot
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
+)
+
+func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) {
+	log.Debug("Resolving Driver Type")
+	DbDriver, err := postgres.ResolveDriverType(driverName)
+	if err != nil {
+		log.Fatal("Can't Connect to DB")
+	}
+	log.Info("Using Driver:", DbDriver)
+
+	postgresConfig := postgres.Config{
+		Hostname:     dbHostname,
+		Port:         dbPort,
+		DatabaseName: dbName,
+		Username:     dbUsername,
+		Password:     dbPassword,
+		Driver:       DbDriver,
+	}
+	DB, err := postgres.NewPostgresDB(postgresConfig)
+	return DB, err
+
+}

internal/boot/startup.md

@@ -0,0 +1,3 @@
+# Overview
+
+This small package will group all the functions needed to start the application. The functions listed here can be utilized regardless of whether the application is meant to track `head` or for `historic` processing.

pkg/database/sql/postgres/config.go

@@ -53,10 +53,6 @@ type Config struct {
 	MaxConnLifetime time.Duration
 	ConnTimeout     time.Duration
 
-	// node info params
-	ID         string
-	ClientName string
-
 	// driver type
 	Driver DriverType
 }

pkg/database/sql/postgres/database.go

@@ -4,7 +4,8 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
 )
 
 var _ sql.Database = &DB{}
@@ -14,15 +15,10 @@ var _ sql.Database = &DB{}
 // This will make sure that if you want a driver, it conforms to the interface.
 
 // NewPostgresDB returns a postgres.DB using the provided Config and driver type.
-func NewPostgresDB(c Config, driverName string) (*DB, error) {
+func NewPostgresDB(c Config) (*DB, error) {
 	var driver *pgxDriver
 
-	driverType, err := ResolveDriverType(driverName)
-	if err != nil {
-		return nil, err
-	}
-
-	driver, err = createDriver(c, driverType)
+	driver, err := createDriver(c)
 
 	if err != nil {
 		return nil, err
@@ -31,15 +27,18 @@ func NewPostgresDB(c Config, driverName string) (*DB, error) {
 	return &DB{driver}, nil
 }
 
-func createDriver(c Config, driverType DriverType) (*pgxDriver, error) {
-	switch driverType {
+func createDriver(c Config) (*pgxDriver, error) {
+	switch c.Driver {
 	case PGX:
+		log.Debug("Creating New Driver")
 		driver, err := newPGXDriver(context.Background(), c)
 		if err != nil {
 			return nil, fmt.Errorf("Error Creating Driver, err: %e", err)
 		}
+		log.Info("Successfully created a driver for PGX")
 		return driver, nil
 	default:
+		log.Fatal("Couldnt find a driver to create for: ", c.Driver)
 		return nil, fmt.Errorf("Can't find a driver to create")
 	}
 

pkg/database/sql/postgres/pgx.go

@@ -8,7 +8,7 @@ import (
 	"github.com/jackc/pgconn"
 	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
 )
 
 // pgxDriver driver, implements sql.Driver

pkg/database/sql/postgres/pgx_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/jackc/pgx/v4/pgxpool"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
 	"github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers"
 )