Code clean up + Beacon Chain Connection

This concludes all the code needed to connect to the DB and beacon node. We will no longer reference the lighthouse client because this application should work interchangeably with any beacon node.

I have also standardized logging.

README.md

@@ -1,6 +1,6 @@
 # ipld-ethcl-indexer
 
-This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum.
+This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node.
 
 # Running the Application
 
@@ -10,11 +10,36 @@ To run the application, utilize the following command, and update the values as
 go run main.go capture head --db.address localhost \
   --db.password password \
   --db.port 8077 \
-  --db.username username \
+  --db.username vdbm \
-  --lh.address localhost \
+  --db.name vulcanize_testing \
-  --lh.port 5052
+  --db.driver PGX \
+  --bc.address localhost \
+  --bc.port 5052 \
+  --log.level info
 ```
 
+# Development Patterns
+
+This section will cover some generic development patterns utilizes.
+
+## Logging
+
+For logging, please keep the following in mind:
+
+- Utilize logrus.
+- Use `log.Debug` to highlight that you are **about** to do something.
+- Use `log.Info-Fatal` when the thing you were about to do has been completed, along with the result.
+
+```
+log.Debug("Adding 1 + 2")
+a := 1 + 2
+log.Info("1 + 2 successfully Added, outcome is: ", a)
+```
+
+## Boot
+
+The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start.
+
 # Contribution
 
 If you want to contribute please make sure you do the following:

cmd/capture.go

@@ -14,7 +14,7 @@ var captureCmd = &cobra.Command{
 	Short: "Capture the SignedBeaconBlocks and BeaconStates from the Beacon Chain",
 	Long: `Capture SignedBeaconBlocks and BeaconStates from the Beacon Chain.
 	These blocks and states will be captured in
-	Postgres. They require a lighthouse client to be connected. You can run this to
+	Postgres. They require a beacon client to be connected. You can run this to
 	capture blocks and states at head or historic blocks.`,
 }
 

cmd/head.go

@@ -5,12 +5,9 @@ Copyright © 2022 NAME HERE <EMAIL ADDRESS>
 package cmd
 
 import (
-	"fmt"
-
-	log "github.com/sirupsen/logrus"
-
 	"github.com/spf13/cobra"
 	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
 )
 
 // headCmd represents the head command
@@ -19,15 +16,14 @@ var headCmd = &cobra.Command{
 	Short: "Capture only the blocks and state at head.",
 	Long:  `Capture only the blocks and state at head.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("head called")
 		startHeadTracking()
 	},
 }
 
 func startHeadTracking() {
-	_, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver)
+	_, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort)
 	if err != nil {
-		log.Fatal("Unable to Start application with error: ", err)
+		loghelper.LogError(err).Error("Unable to Start application")
 	}
 }
 

cmd/root.go

@@ -6,31 +6,32 @@ package cmd
 
 import (
 	"fmt"
+	"io"
 	"os"
 
+	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 var (
-	cfgFile        string
+	cfgFile    string
-	dbUsername     string
+	dbUsername string
-	dbPassword     string
+	dbPassword string
-	dbName         string
+	dbName     string
-	dbAddress      string
+	dbAddress  string
-	dbDriver       string
+	dbDriver   string
-	dbPort         int
+	dbPort     int
-	lhAddress      string
+	bcAddress  string
-	lhPort         uint16
+	bcPort     int
-	logWithCommand log.Entry
 )
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   "ipld-ethcl-indexer",
-	Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.",
+	Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.",
-	Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain.
+	Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain.
 It can either do this will keeping track of head, or backfilling historic data.`,
 	PersistentPreRun: initFuncs,
 	// Uncomment the following line if your bare application
@@ -49,23 +50,10 @@ func Execute() {
 
 // Prerun for Cobra
 func initFuncs(cmd *cobra.Command, args []string) {
-	viper.BindEnv("log.file", "LOGRUS_FILE")
+	logFormat()
-	logfile := viper.GetString("log.file")
+	logFile()
-	if logfile != "" {
-		file, err := os.OpenFile(logfile,
-			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
-		if err == nil {
-			log.Infof("Directing output to %s", logfile)
-			log.SetOutput(file)
-		} else {
-			log.SetOutput(os.Stdout)
-			log.Info("Failed to log to file, using default stdout")
-		}
-	} else {
-		log.SetOutput(os.Stdout)
-	}
 	if err := logLevel(); err != nil {
-		log.Fatal("Could not set log level: ", err)
+		log.WithField("err", err).Error("Could not set log level")
 	}
 }
 
@@ -84,6 +72,35 @@ func logLevel() error {
 	return nil
 }
 
+// Create a log file
+func logFile() {
+	viper.BindEnv("log.file", "LOGRUS_FILE")
+	logfile := viper.GetString("log.file")
+	if logfile != "" {
+		file, err := os.OpenFile(logfile,
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err == nil {
+			log.Infof("Directing output to %s", logfile)
+			mw := io.MultiWriter(os.Stdout, file)
+			logrus.SetOutput(mw)
+		} else {
+			log.SetOutput(os.Stdout)
+			log.Info("Failed to log to file, using default stdout")
+		}
+	} else {
+		log.SetOutput(os.Stdout)
+	}
+}
+
+func logFormat() {
+	logFormat := viper.GetString("log.format")
+
+	if logFormat == "json" {
+		log.SetFormatter(&log.JSONFormatter{})
+
+	}
+}
+
 func init() {
 	cobra.OnInitialize(initConfig)
 
@@ -93,8 +110,9 @@ func init() {
 
 	// Optional Flags
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)")
-	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
+	rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
-	rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging")
+	rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging")
+	rootCmd.PersistentFlags().String("log.format", "json", "json or text")
 
 	// Required Flags
 
@@ -112,16 +130,17 @@ func init() {
 	rootCmd.MarkPersistentFlagRequired("db.name")
 	rootCmd.MarkPersistentFlagRequired("db.driver")
 
-	//// Lighthouse Specific
+	//// Beacon Client Specific
-	rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)")
+	rootCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)")
-	rootCmd.PersistentFlags().Uint16VarP(&lhPort, "lh.port", "r", 0, "Port to connect to lighthouse node (required if username is set)")
+	rootCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)")
-	rootCmd.MarkPersistentFlagRequired("lh.address")
+	rootCmd.MarkPersistentFlagRequired("bc.address")
-	rootCmd.MarkPersistentFlagRequired("lh.port")
+	rootCmd.MarkPersistentFlagRequired("bc.port")
 
 	// Bind Flags with Viper
 	// Optional
-	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level"))
-	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
+	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file"))
+	viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format"))
 
 	//// DB Flags
 	viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username"))
@@ -132,8 +151,8 @@ func init() {
 	viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver"))
 
 	// LH specific
-	viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address"))
+	viper.BindPFlag("bc.address", rootCmd.PersistentFlags().Lookup("bc.address"))
-	viper.BindPFlag("lh.port", rootCmd.PersistentFlags().Lookup("lh.port"))
+	viper.BindPFlag("bc.port", rootCmd.PersistentFlags().Lookup("bc.port"))
 
 	// Cobra also supports local flags, which will only run
 	// when this action is called directly.

internal/boot/boot.go

@@ -1,19 +1,88 @@
 package boot
 
 import (
+	"fmt"
+	"net/http"
+	"strconv"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
 )
 
-func setUpLightHouse() {
+var (
+	bcHealthEndpoint = "/eth/v1/node/health"
+)
+
+// This function will ensure that we can connect to the beacon client.
+// Keep in mind, the beacon client will allow you to connect to it but it might
+// Not allow you to make http requests. This is part of its built in logic, and you will have
+// to follow their provided guidelines. https://lighthouse-book.sigmaprime.io/api-bn.html#security
+func checkBeaconClient(bcAddress string, bcPort int) error {
+	log.Debug("Attempting to connect to the beacon client")
+	bcEndpoint := "http://" + bcAddress + ":" + strconv.Itoa(bcPort) + bcHealthEndpoint
+	resp, err := http.Get(bcEndpoint)
+	if err != nil {
+		loghelper.LogError(err).Error("Unable to get bc endpoint: ", bcEndpoint)
+		return err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		log.Error("We recieved a non 2xx status code when checking the health of the beacon node.")
+		log.Error("Health Endpoint Status Code: ", resp.StatusCode)
+		return fmt.Errorf("beacon Node Provided a non 2xx status code, code provided: %d", resp.StatusCode)
+	}
+
+	log.Info("We can successfully reach the beacon client.")
+	return nil
 
 }
 
-// This function will perform some boot operations.
+// A simple wrapper to create a DB object to use.
-// 1. Setup a logger
+func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) {
+	log.Debug("Resolving Driver Type")
+	DbDriver, err := postgres.ResolveDriverType(driverName)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"err":                  err,
+			"driver_name_provided": driverName,
+		}).Error("Can't resolve driver type")
+	}
+	log.Info("Using Driver:", DbDriver)
+
+	postgresConfig := postgres.Config{
+		Hostname:     dbHostname,
+		Port:         dbPort,
+		DatabaseName: dbName,
+		Username:     dbUsername,
+		Password:     dbPassword,
+		Driver:       DbDriver,
+	}
+	DB, err := postgres.NewPostgresDB(postgresConfig)
+
+	if err != nil {
+		loghelper.LogError(err).Error("Unable to connect to the DB")
+		return nil, err
+	}
+	return DB, err
+
+}
+
+// This function will perform some boot operations. If any steps fail, the application will fail to start.
+// Keep in mind that the DB connection can be lost later in the lifecycle of the application or
+// it might not be able to connect to the beacon client.
+//
+// 1. Make sure the Beacon client is up.
+//
 // 2. Connect to the database.
-// 3. Connect to to the lighthouse client.
+//
-func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) {
+func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) {
+	log.Debug("Checking beacon Client")
+	err := checkBeaconClient(bcAddress, bcPort)
+	if err != nil {
+		return nil, err
+	}
+
 	log.Debug("Setting up DB connection")
 	DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName)
 	if err != nil {

internal/boot/setup_database.go

@@ -1,29 +0,0 @@
-// This file will allow users to setup a new DB based on the user provided inputs.
-
-package boot
-
-import (
-	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
-)
-
-func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) {
-	log.Debug("Resolving Driver Type")
-	DbDriver, err := postgres.ResolveDriverType(driverName)
-	if err != nil {
-		log.Fatal("Can't Connect to DB")
-	}
-	log.Info("Using Driver:", DbDriver)
-
-	postgresConfig := postgres.Config{
-		Hostname:     dbHostname,
-		Port:         dbPort,
-		DatabaseName: dbName,
-		Username:     dbUsername,
-		Password:     dbPassword,
-		Driver:       DbDriver,
-	}
-	DB, err := postgres.NewPostgresDB(postgresConfig)
-	return DB, err
-
-}

pkg/database/sql/postgres/config.go

@@ -22,6 +22,7 @@ var DefaultConfig = Config{
 	DatabaseName: "vulcanize_testing",
 	Username:     "vdbm",
 	Password:     "password",
+	Driver:       "PGX",
 }
 
 // ResolveDriverType resolves a DriverType from a provided string

pkg/database/sql/postgres/database.go

@@ -38,7 +38,7 @@ func createDriver(c Config) (*pgxDriver, error) {
 		log.Info("Successfully created a driver for PGX")
 		return driver, nil
 	default:
-		log.Fatal("Couldnt find a driver to create for: ", c.Driver)
+		log.Error("Couldnt find a driver to create for: ", c.Driver)
 		return nil, fmt.Errorf("Can't find a driver to create")
 	}
 

pkg/database/sql/postgres/pgx_test.go

@@ -79,7 +79,9 @@ func TestPostgresPGX(t *testing.T) {
 	})
 
 	t.Run("throws error when can't connect to the database", func(t *testing.T) {
-		_, err := NewPostgresDB(Config{}, "PGX")
+		_, err := NewPostgresDB(Config{
+			Driver: "PGX",
+		})
 		if err == nil {
 			t.Fatal("Expected an error")
 		}
@@ -87,7 +89,7 @@ func TestPostgresPGX(t *testing.T) {
 		expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg)
 	})
 	t.Run("Connect to the database", func(t *testing.T) {
-		driver, err := NewPostgresDB(DefaultConfig, "pgx")
+		driver, err := NewPostgresDB(DefaultConfig)
 		defer driver.Close()
 
 		if err != nil {

pkg/loghelper/log_error.go

@@ -0,0 +1,13 @@
+// A simple function to help with logging errors.
+package loghelper
+
+import (
+	log "github.com/sirupsen/logrus"
+)
+
+// A simple helper function that will help wrap the error message.
+func LogError(err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err": err,
+	})
+}