README.md

@@ -1,6 +1,6 @@
 # ipld-ethcl-indexer
 
-This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum.
+This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node.
 
 # Running the Application
 
@@ -10,11 +10,36 @@ To run the application, utilize the following command, and update the values as
 go run main.go capture head --db.address localhost \
   --db.password password \
   --db.port 8077 \
-  --db.username username \
-  --lh.address localhost \
-  --lh.port 5052
+  --db.username vdbm \
+  --db.name vulcanize_testing \
+  --db.driver PGX \
+  --bc.address localhost \
+  --bc.port 5052 \
+  --log.level info
 ```
 
+# Development Patterns
+
+This section will cover some generic development patterns utilizes.
+
+## Logging
+
+For logging, please keep the following in mind:
+
+- Utilize logrus.
+- Use `log.Debug` to highlight that you are **about** to do something.
+- Use `log.Info-Fatal` when the thing you were about to do has been completed, along with the result.
+
+```
+log.Debug("Adding 1 + 2")
+a := 1 + 2
+log.Info("1 + 2 successfully Added, outcome is: ", a)
+```
+
+## Boot
+
+The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start.
+
 # Contribution
 
 If you want to contribute please make sure you do the following:

cmd/capture.go

@@ -14,7 +14,7 @@ var captureCmd = &cobra.Command{
 	Short: "Capture the SignedBeaconBlocks and BeaconStates from the Beacon Chain",
 	Long: `Capture SignedBeaconBlocks and BeaconStates from the Beacon Chain.
 	These blocks and states will be captured in
-	Postgres. They require a lighthouse client to be connected. You can run this to
+	Postgres. They require a beacon client to be connected. You can run this to
 	capture blocks and states at head or historic blocks.`,
 }
 

cmd/head.go

@@ -5,12 +5,9 @@ Copyright © 2022 NAME HERE <EMAIL ADDRESS>
 package cmd
 
 import (
-	"fmt"
-
-	log "github.com/sirupsen/logrus"
-
 	"github.com/spf13/cobra"
 	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
 )
 
 // headCmd represents the head command
@@ -19,15 +16,14 @@ var headCmd = &cobra.Command{
 	Short: "Capture only the blocks and state at head.",
 	Long:  `Capture only the blocks and state at head.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("head called")
 		startHeadTracking()
 	},
 }
 
 func startHeadTracking() {
-	_, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver)
+	_, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort)
 	if err != nil {
-		log.Fatal("Unable to Start application with error: ", err)
+		loghelper.LogError(err).Error("Unable to Start application")
 	}
 }
 

cmd/root.go

@@ -6,31 +6,32 @@ package cmd
 
 import (
 	"fmt"
+	"io"
 	"os"
 
+	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 var (
-	cfgFile        string
-	dbUsername     string
-	dbPassword     string
-	dbName         string
-	dbAddress      string
-	dbDriver       string
-	dbPort         int
-	lhAddress      string
-	lhPort         uint16
-	logWithCommand log.Entry
+	cfgFile    string
+	dbUsername string
+	dbPassword string
+	dbName     string
+	dbAddress  string
+	dbDriver   string
+	dbPort     int
+	bcAddress  string
+	bcPort     int
 )
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   "ipld-ethcl-indexer",
-	Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.",
-	Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain.
+	Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.",
+	Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain.
 It can either do this will keeping track of head, or backfilling historic data.`,
 	PersistentPreRun: initFuncs,
 	// Uncomment the following line if your bare application
@@ -49,23 +50,10 @@ func Execute() {
 
 // Prerun for Cobra
 func initFuncs(cmd *cobra.Command, args []string) {
-	viper.BindEnv("log.file", "LOGRUS_FILE")
-	logfile := viper.GetString("log.file")
-	if logfile != "" {
-		file, err := os.OpenFile(logfile,
-			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
-		if err == nil {
-			log.Infof("Directing output to %s", logfile)
-			log.SetOutput(file)
-		} else {
-			log.SetOutput(os.Stdout)
-			log.Info("Failed to log to file, using default stdout")
-		}
-	} else {
-		log.SetOutput(os.Stdout)
-	}
+	logFormat()
+	logFile()
 	if err := logLevel(); err != nil {
-		log.Fatal("Could not set log level: ", err)
+		log.WithField("err", err).Error("Could not set log level")
 	}
 }
 
@@ -84,6 +72,35 @@ func logLevel() error {
 	return nil
 }
 
+// Create a log file
+func logFile() {
+	viper.BindEnv("log.file", "LOGRUS_FILE")
+	logfile := viper.GetString("log.file")
+	if logfile != "" {
+		file, err := os.OpenFile(logfile,
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err == nil {
+			log.Infof("Directing output to %s", logfile)
+			mw := io.MultiWriter(os.Stdout, file)
+			logrus.SetOutput(mw)
+		} else {
+			log.SetOutput(os.Stdout)
+			log.Info("Failed to log to file, using default stdout")
+		}
+	} else {
+		log.SetOutput(os.Stdout)
+	}
+}
+
+func logFormat() {
+	logFormat := viper.GetString("log.format")
+
+	if logFormat == "json" {
+		log.SetFormatter(&log.JSONFormatter{})
+
+	}
+}
+
 func init() {
 	cobra.OnInitialize(initConfig)
 
@@ -93,8 +110,9 @@ func init() {
 
 	// Optional Flags
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)")
-	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
-	rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging")
+	rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
+	rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging")
+	rootCmd.PersistentFlags().String("log.format", "json", "json or text")
 
 	// Required Flags
 
@@ -112,16 +130,17 @@ func init() {
 	rootCmd.MarkPersistentFlagRequired("db.name")
 	rootCmd.MarkPersistentFlagRequired("db.driver")
 
-	//// Lighthouse Specific
-	rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)")
-	rootCmd.PersistentFlags().Uint16VarP(&lhPort, "lh.port", "r", 0, "Port to connect to lighthouse node (required if username is set)")
-	rootCmd.MarkPersistentFlagRequired("lh.address")
-	rootCmd.MarkPersistentFlagRequired("lh.port")
+	//// Beacon Client Specific
+	rootCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)")
+	rootCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)")
+	rootCmd.MarkPersistentFlagRequired("bc.address")
+	rootCmd.MarkPersistentFlagRequired("bc.port")
 
 	// Bind Flags with Viper
 	// Optional
-	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
-	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
+	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level"))
+	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file"))
+	viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format"))
 
 	//// DB Flags
 	viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username"))
@@ -132,8 +151,8 @@ func init() {
 	viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver"))
 
 	// LH specific
-	viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address"))
-	viper.BindPFlag("lh.port", rootCmd.PersistentFlags().Lookup("lh.port"))
+	viper.BindPFlag("bc.address", rootCmd.PersistentFlags().Lookup("bc.address"))
+	viper.BindPFlag("bc.port", rootCmd.PersistentFlags().Lookup("bc.port"))
 
 	// Cobra also supports local flags, which will only run
 	// when this action is called directly.

internal/boot/boot.go

@@ -1,19 +1,88 @@
 package boot
 
 import (
+	"fmt"
+	"net/http"
+	"strconv"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
 )
 
-func setUpLightHouse() {
+var (
+	bcHealthEndpoint = "/eth/v1/node/health"
+)
+
+// This function will ensure that we can connect to the beacon client.
+// Keep in mind, the beacon client will allow you to connect to it but it might
+// Not allow you to make http requests. This is part of its built in logic, and you will have
+// to follow their provided guidelines. https://lighthouse-book.sigmaprime.io/api-bn.html#security
+func checkBeaconClient(bcAddress string, bcPort int) error {
+	log.Debug("Attempting to connect to the beacon client")
+	bcEndpoint := "http://" + bcAddress + ":" + strconv.Itoa(bcPort) + bcHealthEndpoint
+	resp, err := http.Get(bcEndpoint)
+	if err != nil {
+		loghelper.LogError(err).Error("Unable to get bc endpoint: ", bcEndpoint)
+		return err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		log.Error("We recieved a non 2xx status code when checking the health of the beacon node.")
+		log.Error("Health Endpoint Status Code: ", resp.StatusCode)
+		return fmt.Errorf("beacon Node Provided a non 2xx status code, code provided: %d", resp.StatusCode)
+	}
+
+	log.Info("We can successfully reach the beacon client.")
+	return nil
 
 }
 
-// This function will perform some boot operations.
-// 1. Setup a logger
+// A simple wrapper to create a DB object to use.
+func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) {
+	log.Debug("Resolving Driver Type")
+	DbDriver, err := postgres.ResolveDriverType(driverName)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"err":                  err,
+			"driver_name_provided": driverName,
+		}).Error("Can't resolve driver type")
+	}
+	log.Info("Using Driver:", DbDriver)
+
+	postgresConfig := postgres.Config{
+		Hostname:     dbHostname,
+		Port:         dbPort,
+		DatabaseName: dbName,
+		Username:     dbUsername,
+		Password:     dbPassword,
+		Driver:       DbDriver,
+	}
+	DB, err := postgres.NewPostgresDB(postgresConfig)
+
+	if err != nil {
+		loghelper.LogError(err).Error("Unable to connect to the DB")
+		return nil, err
+	}
+	return DB, err
+
+}
+
+// This function will perform some boot operations. If any steps fail, the application will fail to start.
+// Keep in mind that the DB connection can be lost later in the lifecycle of the application or
+// it might not be able to connect to the beacon client.
+//
+// 1. Make sure the Beacon client is up.
+//
 // 2. Connect to the database.
-// 3. Connect to to the lighthouse client.
-func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) {
+//
+func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) {
+	log.Debug("Checking beacon Client")
+	err := checkBeaconClient(bcAddress, bcPort)
+	if err != nil {
+		return nil, err
+	}
+
 	log.Debug("Setting up DB connection")
 	DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName)
 	if err != nil {

internal/boot/setup_database.go

@@ -1,29 +0,0 @@
-// This file will allow users to setup a new DB based on the user provided inputs.
-
-package boot
-
-import (
-	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
-)
-
-func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) {
-	log.Debug("Resolving Driver Type")
-	DbDriver, err := postgres.ResolveDriverType(driverName)
-	if err != nil {
-		log.Fatal("Can't Connect to DB")
-	}
-	log.Info("Using Driver:", DbDriver)
-
-	postgresConfig := postgres.Config{
-		Hostname:     dbHostname,
-		Port:         dbPort,
-		DatabaseName: dbName,
-		Username:     dbUsername,
-		Password:     dbPassword,
-		Driver:       DbDriver,
-	}
-	DB, err := postgres.NewPostgresDB(postgresConfig)
-	return DB, err
-
-}

pkg/database/sql/postgres/config.go

@@ -22,6 +22,7 @@ var DefaultConfig = Config{
 	DatabaseName: "vulcanize_testing",
 	Username:     "vdbm",
 	Password:     "password",
+	Driver:       "PGX",
 }
 
 // ResolveDriverType resolves a DriverType from a provided string

pkg/database/sql/postgres/database.go

@@ -38,7 +38,7 @@ func createDriver(c Config) (*pgxDriver, error) {
 		log.Info("Successfully created a driver for PGX")
 		return driver, nil
 	default:
-		log.Fatal("Couldnt find a driver to create for: ", c.Driver)
+		log.Error("Couldnt find a driver to create for: ", c.Driver)
 		return nil, fmt.Errorf("Can't find a driver to create")
 	}
 

pkg/database/sql/postgres/pgx_test.go

@@ -79,7 +79,9 @@ func TestPostgresPGX(t *testing.T) {
 	})
 
 	t.Run("throws error when can't connect to the database", func(t *testing.T) {
-		_, err := NewPostgresDB(Config{}, "PGX")
+		_, err := NewPostgresDB(Config{
+			Driver: "PGX",
+		})
 		if err == nil {
 			t.Fatal("Expected an error")
 		}
@@ -87,7 +89,7 @@ func TestPostgresPGX(t *testing.T) {
 		expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg)
 	})
 	t.Run("Connect to the database", func(t *testing.T) {
-		driver, err := NewPostgresDB(DefaultConfig, "pgx")
+		driver, err := NewPostgresDB(DefaultConfig)
 		defer driver.Close()
 
 		if err != nil {

pkg/loghelper/log_error.go

@@ -0,0 +1,13 @@
+// A simple function to help with logging errors.
+package loghelper
+
+import (
+	log "github.com/sirupsen/logrus"
+)
+
+// A simple helper function that will help wrap the error message.
+func LogError(err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err": err,
+	})
+}