From d7ad4108a7b28624f00d141debbead0dbe20bab9 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 18:12:44 -0400 Subject: [PATCH] Code clean up + Beacon Chain Connection This concludes all the code needed to connect to the DB and beacon node. We will no longer reference the lighthouse client because this application should work interchangeably with any beacon node. I have also standardized logging. --- README.md | 33 +++++++-- cmd/capture.go | 2 +- cmd/head.go | 10 +-- cmd/root.go | 97 ++++++++++++++++----------- internal/boot/boot.go | 79 ++++++++++++++++++++-- internal/boot/{startup.md => boot.md} | 0 internal/boot/setup_database.go | 29 -------- pkg/database/sql/postgres/config.go | 1 + pkg/database/sql/postgres/database.go | 2 +- pkg/database/sql/postgres/pgx_test.go | 6 +- pkg/loghelper/log_error.go | 13 ++++ 11 files changed, 184 insertions(+), 88 deletions(-) rename internal/boot/{startup.md => boot.md} (100%) delete mode 100644 internal/boot/setup_database.go create mode 100644 pkg/loghelper/log_error.go diff --git a/README.md b/README.md index 88998e9..1025ccc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ipld-ethcl-indexer -This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. +This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node. # Running the Application @@ -10,11 +10,36 @@ To run the application, utilize the following command, and update the values as go run main.go capture head --db.address localhost \ --db.password password \ --db.port 8077 \ - --db.username username \ - --lh.address localhost \ - --lh.port 5052 + --db.username vdbm \ + --db.name vulcanize_testing \ + --db.driver PGX \ + --bc.address localhost \ + --bc.port 5052 \ + --log.level info ``` +# Development Patterns + +This section will cover some generic development patterns utilizes. + +## Logging + +For logging, please keep the following in mind: + +- Utilize logrus. +- Use `log.Debug` to highlight that you are **about** to do something. +- Use `log.Info-Fatal` when the thing you were about to do has been completed, along with the result. + +``` +log.Debug("Adding 1 + 2") +a := 1 + 2 +log.Info("1 + 2 successfully Added, outcome is: ", a) +``` + +## Boot + +The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start. + # Contribution If you want to contribute please make sure you do the following: diff --git a/cmd/capture.go b/cmd/capture.go index 52b5315..6bda50a 100644 --- a/cmd/capture.go +++ b/cmd/capture.go @@ -14,7 +14,7 @@ var captureCmd = &cobra.Command{ Short: "Capture the SignedBeaconBlocks and BeaconStates from the Beacon Chain", Long: `Capture SignedBeaconBlocks and BeaconStates from the Beacon Chain. These blocks and states will be captured in - Postgres. They require a lighthouse client to be connected. You can run this to + Postgres. They require a beacon client to be connected. You can run this to capture blocks and states at head or historic blocks.`, } diff --git a/cmd/head.go b/cmd/head.go index 2d297a3..86e4f9f 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -5,12 +5,9 @@ Copyright © 2022 NAME HERE package cmd import ( - "fmt" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) // headCmd represents the head command @@ -19,15 +16,14 @@ var headCmd = &cobra.Command{ Short: "Capture only the blocks and state at head.", Long: `Capture only the blocks and state at head.`, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("head called") startHeadTracking() }, } func startHeadTracking() { - _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver) + _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) if err != nil { - log.Fatal("Unable to Start application with error: ", err) + loghelper.LogError(err).Error("Unable to Start application") } } diff --git a/cmd/root.go b/cmd/root.go index 05b9682..b06c224 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,31 +6,32 @@ package cmd import ( "fmt" + "io" "os" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - dbUsername string - dbPassword string - dbName string - dbAddress string - dbDriver string - dbPort int - lhAddress string - lhPort uint16 - logWithCommand log.Entry + cfgFile string + dbUsername string + dbPassword string + dbName string + dbAddress string + dbDriver string + dbPort int + bcAddress string + bcPort int ) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "ipld-ethcl-indexer", - Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.", - Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain. + Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.", + Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain. It can either do this will keeping track of head, or backfilling historic data.`, PersistentPreRun: initFuncs, // Uncomment the following line if your bare application @@ -49,23 +50,10 @@ func Execute() { // Prerun for Cobra func initFuncs(cmd *cobra.Command, args []string) { - viper.BindEnv("log.file", "LOGRUS_FILE") - logfile := viper.GetString("log.file") - if logfile != "" { - file, err := os.OpenFile(logfile, - os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err == nil { - log.Infof("Directing output to %s", logfile) - log.SetOutput(file) - } else { - log.SetOutput(os.Stdout) - log.Info("Failed to log to file, using default stdout") - } - } else { - log.SetOutput(os.Stdout) - } + logFormat() + logFile() if err := logLevel(); err != nil { - log.Fatal("Could not set log level: ", err) + log.WithField("err", err).Error("Could not set log level") } } @@ -84,6 +72,35 @@ func logLevel() error { return nil } +// Create a log file +func logFile() { + viper.BindEnv("log.file", "LOGRUS_FILE") + logfile := viper.GetString("log.file") + if logfile != "" { + file, err := os.OpenFile(logfile, + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + log.Infof("Directing output to %s", logfile) + mw := io.MultiWriter(os.Stdout, file) + logrus.SetOutput(mw) + } else { + log.SetOutput(os.Stdout) + log.Info("Failed to log to file, using default stdout") + } + } else { + log.SetOutput(os.Stdout) + } +} + +func logFormat() { + logFormat := viper.GetString("log.format") + + if logFormat == "json" { + log.SetFormatter(&log.JSONFormatter{}) + + } +} + func init() { cobra.OnInitialize(initConfig) @@ -93,8 +110,9 @@ func init() { // Optional Flags rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") - rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") - rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") + rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().String("log.format", "json", "json or text") // Required Flags @@ -112,16 +130,17 @@ func init() { rootCmd.MarkPersistentFlagRequired("db.name") rootCmd.MarkPersistentFlagRequired("db.driver") - //// Lighthouse Specific - rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)") - rootCmd.PersistentFlags().Uint16VarP(&lhPort, "lh.port", "r", 0, "Port to connect to lighthouse node (required if username is set)") - rootCmd.MarkPersistentFlagRequired("lh.address") - rootCmd.MarkPersistentFlagRequired("lh.port") + //// Beacon Client Specific + rootCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)") + rootCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)") + rootCmd.MarkPersistentFlagRequired("bc.address") + rootCmd.MarkPersistentFlagRequired("bc.port") // Bind Flags with Viper // Optional - viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) - viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file")) + viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level")) + viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file")) + viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format")) //// DB Flags viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username")) @@ -132,8 +151,8 @@ func init() { viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver")) // LH specific - viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address")) - viper.BindPFlag("lh.port", rootCmd.PersistentFlags().Lookup("lh.port")) + viper.BindPFlag("bc.address", rootCmd.PersistentFlags().Lookup("bc.address")) + viper.BindPFlag("bc.port", rootCmd.PersistentFlags().Lookup("bc.port")) // Cobra also supports local flags, which will only run // when this action is called directly. diff --git a/internal/boot/boot.go b/internal/boot/boot.go index e251a5b..13420ef 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -1,19 +1,88 @@ package boot import ( + "fmt" + "net/http" + "strconv" + log "github.com/sirupsen/logrus" "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) -func setUpLightHouse() { +var ( + bcHealthEndpoint = "/eth/v1/node/health" +) + +// This function will ensure that we can connect to the beacon client. +// Keep in mind, the beacon client will allow you to connect to it but it might +// Not allow you to make http requests. This is part of its built in logic, and you will have +// to follow their provided guidelines. https://lighthouse-book.sigmaprime.io/api-bn.html#security +func checkBeaconClient(bcAddress string, bcPort int) error { + log.Debug("Attempting to connect to the beacon client") + bcEndpoint := "http://" + bcAddress + ":" + strconv.Itoa(bcPort) + bcHealthEndpoint + resp, err := http.Get(bcEndpoint) + if err != nil { + loghelper.LogError(err).Error("Unable to get bc endpoint: ", bcEndpoint) + return err + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + log.Error("We recieved a non 2xx status code when checking the health of the beacon node.") + log.Error("Health Endpoint Status Code: ", resp.StatusCode) + return fmt.Errorf("beacon Node Provided a non 2xx status code, code provided: %d", resp.StatusCode) + } + + log.Info("We can successfully reach the beacon client.") + return nil } -// This function will perform some boot operations. -// 1. Setup a logger +// A simple wrapper to create a DB object to use. +func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { + log.Debug("Resolving Driver Type") + DbDriver, err := postgres.ResolveDriverType(driverName) + if err != nil { + log.WithFields(log.Fields{ + "err": err, + "driver_name_provided": driverName, + }).Error("Can't resolve driver type") + } + log.Info("Using Driver:", DbDriver) + + postgresConfig := postgres.Config{ + Hostname: dbHostname, + Port: dbPort, + DatabaseName: dbName, + Username: dbUsername, + Password: dbPassword, + Driver: DbDriver, + } + DB, err := postgres.NewPostgresDB(postgresConfig) + + if err != nil { + loghelper.LogError(err).Error("Unable to connect to the DB") + return nil, err + } + return DB, err + +} + +// This function will perform some boot operations. If any steps fail, the application will fail to start. +// Keep in mind that the DB connection can be lost later in the lifecycle of the application or +// it might not be able to connect to the beacon client. +// +// 1. Make sure the Beacon client is up. +// // 2. Connect to the database. -// 3. Connect to to the lighthouse client. -func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { +// +func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { + log.Debug("Checking beacon Client") + err := checkBeaconClient(bcAddress, bcPort) + if err != nil { + return nil, err + } + log.Debug("Setting up DB connection") DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) if err != nil { diff --git a/internal/boot/startup.md b/internal/boot/boot.md similarity index 100% rename from internal/boot/startup.md rename to internal/boot/boot.md diff --git a/internal/boot/setup_database.go b/internal/boot/setup_database.go deleted file mode 100644 index a1efaf0..0000000 --- a/internal/boot/setup_database.go +++ /dev/null @@ -1,29 +0,0 @@ -// This file will allow users to setup a new DB based on the user provided inputs. - -package boot - -import ( - log "github.com/sirupsen/logrus" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" -) - -func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { - log.Debug("Resolving Driver Type") - DbDriver, err := postgres.ResolveDriverType(driverName) - if err != nil { - log.Fatal("Can't Connect to DB") - } - log.Info("Using Driver:", DbDriver) - - postgresConfig := postgres.Config{ - Hostname: dbHostname, - Port: dbPort, - DatabaseName: dbName, - Username: dbUsername, - Password: dbPassword, - Driver: DbDriver, - } - DB, err := postgres.NewPostgresDB(postgresConfig) - return DB, err - -} diff --git a/pkg/database/sql/postgres/config.go b/pkg/database/sql/postgres/config.go index a518bed..efe5eb9 100644 --- a/pkg/database/sql/postgres/config.go +++ b/pkg/database/sql/postgres/config.go @@ -22,6 +22,7 @@ var DefaultConfig = Config{ DatabaseName: "vulcanize_testing", Username: "vdbm", Password: "password", + Driver: "PGX", } // ResolveDriverType resolves a DriverType from a provided string diff --git a/pkg/database/sql/postgres/database.go b/pkg/database/sql/postgres/database.go index 9d44823..d79821a 100644 --- a/pkg/database/sql/postgres/database.go +++ b/pkg/database/sql/postgres/database.go @@ -38,7 +38,7 @@ func createDriver(c Config) (*pgxDriver, error) { log.Info("Successfully created a driver for PGX") return driver, nil default: - log.Fatal("Couldnt find a driver to create for: ", c.Driver) + log.Error("Couldnt find a driver to create for: ", c.Driver) return nil, fmt.Errorf("Can't find a driver to create") } diff --git a/pkg/database/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go index ab75a7e..b66deeb 100644 --- a/pkg/database/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -79,7 +79,9 @@ func TestPostgresPGX(t *testing.T) { }) t.Run("throws error when can't connect to the database", func(t *testing.T) { - _, err := NewPostgresDB(Config{}, "PGX") + _, err := NewPostgresDB(Config{ + Driver: "PGX", + }) if err == nil { t.Fatal("Expected an error") } @@ -87,7 +89,7 @@ func TestPostgresPGX(t *testing.T) { expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) }) t.Run("Connect to the database", func(t *testing.T) { - driver, err := NewPostgresDB(DefaultConfig, "pgx") + driver, err := NewPostgresDB(DefaultConfig) defer driver.Close() if err != nil { diff --git a/pkg/loghelper/log_error.go b/pkg/loghelper/log_error.go new file mode 100644 index 0000000..41d0149 --- /dev/null +++ b/pkg/loghelper/log_error.go @@ -0,0 +1,13 @@ +// A simple function to help with logging errors. +package loghelper + +import ( + log "github.com/sirupsen/logrus" +) + +// A simple helper function that will help wrap the error message. +func LogError(err error) *log.Entry { + return log.WithFields(log.Fields{ + "err": err, + }) +}