diff --git a/README.md b/README.md index 88998e9..1025ccc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ipld-ethcl-indexer -This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. +This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node. # Running the Application @@ -10,11 +10,36 @@ To run the application, utilize the following command, and update the values as go run main.go capture head --db.address localhost \ --db.password password \ --db.port 8077 \ - --db.username username \ - --lh.address localhost \ - --lh.port 5052 + --db.username vdbm \ + --db.name vulcanize_testing \ + --db.driver PGX \ + --bc.address localhost \ + --bc.port 5052 \ + --log.level info ``` +# Development Patterns + +This section will cover some generic development patterns utilizes. + +## Logging + +For logging, please keep the following in mind: + +- Utilize logrus. +- Use `log.Debug` to highlight that you are **about** to do something. +- Use `log.Info-Fatal` when the thing you were about to do has been completed, along with the result. + +``` +log.Debug("Adding 1 + 2") +a := 1 + 2 +log.Info("1 + 2 successfully Added, outcome is: ", a) +``` + +## Boot + +The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start. + # Contribution If you want to contribute please make sure you do the following: diff --git a/cmd/capture.go b/cmd/capture.go index 52b5315..6bda50a 100644 --- a/cmd/capture.go +++ b/cmd/capture.go @@ -14,7 +14,7 @@ var captureCmd = &cobra.Command{ Short: "Capture the SignedBeaconBlocks and BeaconStates from the Beacon Chain", Long: `Capture SignedBeaconBlocks and BeaconStates from the Beacon Chain. These blocks and states will be captured in - Postgres. They require a lighthouse client to be connected. You can run this to + Postgres. They require a beacon client to be connected. You can run this to capture blocks and states at head or historic blocks.`, } diff --git a/cmd/head.go b/cmd/head.go index 2d297a3..86e4f9f 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -5,12 +5,9 @@ Copyright © 2022 NAME HERE package cmd import ( - "fmt" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) // headCmd represents the head command @@ -19,15 +16,14 @@ var headCmd = &cobra.Command{ Short: "Capture only the blocks and state at head.", Long: `Capture only the blocks and state at head.`, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("head called") startHeadTracking() }, } func startHeadTracking() { - _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver) + _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) if err != nil { - log.Fatal("Unable to Start application with error: ", err) + loghelper.LogError(err).Error("Unable to Start application") } } diff --git a/cmd/root.go b/cmd/root.go index 05b9682..b06c224 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,31 +6,32 @@ package cmd import ( "fmt" + "io" "os" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - dbUsername string - dbPassword string - dbName string - dbAddress string - dbDriver string - dbPort int - lhAddress string - lhPort uint16 - logWithCommand log.Entry + cfgFile string + dbUsername string + dbPassword string + dbName string + dbAddress string + dbDriver string + dbPort int + bcAddress string + bcPort int ) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "ipld-ethcl-indexer", - Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.", - Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain. + Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.", + Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain. It can either do this will keeping track of head, or backfilling historic data.`, PersistentPreRun: initFuncs, // Uncomment the following line if your bare application @@ -49,23 +50,10 @@ func Execute() { // Prerun for Cobra func initFuncs(cmd *cobra.Command, args []string) { - viper.BindEnv("log.file", "LOGRUS_FILE") - logfile := viper.GetString("log.file") - if logfile != "" { - file, err := os.OpenFile(logfile, - os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err == nil { - log.Infof("Directing output to %s", logfile) - log.SetOutput(file) - } else { - log.SetOutput(os.Stdout) - log.Info("Failed to log to file, using default stdout") - } - } else { - log.SetOutput(os.Stdout) - } + logFormat() + logFile() if err := logLevel(); err != nil { - log.Fatal("Could not set log level: ", err) + log.WithField("err", err).Error("Could not set log level") } } @@ -84,6 +72,35 @@ func logLevel() error { return nil } +// Create a log file +func logFile() { + viper.BindEnv("log.file", "LOGRUS_FILE") + logfile := viper.GetString("log.file") + if logfile != "" { + file, err := os.OpenFile(logfile, + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + log.Infof("Directing output to %s", logfile) + mw := io.MultiWriter(os.Stdout, file) + logrus.SetOutput(mw) + } else { + log.SetOutput(os.Stdout) + log.Info("Failed to log to file, using default stdout") + } + } else { + log.SetOutput(os.Stdout) + } +} + +func logFormat() { + logFormat := viper.GetString("log.format") + + if logFormat == "json" { + log.SetFormatter(&log.JSONFormatter{}) + + } +} + func init() { cobra.OnInitialize(initConfig) @@ -93,8 +110,9 @@ func init() { // Optional Flags rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") - rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") - rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") + rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().String("log.format", "json", "json or text") // Required Flags @@ -112,16 +130,17 @@ func init() { rootCmd.MarkPersistentFlagRequired("db.name") rootCmd.MarkPersistentFlagRequired("db.driver") - //// Lighthouse Specific - rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)") - rootCmd.PersistentFlags().Uint16VarP(&lhPort, "lh.port", "r", 0, "Port to connect to lighthouse node (required if username is set)") - rootCmd.MarkPersistentFlagRequired("lh.address") - rootCmd.MarkPersistentFlagRequired("lh.port") + //// Beacon Client Specific + rootCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)") + rootCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)") + rootCmd.MarkPersistentFlagRequired("bc.address") + rootCmd.MarkPersistentFlagRequired("bc.port") // Bind Flags with Viper // Optional - viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) - viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file")) + viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level")) + viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file")) + viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format")) //// DB Flags viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username")) @@ -132,8 +151,8 @@ func init() { viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver")) // LH specific - viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address")) - viper.BindPFlag("lh.port", rootCmd.PersistentFlags().Lookup("lh.port")) + viper.BindPFlag("bc.address", rootCmd.PersistentFlags().Lookup("bc.address")) + viper.BindPFlag("bc.port", rootCmd.PersistentFlags().Lookup("bc.port")) // Cobra also supports local flags, which will only run // when this action is called directly. diff --git a/internal/boot/boot.go b/internal/boot/boot.go index e251a5b..13420ef 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -1,19 +1,88 @@ package boot import ( + "fmt" + "net/http" + "strconv" + log "github.com/sirupsen/logrus" "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) -func setUpLightHouse() { +var ( + bcHealthEndpoint = "/eth/v1/node/health" +) + +// This function will ensure that we can connect to the beacon client. +// Keep in mind, the beacon client will allow you to connect to it but it might +// Not allow you to make http requests. This is part of its built in logic, and you will have +// to follow their provided guidelines. https://lighthouse-book.sigmaprime.io/api-bn.html#security +func checkBeaconClient(bcAddress string, bcPort int) error { + log.Debug("Attempting to connect to the beacon client") + bcEndpoint := "http://" + bcAddress + ":" + strconv.Itoa(bcPort) + bcHealthEndpoint + resp, err := http.Get(bcEndpoint) + if err != nil { + loghelper.LogError(err).Error("Unable to get bc endpoint: ", bcEndpoint) + return err + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + log.Error("We recieved a non 2xx status code when checking the health of the beacon node.") + log.Error("Health Endpoint Status Code: ", resp.StatusCode) + return fmt.Errorf("beacon Node Provided a non 2xx status code, code provided: %d", resp.StatusCode) + } + + log.Info("We can successfully reach the beacon client.") + return nil } -// This function will perform some boot operations. -// 1. Setup a logger +// A simple wrapper to create a DB object to use. +func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { + log.Debug("Resolving Driver Type") + DbDriver, err := postgres.ResolveDriverType(driverName) + if err != nil { + log.WithFields(log.Fields{ + "err": err, + "driver_name_provided": driverName, + }).Error("Can't resolve driver type") + } + log.Info("Using Driver:", DbDriver) + + postgresConfig := postgres.Config{ + Hostname: dbHostname, + Port: dbPort, + DatabaseName: dbName, + Username: dbUsername, + Password: dbPassword, + Driver: DbDriver, + } + DB, err := postgres.NewPostgresDB(postgresConfig) + + if err != nil { + loghelper.LogError(err).Error("Unable to connect to the DB") + return nil, err + } + return DB, err + +} + +// This function will perform some boot operations. If any steps fail, the application will fail to start. +// Keep in mind that the DB connection can be lost later in the lifecycle of the application or +// it might not be able to connect to the beacon client. +// +// 1. Make sure the Beacon client is up. +// // 2. Connect to the database. -// 3. Connect to to the lighthouse client. -func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { +// +func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { + log.Debug("Checking beacon Client") + err := checkBeaconClient(bcAddress, bcPort) + if err != nil { + return nil, err + } + log.Debug("Setting up DB connection") DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) if err != nil { diff --git a/internal/boot/startup.md b/internal/boot/boot.md similarity index 100% rename from internal/boot/startup.md rename to internal/boot/boot.md diff --git a/internal/boot/setup_database.go b/internal/boot/setup_database.go deleted file mode 100644 index a1efaf0..0000000 --- a/internal/boot/setup_database.go +++ /dev/null @@ -1,29 +0,0 @@ -// This file will allow users to setup a new DB based on the user provided inputs. - -package boot - -import ( - log "github.com/sirupsen/logrus" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" -) - -func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { - log.Debug("Resolving Driver Type") - DbDriver, err := postgres.ResolveDriverType(driverName) - if err != nil { - log.Fatal("Can't Connect to DB") - } - log.Info("Using Driver:", DbDriver) - - postgresConfig := postgres.Config{ - Hostname: dbHostname, - Port: dbPort, - DatabaseName: dbName, - Username: dbUsername, - Password: dbPassword, - Driver: DbDriver, - } - DB, err := postgres.NewPostgresDB(postgresConfig) - return DB, err - -} diff --git a/pkg/database/sql/postgres/config.go b/pkg/database/sql/postgres/config.go index a518bed..efe5eb9 100644 --- a/pkg/database/sql/postgres/config.go +++ b/pkg/database/sql/postgres/config.go @@ -22,6 +22,7 @@ var DefaultConfig = Config{ DatabaseName: "vulcanize_testing", Username: "vdbm", Password: "password", + Driver: "PGX", } // ResolveDriverType resolves a DriverType from a provided string diff --git a/pkg/database/sql/postgres/database.go b/pkg/database/sql/postgres/database.go index 9d44823..d79821a 100644 --- a/pkg/database/sql/postgres/database.go +++ b/pkg/database/sql/postgres/database.go @@ -38,7 +38,7 @@ func createDriver(c Config) (*pgxDriver, error) { log.Info("Successfully created a driver for PGX") return driver, nil default: - log.Fatal("Couldnt find a driver to create for: ", c.Driver) + log.Error("Couldnt find a driver to create for: ", c.Driver) return nil, fmt.Errorf("Can't find a driver to create") } diff --git a/pkg/database/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go index ab75a7e..b66deeb 100644 --- a/pkg/database/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -79,7 +79,9 @@ func TestPostgresPGX(t *testing.T) { }) t.Run("throws error when can't connect to the database", func(t *testing.T) { - _, err := NewPostgresDB(Config{}, "PGX") + _, err := NewPostgresDB(Config{ + Driver: "PGX", + }) if err == nil { t.Fatal("Expected an error") } @@ -87,7 +89,7 @@ func TestPostgresPGX(t *testing.T) { expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) }) t.Run("Connect to the database", func(t *testing.T) { - driver, err := NewPostgresDB(DefaultConfig, "pgx") + driver, err := NewPostgresDB(DefaultConfig) defer driver.Close() if err != nil { diff --git a/pkg/loghelper/log_error.go b/pkg/loghelper/log_error.go new file mode 100644 index 0000000..41d0149 --- /dev/null +++ b/pkg/loghelper/log_error.go @@ -0,0 +1,13 @@ +// A simple function to help with logging errors. +package loghelper + +import ( + log "github.com/sirupsen/logrus" +) + +// A simple helper function that will help wrap the error message. +func LogError(err error) *log.Entry { + return log.WithFields(log.Fields{ + "err": err, + }) +}