diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index f642407..bdb347e 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,7 +7,7 @@ assignees: "" --- - [Request](#request) - - [Potentialy Solution](#potentialy-solution) + - [Potential Solution](#potential-solution) - [Alternative Solutions](#alternative-solutions) - [Additional Context](#additional-context) @@ -17,7 +17,7 @@ assignees: "" **Explain what you want and why. If this feature is related to a problem please highlight it here.** -## Potentialy Solution +## Potential Solution **Provide any details for a potential solution.** diff --git a/.github/workflows/on-pr-automated.yaml b/.github/workflows/on-pr-automated.yaml deleted file mode 100644 index 2f196de..0000000 --- a/.github/workflows/on-pr-automated.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: Test Application - -on: - pull_request: - -jobs: - build: - name: Run docker build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Run docker build - run: make docker-build - test: - name: Run unit tests - runs-on: ubuntu-latest - env: - foundry-test-ref: feature/build-stack - ipld-eth-db-ref: main - GOPATH: /tmp/go - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - uses: actions/checkout@v2 - with: - path: "./ipld-ethcl-indexer" - - - uses: actions/checkout@v3 - with: - ref: ${{ env.foundry-test-ref }} - path: "./foundry-test/" - repository: vulcanize/foundry-test - fetch-depth: 0 - - - uses: actions/checkout@v3 - with: - ref: ${{ env.ipld-eth-db-ref }} - repository: vulcanize/ipld-eth-db - path: "./ipld-eth-db/" - fetch-depth: 0 - - - name: Create config file - run: | - echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh - echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh - cat ./config.sh - - - name: Run docker compose - run: | - docker-compose \ - -f "$GITHUB_WORKSPACE/foundry-test/docker/local/docker-compose-db.yml" \ - -f "$GITHUB_WORKSPACE/foundry-test/docker/latest/docker-compose-lighthouse.yml" \ - --env-file ./config.sh \ - up -d --build - - - uses: actions/setup-go@v3 - with: - go-version: ">=1.17.0" - check-latest: true - - - name: Install packages - run: | - go install github.com/onsi/ginkgo/v2/ginkgo@latest - which ginkgo - - - name: Run the tests using Make - run: | - cd ipld-ethcl-indexer - make test diff --git a/.github/workflows/on-pr-manual.yml b/.github/workflows/on-pr-manual.yml deleted file mode 100644 index 26807cb..0000000 --- a/.github/workflows/on-pr-manual.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Test Application Manually - -on: - workflow_dispatch: - inputs: - foundry-test-ref: - description: "The branch, commit or sha from foundry-test to checkout" - required: false - default: "main" - ipld-eth-db-ref: - description: "The branch, commit or sha from ipld-eth-db to checkout" - required: false - default: "main" - -jobs: - build: - name: Run docker build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Run docker build - run: make docker-build - test: - name: Run unit tests - runs-on: ubuntu-latest - env: - GOPATH: /tmp/go - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - uses: actions/checkout@v2 - with: - path: "./ipld-ethcl-indexer" - - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.inputs.foundry-test-ref }} - path: "./foundry-test/" - repository: vulcanize/foundry-test - fetch-depth: 0 - - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.inputs.ipld-eth-db-ref }} - repository: vulcanize/ipld-eth-db - path: "./ipld-eth-db/" - fetch-depth: 0 - - - name: Create config file - run: | - echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh - echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh - cat ./config.sh - - - name: Run docker compose - run: | - docker-compose \ - -f "$GITHUB_WORKSPACE/foundry-test/docker/local/docker-compose-db.yml" \ - -f "$GITHUB_WORKSPACE/foundry-test/docker/latest/docker-compose-lighthouse.yml" \ - --env-file ./config.sh \ - up -d --build - - - uses: actions/setup-go@v3 - with: - go-version: ">=1.17.0" - check-latest: true - - - name: Install packages - run: | - go install github.com/onsi/ginkgo/v2/ginkgo@latest - which ginkgo - - - name: Run the tests using Make - run: | - cd ipld-ethcl-indexer - make test diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml new file mode 100644 index 0000000..a033463 --- /dev/null +++ b/.github/workflows/on-pr.yml @@ -0,0 +1,150 @@ +name: Test Application On PR + +on: + workflow_dispatch: + inputs: + foundry-test-ref: + description: "The branch, commit or sha from foundry-test to checkout" + required: false + default: "main" + ipld-eth-db-ref: + description: "The branch, commit or sha from ipld-eth-db to checkout" + required: false + default: "main" + pull_request: + paths: + - "!**.md" + - ".gitignore" + - "!LICENSE" + - "!.github/workflows/**" + - ".github/workflows/on-pr.yml" + +jobs: + build: + name: Run Docker Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker build + run: make docker-build + + unit-test: + name: Run Unit Tests + runs-on: ubuntu-latest + ## IF you want to update the default branch for `pull_request runs, do it after the ||` + env: + foundry-test-ref: ${{ github.event.inputs.foundry-test-ref || 'feature/build-stack'}} + ipld-eth-db-ref: ${{ github.event.inputs.ipld-eth-db-ref || 'main' }} + GOPATH: /tmp/go + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.foundry-test-ref }} + path: "./foundry-test/" + repository: vulcanize/foundry-test + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-eth-db-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + cat ./config.sh + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + check-latest: true + + - name: Install packages + run: | + go install github.com/onsi/ginkgo/v2/ginkgo@latest + which ginkgo + + - name: Run the tests using Make + run: | + cd ipld-ethcl-indexer + make unit-test-ci + + integration-test: + name: Run Integration Tests + runs-on: ubuntu-latest + env: + foundry-test-ref: feature/build-stack + ipld-eth-db-ref: main + GOPATH: /tmp/go + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.foundry-test-ref }} + path: "./foundry-test/" + repository: vulcanize/foundry-test + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-eth-db-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + cat ./config.sh + + - name: Run docker compose + run: | + docker-compose \ + -f "$GITHUB_WORKSPACE/foundry-test/docker/local/docker-compose-db.yml" \ + -f "$GITHUB_WORKSPACE/foundry-test/docker/latest/docker-compose-lighthouse.yml" \ + --env-file ./config.sh \ + up -d --build + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + check-latest: true + + - name: Install packages + run: | + go install github.com/onsi/ginkgo/v2/ginkgo@latest + which ginkgo + + - name: Run the tests using Make + run: | + cd ipld-ethcl-indexer + make integration-test-ci + + golangci: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + args: --disable errcheck diff --git a/Makefile b/Makefile index 36f985f..7b97424 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,28 @@ test: go fmt ./... $(GINKGO) -r -#.PHONY: integrationtest -#integrationtest: | $(GINKGO) $(GOOSE) -# go vet ./... -# go fmt ./... -# $(GINKGO) -r test/ -v +.PHONY: integration-test-ci +integration-test-ci: + go vet ./... + go fmt ./... + $(GINKGO) -r --label-filter integration \ + --procs=4 --compilers=4 \ + --randomize-all --randomize-suites \ + --fail-on-pending --keep-going \ + --cover --coverprofile=cover.profile \ + --race --trace --json-report=report.json + +.PHONY: unit-test-ci +unit-test-ci: + go vet ./... + go fmt ./... + $(GINKGO) -r --label-filter unit \ + --procs=4 --compilers=4 \ + --randomize-all --randomize-suites \ + --fail-on-pending --keep-going \ + --cover --coverprofile=cover.profile \ + --race --trace --json-report=report.json + .PHONY: build build: diff --git a/README.md b/README.md index d3b7892..f24ade2 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,9 @@ This project utilizes `ginkgo` for testing. A few notes on testing: - All tests within this code base will test **public methods only**. - All test packages are named `{base_package}_test`. This ensures we only test the public methods. - If there is a need to test a private method, please include why in the testing file. +- Unit tests must contain the `Label("unit")`. +- Unit tests should not rely on any running service. If a running service is needed. Utilize an integration test. +- Integration tests must contain the `Label("integration")`. # Contribution diff --git a/cmd/capture.go b/cmd/capture.go index 6bda50a..54d3fb9 100644 --- a/cmd/capture.go +++ b/cmd/capture.go @@ -5,7 +5,21 @@ Copyright © 2022 NAME HERE package cmd import ( + "os" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + dbUsername string + dbPassword string + dbName string + dbAddress string + dbDriver string + dbPort int + bcAddress string + bcPort int ) // captureCmd represents the capture command @@ -21,13 +35,64 @@ var captureCmd = &cobra.Command{ func init() { rootCmd.AddCommand(captureCmd) + // Required Flags + + //// DB Specific + captureCmd.PersistentFlags().StringVarP(&dbUsername, "db.username", "", "", "Database username (required)") + captureCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "", "", "Database Password (required)") + captureCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "", "", "Port to connect to DB(required)") + captureCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)") + captureCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)") + captureCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)") + err := captureCmd.MarkPersistentFlagRequired("db.username") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.password") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.address") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.port") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.name") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.driver") + exitErr(err) + + //// Beacon Client Specific + captureCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)") + captureCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)") + err = captureCmd.MarkPersistentFlagRequired("bc.address") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("bc.port") + exitErr(err) + + // Bind Flags with Viper + //// DB Flags + err = viper.BindPFlag("db.username", captureCmd.PersistentFlags().Lookup("db.username")) + exitErr(err) + err = viper.BindPFlag("db.password", captureCmd.PersistentFlags().Lookup("db.password")) + exitErr(err) + err = viper.BindPFlag("db.address", captureCmd.PersistentFlags().Lookup("db.address")) + exitErr(err) + err = viper.BindPFlag("db.port", captureCmd.PersistentFlags().Lookup("db.port")) + exitErr(err) + err = viper.BindPFlag("db.name", captureCmd.PersistentFlags().Lookup("db.name")) + exitErr(err) + err = viper.BindPFlag("db.driver", captureCmd.PersistentFlags().Lookup("db.driver")) + exitErr(err) + + // LH specific + err = viper.BindPFlag("bc.address", captureCmd.PersistentFlags().Lookup("bc.address")) + exitErr(err) + err = viper.BindPFlag("bc.port", captureCmd.PersistentFlags().Lookup("bc.port")) + exitErr(err) // Here you will define your flags and configuration settings. - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // captureCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // captureCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} + +// Helper function to catch any errors. +// We need to capture these errors for the linter. +func exitErr(err error) { + if err != nil { + os.Exit(1) + } } diff --git a/cmd/head.go b/cmd/head.go index 86e4f9f..12b432a 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -20,8 +20,9 @@ var headCmd = &cobra.Command{ }, } +// Start the application to track at head. func startHeadTracking() { - _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) + _, err := boot.BootApplicationWithRetry(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) if err != nil { loghelper.LogError(err).Error("Unable to Start application") } diff --git a/cmd/root.go b/cmd/root.go index 860dc78..6fc17c6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,22 +9,13 @@ import ( "io" "os" - "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - dbUsername string - dbPassword string - dbName string - dbAddress string - dbDriver string - dbPort int - bcAddress string - bcPort int + cfgFile string ) // rootCmd represents the base command when called without any subcommands @@ -51,7 +42,9 @@ func Execute() { // Prerun for Cobra func initFuncs(cmd *cobra.Command, args []string) { logFormat() - logFile() + if err := logFile(); err != nil { + log.WithField("err", err).Error("Could not set log file") + } if err := logLevel(); err != nil { log.WithField("err", err).Error("Could not set log level") } @@ -59,7 +52,10 @@ func initFuncs(cmd *cobra.Command, args []string) { // Set the log level for the application func logLevel() error { - viper.BindEnv("log.level", "LOGRUS_LEVEL") + err := viper.BindEnv("log.level", "LOGRUS_LEVEL") + if err != nil { + return err + } lvl, err := log.ParseLevel(viper.GetString("log.level")) if err != nil { return err @@ -73,16 +69,23 @@ func logLevel() error { } // Create a log file -func logFile() { - viper.BindEnv("log.file", "LOGRUS_FILE") +func logFile() error { + err := viper.BindEnv("log.file", "LOGRUS_FILE") + if err != nil { + return err + } logfile := viper.GetString("log.file") if logfile != "" { file, err := os.OpenFile(logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err == nil { - log.Infof("Directing output to %s", logfile) - mw := io.MultiWriter(os.Stdout, file) - logrus.SetOutput(mw) + if viper.GetBool("log.output") { + log.Infof("Directing output to %s", logfile) + mw := io.MultiWriter(os.Stdout, file) + log.SetOutput(mw) + } else { + log.SetOutput(file) + } } else { log.SetOutput(os.Stdout) log.Info("Failed to log to file, using default stdout") @@ -90,6 +93,7 @@ func logFile() { } else { log.SetOutput(os.Stdout) } + return nil } // Format the logger @@ -113,47 +117,19 @@ func init() { rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().Bool("log.output", true, "Should we log to STDOUT") rootCmd.PersistentFlags().String("log.format", "json", "json or text") - // Required Flags - - //// DB Specific - rootCmd.PersistentFlags().StringVarP(&dbUsername, "db.username", "", "", "Database username (required)") - rootCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "", "", "Database Password (required)") - rootCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "", "", "Port to connect to DB(required)") - rootCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)") - rootCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)") - rootCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)") - rootCmd.MarkPersistentFlagRequired("db.username") - rootCmd.MarkPersistentFlagRequired("db.password") - rootCmd.MarkPersistentFlagRequired("db.address") - rootCmd.MarkPersistentFlagRequired("db.port") - rootCmd.MarkPersistentFlagRequired("db.name") - rootCmd.MarkPersistentFlagRequired("db.driver") - - //// Beacon Client Specific - rootCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)") - rootCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)") - rootCmd.MarkPersistentFlagRequired("bc.address") - rootCmd.MarkPersistentFlagRequired("bc.port") - // Bind Flags with Viper // Optional - viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level")) - viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file")) - viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format")) - - //// DB Flags - viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username")) - viper.BindPFlag("db.password", rootCmd.PersistentFlags().Lookup("db.password")) - viper.BindPFlag("db.address", rootCmd.PersistentFlags().Lookup("db.address")) - viper.BindPFlag("db.port", rootCmd.PersistentFlags().Lookup("db.port")) - viper.BindPFlag("db.name", rootCmd.PersistentFlags().Lookup("db.name")) - viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver")) - - // LH specific - viper.BindPFlag("bc.address", rootCmd.PersistentFlags().Lookup("bc.address")) - viper.BindPFlag("bc.port", rootCmd.PersistentFlags().Lookup("bc.port")) + err := viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level")) + exitErr(err) + err = viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file")) + exitErr(err) + err = viper.BindPFlag("log.output", rootCmd.PersistentFlags().Lookup("log.output")) + exitErr(err) + err = viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format")) + exitErr(err) // Cobra also supports local flags, which will only run // when this action is called directly. diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 0000000..7b70eb4 --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + v "github.com/vulcanize/ipld-ethcl-indexer/pkg/version" +) + +var ( + Major = 0 // Major version component of the current release + Minor = 0 // Minor version component of the current release + Patch = 0 // Patch version component of the current release + Meta = "" // Version metadata to append to the version string +) + +// versionCmd represents the version command +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Prints the version of ipld-eth-server", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + version := v.Version{ + Major: Major, + Minor: Minor, + Patch: Patch, + Meta: Meta, + } + log.Infof("ipld-ethcl-indexer version: %s", version.GetVersionWithMeta()) + fmt.Println(version.GetVersionWithMeta()) + }, +} + +func init() { + rootCmd.AddCommand(versionCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // versionCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // versionCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/go.mod b/go.mod index cb608b1..de583a9 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,12 @@ require ( github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.10.0 // indirect github.com/jackc/puddle v1.2.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lib/pq v1.10.4 // indirect github.com/pkg/errors v0.9.1 // indirect golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) require ( diff --git a/go.sum b/go.sum index 614e88e..fa80980 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -223,19 +224,22 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -657,8 +661,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= diff --git a/internal/boot/boot.go b/internal/boot/boot.go index 13420ef..46e3b02 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -4,14 +4,19 @@ import ( "fmt" "net/http" "strconv" + "time" log "github.com/sirupsen/logrus" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) var ( - bcHealthEndpoint = "/eth/v1/node/health" + bcHealthEndpoint = "/eth/v1/node/health" + maxRetry = 5 // Max times to try to connect to the DB or BC at boot. + retryInterval = 30 // The time to wait between each try. + DB sql.Database = &postgres.DB{} ) // This function will ensure that we can connect to the beacon client. @@ -39,7 +44,7 @@ func checkBeaconClient(bcAddress string, bcPort int) error { } // A simple wrapper to create a DB object to use. -func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { +func SetupPostgresDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (sql.Database, error) { log.Debug("Resolving Driver Type") DbDriver, err := postgres.ResolveDriverType(driverName) if err != nil { @@ -58,7 +63,7 @@ func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, db Password: dbPassword, Driver: DbDriver, } - DB, err := postgres.NewPostgresDB(postgresConfig) + DB, err = postgres.NewPostgresDB(postgresConfig) if err != nil { loghelper.LogError(err).Error("Unable to connect to the DB") @@ -76,7 +81,9 @@ func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, db // // 2. Connect to the database. // -func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { +func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (sql.Database, error) { + log.Info("Booting the Application") + log.Debug("Checking beacon Client") err := checkBeaconClient(bcAddress, bcPort) if err != nil { @@ -84,9 +91,25 @@ func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername st } log.Debug("Setting up DB connection") - DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) + DB, err := SetupPostgresDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) if err != nil { return nil, err } return DB, nil } + +// Add retry logic to ensure that we are give the Beacon Client and the DB time to start. +func BootApplicationWithRetry(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (sql.Database, error) { + var err error + for i := 0; i < maxRetry; i++ { + DB, err = BootApplication(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName, bcAddress, bcPort) + if err != nil { + log.WithFields(log.Fields{ + "retryNumber": i, + }).Warn("Unable to boot application. Going to try again") + time.Sleep(time.Duration(retryInterval) * time.Second) + continue + } + } + return DB, err +} diff --git a/internal/boot/boot_suite_test.go b/internal/boot/boot_suite_test.go new file mode 100644 index 0000000..e7bcdf3 --- /dev/null +++ b/internal/boot/boot_suite_test.go @@ -0,0 +1,13 @@ +package boot_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBoot(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Boot Suite") +} diff --git a/internal/boot/boot_test.go b/internal/boot/boot_test.go new file mode 100644 index 0000000..729f30b --- /dev/null +++ b/internal/boot/boot_test.go @@ -0,0 +1,48 @@ +package boot_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" +) + +var _ = Describe("Boot", func() { + var ( + dbAddress string = "localhost" + dbPort int = 8077 + dbName string = "vulcanize_testing" + dbUsername string = "vdbm" + dbPassword string = "password" + dbDriver string = "PGX" + bcAddress string = "localhost" + bcPort int = 5052 + ) + Describe("Booting the application", Label("integration"), func() { + Context("When the DB and BC are both up and running", func() { + It("Should connect successfully", func() { + db, err := boot.BootApplicationWithRetry(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) + defer db.Close() + Expect(err).To(BeNil()) + }) + }) + Context("When the DB is running but not the BC", func() { + It("Should not connect successfully", func() { + _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, "hi", 100) + Expect(err).ToNot(BeNil()) + }) + }) + Context("When the BC is running but not the DB", func() { + It("Should not connect successfully", func() { + _, err := boot.BootApplication("hi", 10, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) + Expect(err).ToNot(BeNil()) + }) + }) + Context("When neither the BC or DB are running", func() { + It("Should not connect successfully", func() { + _, err := boot.BootApplication("hi", 10, dbName, dbUsername, dbPassword, dbDriver, "hi", 100) + Expect(err).ToNot(BeNil()) + }) + }) + }) + +}) diff --git a/pkg/database/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go index c8c1bae..c5c5422 100644 --- a/pkg/database/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -23,7 +23,7 @@ var _ = Describe("Pgx", func() { ctx = context.Background() }) - Describe("Connecting to the DB", func() { + Describe("Connecting to the DB", Label("integration"), func() { Context("But connection is unsucessful", func() { It("throws error when can't connect to the database", func() { _, err := postgres.NewPostgresDB(postgres.Config{ @@ -43,7 +43,7 @@ var _ = Describe("Pgx", func() { }) }) }) - Describe("Write to the DB", func() { + Describe("Write to the DB", Label("integration"), func() { Context("Serialize big.Int to DB", func() { It("Should serialize successfully", func() { dbPool, err := postgres.NewPostgresDB(postgres.DefaultConfig) @@ -53,9 +53,13 @@ var _ = Describe("Pgx", func() { bi := new(big.Int) bi.SetString("34940183920000000000", 10) isEqual, err := testhelpers.IsEqual(bi.String(), "34940183920000000000") + Expect(err).To(BeNil()) Expect(isEqual).To(BeTrue()) - defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) + defer func() { + _, err := dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) + Expect(err).To(BeNil()) + }() _, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )") Expect(err).To(BeNil()) @@ -71,11 +75,13 @@ var _ = Describe("Pgx", func() { isEqual, err = testhelpers.IsEqual(data, bi.String()) Expect(isEqual).To(BeTrue()) + Expect(err).To(BeNil()) actual := new(big.Int) actual.SetString(data, 10) isEqual, err = testhelpers.IsEqual(actual, bi) Expect(isEqual).To(BeTrue()) + Expect(err).To(BeNil()) }) }) }) diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 0000000..778fc85 --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,25 @@ +package version + +import "fmt" + +// A reusable structure to allow developers to set their application versions. +type Version struct { + Major int // Major version component of the current release + Minor int // Minor version component of the current release + Patch int // Patch version component of the current release + Meta string // Version metadata to append to the version string +} + +// Provides a string with the version +func (version *Version) GetVersion() string { + return fmt.Sprintf("%d.%d.%d", version.Major, version.Minor, version.Patch) +} + +// Provides a string with the version and Meta. +func (version *Version) GetVersionWithMeta() string { + v := version.GetVersion() + if version.Meta != "" { + v += "-" + version.Meta + } + return v +}