From 90a16d32f951ca00866cb81bc83abe3ebef15ec4 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Thu, 5 May 2022 12:44:44 -0400 Subject: [PATCH] Boot Application on PR --- .github/workflows/on-pr.yml | 39 +++++++++++- cmd/boot.go | 80 ++++++++++++++++++++++++ cmd/head.go | 4 +- entrypoint.sh | 11 +++- internal/boot/boot.go | 1 + internal/shutdown/shutdown.go | 5 +- internal/shutdown/shutdown_test.go | 15 +++-- pkg/gracefulshutdown/gracefulshutdown.go | 7 +-- 8 files changed, 145 insertions(+), 17 deletions(-) create mode 100644 cmd/boot.go diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 67bf3e8..b5a122e 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -21,7 +21,7 @@ on: - "**" env: - stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || 'fcbc74451c5494664fe21f765e89c9c6565c07cb'}} + stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || '21d076268730e3f25fcec6371c1aca1bf48040d8'}} ipld-eth-db-ref: ${{ github.event.inputs.ipld-eth-db-ref || '05600e51d2163e1c5e2a872cb54606bc0a380d12' }} GOPATH: /tmp/go jobs: @@ -33,6 +33,42 @@ jobs: - name: Run docker build run: make docker-build + - uses: actions/checkout@v3 + with: + ref: ${{ env.stack-orchestrator-ref }} + path: "./stack-orchestrator/" + repository: vulcanize/stack-orchestrator + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-eth-db-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + cat ./config.sh + + - name: Run docker compose + run: | + docker-compose \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \ + --env-file ./config.sh \ + up -d --build + + - name: Check to make sure HEALTH file is present + run: | + docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/HEALTH ./HEALTH + while [ $? -ne 0 ]; do !!; done + cat ./HEALTH + if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && exit 0; else exit 1; fi + unit-test: name: Run Unit Tests runs-on: ubuntu-latest @@ -109,6 +145,7 @@ jobs: run: | echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + echo ethcl_capture_mode=boot >> ./config.sh cat ./config.sh - name: Run docker compose diff --git a/cmd/boot.go b/cmd/boot.go new file mode 100644 index 0000000..8a0ffa1 --- /dev/null +++ b/cmd/boot.go @@ -0,0 +1,80 @@ +/* +Copyright © 2022 Abdul Rabbani + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package cmd + +import ( + "context" + "os" + "syscall" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" + "github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" +) + +// bootCmd represents the boot command +var bootCmd = &cobra.Command{ + Use: "boot", + Short: "Run the boot command then exit", + Long: `Run the application to boot and exit. Primarily used for testing.`, + Run: func(cmd *cobra.Command, args []string) { + bootApp() + }, +} + +func bootApp() { + + // Boot the application + log.Info("Starting the application in boot mode.") + ctx := context.Background() + + BC, DB, err := boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol) + if err != nil { + loghelper.LogError(err).Error("Unable to Start application") + } + + log.Info("Boot complete, we are going to shutdown.") + + notifierCh := make(chan os.Signal, 1) + + go func() { + notifierCh <- syscall.SIGTERM + }() + + err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC) + if err != nil { + loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!") + } else { + log.Info("Gracefully shutdown ipld-ethcl-indexer") + } +} + +func init() { + captureCmd.AddCommand(bootCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // bootCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // bootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/head.go b/cmd/head.go index 99ab20b..2bf292d 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -6,6 +6,7 @@ package cmd import ( "context" + "os" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -39,7 +40,8 @@ func startHeadTracking() { go BC.CaptureHead(DB) // Shutdown when the time is right. - err = shutdown.ShutdownServices(ctx, maxWaitSecondsShutdown, DB, BC) + notifierCh := make(chan os.Signal, 1) + err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC) if err != nil { loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!") } else { diff --git a/entrypoint.sh b/entrypoint.sh index 6e4608e..c9923e3 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,7 +1,9 @@ #!/bin/bash + +sleep 10 echo "Starting ipld-ethcl-indexer" -echo /root/ipld-ethcl-indexer capture head --db.address $DB_ADDRESS \ +echo /root/ipld-ethcl-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \ --db.password $DB_PASSWORD \ --db.port $DB_PORT \ --db.username $DB_USER \ @@ -11,7 +13,7 @@ echo /root/ipld-ethcl-indexer capture head --db.address $DB_ADDRESS \ --bc.port $BC_PORT \ --log.level $LOG_LEVEL -/root/ipld-ethcl-indexer capture head --db.address $DB_ADDRESS \ +/root/ipld-ethcl-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \ --db.password $DB_PASSWORD \ --db.port $DB_PORT \ --db.username $DB_USER \ @@ -25,7 +27,10 @@ rv=$? if [ $rv != 0 ]; then echo "ipld-ethcl-indexer startup failed" - exit 1 + echo 1 > /root/HEALTH +else + echo "ipld-ethcl-indexer startup succeeded" + echo 0 > /root/HEALTH fi tail -f /dev/null \ No newline at end of file diff --git a/internal/boot/boot.go b/internal/boot/boot.go index ce9d61a..d318941 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -86,6 +86,7 @@ func BootApplicationWithRetry(ctx context.Context, dbHostname string, dbPort int "retryNumber": i, }).Warn("Unable to boot application. Going to try again") time.Sleep(time.Duration(retryInterval) * time.Second) + continue } break } diff --git a/internal/shutdown/shutdown.go b/internal/shutdown/shutdown.go index b45b3d2..299a027 100644 --- a/internal/shutdown/shutdown.go +++ b/internal/shutdown/shutdown.go @@ -2,6 +2,7 @@ package shutdown import ( "context" + "os" "time" "github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient" @@ -11,8 +12,8 @@ import ( ) // Shutdown all the internal services for the application. -func ShutdownServices(ctx context.Context, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error { - successCh, errCh := gracefulshutdown.Shutdown(ctx, waitTime, map[string]gracefulshutdown.Operation{ +func ShutdownServices(ctx context.Context, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error { + successCh, errCh := gracefulshutdown.Shutdown(ctx, notifierCh, waitTime, map[string]gracefulshutdown.Operation{ // Combining DB shutdown with BC because BC needs DB open to cleanly shutdown. "beaconClient": func(ctx context.Context) error { defer DB.Close() diff --git a/internal/shutdown/shutdown_test.go b/internal/shutdown/shutdown_test.go index b36d814..c978ac4 100644 --- a/internal/shutdown/shutdown_test.go +++ b/internal/shutdown/shutdown_test.go @@ -5,6 +5,7 @@ package shutdown_test import ( "context" + "os" "syscall" "time" @@ -35,10 +36,12 @@ var _ = Describe("Shutdown", func() { BC *beaconclient.BeaconClient err error ctx context.Context + notifierCh chan os.Signal ) BeforeEach(func() { ctx = context.Background() BC, DB, err = boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol) + notifierCh = make(chan os.Signal, 1) Expect(err).To(BeNil()) }) @@ -47,7 +50,7 @@ var _ = Describe("Shutdown", func() { It("Should Shutdown Successfully.", func() { go func() { log.Debug("Starting shutdown chan") - err = shutdown.ShutdownServices(ctx, maxWaitSecondsShutdown, DB, BC) + err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC) log.Debug("We have completed the shutdown...") Expect(err).ToNot(HaveOccurred()) }() @@ -59,7 +62,7 @@ var _ = Describe("Shutdown", func() { //log.SetLevel(log.DebugLevel) go func() { log.Debug("Starting shutdown chan") - err = shutdown.ShutdownServices(ctx, maxWaitSecondsShutdown, DB, BC) + err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC) log.Debug("We have completed the shutdown...") Expect(err).ToNot(HaveOccurred()) shutdownCh <- true @@ -77,8 +80,8 @@ var _ = Describe("Shutdown", func() { go func() { <-messageAddCh - log.Debug("Calling SIGHUP") - syscall.Kill(syscall.Getpid(), syscall.SIGHUP) + log.Debug("Calling SIGTERM") + notifierCh <- syscall.SIGTERM log.Debug("Reading messages from channel") <-BC.HeadTracking.MessagesCh //<-BC.FinalizationTracking.MessagesCh @@ -92,7 +95,7 @@ var _ = Describe("Shutdown", func() { //log.SetLevel(log.DebugLevel) go func() { log.Debug("Starting shutdown chan") - err = shutdown.ShutdownServices(ctx, maxWaitSecondsShutdown, DB, BC) + err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC) log.Debug("We have completed the shutdown...") Expect(err).To(MatchError(gracefulshutdown.TimeoutErr(maxWaitSecondsShutdown.String()))) shutdownCh <- true @@ -105,7 +108,7 @@ var _ = Describe("Shutdown", func() { BC.ReOrgTracking.MessagesCh <- &sse.Event{} log.Debug("Message adding complete") log.Debug("Calling SIGHUP") - syscall.Kill(syscall.Getpid(), syscall.SIGHUP) + notifierCh <- syscall.SIGTERM }() <-shutdownCh diff --git a/pkg/gracefulshutdown/gracefulshutdown.go b/pkg/gracefulshutdown/gracefulshutdown.go index 1b8b52a..a8ba972 100644 --- a/pkg/gracefulshutdown/gracefulshutdown.go +++ b/pkg/gracefulshutdown/gracefulshutdown.go @@ -23,15 +23,14 @@ var ( ) // gracefulShutdown waits for termination syscalls and doing clean up operations after received it -func Shutdown(ctx context.Context, timeout time.Duration, ops map[string]Operation) (<-chan struct{}, <-chan error) { +func Shutdown(ctx context.Context, notifierCh chan os.Signal, timeout time.Duration, ops map[string]Operation) (<-chan struct{}, <-chan error) { waitCh := make(chan struct{}) errCh := make(chan error) go func() { - s := make(chan os.Signal, 1) // add any other syscalls that you want to be notified with - signal.Notify(s, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - <-s + signal.Notify(notifierCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + <-notifierCh log.Info("Shutting Down your application")