Slight Performance improvements with Logging (#67)

* Slight Performance improvements with Logging

* Use exec in docker container

* Update packages for dockerfile

* Add flakey test and boot command specific logic

* Remove excess logging and add config val

.github/workflows/generic-testing.yml

@@ -0,0 +1,199 @@
+name: Test the stack.
+on:
+  workflow_call:
+    inputs:
+      stack-orchestrator-ref:
+        required: false
+        type: string
+      ipld-eth-beacon-db-ref:
+        required: false
+        type: string
+      ssz-data-ref:
+        required: false
+        type: string
+    secrets:
+      GHA_KEY:
+        required: true
+
+env:
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '7fb664270a0ba09e2caa3095e8c91f3fdb5b38af' }}
+  ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref  || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
+  ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }}
+  GOPATH: /tmp/go
+jobs:
+  build:
+    name: Run Docker Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-beacon-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-beacon-db-ref }}
+          repository: vulcanize/ipld-eth-beacon-db
+          path: "./ipld-eth-beacon-db/"
+          ssh-key: ${{secrets.GHA_KEY}}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
+          echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
+          echo eth_beacon_config_file=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer/config/cicd/boot.ipld-eth-beacon-indexer.json >> ./config.sh
+          echo eth_beacon_capture_mode=boot >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - name: Check to make sure HEALTH file is present
+        shell: bash
+        run: |
+          until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" --env-file ./config.sh cp ipld-eth-beacon-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
+          cat ./HEALTH
+          if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" cp ipld-eth-beacon-indexer:/root/ipld-eth-beacon-indexer.log . && cat ipld-eth-beacon-indexer.log && (exit 1); fi
+
+  unit-test:
+    name: Run Unit Tests
+    runs-on: ubuntu-latest
+    ## IF you want to update the default branch for `pull_request runs, do it after the ||`
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-beacon-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-beacon-db-ref }}
+          repository: vulcanize/ipld-eth-beacon-db
+          path: "./ipld-eth-beacon-db/"
+          ssh-key: ${{ secrets.GHA_KEY }}
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ssz-data-ref }}
+          repository: vulcanize/ssz-data
+          path: "./ipld-eth-beacon-indexer/pkg/beaconclient/ssz-data"
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
+          echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          cd ipld-eth-beacon-indexer
+          make unit-test-ci
+
+  integration-test:
+    name: Run Integration Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-beacon-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-beacon-db-ref }}
+          repository: vulcanize/ipld-eth-beacon-db
+          path: "./ipld-eth-beacon-db/"
+          ssh-key: ${{secrets.GHA_KEY}}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
+          echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          cd ipld-eth-beacon-indexer
+          make integration-test-ci
+
+  golangci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          args: --timeout 90s --disable deadcode,unused
+#          args: --timeout 90s --disable deadcode,

.github/workflows/on-pr.yml

@@ -3,12 +3,16 @@ name: Test Application On PR
 on:
   workflow_dispatch:
     inputs:
-      foundry-test-ref:
+      stack-orchestrator-ref:
-        description: "The branch, commit or sha from foundry-test to checkout"
+        description: "The branch, commit or sha from stack-orchestrator to checkout"
         required: false
         default: "main"
-      ipld-eth-db-ref:
+      ipld-eth-beacon-db-ref:
-        description: "The branch, commit or sha from ipld-eth-db to checkout"
+        description: "The branch, commit or sha from ipld-eth-beacon-db to checkout"
+        required: false
+        default: "main"
+      ssz-data-ref:
+        description: "The branch, commit or sha from ssz-data to checkout"
         required: false
         default: "main"
   pull_request:
@@ -18,134 +22,26 @@ on:
       - "!LICENSE"
       - "!.github/workflows/**"
       - ".github/workflows/on-pr.yml"
+      - ".github/workflows/tests.yml"
       - "**"
+  schedule:
+    - cron: '0 13 * * *' # Must be single quotes!!
 
 jobs:
-  build:
+  trigger-tests:
-    name: Run Docker Build
+    if: github.event_name != 'schedule'
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/generic-testing.yml
-    steps:
+    with:
-      - uses: actions/checkout@v2
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
-      - name: Run docker build
+      ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
-        run: make docker-build
+      ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }}
-
+    secrets:
-  unit-test:
+      GHA_KEY: ${{secrets.GHA_KEY}}
-    name: Run Unit Tests
+  system-testing:
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/system-tests.yml
-    ## IF you want to update the default branch for `pull_request runs, do it after the ||`
+    with:
-    env:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
-      foundry-test-ref: ${{ github.event.inputs.foundry-test-ref  || 'c17752de64f208f286f02379b80d2a935237c860'}}
+      ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
-      ipld-eth-db-ref: ${{ github.event.inputs.ipld-eth-db-ref  || '05600e51d2163e1c5e2a872cb54606bc0a380d12' }}
+    secrets:
-      GOPATH: /tmp/go
+      GHA_KEY: ${{secrets.GHA_KEY}}
-    steps:
+      BC_ADDRESS: ${{secrets.BC_ADDRESS}}
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-ethcl-indexer"
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.foundry-test-ref }}
-          path: "./foundry-test/"
-          repository: vulcanize/foundry-test
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ipld-eth-db-ref }}
-          repository: vulcanize/ipld-eth-db
-          path: "./ipld-eth-db/"
-          fetch-depth: 0
-
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh
-          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
-          cat ./config.sh
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-          check-latest: true
-
-      - name: Install packages
-        run: |
-          go install github.com/onsi/ginkgo/v2/ginkgo@latest
-          which ginkgo
-
-      - name: Run the tests using Make
-        run: |
-          cd ipld-ethcl-indexer
-          make unit-test-ci
-
-  integration-test:
-    name: Run Integration Tests
-    runs-on: ubuntu-latest
-    env:
-      foundry-test-ref: feature/build-stack
-      ipld-eth-db-ref: main
-      GOPATH: /tmp/go
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-ethcl-indexer"
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.foundry-test-ref }}
-          path: "./foundry-test/"
-          repository: vulcanize/foundry-test
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ipld-eth-db-ref }}
-          repository: vulcanize/ipld-eth-db
-          path: "./ipld-eth-db/"
-          fetch-depth: 0
-
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh
-          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
-          cat ./config.sh
-
-      - name: Run docker compose
-        run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/foundry-test/docker/local/docker-compose-db.yml" \
-          -f "$GITHUB_WORKSPACE/foundry-test/docker/latest/docker-compose-lighthouse.yml" \
-          --env-file ./config.sh \
-          up -d --build
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-          check-latest: true
-
-      - name: Install packages
-        run: |
-          go install github.com/onsi/ginkgo/v2/ginkgo@latest
-          which ginkgo
-
-      - name: Run the tests using Make
-        run: |
-          cd ipld-ethcl-indexer
-          make integration-test-ci
-
-  golangci:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-      - uses: actions/checkout@v3
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          args: --disable errcheck

.github/workflows/on-publish.yml

@@ -0,0 +1,60 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published, edited]
+jobs:
+  trigger-tests:
+    uses: ./.github/workflows/generic-testing.yml
+    with:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
+      ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
+      ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }}
+    secrets:
+      GHA_KEY: ${{secrets.GHA_KEY}}
+  system-testing:
+    uses: ./.github/workflows/system-tests.yml
+    with:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
+      ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
+    secrets:
+      GHA_KEY: ${{secrets.GHA_KEY}}
+      BC_ADDRESS: ${{secrets.BC_ADDRESS}}
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    needs:
+      - trigger-tests
+      - system-testing
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+      - name: Run docker build
+        run: make docker-build
+      - name: Tag docker image
+        run: docker tag vulcanize/ipld-eth-beacon-indexer docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Push
+        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
+  push_to_registries:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Get the version
+        id: vars
+        run: |
+          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+      - name: Docker Login to Github Registry
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Pull
+        run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
+      - name: Docker Login to Docker Registry
+        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+      - name: Tag docker image
+        run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-beacon-indexer:${{steps.vars.outputs.tag}}
+      - name: Docker Push to Docker Hub
+        run: docker push vulcanize/ipld-eth-beacon-indexer:${{steps.vars.outputs.tag}}

.github/workflows/system-tests.yml

@@ -0,0 +1,95 @@
+name: System Testing for the stack.
+on:
+  workflow_call:
+    inputs:
+      stack-orchestrator-ref:
+        required: false
+        type: string
+      ipld-eth-beacon-db-ref:
+        required: false
+        type: string
+    secrets:
+      GHA_KEY:
+        required: true
+      BC_ADDRESS:
+        required: true
+env:
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '7fb664270a0ba09e2caa3095e8c91f3fdb5b38af' }}
+  ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref  || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
+  GOPATH: /tmp/go
+  bc_protocol: "http"
+  bc_address: ${{secrets.BC_ADDRESS}}
+  bc_port: 5052
+  db_host: localhost
+  db_port: 8076
+  db_name: vulcanize_testing
+  db_user: vdbm
+  db_password: password
+  db_driver: "pgx"
+
+jobs:
+  system-testing:
+    runs-on: self-hosted
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-beacon-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-beacon-db-ref }}
+          repository: vulcanize/ipld-eth-beacon-db
+          path: "./ipld-eth-beacon-db/"
+          ssh-key: ${{secrets.GHA_KEY}}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_beacon_db=$(pwd)/ipld-eth-beacon-db > ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        id: compose
+        shell: bash
+        run: |
+          ls "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml"
+          whoami
+          /usr/local/bin/docker-compose  \
+          -f "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          sleep 20
+          cd ipld-eth-beacon-indexer
+          make system-test-ci
+
+      - name: Clean up the docker containers
+        if: always() && steps.compose.outcome == 'success'
+        shell: bash
+        run: |
+          /usr/local/bin/docker-compose \
+          -f "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          --env-file ./config.sh \
+          down -v

.gitignore

@@ -3,3 +3,11 @@ ipld-ethcl-indexer
 ipld-ethcl-indexer.log
 report.json
 cover.profile
+temp/*
+.vscode/*
+pkg/beaconclient/ssz-data/
+*.test
+ipld-eth-beacon-indexer.log
+ipld-eth-beacon-indexer
+config/local.ipld-eth-beacon-indexer-config.json
+config/docker.ipld-eth-beacon-indexer-config.json

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.18-alpine as builder
 
-WORKDIR /go/src/github.com/vulcanize/ipld-ethcl-indexer
+WORKDIR /go/src/github.com/vulcanize/ipld-eth-beacon-indexer
-RUN apk --no-cache add ca-certificates make git g++ linux-headers
+RUN apk --no-cache add ca-certificates make git g++ linux-headers libstdc++
 
 ENV GO111MODULE=on
 COPY go.mod .
@@ -9,12 +9,12 @@ COPY go.sum .
 RUN go mod tidy; go mod download
 COPY . .
 
-RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-ethcl-indexer .
+RUN GCO_ENABLED=0 GOOS=linux go build -race -ldflags="-s -w" -o ipld-eth-beacon-indexer .
-RUN chmod +x ipld-ethcl-indexer
+RUN chmod +x ipld-eth-beacon-indexer
 
 FROM frolvlad/alpine-bash:latest
-RUN apk --no-cache add ca-certificates
+RUN apk --no-cache add ca-certificates libstdc++
 WORKDIR /root/
-COPY --from=builder /go/src/github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer /root/ipld-ethcl-indexer
+COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer /root/ipld-eth-beacon-indexer
 ADD entrypoint.sh .
 ENTRYPOINT ["./entrypoint.sh"]

Makefile

@@ -30,6 +30,17 @@ integration-test-ci:
 	--cover --coverprofile=cover.profile \
 	--race --trace --json-report=report.json
 
+.PHONY: integration-test-ci-no-race
+integration-test-ci-no-race:
+	go vet ./...
+	go fmt ./...
+	$(GINKGO) -r --label-filter integration \
+	--procs=4 --compilers=4 \
+	--randomize-all --randomize-suites \
+	--fail-on-pending --keep-going \
+	--cover --coverprofile=cover.profile \
+	--trace --json-report=report.json
+
 .PHONY: integration-test-local
 integration-test-local:
 	go vet ./...
@@ -38,29 +49,57 @@ integration-test-local:
 	--procs=4 --compilers=4 \
 	--randomize-all --randomize-suites \
 	--fail-on-pending --keep-going \
-	--race --trace
+	--trace --race
+
+.PHONY: integration-test-local-no-race
+integration-test-local-no-race:
+	go vet ./...
+	go fmt ./...
+	$(GINKGO) -r --label-filter integration \
+	--procs=4 --compilers=4 \
+	--randomize-all --randomize-suites \
+	--fail-on-pending --keep-going \
+	--trace
 
 .PHONY: unit-test-local
 unit-test-local:
 	go vet ./...
 	go fmt ./...
 	$(GINKGO) -r --label-filter unit \
-	--procs=4 --compilers=4 \
 	--randomize-all --randomize-suites \
+	--flake-attempts=3 \
 	--fail-on-pending --keep-going \
-	--race --trace
+	--trace
 
 .PHONY: unit-test-ci
 unit-test-ci:
 	go vet ./...
 	go fmt ./...
 	$(GINKGO) -r --label-filter unit \
-	--procs=4 --compilers=4 \
+	--randomize-all --randomize-suites
+	--flake-attempts=3 \
+	--fail-on-pending --keep-going \
+	--cover --coverprofile=cover.profile \
+	--trace --json-report=report.json
+
+.PHONY: system-test-ci
+system-test-ci:
+	go vet ./...
+	go fmt ./...
+	$(GINKGO) -r --label-filter system \
 	--randomize-all --randomize-suites \
 	--fail-on-pending --keep-going \
 	--cover --coverprofile=cover.profile \
-	--race --trace --json-report=report.json
+	--trace --json-report=report.json
 
+.PHONY: system-test-local
+system-test-local:
+	go vet ./...
+	go fmt ./...
+	$(GINKGO) -r --label-filter system \
+	--randomize-all --randomize-suites \
+	--fail-on-pending --keep-going \
+	--trace
 
 .PHONY: build
 build:
@@ -70,4 +109,4 @@ build:
 ## Build docker image
 .PHONY: docker-build
 docker-build:
-	docker build -t vulcanize/ipld-ethcl-indexer .
+	docker build -t vulcanize/ipld-eth-beacon-indexer .

README.md

@@ -1,4 +1,4 @@
-- [ipld-ethcl-indexer](#ipld-ethcl-indexer)
+- [ipld-eth-beacon-indexer](#ipld-eth-beacon-indexer)
 - [Running the Application](#running-the-application)
 - [Development Patterns](#development-patterns)
   - [Logging](#logging)
@@ -8,28 +8,46 @@
 
 <small><i><a href='http://ecotrust-canada.github.io/markdown-toc/'>Table of contents generated with markdown-toc</a></i></small>
 
-# ipld-ethcl-indexer
+# ipld-eth-beacon-indexer
 
 This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node.
 
 To learn more about the applications individual components, please read the [application components](/application_component.md).
 
-# Running the Application
+# Quick Start
 
-To run the application, utilize the following command, and update the values as needed.
+## Running the Application
+
+To run the application, do as follows:
+
+1. Setup the prerequisite applications.
+   a. Run a beacon client (such as lighthouse).
+   b. Run a postgres DB for eth-beacon.
+   c. You can utilize the `stack-orchestrator` [repository](https://github.com/vulcanize/stack-orchestrato).
+
+   ```
+   ./wrapper.sh -e skip \
+   -d ../docker/local/docker-compose-ipld-eth-beacon-db.yml \
+   -d ../docker/latest/docker-compose-lighthouse.yml \
+   -v remove \
+   -p ../local-config.sh
+
+   ```
+
+2. Run the start up command.
 
 ```
-go run main.go capture head --db.address localhost \
+go run -race main.go capture full --config ./example.ipld-eth-beacon-indexer-config.json
-  --db.password password \
-  --db.port 8077 \
-  --db.username vdbm \
-  --db.name vulcanize_testing \
-  --db.driver PGX \
-  --bc.address localhost \
-  --bc.port 5052 \
-  --log.level info
 ```
 
+## Running Tests
+
+To run tests, you will need to clone another repository which contains all the ssz files.
+
+1. `git clone git@github.com:vulcanize/ssz-data.git pkg/beaconclient/ssz-data`
+2. To run unit tests, make sure you have a DB running: `make unit-test-local`
+3. To run integration tests, make sure you have a lighthouse client and a DB running: `make integration-test-local-no-race` .
+
 # Development Patterns
 
 This section will cover some generic development patterns utilizes.
@@ -58,9 +76,18 @@ This project utilizes `ginkgo` for testing. A few notes on testing:
 - All test packages are named `{base_package}_test`. This ensures we only test the public methods.
 - If there is a need to test a private method, please include why in the testing file.
 - Unit tests must contain the `Label("unit")`.
-- Unit tests should not rely on any running service. If a running service is needed. Utilize an integration test.
+- Unit tests should not rely on any running service (except for a postgres DB). If a running service is needed. Utilize an integration test.
 - Integration tests must contain the `Label("integration")`.
 
+#### Understanding Testing Components
+
+A few notes about the testing components.
+
+- The `TestEvents` map contains several events for testers to leverage when testing.
+- Any object ending in `-dummy` is not a real object. You will also notice it has a present field called `MimicConfig`. This object will use an existing SSZ object, and update the parameters from the `Head` and `MimicConfig`.
+  - This is done because creating an empty or minimal `SignedBeaconBlock` and `BeaconState` is fairly challenging.
+  - By slightly modifying an existing object, we can test re-org, malformed objects, and other negative conditions.
+
 # Contribution
 
 If you want to contribute please make sure you do the following:

application_component.md

@@ -22,6 +22,33 @@ The `database` package allows us to interact with a postgres DB. We utilize the
 
 This package will contain code to interact with the beacon client.
 
+### Known Gaps
+
+Known Gaps tracking is handled within this package. The columns are as follows:
+
+- StartSlot - The start slot for known_gaps, inclusive.
+- EndSlot - The end slot for known_gaps, inclusive.
+- CheckedOut - Indicates if any process is currently processing this entry.
+- ErrorMessage - Captures any error message that might have occurred when previously processing this entry.
+- EntryTime - The time this range was added to the DB. This can help us catch ranges that have not been processed for a long time due to some error.
+- EntryProcess - The entry process that added this process. Potential options are StartUp, Error, HeadGap.
+  - This can help us understand how a specific entry was added. It can be useful for debugging the application.
+  - StartUp - Gaps found when we started the application.
+  - Error - Indicates that the entry was added due to an error with processing.
+  - HeadGap - Indicates that gaps where found when keeping up with Head.
+
 ## `pkg/version`
 
 A generic package which can be utilized to easily version our applications.
+
+## `pkg/gracefulshutdown`
+
+A generic package that can be used to shutdown various services within an application.
+
+## `pkg/loghelper`
+
+This package contains useful functions for logging.
+
+## `internal/shutdown`
+
+This package is used to shutdown the `ipld-eth-beacon-indexer`. It calls the `pkg/gracefulshutdown` package.

cmd/boot.go

@@ -0,0 +1,83 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"os"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+// bootCmd represents the boot command
+var bootCmd = &cobra.Command{
+	Use:   "boot",
+	Short: "Run the boot command then exit",
+	Long:  `Run the application to boot and exit. Primarily used for testing.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		bootApp()
+	},
+}
+
+func bootApp() {
+
+	// Boot the application
+	log.Info("Starting the application in boot mode.")
+	ctx := context.Background()
+
+	Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
+		viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
+		viper.GetInt("kg.increment"), "boot", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
+	if err != nil {
+		StopApplicationPreBoot(err, Db)
+	}
+
+	log.Info("Boot complete, we are going to shutdown.")
+
+	notifierCh := make(chan os.Signal, 1)
+
+	go func() {
+		notifierCh <- syscall.SIGTERM
+	}()
+
+	err = shutdown.ShutdownBoot(ctx, notifierCh, maxWaitSecondsShutdown, Db, Bc)
+	if err != nil {
+		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
+	} else {
+		log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
+	}
+}
+
+func init() {
+	captureCmd.AddCommand(bootCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// bootCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// bootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

cmd/capture.go

@@ -1,10 +1,23 @@
-/*
+// VulcanizeDB
-Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-*/
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"time"
 
@@ -13,16 +26,30 @@ import (
 )
 
 var (
-	dbUsername             string
+	dbUsername                 string
-	dbPassword             string
+	dbPassword                 string
-	dbName                 string
+	dbName                     string
-	dbAddress              string
+	dbAddress                  string
-	dbDriver               string
+	dbDriver                   string
-	dbPort                 int
+	dbPort                     int
-	bcAddress              string
+	bcAddress                  string
-	bcPort                 int
+	bcPort                     int
-	bcConnectionProtocol   string
+	bcBootRetryInterval        int
-	maxWaitSecondsShutdown time.Duration = time.Duration(5) * time.Second
+	bcBootMaxRetry             int
+	bcConnectionProtocol       string
+	bcType                     string
+	bcMaxHistoricProcessWorker int
+	bcUniqueNodeIdentifier     int
+	bcCheckDb                  bool
+	kgMaxWorker                int
+	kgTableIncrement           int
+	kgProcessGaps              bool
+	pmMetrics                  bool
+	pmAddress                  string
+	pmPort                     int
+	maxWaitSecondsShutdown     time.Duration  = time.Duration(20) * time.Second
+	notifierCh                 chan os.Signal = make(chan os.Signal, 1)
+	testDisregardSync          bool
 )
 
 // captureCmd represents the capture command
@@ -47,31 +74,50 @@ func init() {
 	captureCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)")
 	captureCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)")
 	captureCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)")
-	err := captureCmd.MarkPersistentFlagRequired("db.username")
+	//err := captureCmd.MarkPersistentFlagRequired("db.username")
-	exitErr(err)
+	// exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.password")
+	// err = captureCmd.MarkPersistentFlagRequired("db.password")
-	exitErr(err)
+	// exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.address")
+	// err = captureCmd.MarkPersistentFlagRequired("db.address")
-	exitErr(err)
+	// exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.port")
+	// err = captureCmd.MarkPersistentFlagRequired("db.port")
-	exitErr(err)
+	// exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.name")
+	// err = captureCmd.MarkPersistentFlagRequired("db.name")
-	exitErr(err)
+	// exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.driver")
+	// err = captureCmd.MarkPersistentFlagRequired("db.driver")
-	exitErr(err)
+	// exitErr(err)
 
 	//// Beacon Client Specific
-	captureCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)")
+	captureCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required)")
-	captureCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)")
+	captureCmd.PersistentFlags().StringVarP(&bcType, "bc.type", "", "lighthouse", "The beacon client we are using, options are prysm and lighthouse.")
+	captureCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required )")
 	captureCmd.PersistentFlags().StringVarP(&bcConnectionProtocol, "bc.connectionProtocol", "", "http", "protocol for connecting to the beacon node.")
-	err = captureCmd.MarkPersistentFlagRequired("bc.address")
+	captureCmd.PersistentFlags().IntVarP(&bcBootRetryInterval, "bc.bootRetryInterval", "", 30, "The amount of time to wait between retries while booting the application")
-	exitErr(err)
+	captureCmd.PersistentFlags().IntVarP(&bcBootMaxRetry, "bc.bootMaxRetry", "", 5, "The amount of time to wait between retries while booting the application")
-	err = captureCmd.MarkPersistentFlagRequired("bc.port")
+	captureCmd.PersistentFlags().IntVarP(&bcMaxHistoricProcessWorker, "bc.maxHistoricProcessWorker", "", 30, "The number of workers that should be actively processing slots from the eth-beacon.historic_process table. Be careful of system memory.")
-	exitErr(err)
+	captureCmd.PersistentFlags().IntVarP(&bcUniqueNodeIdentifier, "bc.uniqueNodeIdentifier", "", 0, "The unique identifier of this application. Each application connecting to the DB should have a unique identifier.")
+	captureCmd.PersistentFlags().BoolVarP(&bcCheckDb, "bc.checkDb", "", true, "Should we check to see if the slot exists in the DB before writing it?")
+	// err = captureCmd.MarkPersistentFlagRequired("bc.address")
+	// exitErr(err)
+	// err = captureCmd.MarkPersistentFlagRequired("bc.port")
+	// exitErr(err)
+
+	//// Known Gaps specific
+	captureCmd.PersistentFlags().BoolVarP(&kgProcessGaps, "kg.processKnownGaps", "", true, "Should we process the slots within the eth-beacon.known_gaps table.")
+	captureCmd.PersistentFlags().IntVarP(&kgTableIncrement, "kg.increment", "", 10000, "The max slots within a single entry to the known_gaps table.")
+	captureCmd.PersistentFlags().IntVarP(&kgMaxWorker, "kg.maxKnownGapsWorker", "", 30, "The number of workers that should be actively processing slots from the eth-beacon.known_gaps table. Be careful of system memory.")
+
+	// Prometheus Specific
+	captureCmd.PersistentFlags().BoolVarP(&pmMetrics, "pm.metrics", "", true, "Should we capture prometheus metrics.")
+	captureCmd.PersistentFlags().StringVarP(&pmAddress, "pm.address", "", "localhost", "Address to send the prometheus metrics.")
+	captureCmd.PersistentFlags().IntVarP(&pmPort, "pm.port", "", 9000, "The port to send prometheus metrics.")
+
+	//// Testing Specific
+	captureCmd.PersistentFlags().BoolVar(&testDisregardSync, "t.skipSync", false, "Should we disregard the head sync?")
 
 	// Bind Flags with Viper
 	//// DB Flags
-	err = viper.BindPFlag("db.username", captureCmd.PersistentFlags().Lookup("db.username"))
+	err := viper.BindPFlag("db.username", captureCmd.PersistentFlags().Lookup("db.username"))
 	exitErr(err)
 	err = viper.BindPFlag("db.password", captureCmd.PersistentFlags().Lookup("db.password"))
 	exitErr(err)
@@ -84,21 +130,53 @@ func init() {
 	err = viper.BindPFlag("db.driver", captureCmd.PersistentFlags().Lookup("db.driver"))
 	exitErr(err)
 
-	// LH specific
+	//// Testing Specific
+	err = viper.BindPFlag("t.skipSync", captureCmd.PersistentFlags().Lookup("t.skipSync"))
+	exitErr(err)
+
+	//// LH specific
 	err = viper.BindPFlag("bc.address", captureCmd.PersistentFlags().Lookup("bc.address"))
 	exitErr(err)
+	err = viper.BindPFlag("bc.type", captureCmd.PersistentFlags().Lookup("bc.type"))
+	exitErr(err)
 	err = viper.BindPFlag("bc.port", captureCmd.PersistentFlags().Lookup("bc.port"))
 	exitErr(err)
 	err = viper.BindPFlag("bc.connectionProtocol", captureCmd.PersistentFlags().Lookup("bc.connectionProtocol"))
 	exitErr(err)
+	err = viper.BindPFlag("bc.bootRetryInterval", captureCmd.PersistentFlags().Lookup("bc.bootRetryInterval"))
+	exitErr(err)
+	err = viper.BindPFlag("bc.bootMaxRetry", captureCmd.PersistentFlags().Lookup("bc.bootMaxRetry"))
+	exitErr(err)
+	err = viper.BindPFlag("bc.maxHistoricProcessWorker", captureCmd.PersistentFlags().Lookup("bc.maxHistoricProcessWorker"))
+	exitErr(err)
+	err = viper.BindPFlag("bc.uniqueNodeIdentifier", captureCmd.PersistentFlags().Lookup("bc.uniqueNodeIdentifier"))
+	exitErr(err)
+	err = viper.BindPFlag("bc.checkDb", captureCmd.PersistentFlags().Lookup("bc.checkDb"))
+	exitErr(err)
 	// Here you will define your flags and configuration settings.
 
+	//// Known Gap Specific
+	err = viper.BindPFlag("kg.processKnownGaps", captureCmd.PersistentFlags().Lookup("kg.processKnownGaps"))
+	exitErr(err)
+	err = viper.BindPFlag("kg.increment", captureCmd.PersistentFlags().Lookup("kg.increment"))
+	exitErr(err)
+	err = viper.BindPFlag("kg.processKnownGaps", captureCmd.PersistentFlags().Lookup("kg.maxKnownGapsWorker"))
+	exitErr(err)
+
+	// Prometheus Specific
+	err = viper.BindPFlag("pm.metrics", captureCmd.PersistentFlags().Lookup("pm.metrics"))
+	exitErr(err)
+	err = viper.BindPFlag("pm.address", captureCmd.PersistentFlags().Lookup("pm.address"))
+	exitErr(err)
+	err = viper.BindPFlag("pm.port", captureCmd.PersistentFlags().Lookup("pm.port"))
+	exitErr(err)
 }
 
 // Helper function to catch any errors.
 // We need to capture these errors for the linter.
 func exitErr(err error) {
 	if err != nil {
+		fmt.Println("Error: ", err)
 		os.Exit(1)
 	}
 }

cmd/full.go

@@ -0,0 +1,118 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
+)
+
+// fullCmd represents the full command
+var fullCmd = &cobra.Command{
+	Use:   "full",
+	Short: "Capture all components of the application (head and historical)",
+	Long:  `Capture all components of the application (head and historical`,
+	Run: func(cmd *cobra.Command, args []string) {
+		startFullProcessing()
+	},
+}
+
+func init() {
+	captureCmd.AddCommand(fullCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// fullCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// fullCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}
+
+// Start the application to track at head and historical processing.
+func startFullProcessing() {
+	// Boot the application
+	log.Info("Starting the application in head tracking mode.")
+	ctx := context.Background()
+
+	Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
+		viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
+		viper.GetInt("kg.increment"), "head", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
+	if err != nil {
+		StopApplicationPreBoot(err, Db)
+	}
+
+	if viper.GetBool("pm.metrics") {
+		addr := viper.GetString("pm.address") + ":" + strconv.Itoa(viper.GetInt("pm.port"))
+		serveProm(addr)
+	}
+
+	log.Info("The Beacon Client has booted successfully!")
+	// Capture head blocks
+	go Bc.CaptureHead()
+
+	hpContext, hpCancel := context.WithCancel(context.Background())
+
+	errG, _ := errgroup.WithContext(context.Background())
+	errG.Go(func() error {
+		errs := Bc.CaptureHistoric(hpContext, viper.GetInt("bc.maxHistoricProcessWorker"))
+		if len(errs) != 0 {
+			if len(errs) != 0 {
+				log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing historic events")
+				return fmt.Errorf("Application ended because there were too many error when attempting to process historic")
+			}
+		}
+		return nil
+	})
+	kgCtx, KgCancel := context.WithCancel(context.Background())
+	if viper.GetBool("kg.processKnownGaps") {
+		go func() {
+			errG := new(errgroup.Group)
+			errG.Go(func() error {
+				errs := Bc.ProcessKnownGaps(kgCtx, viper.GetInt("kg.maxKnownGapsWorker"))
+				if len(errs) != 0 {
+					log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
+					return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
+				}
+				return nil
+			})
+			if err := errG.Wait(); err != nil {
+				loghelper.LogError(err).Error("Error with knownGaps processing")
+			}
+		}()
+	}
+
+	// Shutdown when the time is right.
+	err = shutdown.ShutdownFull(ctx, KgCancel, hpCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
+	if err != nil {
+		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
+	} else {
+		log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
+	}
+
+}

cmd/head.go

@@ -1,18 +1,35 @@
-/*
+// VulcanizeDB
-Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-*/
 package cmd
 
 import (
 	"context"
-	"time"
+	"fmt"
+	"net/http"
+	"strconv"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
+	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
 )
 
 // headCmd represents the head command
@@ -31,28 +48,65 @@ func startHeadTracking() {
 	log.Info("Starting the application in head tracking mode.")
 	ctx := context.Background()
 
-	BC, DB, err := boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol)
+	Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
+		viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
+		viper.GetInt("kg.increment"), "head", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
 	if err != nil {
-		loghelper.LogError(err).Error("Unable to Start application")
+		StopApplicationPreBoot(err, Db)
 	}
 
+	if viper.GetBool("pm.metrics") {
+		addr := viper.GetString("pm.address") + ":" + strconv.Itoa(viper.GetInt("pm.port"))
+		serveProm(addr)
+	}
+
+	log.Info("The Beacon Client has booted successfully!")
 	// Capture head blocks
-	go BC.CaptureHead()
+	go Bc.CaptureHead()
+	kgCtx, KgCancel := context.WithCancel(context.Background())
+	if viper.GetBool("kg.processKnownGaps") {
+		go func() {
+			errG := new(errgroup.Group)
+			errG.Go(func() error {
+				errs := Bc.ProcessKnownGaps(kgCtx, viper.GetInt("kg.maxKnownGapsWorker"))
+				if len(errs) != 0 {
+					log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
+					return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
+				}
+				return nil
+			})
+			if err := errG.Wait(); err != nil {
+				loghelper.LogError(err).Error("Error with knownGaps processing")
+			}
+		}()
+	}
 
 	// Shutdown when the time is right.
-	shutdown.ShutdownServices(ctx, time.Duration(maxWaitSecondsShutdown), DB, BC)
+	err = shutdown.ShutdownHeadTracking(ctx, KgCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
+	if err != nil {
+		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
+	} else {
+		log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
+	}
+
 }
 
 func init() {
 	captureCmd.AddCommand(headCmd)
-
+}
-	// Here you will define your flags and configuration settings.
+
-
+// Start prometheus server
-	// Cobra supports Persistent Flags which will work for this command
+func serveProm(addr string) {
-	// and all subcommands, e.g.:
+	mux := http.NewServeMux()
-	// headCmd.PersistentFlags().String("foo", "", "A help for foo")
+	mux.Handle("/metrics", promhttp.Handler())
-
+
-	// Cobra supports local flags which will only run when this command
+	srv := http.Server{
-	// is called directly, e.g.:
+		Addr:    addr,
-	// headCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+		Handler: mux,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			loghelper.LogError(err).WithField("endpoint", addr).Error("Error with prometheus")
+		}
+	}()
 }

cmd/historic.go

@@ -1,13 +1,35 @@
-/*
+// VulcanizeDB
-Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-*/
 package cmd
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"strconv"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
 )
 
 // historicCmd represents the historic command
@@ -16,10 +38,69 @@ var historicCmd = &cobra.Command{
 	Short: "Capture the historic blocks and states.",
 	Long:  `Capture the historic blocks and states.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("historic called")
+		startHistoricProcessing()
 	},
 }
 
+// Start the application to process historical slots.
+func startHistoricProcessing() {
+	// Boot the application
+	log.Info("Starting the application in head tracking mode.")
+	ctx := context.Background()
+
+	Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
+		viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
+		viper.GetInt("kg.increment"), "historic", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
+	if err != nil {
+		StopApplicationPreBoot(err, Db)
+	}
+
+	if viper.GetBool("pm.metrics") {
+		addr := viper.GetString("pm.address") + ":" + strconv.Itoa(viper.GetInt("pm.port"))
+		serveProm(addr)
+	}
+
+	hpContext, hpCancel := context.WithCancel(context.Background())
+
+	errG, _ := errgroup.WithContext(context.Background())
+	errG.Go(func() error {
+		errs := Bc.CaptureHistoric(hpContext, viper.GetInt("bc.maxHistoricProcessWorker"))
+		if len(errs) != 0 {
+			if len(errs) != 0 {
+				log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing historic events")
+				return fmt.Errorf("Application ended because there were too many error when attempting to process historic")
+			}
+		}
+		return nil
+	})
+
+	kgContext, kgCancel := context.WithCancel(context.Background())
+	if viper.GetBool("kg.processKnownGaps") {
+		go func() {
+			errG := new(errgroup.Group)
+			errG.Go(func() error {
+				errs := Bc.ProcessKnownGaps(kgContext, viper.GetInt("kg.maxKnownGapsWorker"))
+				if len(errs) != 0 {
+					log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
+					return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
+				}
+				return nil
+			})
+			if err := errG.Wait(); err != nil {
+				loghelper.LogError(err).Error("Error with knownGaps processing")
+			}
+		}()
+	}
+
+	// Shutdown when the time is right.
+	err = shutdown.ShutdownHistoricProcessing(ctx, kgCancel, hpCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
+	if err != nil {
+		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
+	} else {
+		log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
+	}
+}
+
 func init() {
 	captureCmd.AddCommand(historicCmd)
 
@@ -33,3 +114,12 @@ func init() {
 	// is called directly, e.g.:
 	// historicCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 }
+
+// Stop the application during its initial boot phases.
+func StopApplicationPreBoot(startErr error, db sql.Database) {
+	loghelper.LogError(startErr).Error("Unable to Start application")
+	if db != nil {
+		db.Close()
+	}
+	os.Exit(1)
+}

cmd/root.go

@@ -1,7 +1,19 @@
-/*
+// VulcanizeDB
-Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-*/
 package cmd
 
 import (
@@ -20,7 +32,7 @@ var (
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:   "ipld-ethcl-indexer",
+	Use:   "ipld-eth-beacon-indexer",
 	Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.",
 	Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain.
 It can either do this will keeping track of head, or backfilling historic data.`,
@@ -35,6 +47,7 @@ It can either do this will keeping track of head, or backfilling historic data.`
 func Execute() {
 	err := rootCmd.Execute()
 	if err != nil {
+		fmt.Println("Err when executing rootCmd", err)
 		os.Exit(1)
 	}
 }
@@ -114,9 +127,9 @@ func init() {
 	// will be global for your application.
 
 	// Optional Flags
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)")
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-eth-beacon-indexer.yaml)")
 	rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
-	rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging")
+	rootCmd.PersistentFlags().String("log.file", "ipld-eth-beacon-indexer.log", "file path for logging")
 	rootCmd.PersistentFlags().Bool("log.output", true, "Should we log to STDOUT")
 	rootCmd.PersistentFlags().String("log.format", "json", "json or text")
 
@@ -147,10 +160,10 @@ func initConfig() {
 		home, err := os.UserHomeDir()
 		cobra.CheckErr(err)
 
-		// Search config in home directory with name ".ipld-ethcl-indexer" (without extension).
+		// Search config in home directory with name ".ipld-eth-beacon-indexer" (without extension).
 		viper.AddConfigPath(home)
 		viper.SetConfigType("yaml")
-		viper.SetConfigName(".ipld-ethcl-indexer")
+		viper.SetConfigName(".ipld-eth-beacon-indexer")
 	}
 
 	viper.AutomaticEnv() // read in environment variables that match

cmd/version.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package cmd
 
 import (
@@ -5,7 +20,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	v "github.com/vulcanize/ipld-ethcl-indexer/pkg/version"
+	v "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/version"
 )
 
 var (
@@ -32,7 +47,7 @@ to quickly create a Cobra application.`,
 			Patch: Patch,
 			Meta:  Meta,
 		}
-		log.Infof("ipld-ethcl-indexer version: %s", version.GetVersionWithMeta())
+		log.Infof("ipld-eth-beacon-indexer version: %s", version.GetVersionWithMeta())
 		fmt.Println(version.GetVersionWithMeta())
 	},
 }

config/cicd/boot.ipld-eth-beacon-indexer.json

@@ -0,0 +1,40 @@
+{
+  "db": {
+    "address": "ipld-eth-beacon-db",
+    "password": "password",
+    "port": 5432,
+    "username": "vdbm",
+    "name": "vulcanize_testing",
+    "driver": "PGX"
+  },
+  "bc": {
+    "address": "lighthouse",
+    "port": 5052,
+    "type": "lighthouse",
+    "bootRetryInterval": 30,
+    "bootMaxRetry": 5,
+    "maxHistoricProcessWorker": 2,
+    "connectionProtocol": "http",
+    "uniqueNodeIdentifier": 100,
+    "checkDb": true
+  },
+  "t": {
+    "skipSync": true
+  },
+  "log": {
+    "level": "debug",
+    "output": true,
+    "file": "./ipld-eth-beacon-indexer.log",
+    "format": "json"
+  },
+  "kg": {
+    "increment": 10000,
+    "processKnownGaps": true,
+    "maxKnownGapsWorker": 2
+  },
+  "pm": {
+    "address": "localhost",
+    "port": 9000,
+    "metrics": true
+  }
+}

config/example.ipld-eth-beacon-indexer-config.json

@@ -0,0 +1,40 @@
+{
+  "db": {
+    "address": "localhost",
+    "password": "password",
+    "port": 8076,
+    "username": "vdbm",
+    "name": "vulcanize_testing",
+    "driver": "PGX"
+  },
+  "bc": {
+    "address": "localhost",
+    "port": 5052,
+    "type": "lighthouse",
+    "bootRetryInterval": 30,
+    "bootMaxRetry": 5,
+    "maxHistoricProcessWorker": 2,
+    "connectionProtocol": "http",
+    "uniqueNodeIdentifier": 100,
+    "checkDb": true
+  },
+  "t": {
+    "skipSync": true
+  },
+  "log": {
+    "level": "debug",
+    "output": true,
+    "file": "./ipld-eth-beacon-indexer.log",
+    "format": "json"
+  },
+  "kg": {
+    "increment": 10000,
+    "processKnownGaps": true,
+    "maxKnownGapsWorker": 2
+  },
+  "pm": {
+    "address": "localhost",
+    "port": 9000,
+    "metrics": true
+  }
+}

entrypoint.sh

@@ -1,31 +1,24 @@
 #!/bin/bash
-echo "Starting ipld-ethcl-indexer"
 
-echo /root/ipld-ethcl-indexer capture head --db.address $DB_ADDRESS \
+sleep 10
-  --db.password $DB_PASSWORD \
+echo "Starting ipld-eth-beacon-indexer"
-  --db.port $DB_PORT \
-  --db.username $DB_USER \
-  --db.name $DB_NAME \
-  --db.driver $DB_DRIVER \
-  --bc.address $BC_ADDRESS \
-  --bc.port $BC_PORT \
-  --log.level $LOG_LEVEL
 
-/root/ipld-ethcl-indexer capture head --db.address $DB_ADDRESS \
+echo /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
-  --db.password $DB_PASSWORD \
-  --db.port $DB_PORT \
-  --db.username $DB_USER \
-  --db.name $DB_NAME \
-  --db.driver $DB_DRIVER \
-  --bc.address $BC_ADDRESS \
-  --bc.port $BC_PORT \
-  --log.level $LOG_LEVEL
 
-rv=$?
+if [ ${CAPTURE_MODE} == "boot" ]; then
+    /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
+    rv=$?
 
-if [ $rv != 0 ]; then
+    if [ $rv != 0 ]; then
-  echo "ipld-ethcl-indexer startup failed"
+      echo "ipld-eth-beacon-indexer boot failed"
-  exit 1
+    else
+      echo "ipld-eth-beacon-indexer boot succeeded"
+    fi
+      echo $rv > /root/HEALTH
+      echo $rv
+      cat /root/ipld-eth-beacon-indexer.output
+
+    tail -f /dev/null
+else
+    exec /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
 fi
-
-tail -f /dev/null

go.mod

@@ -1,44 +1,99 @@
-module github.com/vulcanize/ipld-ethcl-indexer
+module github.com/vulcanize/ipld-eth-beacon-indexer
 
 go 1.18
 
 require (
-	github.com/jackc/pgconn v1.11.0
+	github.com/ipfs/go-ipfs-blockstore v1.2.0
-	github.com/onsi/ginkgo/v2 v2.1.3
+	github.com/ipfs/go-ipfs-ds-help v1.1.0
+	github.com/jackc/pgconn v1.12.0
+	github.com/multiformats/go-multihash v0.1.0
+	github.com/onsi/ginkgo/v2 v2.1.4
 	github.com/onsi/gomega v1.19.0
+	github.com/prometheus/client_golang v1.12.1
+	github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc
 	github.com/sirupsen/logrus v1.8.1
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/ethereum/go-ethereum v1.10.17 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.0.3 // indirect
+	github.com/ipfs/go-cid v0.1.0 // indirect
+	github.com/ipfs/go-datastore v0.5.0 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-format v0.3.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.5.0 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.10.0 // indirect
+	github.com/jackc/pgtype v1.11.0 // indirect
 	github.com/jackc/puddle v1.2.1 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
-	github.com/lib/pq v1.10.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
-	github.com/minio/sha256-simd v0.1.1 // indirect
+	github.com/lib/pq v1.10.5 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/minio/highwayhash v1.0.1 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.0.4 // indirect
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.0.3 // indirect
+	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
-	golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect
+	github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220303211031-f753e083138c // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/supranational/blst v0.3.5 // indirect
+	github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect
+	github.com/urfave/cli/v2 v2.3.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/zap v1.21.0 // indirect
+	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
+	google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 // indirect
+	google.golang.org/grpc v1.46.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	lukechampine.com/blake3 v1.1.7 // indirect
 )
 
 require (
-	github.com/ferranbt/fastssz v0.0.0-20220303160658-88bb965b6747
+	github.com/ferranbt/fastssz v0.0.0-20220303160658-88bb965b6747 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/georgysavva/scany v0.3.0
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jackc/pgx/v4 v4.15.0
+	github.com/jackc/pgx/v4 v4.16.0
+	github.com/jarcoal/httpmock v1.2.0
 	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.0 // indirect
-	github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc
+	github.com/prysmaticlabs/prysm v1.4.2-0.20220504145118-df695346a53c
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.4.0
@@ -46,7 +101,8 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.11.0
 	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

internal/boot/boot.go

@@ -1,53 +1,37 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package boot
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql/postgres"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
 )
 
 var (
-	maxRetry                                 = 5  // Max times to try to connect to the DB or BC at boot.
+	DB sql.Database               = &postgres.DB{}
-	retryInterval                            = 30 // The time to wait between each try.
+	BC *beaconclient.BeaconClient = &beaconclient.BeaconClient{}
-	DB            sql.Database               = &postgres.DB{}
-	BC            *beaconclient.BeaconClient = &beaconclient.BeaconClient{}
 )
 
-// A simple wrapper to create a DB object to use.
-func SetupPostgresDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (sql.Database, error) {
-	log.Debug("Resolving Driver Type")
-	DbDriver, err := postgres.ResolveDriverType(driverName)
-	if err != nil {
-		log.WithFields(log.Fields{
-			"err":                  err,
-			"driver_name_provided": driverName,
-		}).Error("Can't resolve driver type")
-	}
-	log.Info("Using Driver:", DbDriver)
-
-	postgresConfig := postgres.Config{
-		Hostname:     dbHostname,
-		Port:         dbPort,
-		DatabaseName: dbName,
-		Username:     dbUsername,
-		Password:     dbPassword,
-		Driver:       DbDriver,
-	}
-	DB, err = postgres.NewPostgresDB(postgresConfig)
-
-	if err != nil {
-		loghelper.LogError(err).Error("Unable to connect to the DB")
-		return nil, err
-	}
-	return DB, err
-
-}
-
 // This function will perform some boot operations. If any steps fail, the application will fail to start.
 // Keep in mind that the DB connection can be lost later in the lifecycle of the application or
 // it might not be able to connect to the beacon client.
@@ -56,38 +40,113 @@ func SetupPostgresDb(dbHostname string, dbPort int, dbName string, dbUsername st
 //
 // 2. Connect to the database.
 //
-func BootApplication(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int, bcConnectionProtocol string) (*beaconclient.BeaconClient, sql.Database, error) {
+// 3. Make sure the node is synced, unless disregardSync is true.
+func BootApplication(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string,
+	bcAddress string, bcPort int, bcConnectionProtocol string, bcKgTableIncrement int, disregardSync bool, uniqueNodeIdentifier int, checkDb bool) (*beaconclient.BeaconClient, sql.Database, error) {
 	log.Info("Booting the Application")
 
 	log.Debug("Creating the Beacon Client")
-	BC = beaconclient.CreateBeaconClient(ctx, bcConnectionProtocol, bcAddress, bcPort)
+	Bc, err := beaconclient.CreateBeaconClient(ctx, bcConnectionProtocol, bcAddress, bcPort, bcKgTableIncrement, uniqueNodeIdentifier, checkDb)
+	if err != nil {
+		return Bc, nil, err
+	}
 
 	log.Debug("Checking Beacon Client")
-	err := BC.CheckBeaconClient()
+	err = Bc.CheckBeaconClient()
 	if err != nil {
 		return nil, nil, err
 	}
 
 	log.Debug("Setting up DB connection")
-	DB, err := SetupPostgresDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName)
+	DB, err = postgres.SetupPostgresDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName)
 	if err != nil {
 		return nil, nil, err
 	}
-	return BC, DB, nil
+
+	Bc.Db = DB
+
+	var status bool
+	if !disregardSync {
+		status, err = Bc.CheckHeadSync()
+		if err != nil {
+			log.Error("Unable to get the nodes sync status")
+			return Bc, DB, err
+		}
+		if status {
+			log.Error("The node is still syncing..")
+			err = fmt.Errorf("The node is still syncing.")
+			return Bc, DB, err
+		}
+	} else {
+		log.Warn("We are not checking to see if the node has synced to head.")
+	}
+	return Bc, DB, nil
 }
 
 // Add retry logic to ensure that we are give the Beacon Client and the DB time to start.
-func BootApplicationWithRetry(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int, bcConnectionProtocol string) (*beaconclient.BeaconClient, sql.Database, error) {
+func BootApplicationWithRetry(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string,
+	bcAddress string, bcPort int, bcConnectionProtocol string, bcType string, bcRetryInterval int, bcMaxRetry int, bcKgTableIncrement int,
+	startUpMode string, disregardSync bool, uniqueNodeIdentifier int, checkDb bool) (*beaconclient.BeaconClient, sql.Database, error) {
 	var err error
-	for i := 0; i < maxRetry; i++ {
+
-		BC, DB, err = BootApplication(ctx, dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName, bcAddress, bcPort, bcConnectionProtocol)
+	if bcMaxRetry < 0 {
-		if err != nil {
+		i := 0
-			log.WithFields(log.Fields{
+		for {
-				"retryNumber": i,
+			BC, DB, err = BootApplication(ctx, dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName,
-			}).Warn("Unable to boot application. Going to try again")
+				bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, disregardSync, uniqueNodeIdentifier, checkDb)
-			time.Sleep(time.Duration(retryInterval) * time.Second)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"retryNumber": i,
+					"err":         err,
+				}).Warn("Unable to boot application. Going to try again")
+				time.Sleep(time.Duration(bcRetryInterval) * time.Second)
+				i = i + 1
+				continue
+			}
+			break
+		}
+	} else {
+		for i := 0; i < bcMaxRetry; i++ {
+			BC, DB, err = BootApplication(ctx, dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName,
+				bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, disregardSync, uniqueNodeIdentifier, checkDb)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"retryNumber": i,
+					"err":         err,
+				}).Warn("Unable to boot application. Going to try again")
+				time.Sleep(time.Duration(bcRetryInterval) * time.Second)
+				continue
+			}
+			break
 		}
-		break
 	}
+
+	switch strings.ToLower(startUpMode) {
+	case "head":
+		BC.PerformHeadTracking = true
+	case "historic":
+		log.Debug("Performing additional boot steps for historical processing")
+		BC.PerformHistoricalProcessing = true
+		// This field is not currently used.
+		// The idea is, that if we are doing historially processing and we get a slot
+		// greater than this slot, then we would rerun this function.
+		// this would ensure that we have the slots necessary for processing
+		// within the beacon server.
+
+		// We can implement this feature if we notice any errors.
+		headSlot, err := BC.GetLatestSlotInBeaconServer(bcType)
+		if err != nil {
+			return BC, DB, err
+		}
+		BC.UpdateLatestSlotInBeaconServer(int64(headSlot))
+		// Add another switch case for bcType if its ever needed.
+	case "boot":
+		log.Debug("Running application in boot mode.")
+	default:
+		log.WithFields(log.Fields{
+			"startUpMode": startUpMode,
+		}).Error("The startUpMode provided is not handled.")
+	}
+
 	return BC, DB, err
 }

internal/boot/boot_suite_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package boot_test
 
 import (

internal/boot/boot_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package boot_test
 
 import (
@@ -5,13 +20,13 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
 )
 
 var _ = Describe("Boot", func() {
 	var (
 		dbAddress            string = "localhost"
-		dbPort               int    = 8077
+		dbPort               int    = 8076
 		dbName               string = "vulcanize_testing"
 		dbUsername           string = "vdbm"
 		dbPassword           string = "password"
@@ -19,31 +34,58 @@ var _ = Describe("Boot", func() {
 		bcAddress            string = "localhost"
 		bcPort               int    = 5052
 		bcConnectionProtocol string = "http"
+		bcType               string = "lighthouse"
+		bcBootRetryInterval  int    = 1
+		bcBootMaxRetry       int    = 5
+		bcKgTableIncrement   int    = 10
+		bcUniqueIdentifier   int    = 100
+		bcCheckDb            bool   = false
 	)
 	Describe("Booting the application", Label("integration"), func() {
-		Context("When the DB and BC are both up and running", func() {
+		Context("When the DB and BC are both up and running, we skip checking for a synced head, and we are processing head", func() {
 			It("Should connect successfully", func() {
-				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol)
+				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", true, bcUniqueIdentifier, bcCheckDb)
 				defer db.Close()
-				Expect(err).To(BeNil())
+				Expect(err).ToNot(HaveOccurred())
+			})
+		})
+		Context("When the DB and BC are both up and running, we skip checking for a synced head, and we are processing historic ", func() {
+			It("Should connect successfully", func() {
+				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "historic", true, bcUniqueIdentifier, bcCheckDb)
+				defer db.Close()
+				Expect(err).ToNot(HaveOccurred())
+			})
+		})
+		Context("When the DB and BC are both up and running, and we check for a synced head", func() {
+			It("Should not connect successfully", func() {
+				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", false, bcUniqueIdentifier, bcCheckDb)
+				defer db.Close()
+				Expect(err).To(HaveOccurred())
+			})
+		})
+		Context("When the DB and BC are both up and running, we skip checking for a synced head, but the unique identifier is 0", func() {
+			It("Should not connect successfully", func() {
+				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", false, 0, bcCheckDb)
+				defer db.Close()
+				Expect(err).To(HaveOccurred())
 			})
 		})
 		Context("When the DB is running but not the BC", func() {
 			It("Should not connect successfully", func() {
-				_, _, err := boot.BootApplication(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol)
+				_, _, err := boot.BootApplication(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
-				Expect(err).ToNot(BeNil())
+				Expect(err).To(HaveOccurred())
 			})
 		})
 		Context("When the BC is running but not the DB", func() {
 			It("Should not connect successfully", func() {
-				_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol)
+				_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
-				Expect(err).ToNot(BeNil())
+				Expect(err).To(HaveOccurred())
 			})
 		})
 		Context("When neither the BC or DB are running", func() {
 			It("Should not connect successfully", func() {
-				_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol)
+				_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
-				Expect(err).ToNot(BeNil())
+				Expect(err).To(HaveOccurred())
 			})
 		})
 	})

internal/shutdown/shutdown.go

@@ -1,39 +1,119 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package shutdown
 
 import (
 	"context"
+	"os"
 	"time"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/gracefulshutdown"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/gracefulshutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
 )
 
 // Shutdown all the internal services for the application.
-func ShutdownServices(ctx context.Context, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) {
+func ShutdownServices(ctx context.Context, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient, shutdownOperations map[string]gracefulshutdown.Operation) error {
-	successCh, errCh := gracefulshutdown.Shutdown(ctx, waitTime, map[string]gracefulshutdown.Operation{
+	//successCh, errCh := gracefulshutdown.Shutdown(ctx, notifierCh, waitTime, )
-		"database": func(ctx context.Context) error {
+	successCh, errCh := gracefulshutdown.Shutdown(ctx, notifierCh, waitTime, shutdownOperations)
-			err := DB.Close()
+
-			if err != nil {
+	select {
-				loghelper.LogError(err).Error("Unable to close the DB")
+	case <-successCh:
-			}
+		return nil
-			return err
+	case err := <-errCh:
-		},
+		return err
+	}
+}
+
+// Wrapper function for shutting down the head tracking process.
+func ShutdownHeadTracking(ctx context.Context, kgCancel context.CancelFunc, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
+	return ShutdownServices(ctx, notifierCh, waitTime, DB, BC, map[string]gracefulshutdown.Operation{
+		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
 		"beaconClient": func(ctx context.Context) error {
+			defer DB.Close()
 			err := BC.StopHeadTracking()
 			if err != nil {
 				loghelper.LogError(err).Error("Unable to trigger shutdown of head tracking")
 			}
+			if BC.KnownGapsProcess != (beaconclient.KnownGapsProcessing{}) {
+				err = BC.StopKnownGapsProcessing(kgCancel)
+				if err != nil {
+					loghelper.LogError(err).Error("Unable to stop processing known gaps")
+				}
+			}
+			return err
+		},
+	})
+}
+
+// Wrapper function for shutting down the head tracking process.
+func ShutdownHistoricProcessing(ctx context.Context, kgCancel, hpCancel context.CancelFunc, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
+	return ShutdownServices(ctx, notifierCh, waitTime, DB, BC, map[string]gracefulshutdown.Operation{
+		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
+		"beaconClient": func(ctx context.Context) error {
+			defer DB.Close()
+			err := BC.StopHistoric(hpCancel)
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to stop processing historic")
+			}
+			if BC.KnownGapsProcess != (beaconclient.KnownGapsProcessing{}) {
+				err = BC.StopKnownGapsProcessing(kgCancel)
+				if err != nil {
+					loghelper.LogError(err).Error("Unable to stop processing known gaps")
+				}
+			}
+			return err
+		},
+	})
+}
+
+// Shutdown the head and historical processing
+func ShutdownFull(ctx context.Context, kgCancel, hpCancel context.CancelFunc, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
+	return ShutdownServices(ctx, notifierCh, waitTime, DB, BC, map[string]gracefulshutdown.Operation{
+		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
+		"beaconClient": func(ctx context.Context) error {
+			defer DB.Close()
+			err := BC.StopHistoric(hpCancel)
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to stop processing historic")
+			}
+			if BC.KnownGapsProcess != (beaconclient.KnownGapsProcessing{}) {
+				err = BC.StopKnownGapsProcessing(kgCancel)
+				if err != nil {
+					loghelper.LogError(err).Error("Unable to stop processing known gaps")
+				}
+			}
+			err = BC.StopHeadTracking()
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to trigger shutdown of head tracking")
+			}
+
 			return err
 		},
 	})
 
-	select {
+}
-	case _ = <-successCh:
+
-		log.Info("Gracefully Shutdown ipld-ethcl-indexer!")
+// Wrapper function for shutting down the application in boot mode.
-	case err := <-errCh:
+func ShutdownBoot(ctx context.Context, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
-		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!")
+	return ShutdownServices(ctx, notifierCh, waitTime, DB, BC, map[string]gracefulshutdown.Operation{
-	}
+		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
+		"Database": func(ctx context.Context) error {
+			return DB.Close()
+		},
+	})
 }

internal/shutdown/shutdown_suite_test.go

@@ -0,0 +1,28 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package shutdown_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestShutdown(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Shutdown Suite")
+}

internal/shutdown/shutdown_test.go

@@ -0,0 +1,144 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//go:build !race
+// +build !race
+
+package shutdown_test
+
+import (
+	"context"
+	"os"
+	"syscall"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/r3labs/sse"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/gracefulshutdown"
+)
+
+var (
+	dbAddress              string        = "localhost"
+	dbPort                 int           = 8076
+	dbName                 string        = "vulcanize_testing"
+	dbUsername             string        = "vdbm"
+	dbPassword             string        = "password"
+	dbDriver               string        = "PGX"
+	bcAddress              string        = "localhost"
+	bcPort                 int           = 5052
+	bcConnectionProtocol   string        = "http"
+	bcType                 string        = "lighthouse"
+	bcBootRetryInterval    int           = 1
+	bcBootMaxRetry         int           = 5
+	bcKgTableIncrement     int           = 10
+	bcUniqueIdentifier     int           = 100
+	bcCheckDb              bool          = false
+	maxWaitSecondsShutdown time.Duration = time.Duration(1) * time.Second
+	DB                     sql.Database
+	BC                     *beaconclient.BeaconClient
+	err                    error
+	ctx                    context.Context
+	notifierCh             chan os.Signal
+)
+
+var _ = Describe("Shutdown", func() {
+	BeforeEach(func() {
+		ctx = context.Background()
+		BC, DB, err = boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress,
+			bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", true, bcUniqueIdentifier, bcCheckDb)
+		notifierCh = make(chan os.Signal, 1)
+		Expect(err).To(BeNil())
+	})
+
+	Describe("Run Shutdown Function for head tracking,", Label("integration"), func() {
+		Context("When Channels are empty,", func() {
+			It("Should Shutdown Successfully.", func() {
+				go func() {
+					_, cancel := context.WithCancel(context.Background())
+					log.Debug("Starting shutdown chan")
+					err = shutdown.ShutdownHeadTracking(ctx, cancel, notifierCh, maxWaitSecondsShutdown, DB, BC)
+					log.Debug("We have completed the shutdown...")
+					Expect(err).ToNot(HaveOccurred())
+				}()
+			})
+		})
+		Context("When the Channels are not empty,", func() {
+			It("Should try to clear them and shutdown gracefully.", func() {
+				shutdownCh := make(chan bool)
+				//log.SetLevel(log.DebugLevel)
+				go func() {
+					_, cancel := context.WithCancel(context.Background())
+					log.Debug("Starting shutdown chan")
+					err = shutdown.ShutdownHeadTracking(ctx, cancel, notifierCh, maxWaitSecondsShutdown, DB, BC)
+					log.Debug("We have completed the shutdown...")
+					Expect(err).ToNot(HaveOccurred())
+					shutdownCh <- true
+				}()
+
+				messageAddCh := make(chan bool)
+				go func() {
+					log.Debug("Adding messages to Channels")
+					BC.HeadTracking.MessagesCh <- &sse.Event{}
+					//BC.FinalizationTracking.MessagesCh <- &sse.Event{}
+					BC.ReOrgTracking.MessagesCh <- &sse.Event{}
+					log.Debug("Message adding complete")
+					messageAddCh <- true
+				}()
+
+				go func() {
+					<-messageAddCh
+					log.Debug("Calling SIGTERM")
+					notifierCh <- syscall.SIGTERM
+					log.Debug("Reading messages from channel")
+					<-BC.HeadTracking.MessagesCh
+					//<-BC.FinalizationTracking.MessagesCh
+					<-BC.ReOrgTracking.MessagesCh
+				}()
+				<-shutdownCh
+
+			})
+			It("Should try to clear them, if it can't, shutdown within a given time frame.", func() {
+				shutdownCh := make(chan bool)
+				//log.SetLevel(log.DebugLevel)
+				go func() {
+					log.Debug("Starting shutdown chan")
+					_, cancel := context.WithCancel(context.Background())
+					err = shutdown.ShutdownHeadTracking(ctx, cancel, notifierCh, maxWaitSecondsShutdown, DB, BC)
+					log.Debug("We have completed the shutdown...")
+					Expect(err).To(MatchError(gracefulshutdown.TimeoutErr(maxWaitSecondsShutdown.String())))
+					shutdownCh <- true
+				}()
+
+				go func() {
+					log.Debug("Adding messages to Channels")
+					BC.HeadTracking.MessagesCh <- &sse.Event{}
+					//BC.FinalizationTracking.MessagesCh <- &sse.Event{}
+					BC.ReOrgTracking.MessagesCh <- &sse.Event{}
+					log.Debug("Message adding complete")
+					log.Debug("Calling SIGHUP")
+					notifierCh <- syscall.SIGTERM
+				}()
+
+				<-shutdownCh
+			})
+		})
+	})
+})

main.go

@@ -1,10 +1,25 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 /*
 Copyright © 2022 NAME HERE <EMAIL ADDRESS>
 
 */
 package main
 
-import "github.com/vulcanize/ipld-ethcl-indexer/cmd"
+import "github.com/vulcanize/ipld-eth-beacon-indexer/cmd"
 
 func main() {
 	cmd.Execute()

pkg/beaconclient/beaconclient.go

@@ -1,31 +1,75 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
 import (
 	"context"
 	"fmt"
+	"math/rand"
 
 	"github.com/r3labs/sse"
 	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
 )
 
+// TODO: Use prysms config values instead of hardcoding them here.
 var (
-	bcHealthEndpoint         = "/eth/v1/node/health"                        // Endpoint used for the healthcheck
+	bcHealthEndpoint     = "/eth/v1/node/health"               // Endpoint used for the healthcheck
-	bcHeadTopicEndpoint      = "/eth/v1/events?topics=head"                 // Endpoint used to subscribe to the head of the chain
+	BcHeadTopicEndpoint  = "/eth/v1/events?topics=head"        // Endpoint used to subscribe to the head of the chain
-	bcReorgTopicEndpoint     = "/eth/v1/events?topics=chain_reorg"          // Endpoint used to subscribe to the head of the chain
+	bcReorgTopicEndpoint = "/eth/v1/events?topics=chain_reorg" // Endpoint used to subscribe to the head of the chain
-	bcFinalizedTopicEndpoint = "/eth/v1/events?topics=finalized_checkpoint" // Endpoint used to subscribe to the head of the chain
+	BcBlockQueryEndpoint = "/eth/v2/beacon/blocks/"            // Endpoint to query individual Blocks
-	bcBlockQueryEndpoint     = "/eth/v2/beacon/blocks/"                     // Endpoint to query individual Blocks
+	BcStateQueryEndpoint = "/eth/v2/debug/beacon/states/"      // Endpoint to query individual States
-	bcStateQueryEndpoint     = "/eth/v2/debug/beacon/states/"               // Endpoint to query individual States
+	BcSyncStatusEndpoint = "/eth/v1/node/syncing"              // The endpoint to check to see if the beacon server is still trying to sync to head.
+	LhDbInfoEndpoint     = "/lighthouse/database/info"         // The endpoint for the LIGHTHOUSE server to get the database information.
+	BcBlockRootEndpoint  = func(slot string) string {
+		return "/eth/v1/beacon/blocks/" + slot + "/root"
+	}
+	bcSlotsPerEpoch = 32 // Number of slots in a single Epoch
+	//bcSlotPerHistoricalVector = 8192                                // The number of slots in a historic vector.
+	//bcFinalizedTopicEndpoint  = "/eth/v1/events?topics=finalized_checkpoint" // Endpoint used to subscribe to the head of the chain
 )
 
 // A struct that capture the Beacon Server that the Beacon Client will be interacting with and querying.
 type BeaconClient struct {
-	Context                     context.Context                 // A context generic context with multiple uses.
+	Context                context.Context      // A context generic context with multiple uses.
-	ServerEndpoint              string                          // What is the endpoint of the beacon server.
+	ServerEndpoint         string               // What is the endpoint of the beacon server.
-	PerformHeadTracking         bool                            // Should we track head?
+	Db                     sql.Database         // Database object used for reads and writes.
-	PerformHistoricalProcessing bool                            // Should we perform historical processing?
+	Metrics                *BeaconClientMetrics // An object used to keep track of certain BeaconClient Metrics.
-	HeadTracking                *SseEvents[Head]                // Track the head block
+	KnownGapTableIncrement int                  // The max number of slots within a single known_gaps table entry.
-	ReOrgTracking               *SseEvents[ChainReorg]          // Track all Reorgs
+	UniqueNodeIdentifier   int                  // The unique identifier within the cluster of this individual node.
-	FinalizationTracking        *SseEvents[FinalizedCheckpoint] // Track all finalization checkpoints
+	KnownGapsProcess       KnownGapsProcessing  // object keeping track of knowngaps processing
+	CheckDb                bool                 // Should we check the DB to see if the slot exists before processing it?
+
+	// Used for Head Tracking
+
+	PerformHeadTracking bool                   // Should we track head?
+	StartingSlot        int                    // If we're performing head tracking. What is the first slot we processed.
+	PreviousSlot        int                    // Whats the previous slot we processed
+	PreviousBlockRoot   string                 // Whats the previous block root, used to check the next blocks parent.
+	HeadTracking        *SseEvents[Head]       // Track the head block
+	ReOrgTracking       *SseEvents[ChainReorg] // Track all Reorgs
+	//FinalizationTracking        *SseEvents[FinalizedCheckpoint] // Track all finalization checkpoints
+
+	// Used for Historical Processing
+
+	// The latest available slot within the Beacon Server. We can't query any slot greater than this.
+	// This value is lazily updated. Therefore at times it will be outdated.
+	LatestSlotInBeaconServer    int64
+	PerformHistoricalProcessing bool               // Should we perform historical processing?
+	HistoricalProcess           HistoricProcessing // object keeping track of historical processing
 }
 
 // A struct to keep track of relevant the head event topic.
@@ -44,16 +88,30 @@ type SseError struct {
 }
 
 // A Function to create the BeaconClient.
-func CreateBeaconClient(ctx context.Context, connectionProtocol string, bcAddress string, bcPort int) *BeaconClient {
+func CreateBeaconClient(ctx context.Context, connectionProtocol string, bcAddress string, bcPort int, bcKgTableIncrement int, uniqueNodeIdentifier int, checkDb bool) (*BeaconClient, error) {
+	if uniqueNodeIdentifier == 0 {
+		uniqueNodeIdentifier := rand.Int()
+		log.WithField("randomUniqueNodeIdentifier", uniqueNodeIdentifier).Warn("No uniqueNodeIdentifier provided, we are going to use a randomly generated one.")
+	}
+
+	metrics, err := CreateBeaconClientMetrics()
+	if err != nil {
+		return nil, err
+	}
+
 	endpoint := fmt.Sprintf("%s://%s:%d", connectionProtocol, bcAddress, bcPort)
 	log.Info("Creating the BeaconClient")
 	return &BeaconClient{
-		Context:              ctx,
+		Context:                ctx,
-		ServerEndpoint:       endpoint,
+		ServerEndpoint:         endpoint,
-		HeadTracking:         createSseEvent[Head](endpoint, bcHeadTopicEndpoint),
+		KnownGapTableIncrement: bcKgTableIncrement,
-		ReOrgTracking:        createSseEvent[ChainReorg](endpoint, bcReorgTopicEndpoint),
+		HeadTracking:           createSseEvent[Head](endpoint, BcHeadTopicEndpoint),
-		FinalizationTracking: createSseEvent[FinalizedCheckpoint](endpoint, bcFinalizedTopicEndpoint),
+		ReOrgTracking:          createSseEvent[ChainReorg](endpoint, bcReorgTopicEndpoint),
-	}
+		Metrics:                metrics,
+		UniqueNodeIdentifier:   uniqueNodeIdentifier,
+		CheckDb:                checkDb,
+		//FinalizationTracking: createSseEvent[FinalizedCheckpoint](endpoint, bcFinalizedTopicEndpoint),
+	}, nil
 }
 
 // Create all the channels to handle a SSE events
@@ -61,7 +119,7 @@ func createSseEvent[P ProcessedEvents](baseEndpoint string, path string) *SseEve
 	endpoint := baseEndpoint + path
 	sseEvents := &SseEvents[P]{
 		Endpoint:   endpoint,
-		MessagesCh: make(chan *sse.Event),
+		MessagesCh: make(chan *sse.Event, 1),
 		ErrorCh:    make(chan *SseError),
 		ProcessCh:  make(chan *P),
 		SseClient: func(endpoint string) *sse.Client {

pkg/beaconclient/beaconclient_suite_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient_test
 
 import (

pkg/beaconclient/capturehead.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This file will call all the functions to start and stop capturing the head of the beacon chain.
 
 package beaconclient
@@ -5,48 +20,16 @@ package beaconclient
 import (
 	"time"
 
-	"github.com/ferranbt/fastssz/spectests"
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // This function will perform all the heavy lifting for tracking the head of the chain.
 func (bc *BeaconClient) CaptureHead() {
 	log.Info("We are tracking the head of the chain.")
-	bc.tempHelper()
+	go bc.handleHead()
-	// go bc.handleHead()
+	go bc.handleReorg()
-	// go bc.handleFinalizedCheckpoint()
+	bc.captureEventTopic()
-	// go bc.handleReorgs()
-	// bc.captureEventTopic()
-}
-
-// A temporary helper function to see the output of beacon block and states.
-func (bc *BeaconClient) tempHelper() {
-	slot := "3200"
-	blockEndpoint := bc.ServerEndpoint + bcBlockQueryEndpoint + slot
-	stateEndpoint := bc.ServerEndpoint + bcStateQueryEndpoint + slot
-	// Query
-	log.Info("Get")
-	blockSsz, _ := querySsz(blockEndpoint, slot)
-	stateSsz, _ := querySsz(stateEndpoint, slot)
-	// Transform
-	log.Info("Tranform")
-	stateObj := new(spectests.BeaconState)
-	err := stateObj.UnmarshalSSZ(stateSsz)
-	if err != nil {
-		loghelper.LogSlotError(slot, err).Error("Unable to unmarshal the SSZ response from the Beacon Node Successfully!")
-	}
-
-	blockObj := new(spectests.SignedBeaconBlock)
-	err = blockObj.UnmarshalSSZ(blockSsz)
-	if err != nil {
-		loghelper.LogSlotError(slot, err).Error("Unable to unmarshal the SSZ response from the Beacon Node Successfully!")
-	}
-
-	// Check
-	log.Info("Check")
-	log.Info("State Slot: ", stateObj.Slot)
-	log.Info("Block Slot: ", blockObj.Block.Slot)
 }
 
 // Stop the head tracking service.
@@ -54,14 +37,11 @@ func (bc *BeaconClient) StopHeadTracking() error {
 	log.Info("We are going to stop tracking the head of chain because of the shutdown signal.")
 	chHead := make(chan bool)
 	chReorg := make(chan bool)
-	chFinal := make(chan bool)
 
 	go bc.HeadTracking.finishProcessingChannel(chHead)
 	go bc.ReOrgTracking.finishProcessingChannel(chReorg)
-	go bc.FinalizationTracking.finishProcessingChannel(chFinal)
 
 	<-chHead
-	<-chFinal
 	<-chReorg
 	log.Info("Successfully stopped the head tracking service.")
 	return nil
@@ -71,7 +51,7 @@ func (bc *BeaconClient) StopHeadTracking() error {
 func (se *SseEvents[ProcessedEvents]) finishProcessingChannel(finish chan<- bool) {
 	loghelper.LogEndpoint(se.Endpoint).Info("Received a close event.")
 	se.SseClient.Unsubscribe(se.MessagesCh)
-	for len(se.MessagesCh) != 0 && len(se.ProcessCh) != 0 {
+	for len(se.MessagesCh) != 0 || len(se.ProcessCh) != 0 {
 		time.Sleep(time.Duration(shutdownWaitInterval) * time.Millisecond)
 	}
 	loghelper.LogEndpoint(se.Endpoint).Info("Done processing all messages, ready for shutdown")

pkg/beaconclient/capturehead_test.go

@@ -0,0 +1,993 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package beaconclient_test
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync/atomic"
+	"time"
+
+	"github.com/jarcoal/httpmock"
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	si "github.com/prysmaticlabs/prysm/consensus-types/interfaces"
+	types "github.com/prysmaticlabs/prysm/consensus-types/primitives"
+	dt "github.com/prysmaticlabs/prysm/encoding/ssz/detect"
+	st "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/r3labs/sse"
+	log "github.com/sirupsen/logrus"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql/postgres"
+)
+
+var (
+	address                 string = "localhost"
+	port                    int    = 8080
+	protocol                string = "http"
+	dbHost                  string = "localhost"
+	dbPort                  int    = 8076
+	dbName                  string = "vulcanize_testing"
+	dbUser                  string = "vdbm"
+	dbPassword              string = "password"
+	dbDriver                string = "pgx"
+	bcUniqueIdentifier      int    = 100
+	dummyParentRoot         string = "46f98c08b54a71dfda4d56e29ec3952b8300cd8d6b67a9b6c562ae96a7a25a42"
+	knownGapsTableIncrement int    = 100000
+	maxRetry                int    = 160
+
+	TestEvents = map[string]Message{
+		"0": {
+			HeadMessage: beaconclient.Head{
+				Slot:                      "0",
+				Block:                     "0x4d611d5b93fdab69013a7f0a2f961caca0c853f87cfe9595fe50038163079360",
+				State:                     "0x7e76880eb67bbdc86250aa578958e9d0675e64e714337855204fb5abaaf82c2b",
+				CurrentDutyDependentRoot:  "",
+				PreviousDutyDependentRoot: "",
+				EpochTransition:           false,
+				ExecutionOptimistic:       false,
+			},
+			SignedBeaconBlock:             filepath.Join("ssz-data", "0", "signed-beacon-block.ssz"),
+			BeaconState:                   filepath.Join("ssz-data", "0", "beacon-state.ssz"),
+			CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA2GINRRGFSDKYRZGNTGIYLCGY4TAMJTME3WMMDBGJTDSNRRMNQWGYJQMM4DKM3GHA3WGZTFHE2TSNLGMU2TAMBTHAYTMMZQG44TGNRQ",
+			CorrectBeaconStateMhKey:       "/blocks/QLVAEQRQPA3WKNZWHA4DAZLCGY3WEYTEMM4DMMRVGBQWCNJXHA4TKODFHFSDANRXGVSTMNDFG4YTIMZTG44DKNJSGA2GMYRVMFRGCYLGHAZGGMTC",
+			CorrectParentRoot:             "0x0000000000000000000000000000000000000000000000000000000000000000",
+			CorrectEth1BlockHash:          "0x0000000000000000000000000000000000000000000000000000000000000000",
+		},
+		"100-dummy": {
+			HeadMessage: beaconclient.Head{
+				Slot:                      "100",
+				Block:                     "04955400371347e26f61d7a4bbda5b23fa0b25d5fc465160f2a92d52a63b919b",
+				State:                     "36d5c9a129979b4502bd9a06e57a742810ecbc3fa55a0361c0723c92c1782bfa",
+				CurrentDutyDependentRoot:  "",
+				PreviousDutyDependentRoot: "",
+				EpochTransition:           false,
+				ExecutionOptimistic:       false,
+			},
+			TestNotes: "A block that is supposed to replicate slot 100, but contains some dummy test information.",
+			MimicConfig: &MimicConfig{
+				ForkVersion: "phase0",
+			},
+			SignedBeaconBlock: filepath.Join("ssz-data", "100", "signed-beacon-block.ssz"),
+			BeaconState:       filepath.Join("ssz-data", "100", "beacon-state.ssz"),
+		},
+		"100-dummy-2": {
+			HeadMessage: beaconclient.Head{
+				Slot:                      "100",
+				Block:                     "04955400371347e26f61d7a4bbda5b23fa0b25d5fc465160f2a9aaaaaaaaaaaa",
+				State:                     "36d5c9a129979b4502bd9a06e57a742810ecbc3fa55a0361c072bbbbbbbbbbbb",
+				CurrentDutyDependentRoot:  "",
+				PreviousDutyDependentRoot: "",
+				EpochTransition:           false,
+				ExecutionOptimistic:       false,
+			},
+			TestNotes: "A block that is supposed to replicate slot 100, but contains some dummy test information.",
+			MimicConfig: &MimicConfig{
+				ForkVersion: "phase0",
+			},
+			SignedBeaconBlock: filepath.Join("ssz-data", "100", "signed-beacon-block.ssz"),
+			BeaconState:       filepath.Join("ssz-data", "100", "beacon-state.ssz"),
+		},
+		"102-wrong-ssz-1": {
+			HeadMessage: beaconclient.Head{
+				Slot:                      "102",
+				Block:                     "0x46f98c08b54a71dfda4d56e29ec3952b8300cd8d6b67a9b6c562ae96a7a25a42",
+				State:                     "0x9b20b114c613c1aa462e02d590b3da902b0a1377e938ed0f94dd3491d763ef67",
+				CurrentDutyDependentRoot:  "",
+				PreviousDutyDependentRoot: "",
+				EpochTransition:           false,
+				ExecutionOptimistic:       false,
+			},
+			TestNotes:         "A bad block that returns the wrong ssz objects, used for testing incorrect SSZ decoding.",
+			BeaconState:       filepath.Join("ssz-data", "102", "signed-beacon-block.ssz"),
+			SignedBeaconBlock: filepath.Join("ssz-data", "102", "beacon-state.ssz"),
+		},
+		"100": {
+			HeadMessage: beaconclient.Head{
+				Slot:                      "100",
+				Block:                     "0x582187e97f7520bb69eea014c3834c964c45259372a0eaaea3f032013797996b",
+				State:                     "0xf286a0379c0386a3c7be28d05d829f8eb7b280cc9ede15449af20ebcd06a7a56",
+				CurrentDutyDependentRoot:  "",
+				PreviousDutyDependentRoot: "",
+				EpochTransition:           false,
+				ExecutionOptimistic:       false,
+			},
+			TestNotes:                     "An easy to process Phase 0 block",
+			SignedBeaconBlock:             filepath.Join("ssz-data", "100", "signed-beacon-block.ssz"),
+			BeaconState:                   filepath.Join("ssz-data", "100", "beacon-state.ssz"),
+			CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA2TQMRRHA3WKOJXMY3TKMRQMJRDMOLFMVQTAMJUMMZTQMZUMM4TMNDDGQ2TENJZGM3TEYJQMVQWCZLBGNTDAMZSGAYTGNZZG44TSNTC",
+			CorrectBeaconStateMhKey:       "/blocks/QLVAEQRQPBTDEOBWMEYDGNZZMMYDGOBWMEZWGN3CMUZDQZBQGVSDQMRZMY4GKYRXMIZDQMDDMM4WKZDFGE2TINBZMFTDEMDFMJRWIMBWME3WCNJW",
+			CorrectParentRoot:             "0x629ae1587895043076500f4f5dcb202a47c2fc95d5b5c548cb83bc97bd2dbfe1",
+			CorrectEth1BlockHash:          "0x8d3f027beef5cbd4f8b29fc831aba67a5d74768edca529f5596f07fd207865e1",
+		},
+		"101": {
+			HeadMessage: beaconclient.Head{
+				Slot:                      "101",
+				Block:                     "0xabe1a972e512182d04f0d4a5c9c25f9ee57c2e9d0ff3f4c4c82fd42d13d31083",
+				State:                     "0xcb04aa2edbf13c7bb7e7bd9b621ced6832e0075e89147352eac3019a824ce847",
+				CurrentDutyDependentRoot:  "",
+				PreviousDutyDependentRoot: "",
+				EpochTransition:           false,
+				ExecutionOptimistic:       false,
+			},
+			TestNotes:                     "An easy to process Phase 0 block",
+			SignedBeaconBlock:             filepath.Join("ssz-data", "101", "signed-beacon-block.ssz"),
+			BeaconState:                   filepath.Join("ssz-data", "101", "beacon-state.ssz"),
+			CorrectEth1BlockHash:          "0x8d3f027beef5cbd4f8b29fc831aba67a5d74768edca529f5596f07fd207865e1",
+			CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPBQWEZJRME4TOMTFGUYTEMJYGJSDANDGGBSDIYJVMM4WGMRVMY4WKZJVG5RTEZJZMQYGMZRTMY2GGNDDHAZGMZBUGJSDCM3EGMYTAOBT",
+			CorrectBeaconStateMhKey:       "/blocks/QLVAEQRQPBRWEMBUMFQTEZLEMJTDCM3DG5RGEN3FG5RGIOLCGYZDCY3FMQ3DQMZSMUYDANZVMU4DSMJUG4ZTKMTFMFRTGMBRHFQTQMRUMNSTQNBX",
+		},
+		"2375703-dummy": {
+			HeadMessage: beaconclient.Head{
+				Slot:                      "2375703",
+				Block:                     "c9fb337b62e2a0dae4f27ab49913132570f7f2cab3f23ad99f4d07508a8e648e",
+				State:                     "0299a145bcda2c8f5e7d2e068ee101861edbee2ec1db2d5e1d850b0d265aef5f",
+				CurrentDutyDependentRoot:  "",
+				PreviousDutyDependentRoot: "",
+				EpochTransition:           false,
+				ExecutionOptimistic:       false,
+			},
+			TestNotes: "This is a dummy message that is used for reorgs",
+			MimicConfig: &MimicConfig{
+				ForkVersion: "altair",
+			},
+			SignedBeaconBlock: filepath.Join("ssz-data", "2375703", "signed-beacon-block.ssz"),
+			BeaconState:       filepath.Join("ssz-data", "2375703", "beacon-state.ssz"),
+		},
+		"2375703-dummy-2": {
+			HeadMessage: beaconclient.Head{
+				Slot:                      "2375703",
+				Block:                     "c9fb337b62e2a0dae4f27ab49913132570f7f2cab3f23ad99f4d07508aaaaaaa",
+				State:                     "0299a145bcda2c8f5e7d2e068ee101861edbee2ec1db2d5e1d850b0d2bbbbbbb",
+				CurrentDutyDependentRoot:  "",
+				PreviousDutyDependentRoot: "",
+				EpochTransition:           false,
+				ExecutionOptimistic:       false,
+			},
+			TestNotes: "This is a dummy message that is used for reorgs",
+			MimicConfig: &MimicConfig{
+				ForkVersion: "altair",
+			},
+			SignedBeaconBlock: filepath.Join("ssz-data", "2375703", "signed-beacon-block.ssz"),
+			BeaconState:       filepath.Join("ssz-data", "2375703", "beacon-state.ssz"),
+		},
+		"2375703": {
+			HeadMessage: beaconclient.Head{
+				Slot:                     "2375703",
+				Block:                    "0x4392372c5f6e39499e31bf924388b5815639103149f0f54f8a453773b1802301",
+				State:                    "0xb6215b560273af63ec7e011572b60ec1ca0b0232f8ff44fcd4ed55c7526e964e",
+				CurrentDutyDependentRoot: "", PreviousDutyDependentRoot: "", EpochTransition: false, ExecutionOptimistic: false},
+			TestNotes:                     "An easy to process Altair Block",
+			SignedBeaconBlock:             filepath.Join("ssz-data", "2375703", "signed-beacon-block.ssz"),
+			BeaconState:                   filepath.Join("ssz-data", "2375703", "beacon-state.ssz"),
+			CorrectEth1BlockHash:          "0xd74b1c60423651624de6bb301ac25808951c167ba6ecdd9b2e79b4315aee8202",
+			CorrectParentRoot:             "0x08736ddc20b77f65d1aa6301f7e6e856a820ff3ce6430ed2c3694ae35580e740",
+			CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA2DGOJSGM3TEYZVMY3GKMZZGQ4TSZJTGFRGMOJSGQZTQODCGU4DCNJWGM4TCMBTGE2DSZRQMY2TIZRYME2DKMZXG4ZWEMJYGAZDGMBR",
+			CorrectBeaconStateMhKey:       "/blocks/QLVAEQRQPBRDMMRRGVRDKNRQGI3TGYLGGYZWKYZXMUYDCMJVG4ZGENRQMVRTCY3BGBRDAMRTGJTDQZTGGQ2GMY3EGRSWINJVMM3TKMRWMU4TMNDF",
+		},
+		"3797056": {
+			HeadMessage: beaconclient.Head{
+				Slot:                     "3797056",
+				Block:                    "",
+				State:                    "",
+				CurrentDutyDependentRoot: "", PreviousDutyDependentRoot: "", EpochTransition: false, ExecutionOptimistic: false},
+			TestNotes: "An easy to process Altair Block",
+			// The file below should not exist, this will trigger an error message and 404 response from the mock.
+			SignedBeaconBlock: filepath.Join("ssz-data", "3797056", "should-not-exist.txt"),
+			BeaconState:       filepath.Join("ssz-data", "3797056", "beacon-state.ssz"),
+		},
+	}
+	TestConfig = Config{
+		protocol:                protocol,
+		address:                 address,
+		port:                    port,
+		dummyParentRoot:         dummyParentRoot,
+		dbHost:                  dbHost,
+		dbPort:                  dbPort,
+		dbName:                  dbName,
+		dbUser:                  dbUser,
+		dbPassword:              dbPassword,
+		dbDriver:                dbDriver,
+		knownGapsTableIncrement: knownGapsTableIncrement,
+		bcUniqueIdentifier:      bcUniqueIdentifier,
+		checkDb:                 true,
+	}
+
+	BeaconNodeTester = TestBeaconNode{
+		TestEvents: TestEvents,
+		TestConfig: TestConfig,
+	}
+)
+
+type Message struct {
+	HeadMessage                   beaconclient.Head // The head messsage that will be streamed to the BeaconClient
+	TestNotes                     string            // A small explanation of the purpose this structure plays in the testing landscape.
+	MimicConfig                   *MimicConfig      // A configuration of parameters that you are trying to
+	SignedBeaconBlock             string            // The file path output of an SSZ encoded SignedBeaconBlock.
+	BeaconState                   string            // The file path output of an SSZ encoded BeaconState.
+	CorrectSignedBeaconBlockMhKey string            // The correct MhKey for the signedBeaconBlock
+	CorrectBeaconStateMhKey       string            // The correct MhKey beaconState
+	CorrectParentRoot             string            // The correct parent root
+	CorrectEth1BlockHash          string            // The correct eth1blockHash
+}
+
+// A structure that can be utilized to mimic and existing SSZ object but change it ever so slightly.
+// This is used because creating your own SSZ object is a headache.
+type MimicConfig struct {
+	ParentRoot  string // The parent root, leave it empty if you want a to use the universal
+	ForkVersion string // Specify the fork version. This is needed as a workaround to create dummy SignedBeaconBlocks.
+}
+
+var _ = Describe("Capturehead", Label("head"), func() {
+
+	Describe("Receiving New Head SSE messages", Label("unit", "behavioral"), func() {
+		Context("Correctly formatted Phase0 Block", func() {
+			It("Should turn it into a struct successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, 3, maxRetry, 1, 0, 0)
+				validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectParentRoot, BeaconNodeTester.TestEvents["100"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["100"].CorrectSignedBeaconBlockMhKey)
+				validateBeaconState(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectBeaconStateMhKey)
+
+			})
+		})
+		Context("Correctly formatted Altair Block", func() {
+			It("Should turn it into a struct successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+				validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectParentRoot, BeaconNodeTester.TestEvents["2375703"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["2375703"].CorrectSignedBeaconBlockMhKey)
+				validateBeaconState(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectBeaconStateMhKey)
+			})
+		})
+		Context("Correctly formatted Altair Test Blocks", func() {
+			It("Should turn it into a struct successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703-dummy"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+
+				bc = setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703-dummy-2"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+
+			})
+		})
+		Context("Correctly formatted Phase0 Test Blocks", func() {
+			It("Should turn it into a struct successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["100-dummy"].HeadMessage, 3, maxRetry, 1, 0, 0)
+
+				bc = setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["100-dummy-2"].HeadMessage, 3, maxRetry, 1, 0, 0)
+			})
+
+		})
+		Context("Two consecutive correct blocks", func() {
+			It("Should handle both blocks correctly, without any reorgs or known_gaps", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, 3, maxRetry, 1, 0, 0)
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, 3, maxRetry, 1, 0, 0)
+			})
+		})
+		Context("Two consecutive blocks with a bad parent", func() {
+			It("Should add the previous block to the knownGaps table.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["100-dummy"].HeadMessage, 3, maxRetry, 1, 0, 0)
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, 3, maxRetry, 1, 1, 1)
+			})
+		})
+		Context("Phase 0: We have a correctly formated SSZ SignedBeaconBlock and BeaconState", func() {
+			It("Should be able to get each objects root hash.", func() {
+				testSszRoot(BeaconNodeTester.TestEvents["100"])
+			})
+		})
+		Context("Altair: We have a correctly formated SSZ SignedBeaconBlock and BeaconState", func() {
+			It("Should be able to get each objects root hash.", func() {
+				testSszRoot(BeaconNodeTester.TestEvents["2375703"])
+			})
+		})
+		//Context("When there is a skipped slot", func() {
+		//	It("Should indicate that the slot was skipped", func() {
+
+		//	})
+		//})
+		Context("When the proper SSZ objects are not served", func() {
+			It("Should return an error, and add the slot to the knownGaps table.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "101")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["102-wrong-ssz-1"].HeadMessage, 3, maxRetry, 0, 1, 0)
+
+				knownGapCount := countKnownGapsTable(bc.Db)
+				Expect(knownGapCount).To(Equal(1))
+
+				start, end := queryKnownGaps(bc.Db, "102", "102")
+				Expect(start).To(Equal(102))
+				Expect(end).To(Equal(102))
+			})
+		})
+	})
+
+	Describe("Known Gaps Scenario", Label("unit", "behavioral"), func() {
+		Context("There is a gap at start up within one incrementing range.", func() {
+			It("Should add only a single entry to the knownGaps table.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "10")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testKnownGapsMessages(bc, 100, 1, maxRetry, BeaconNodeTester.TestEvents["100"].HeadMessage)
+				start, end := queryKnownGaps(bc.Db, "11", "99")
+				Expect(start).To(Equal(11))
+				Expect(end).To(Equal(99))
+			})
+		})
+		Context("There is a gap at start up spanning multiple incrementing range.", func() {
+			It("Should add multiple entries to the knownGaps table.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "5")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testKnownGapsMessages(bc, 10, 10, maxRetry, BeaconNodeTester.TestEvents["100"].HeadMessage)
+
+				start, end := queryKnownGaps(bc.Db, "6", "16")
+				Expect(start).To(Equal(6))
+				Expect(end).To(Equal(16))
+
+				start, end = queryKnownGaps(bc.Db, "96", "99")
+				Expect(start).To(Equal(96))
+				Expect(end).To(Equal(99))
+			})
+		})
+		Context("Gaps between two head messages", func() {
+			It("Should add the slots in-between", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testKnownGapsMessages(bc, 1000000, 3, maxRetry, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].HeadMessage)
+
+				start, end := queryKnownGaps(bc.Db, "101", "1000101")
+				Expect(start).To(Equal(101))
+				Expect(end).To(Equal(1000101))
+
+				start, end = queryKnownGaps(bc.Db, "2000101", "2375702")
+				Expect(start).To(Equal(2000101))
+				Expect(end).To(Equal(2375702))
+			})
+		})
+	})
+
+	Describe("ReOrg Scenario", Label("unit", "behavioral"), func() {
+		Context("Altair: Multiple head messages for the same slot.", func() {
+			It("The previous block should be marked as 'forked', the new block should be the only one marked as 'proposed'.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testMultipleHead(bc, TestEvents["2375703"].HeadMessage, TestEvents["2375703-dummy"].HeadMessage, 74240, maxRetry)
+			})
+		})
+		Context("Phase0: Multiple head messages for the same slot.", func() {
+			It("The previous block should be marked as 'forked', the new block should be the only one marked as 'proposed'.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testMultipleHead(bc, TestEvents["100-dummy"].HeadMessage, TestEvents["100"].HeadMessage, 3, maxRetry)
+			})
+		})
+		Context("Phase 0: Multiple reorgs have occurred on this slot", func() {
+			It("The previous blocks should be marked as 'forked', the new block should be the only one marked as 'proposed'.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testMultipleReorgs(bc, TestEvents["100-dummy"].HeadMessage, TestEvents["100-dummy-2"].HeadMessage, TestEvents["100"].HeadMessage, 3, maxRetry)
+			})
+		})
+		Context("Altair: Multiple reorgs have occurred on this slot", func() {
+			It("The previous blocks should be marked as 'forked', the new block should be the only one marked as 'proposed'.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.testMultipleReorgs(bc, TestEvents["2375703-dummy"].HeadMessage, TestEvents["2375703-dummy-2"].HeadMessage, TestEvents["2375703"].HeadMessage, 74240, maxRetry)
+			})
+		})
+	})
+})
+
+type Config struct {
+	protocol                string
+	address                 string
+	port                    int
+	dummyParentRoot         string
+	dbHost                  string
+	dbPort                  int
+	dbName                  string
+	dbUser                  string
+	dbPassword              string
+	dbDriver                string
+	knownGapsTableIncrement int
+	bcUniqueIdentifier      int
+	checkDb                 bool
+}
+
+//////////////////////////////////////////////////////
+// Helper functions
+//////////////////////////////////////////////////////
+
+// Must run before each test. We can't use the beforeEach because of the way
+// Gingko treats race conditions.
+func setUpTest(config Config, maxSlot string) *beaconclient.BeaconClient {
+	bc, err := beaconclient.CreateBeaconClient(context.Background(), config.protocol, config.address, config.port, config.knownGapsTableIncrement, config.bcUniqueIdentifier, config.checkDb)
+	Expect(err).ToNot(HaveOccurred())
+	db, err := postgres.SetupPostgresDb(config.dbHost, config.dbPort, config.dbName, config.dbUser, config.dbPassword, config.dbDriver)
+	Expect(err).ToNot(HaveOccurred())
+
+	// Drop all records from the DB.
+	clearEthBeaconDbTables(db)
+
+	// Add an slot to the eth_beacon.slots table so it we can control how known_gaps are handled.
+	writeSlot(db, maxSlot)
+	bc.Db = db
+
+	return bc
+}
+
+// A helper function to validate the expected output from the eth_beacon.slots table.
+func validateSlot(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctEpoch int, correctStatus string) {
+	epoch, dbSlot, blockRoot, stateRoot, status := queryDbSlotAndBlock(bc.Db, headMessage.Slot, headMessage.Block)
+	log.Info("validateSlot: ", headMessage)
+	baseSlot, err := strconv.Atoi(headMessage.Slot)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(dbSlot).To(Equal(baseSlot))
+	Expect(epoch).To(Equal(correctEpoch))
+	Expect(blockRoot).To(Equal(headMessage.Block))
+	Expect(stateRoot).To(Equal(headMessage.State))
+	Expect(status).To(Equal(correctStatus))
+}
+
+// A helper function to validate the expected output from the eth_beacon.signed_block table.
+func validateSignedBeaconBlock(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctParentRoot string, correctEth1BlockHash string, correctMhKey string) {
+	dbSlot, blockRoot, parentRoot, eth1BlockHash, mhKey := queryDbSignedBeaconBlock(bc.Db, headMessage.Slot, headMessage.Block)
+	log.Info("validateSignedBeaconBlock: ", headMessage)
+	baseSlot, err := strconv.Atoi(headMessage.Slot)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(dbSlot).To(Equal(baseSlot))
+	Expect(blockRoot).To(Equal(headMessage.Block))
+	Expect(parentRoot).To(Equal(correctParentRoot))
+	Expect(eth1BlockHash).To(Equal(correctEth1BlockHash))
+	Expect(mhKey).To(Equal(correctMhKey))
+
+}
+
+// A helper function to validate the expected output from the eth_beacon.state table.
+func validateBeaconState(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctMhKey string) {
+	dbSlot, stateRoot, mhKey := queryDbBeaconState(bc.Db, headMessage.Slot, headMessage.State)
+	log.Info("validateBeaconState: ", headMessage)
+	baseSlot, err := strconv.Atoi(headMessage.Slot)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(dbSlot).To(Equal(baseSlot))
+	Expect(stateRoot).To(Equal(headMessage.State))
+	Expect(mhKey).To(Equal(correctMhKey))
+
+}
+
+// Wrapper function to send a head message to the beaconclient
+func sendHeadMessage(bc *beaconclient.BeaconClient, head beaconclient.Head, maxRetry int, expectedSuccessfulInserts uint64) {
+
+	data, err := json.Marshal(head)
+	Expect(err).ToNot(HaveOccurred())
+
+	startInserts := atomic.LoadUint64(&bc.Metrics.SlotInserts)
+	bc.HeadTracking.MessagesCh <- &sse.Event{
+		ID:    []byte{},
+		Data:  data,
+		Event: []byte{},
+		Retry: []byte{},
+	}
+	curRetry := 0
+	for atomic.LoadUint64(&bc.Metrics.SlotInserts) != startInserts+expectedSuccessfulInserts {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			log.WithFields(log.Fields{
+				"startInsert":               startInserts,
+				"expectedSuccessfulInserts": expectedSuccessfulInserts,
+				"currentValue":              atomic.LoadUint64(&bc.Metrics.SlotInserts),
+			}).Error("HeadTracking Insert wasn't incremented properly.")
+			Fail("Too many retries have occurred.")
+		}
+	}
+}
+
+// A helper function to query the eth_beacon.slots table based on the slot and block_root
+func queryDbSlotAndBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, int, string, string, string) {
+	sqlStatement := `SELECT epoch, slot, block_root, state_root, status FROM eth_beacon.slots WHERE slot=$1 AND block_root=$2;`
+	var epoch, slot int
+	var blockRoot, stateRoot, status string
+	log.Debug("Starting to query the eth_beacon.slots table, ", querySlot, " ", queryBlockRoot)
+	err := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot).Scan(&epoch, &slot, &blockRoot, &stateRoot, &status)
+	Expect(err).ToNot(HaveOccurred())
+	log.Debug("Querying the eth_beacon.slots table complete")
+	return epoch, slot, blockRoot, stateRoot, status
+}
+
+// A helper function to query the eth_beacon.signed_block table based on the slot and block_root.
+func queryDbSignedBeaconBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, string, string, string, string) {
+	sqlStatement := `SELECT slot, block_root, parent_block_root, eth1_block_hash, mh_key FROM eth_beacon.signed_block WHERE slot=$1 AND block_root=$2;`
+	var slot int
+	var blockRoot, parent_block_root, eth1_block_hash, mh_key string
+	row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot)
+	err := row.Scan(&slot, &blockRoot, &parent_block_root, &eth1_block_hash, &mh_key)
+	Expect(err).ToNot(HaveOccurred())
+	return slot, blockRoot, parent_block_root, eth1_block_hash, mh_key
+}
+
+// A helper function to query the eth_beacon.signed_block table based on the slot and block_root.
+func queryDbBeaconState(db sql.Database, querySlot string, queryStateRoot string) (int, string, string) {
+	sqlStatement := `SELECT slot, state_root, mh_key FROM eth_beacon.state WHERE slot=$1 AND state_root=$2;`
+	var slot int
+	var stateRoot, mh_key string
+	row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryStateRoot)
+	err := row.Scan(&slot, &stateRoot, &mh_key)
+	Expect(err).ToNot(HaveOccurred())
+	return slot, stateRoot, mh_key
+}
+
+// Count the entries in the knownGaps table.
+func countKnownGapsTable(db sql.Database) int {
+	var count int
+	sqlStatement := "SELECT COUNT(*) FROM eth_beacon.known_gaps"
+	err := db.QueryRow(context.Background(), sqlStatement).Scan(&count)
+	Expect(err).ToNot(HaveOccurred())
+	return count
+}
+
+// Return the start and end slot
+func queryKnownGaps(db sql.Database, queryStartGap string, QueryEndGap string) (int, int) {
+	sqlStatement := `SELECT start_slot, end_slot FROM eth_beacon.known_gaps WHERE start_slot=$1 AND end_slot=$2;`
+	var startGap, endGap int
+	row := db.QueryRow(context.Background(), sqlStatement, queryStartGap, QueryEndGap)
+	err := row.Scan(&startGap, &endGap)
+	Expect(err).ToNot(HaveOccurred())
+	return startGap, endGap
+}
+
+// A function that will remove all entries from the eth_beacon tables for you.
+func clearEthBeaconDbTables(db sql.Database) {
+	deleteQueries := []string{"DELETE FROM eth_beacon.slots;", "DELETE FROM eth_beacon.signed_block;", "DELETE FROM eth_beacon.state;", "DELETE FROM eth_beacon.known_gaps;", "DELETE FROM eth_beacon.historic_process;", "DELETE FROM public.blocks;"}
+	for _, queries := range deleteQueries {
+		_, err := db.Exec(context.Background(), queries)
+		Expect(err).ToNot(HaveOccurred())
+	}
+}
+
+// Write an entry to the eth_beacon.slots table with just a slot number
+func writeSlot(db sql.Database, slot string) {
+	_, err := db.Exec(context.Background(), beaconclient.UpsertSlotsStmt, "0", slot, "", "", "")
+	Expect(err).ToNot(HaveOccurred())
+}
+
+// Read a file with the SignedBeaconBlock in SSZ and return the SSZ object. This is used for testing only.
+// We can't use the readSignedBeaconBlockInterface to update struct fields so this is the workaround.
+func readSignedBeaconBlock(slotFile string) (*st.SignedBeaconBlock, error) {
+	dat, err := os.ReadFile(slotFile)
+	if err != nil {
+		return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
+	}
+	block := &st.SignedBeaconBlock{}
+	err = block.UnmarshalSSZ(dat)
+	Expect(err).ToNot(HaveOccurred())
+	return block, nil
+}
+
+// Read a file with the SignedBeaconBlock in SSZ and return the SSZ object. This is used for testing only.
+// We can't use the readSignedBeaconBlockInterface to update struct fields so this is the workaround.
+func readSignedBeaconBlockAltair(slotFile string) (*st.SignedBeaconBlockAltair, error) {
+	dat, err := os.ReadFile(slotFile)
+	if err != nil {
+		return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
+	}
+	block := &st.SignedBeaconBlockAltair{}
+	err = block.UnmarshalSSZ(dat)
+	Expect(err).ToNot(HaveOccurred())
+	return block, nil
+}
+
+// Read a file with the SignedBeaconBlock in SSZ and return the SSZ objects interface. This is production like.
+// It will provide the correct struct for the given fork.
+func readSignedBeaconBlockInterface(slotFile string, vm *dt.VersionedUnmarshaler) (si.SignedBeaconBlock, error) {
+	dat, err := os.ReadFile(slotFile)
+	if err != nil {
+		return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
+	}
+
+	block, err := vm.UnmarshalBeaconBlock(dat)
+	Expect(err).ToNot(HaveOccurred())
+	return block, nil
+
+}
+
+// Read a file with the BeaconState in SSZ and return the SSZ object
+func readBeaconState(slotFile string) (state.BeaconState, *dt.VersionedUnmarshaler, error) {
+	dat, err := os.ReadFile(slotFile)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
+	}
+	versionedUnmarshaler, err := dt.FromState(dat)
+	Expect(err).ToNot(HaveOccurred())
+	state, err := versionedUnmarshaler.UnmarshalBeaconState(dat)
+	Expect(err).ToNot(HaveOccurred())
+	return state, versionedUnmarshaler, nil
+}
+
+// An object that is used to aggregate test functions. Test functions are needed because we need to
+// run the same tests on multiple blocks for multiple forks. So they save us time.
+type TestBeaconNode struct {
+	TestEvents map[string]Message
+	TestConfig Config
+}
+
+// Create a new new mock for the beacon node.
+func (tbc TestBeaconNode) SetupBeaconNodeMock(TestEvents map[string]Message, protocol string, address string, port int, dummyParentRoot string) {
+	httpmock.Activate()
+	stateUrl := `=~^` + protocol + "://" + address + ":" + strconv.Itoa(port) + beaconclient.BcStateQueryEndpoint + `([^/]+)\z`
+	httpmock.RegisterResponder("GET", stateUrl,
+		func(req *http.Request) (*http.Response, error) {
+			// Get ID from request
+			id := httpmock.MustGetSubmatch(req, 1)
+			dat, err := tbc.provideSsz(id, "state", dummyParentRoot)
+			if err != nil {
+				return httpmock.NewStringResponse(404, fmt.Sprintf("Unable to find file for %s", id)), nil
+			}
+			return httpmock.NewBytesResponse(200, dat), nil
+		},
+	)
+
+	blockUrl := `=~^` + protocol + "://" + address + ":" + strconv.Itoa(port) + beaconclient.BcBlockQueryEndpoint + `([^/]+)\z`
+	httpmock.RegisterResponder("GET", blockUrl,
+		func(req *http.Request) (*http.Response, error) {
+			// Get ID from request
+			id := httpmock.MustGetSubmatch(req, 1)
+			dat, err := tbc.provideSsz(id, "block", dummyParentRoot)
+			if err != nil {
+				return httpmock.NewStringResponse(404, fmt.Sprintf("Unable to find file for %s", id)), nil
+			}
+			return httpmock.NewBytesResponse(200, dat), nil
+		},
+	)
+	// Not needed but could be useful to have.
+	blockRootUrl := `=~^` + protocol + "://" + address + ":" + strconv.Itoa(port) + "/eth/v1/beacon/blocks/" + `([^/]+)` + "/root"
+	httpmock.RegisterResponder("GET", blockRootUrl,
+		func(req *http.Request) (*http.Response, error) {
+			// Get ID from request
+			slot := httpmock.MustGetSubmatch(req, 1)
+			dat, err := tbc.provideBlockRoot(slot)
+			if err != nil {
+				Expect(err).NotTo(HaveOccurred())
+				return httpmock.NewStringResponse(404, fmt.Sprintf("Unable to find block root for %s", slot)), err
+			}
+			return httpmock.NewBytesResponse(200, dat), nil
+		},
+	)
+}
+
+// Provide the Block root
+func (tbc TestBeaconNode) provideBlockRoot(slot string) ([]byte, error) {
+	for _, val := range tbc.TestEvents {
+		if val.HeadMessage.Slot == slot && val.MimicConfig == nil {
+			block, err := hex.DecodeString(val.HeadMessage.Block[2:])
+			Expect(err).ToNot(HaveOccurred())
+			return block, nil
+		}
+	}
+	return nil, fmt.Errorf("Unable to find the Blockroot in test object.")
+}
+
+// A function to mimic querying the state from the beacon node. We simply get the SSZ file are return it.
+func (tbc TestBeaconNode) provideSsz(slotIdentifier string, sszIdentifier string, dummyParentRoot string) ([]byte, error) {
+	var slotFile string
+	var Message Message
+
+	for _, val := range tbc.TestEvents {
+		if sszIdentifier == "state" {
+			if (val.HeadMessage.Slot == slotIdentifier && val.MimicConfig == nil) || val.HeadMessage.State == slotIdentifier {
+				slotFile = val.BeaconState
+				Message = val
+			}
+		} else if sszIdentifier == "block" {
+			if (val.HeadMessage.Slot == slotIdentifier && val.MimicConfig == nil) || val.HeadMessage.Block == slotIdentifier {
+				slotFile = val.SignedBeaconBlock
+				Message = val
+			}
+		}
+	}
+
+	if Message.MimicConfig != nil {
+		log.Info("We are going to create a custom SSZ object for testing purposes.")
+		if sszIdentifier == "block" {
+			// A dirty solution to handle different Block Types.
+			// * I was unsuccessful in implementing generics.
+			// * I can't use the interfaces.SignedBeaconBlock
+			// * I was short on time.
+			// * This solution allows us to hardcode the version and create the write block type for it when we
+			// Are mimicing an existing block.
+			switch Message.MimicConfig.ForkVersion {
+			case "phase0":
+				block, err := readSignedBeaconBlock(slotFile)
+				if err != nil {
+					return nil, err
+				}
+				slot, err := strconv.ParseUint(Message.HeadMessage.Slot, 10, 64)
+				Expect(err).ToNot(HaveOccurred())
+				block.Block.Slot = types.Slot(slot)
+
+				block.Block.StateRoot, err = hex.DecodeString(Message.HeadMessage.State)
+				Expect(err).ToNot(HaveOccurred())
+
+				if Message.MimicConfig.ParentRoot == "" {
+					block.Block.ParentRoot, err = hex.DecodeString(dummyParentRoot)
+					Expect(err).ToNot(HaveOccurred())
+				} else {
+					block.Block.ParentRoot, err = hex.DecodeString(Message.MimicConfig.ParentRoot)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				return block.MarshalSSZ()
+			case "altair":
+				block, err := readSignedBeaconBlockAltair(slotFile)
+				if err != nil {
+					return nil, err
+				}
+				slot, err := strconv.ParseUint(Message.HeadMessage.Slot, 10, 64)
+				Expect(err).ToNot(HaveOccurred())
+				block.Block.Slot = types.Slot(slot)
+
+				block.Block.StateRoot, err = hex.DecodeString(Message.HeadMessage.State)
+				Expect(err).ToNot(HaveOccurred())
+
+				if Message.MimicConfig.ParentRoot == "" {
+					block.Block.ParentRoot, err = hex.DecodeString(dummyParentRoot)
+					Expect(err).ToNot(HaveOccurred())
+				} else {
+					block.Block.ParentRoot, err = hex.DecodeString(Message.MimicConfig.ParentRoot)
+					Expect(err).ToNot(HaveOccurred())
+				}
+				return block.MarshalSSZ()
+			}
+		}
+		if sszIdentifier == "state" {
+			state, _, err := readBeaconState(slotFile)
+			if err != nil {
+				return nil, err
+			}
+			slot, err := strconv.ParseUint(Message.HeadMessage.Slot, 10, 64)
+			Expect(err).ToNot(HaveOccurred())
+			err = state.SetSlot(types.Slot(slot))
+			Expect(err).ToNot(HaveOccurred())
+			return state.MarshalSSZ()
+		}
+	}
+
+	if slotFile == "" {
+		return nil, fmt.Errorf("We couldn't find the slot file for %s", slotIdentifier)
+	}
+
+	dat, err := os.ReadFile(slotFile)
+	if err != nil {
+		return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
+	}
+	return dat, nil
+}
+
+// Helper function to test three reorg messages. There are going to be many functions like this,
+// Because we need to test the same logic for multiple phases.
+func (tbc TestBeaconNode) testMultipleReorgs(bc *beaconclient.BeaconClient, firstHead beaconclient.Head, secondHead beaconclient.Head, thirdHead beaconclient.Head, epoch int, maxRetry int) {
+	go bc.CaptureHead()
+	time.Sleep(1 * time.Second)
+
+	log.Info("Sending Messages to BeaconClient")
+	sendHeadMessage(bc, firstHead, maxRetry, 1)
+	sendHeadMessage(bc, secondHead, maxRetry, 1)
+	sendHeadMessage(bc, thirdHead, maxRetry, 1)
+
+	curRetry := 0
+	for atomic.LoadUint64(&bc.Metrics.ReorgInserts) != 2 {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(" Too many retries have occurred.")
+		}
+	}
+
+	log.Info("Checking to make sure the fork was marked properly.")
+	validateSlot(bc, firstHead, epoch, "forked")
+	validateSlot(bc, secondHead, epoch, "forked")
+	validateSlot(bc, thirdHead, epoch, "proposed")
+
+	log.Info("Send the reorg message.")
+
+	data, err := json.Marshal(&beaconclient.ChainReorg{
+		Slot:                firstHead.Slot,
+		Depth:               "1",
+		OldHeadBlock:        thirdHead.Block,
+		NewHeadBlock:        secondHead.Block,
+		OldHeadState:        thirdHead.State,
+		NewHeadState:        secondHead.State,
+		Epoch:               strconv.Itoa(epoch),
+		ExecutionOptimistic: false,
+	})
+	Expect(err).ToNot(HaveOccurred())
+	bc.ReOrgTracking.MessagesCh <- &sse.Event{
+		Data: data,
+	}
+
+	curRetry = 0
+	for atomic.LoadUint64(&bc.Metrics.ReorgInserts) != 3 {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail("Too many retries have occurred.")
+		}
+	}
+
+	if bc.Metrics.KnownGapsInserts != 0 {
+		Fail("We found gaps when processing a single block")
+	}
+
+	log.Info("Make sure the forks were properly updated!")
+
+	validateSlot(bc, firstHead, epoch, "forked")
+	validateSlot(bc, secondHead, epoch, "proposed")
+	validateSlot(bc, thirdHead, epoch, "forked")
+
+}
+
+// A test to validate a single block was processed correctly
+func (tbc TestBeaconNode) testProcessBlock(bc *beaconclient.BeaconClient, head beaconclient.Head, epoch int, maxRetry int, expectedSuccessInsert uint64, expectedKnownGaps uint64, expectedReorgs uint64) {
+	go bc.CaptureHead()
+	time.Sleep(1 * time.Second)
+	sendHeadMessage(bc, head, maxRetry, expectedSuccessInsert)
+
+	curRetry := 0
+	for atomic.LoadUint64(&bc.Metrics.KnownGapsInserts) != expectedKnownGaps {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Wrong gap metrics, got: %d, wanted %d", bc.Metrics.KnownGapsInserts, expectedKnownGaps))
+		}
+	}
+
+	curRetry = 0
+	for atomic.LoadUint64(&bc.Metrics.ReorgInserts) != expectedReorgs {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Wrong reorg metrics, got: %d, wanted %d", bc.Metrics.KnownGapsInserts, expectedKnownGaps))
+		}
+	}
+
+	if expectedSuccessInsert > 0 {
+		validateSlot(bc, head, epoch, "proposed")
+	}
+}
+
+// A test that ensures that if two HeadMessages occur for a single slot they are marked
+// as proposed and forked correctly.
+func (tbc TestBeaconNode) testMultipleHead(bc *beaconclient.BeaconClient, firstHead beaconclient.Head, secondHead beaconclient.Head, epoch int, maxRetry int) {
+	go bc.CaptureHead()
+	time.Sleep(1 * time.Second)
+
+	sendHeadMessage(bc, firstHead, maxRetry, 1)
+	sendHeadMessage(bc, secondHead, maxRetry, 1)
+
+	curRetry := 0
+	for atomic.LoadUint64(&bc.Metrics.ReorgInserts) != 1 {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(" Too many retries have occurred.")
+		}
+	}
+
+	if bc.Metrics.KnownGapsInserts != 0 {
+		Fail("We found gaps when processing a single block")
+	}
+
+	log.Info("Checking Altair to make sure the fork was marked properly.")
+	validateSlot(bc, firstHead, epoch, "forked")
+	validateSlot(bc, secondHead, epoch, "proposed")
+}
+
+// A test that ensures that if two HeadMessages occur for a single slot they are marked
+// as proposed and forked correctly.
+func (tbc TestBeaconNode) testKnownGapsMessages(bc *beaconclient.BeaconClient, tableIncrement int, expectedEntries uint64, maxRetry int, msg ...beaconclient.Head) {
+	bc.KnownGapTableIncrement = tableIncrement
+	go bc.CaptureHead()
+	time.Sleep(1 * time.Second)
+
+	for _, headMsg := range msg {
+		sendHeadMessage(bc, headMsg, maxRetry, 1)
+	}
+
+	curRetry := 0
+	for atomic.LoadUint64(&bc.Metrics.KnownGapsInserts) != expectedEntries {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail("Too many retries have occurred.")
+		}
+	}
+
+	log.Info("Checking to make sure we have the expected number of entries in the knownGaps table.")
+	knownGapCount := countKnownGapsTable(bc.Db)
+	Expect(knownGapCount).To(Equal(int(expectedEntries)))
+
+	if atomic.LoadUint64(&bc.Metrics.ReorgInserts) != 0 {
+		Fail("We found reorgs when we didn't expect it")
+	}
+}
+
+// This function will make sure we are properly able to get the SszRoot of the SignedBeaconBlock and the BeaconState.
+func testSszRoot(msg Message) {
+	state, vm, err := readBeaconState(msg.BeaconState)
+	Expect(err).ToNot(HaveOccurred())
+	stateRoot, err := state.HashTreeRoot(context.Background())
+	Expect(err).ToNot(HaveOccurred())
+	Expect(msg.HeadMessage.State).To(Equal("0x" + hex.EncodeToString(stateRoot[:])))
+
+	block, err := readSignedBeaconBlockInterface(msg.SignedBeaconBlock, vm)
+	Expect(err).ToNot(HaveOccurred())
+	blockRoot, err := block.Block().HashTreeRoot()
+	Expect(err).ToNot(HaveOccurred())
+	Expect(msg.HeadMessage.Block).To(Equal("0x" + hex.EncodeToString(blockRoot[:])))
+}

pkg/beaconclient/capturehistoric.go

@@ -0,0 +1,180 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// This file will call all the functions to start and stop capturing the head of the beacon chain.
+
+package beaconclient
+
+import (
+	"context"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
+)
+
+// This function will perform all the heavy lifting for tracking the head of the chain.
+func (bc *BeaconClient) CaptureHistoric(ctx context.Context, maxWorkers int) []error {
+	log.Info("We are starting the historical processing service.")
+	bc.HistoricalProcess = HistoricProcessing{db: bc.Db, metrics: bc.Metrics, uniqueNodeIdentifier: bc.UniqueNodeIdentifier}
+	errs := handleBatchProcess(ctx, maxWorkers, bc.HistoricalProcess, bc.HistoricalProcess.db, bc.ServerEndpoint, bc.Metrics, bc.CheckDb, bc.Metrics.IncrementHistoricSlotProcessed)
+	log.Debug("Exiting Historical")
+	return errs
+}
+
+// This function will perform all the necessary clean up tasks for stopping historical processing.
+func (bc *BeaconClient) StopHistoric(cancel context.CancelFunc) error {
+	log.Info("We are stopping the historical processing service.")
+	cancel()
+	err := bc.HistoricalProcess.releaseDbLocks()
+	if err != nil {
+		loghelper.LogError(err).WithField("uniqueIdentifier", bc.UniqueNodeIdentifier).Error("We were unable to remove the locks from the eth_beacon.historic_processing table. Manual Intervention is needed!")
+	}
+	return nil
+}
+
+// An interface to enforce any batch processing. Currently there are two use cases for this.
+//
+// 1. Historic Processing
+//
+// 2. Known Gaps Processing
+type BatchProcessing interface {
+	getSlotRange(context.Context, chan<- slotsToProcess) []error       // Write the slots to process in a channel, return an error if you cant get the next slots to write.
+	handleProcessingErrors(context.Context, <-chan batchHistoricError) // Custom logic to handle errors.
+	removeTableEntry(context.Context, <-chan slotsToProcess) error     // With the provided start and end slot, remove the entry from the database.
+	releaseDbLocks() error                                             // Update the checked_out column to false for whatever table is being updated.
+}
+
+/// ^^^
+// Might be better to remove the interface and create a single struct that historicalProcessing
+// and knownGapsProcessing can use. The struct would contain all the SQL strings that they need.
+// And the only difference in logic for processing would be within the error handling.
+// Which can be a function we pass into handleBatchProcess()
+
+// A struct to pass around indicating a table entry for slots to process.
+type slotsToProcess struct {
+	startSlot int // The start slot
+	endSlot   int // The end slot
+}
+
+type batchHistoricError struct {
+	err        error  // The error that occurred when attempting to a slot
+	errProcess string // The process that caused the error.
+	slot       int    // The slot which the error is for.
+}
+
+// Wrapper function for the BatchProcessing interface.
+// This function will take the structure that needs batch processing.
+// It follows a generic format.
+// Get new entries from any given table.
+// 1. Add it to the slotsCh.
+//
+// 2. Run the maximum specified workers to handle individual slots. We need a maximum because we don't want
+// To store too many SSZ objects in memory.
+//
+// 3. Process the slots and send the err to the ErrCh. Each structure can define how it wants its own errors handled.
+//
+// 4. Remove the slot entry from the DB.
+//
+// 5. Handle any errors.
+func handleBatchProcess(ctx context.Context, maxWorkers int, bp BatchProcessing, db sql.Database, serverEndpoint string, metrics *BeaconClientMetrics, checkDb bool, incrementTracker func(uint64)) []error {
+	slotsCh := make(chan slotsToProcess)
+	workCh := make(chan int)
+	processedCh := make(chan slotsToProcess)
+	errCh := make(chan batchHistoricError)
+	finalErrCh := make(chan []error, 1)
+
+	// Checkout Rows with same node Identifier.
+	err := bp.releaseDbLocks()
+	if err != nil {
+		loghelper.LogError(err).Error(("We are unable to un-checkout entries at the start!"))
+	}
+
+	// Start workers
+	for w := 1; w <= maxWorkers; w++ {
+		log.WithFields(log.Fields{"maxWorkers": maxWorkers}).Debug("Starting batch processing workers")
+
+		go processSlotRangeWorker(ctx, workCh, errCh, db, serverEndpoint, metrics, checkDb, incrementTracker)
+	}
+
+	// Process all ranges and send each individual slot to the worker.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case slots := <-slotsCh:
+				if slots.startSlot > slots.endSlot {
+					log.Error("We received a batch process request where the startSlot is greater than the end slot.")
+					errCh <- batchHistoricError{
+						err:        fmt.Errorf("We received a startSlot where the start was greater than the end."),
+						errProcess: "RangeOrder",
+						slot:       slots.startSlot,
+					}
+					errCh <- batchHistoricError{
+						err:        fmt.Errorf("We received a endSlot where the start was greater than the end."),
+						errProcess: "RangeOrder",
+						slot:       slots.endSlot,
+					}
+				} else if slots.startSlot == slots.endSlot {
+					log.WithField("slot", slots.startSlot).Debug("Added new slot to workCh")
+					workCh <- slots.startSlot
+					processedCh <- slots
+				} else {
+					for i := slots.startSlot; i <= slots.endSlot; i++ {
+						workCh <- i
+						log.WithField("slot", i).Debug("Added new slot to workCh")
+					}
+					processedCh <- slots
+				}
+			}
+
+		}
+	}()
+
+	// Remove entries, end the application if a row cannot be removed..
+	go func() {
+		errG := new(errgroup.Group)
+		errG.Go(func() error {
+			return bp.removeTableEntry(ctx, processedCh)
+		})
+		if err := errG.Wait(); err != nil {
+			finalErrCh <- []error{err}
+		}
+	}()
+	// Process errors from slot processing.
+	go bp.handleProcessingErrors(ctx, errCh)
+
+	// Get slots from the DB.
+	go func() {
+		errs := bp.getSlotRange(ctx, slotsCh) // Periodically adds new entries....
+		if errs != nil {
+			finalErrCh <- errs
+		}
+		finalErrCh <- nil
+		log.Debug("We are stopping the processing of adding new entries")
+	}()
+	log.Debug("Waiting for shutdown signal from channel")
+	select {
+	case <-ctx.Done():
+		log.Debug("Received shutdown signal from channel")
+		return nil
+	case errs := <-finalErrCh:
+		log.Debug("Finishing the batchProcess")
+		return errs
+	}
+}

pkg/beaconclient/capturehistoric_test.go

@@ -0,0 +1,290 @@
+package beaconclient_test
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/jarcoal/httpmock"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+)
+
+var (
+	kgCheckCheckedOutStmt = `SELECT * FROM eth_beacon.known_gaps WHERE checked_out=true `
+	hpCheckCheckedOutStmt = `SELECT * FROM eth_beacon.historic_process WHERE checked_out=true `
+)
+
+var _ = Describe("Capturehistoric", func() {
+
+	Describe("Run the application in historic mode", Label("unit", "behavioral", "historical"), func() {
+		Context("Phase0 + Altairs: When we need to process a multiple blocks in a multiple entries in the eth_beacon.historic_process table.", Label("deb"), func() {
+			It("Successfully Process the Blocks", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 100, 101, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 2, 0, 0, 0)
+				// Run Two seperate processes
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 2375703, 2375703, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+		Context("When the start block is greater than the endBlock", func() {
+			It("Should Add two entries to the knownGaps table", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 101, 100, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 0, 0, 2, 0)
+			})
+		})
+		Context("Processing the Genesis block", Label("genesis"), func() {
+			It("Should Process properly", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "100")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 0, 0, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 1, 0, 0, 0)
+				validateSlot(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, 0, "proposed")
+				validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, BeaconNodeTester.TestEvents["0"].CorrectParentRoot, BeaconNodeTester.TestEvents["0"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["0"].CorrectSignedBeaconBlockMhKey)
+				validateBeaconState(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, BeaconNodeTester.TestEvents["0"].CorrectBeaconStateMhKey)
+			})
+		})
+		Context("When there is a skipped slot", func() {
+			It("Should process the slot properly.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "3797055")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 3797056, 3797056, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 1, 0, 0, 0)
+				validateSlot(bc, BeaconNodeTester.TestEvents["3797056"].HeadMessage, 118658, "skipped")
+			})
+		})
+	})
+	Describe("Running the Application to process Known Gaps", Label("unit", "behavioral", "knownGaps"), func() {
+		Context("Phase0 + Altairs: When we need to process a multiple blocks in a multiple entries in the eth_beacon.known_gaps table.", func() {
+			It("Successfully Process the Blocks", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToKnownGaps(bc, 100, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 2, 0, 0, 0)
+				// Run Two seperate processes
+				BeaconNodeTester.writeEventToKnownGaps(bc, 2375703, 2375703)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+		Context("When the start block is greater than the endBlock", func() {
+			It("Should Add two entries to the knownGaps table", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "104")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToKnownGaps(bc, 101, 100)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 2, 0, 2, 0)
+			})
+		})
+		Context("When theres a reprocessing error", Label("reprocessingError"), func() {
+			It("Should update the reprocessing error.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				// We dont have an entry in the BeaconNodeTester for this slot
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 105, 105, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 0, 0, 1, 0)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 0, 0, 1, 1)
+			})
+		})
+	})
+	Describe("Running the application in Historic, Head, and KnownGaps mode", Label("unit", "historical", "full"), func() {
+		Context("When it recieves a head, historic and known Gaps message (in order)", func() {
+			It("Should process them all successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				// Head
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 100, 100, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 2, 0, 0, 0)
+
+				// Known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 101, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+		Context("When it recieves a historic, head and known Gaps message (in order)", func() {
+			It("Should process them all successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 100, 100, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 1, 0, 0, 0)
+
+				// Head
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+
+				// Known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 101, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+		Context("When it recieves a known Gaps, historic and head  message (in order)", func() {
+			It("Should process them all successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				// Known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 101, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 1, 0, 0, 0)
+
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 100, 100, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 2, 0, 0, 0)
+
+				// Head
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+	})
+})
+
+// This function will write an even to the eth_beacon.known_gaps table
+func (tbc TestBeaconNode) writeEventToKnownGaps(bc *beaconclient.BeaconClient, startSlot, endSlot int) {
+	log.Debug("We are writing the necessary events to batch process")
+	insertKnownGapsStmt := `INSERT INTO eth_beacon.known_gaps (start_slot, end_slot)
+	VALUES ($1, $2);`
+	res, err := bc.Db.Exec(context.Background(), insertKnownGapsStmt, startSlot, endSlot)
+	Expect(err).ToNot(HaveOccurred())
+	rows, err := res.RowsAffected()
+	if rows != 1 {
+		Fail("We didnt write...")
+	}
+	Expect(err).ToNot(HaveOccurred())
+}
+
+// This function will write an even to the eth_beacon.known_gaps table
+func (tbc TestBeaconNode) writeEventToHistoricProcess(bc *beaconclient.BeaconClient, startSlot, endSlot, priority int) {
+	log.Debug("We are writing the necessary events to batch process")
+	insertHistoricProcessingStmt := `INSERT INTO eth_beacon.historic_process (start_slot, end_slot, priority)
+	VALUES ($1, $2, $3);`
+	res, err := bc.Db.Exec(context.Background(), insertHistoricProcessingStmt, startSlot, endSlot, priority)
+	Expect(err).ToNot(HaveOccurred())
+	rows, err := res.RowsAffected()
+	if rows != 1 {
+		Fail("We didnt write...")
+	}
+	Expect(err).ToNot(HaveOccurred())
+}
+
+// Start the CaptureHistoric function, and check for the correct inserted slots.
+func (tbc TestBeaconNode) runHistoricalProcess(bc *beaconclient.BeaconClient, maxWorkers int, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	ctx, cancel := context.WithCancel(context.Background())
+	go bc.CaptureHistoric(ctx, maxWorkers)
+	validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
+	log.Debug("Calling the stop function for historical processing..")
+	err := bc.StopHistoric(cancel)
+	time.Sleep(5 * time.Second)
+	Expect(err).ToNot(HaveOccurred())
+	validateAllRowsCheckedOut(bc.Db, hpCheckCheckedOutStmt)
+}
+
+// Wrapper function that processes knownGaps
+func (tbc TestBeaconNode) runKnownGapsProcess(bc *beaconclient.BeaconClient, maxWorkers int, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	ctx, cancel := context.WithCancel(context.Background())
+	go bc.ProcessKnownGaps(ctx, maxWorkers)
+	validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
+	err := bc.StopKnownGapsProcessing(cancel)
+	time.Sleep(5 * time.Second)
+	Expect(err).ToNot(HaveOccurred())
+	validateAllRowsCheckedOut(bc.Db, kgCheckCheckedOutStmt)
+}
+
+func validateMetrics(bc *beaconclient.BeaconClient, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	curRetry := 0
+	value := atomic.LoadUint64(&bc.Metrics.SlotInserts)
+	for value != expectedInserts {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Too many retries have occurred. The number of inserts expected %d, the number that actually occurred, %d", expectedInserts, atomic.LoadUint64(&bc.Metrics.SlotInserts)))
+		}
+		value = atomic.LoadUint64(&bc.Metrics.SlotInserts)
+	}
+	curRetry = 0
+	value = atomic.LoadUint64(&bc.Metrics.KnownGapsInserts)
+	for value != expectedKnownGaps {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Too many retries have occurred. The number of knownGaps expected %d, the number that actually occurred, %d", expectedKnownGaps, atomic.LoadUint64(&bc.Metrics.KnownGapsInserts)))
+		}
+		value = atomic.LoadUint64(&bc.Metrics.KnownGapsInserts)
+	}
+	curRetry = 0
+	value = atomic.LoadUint64(&bc.Metrics.KnownGapsReprocessError)
+	for value != expectedKnownGapsReprocessError {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Too many retries have occurred. The number of knownGapsReprocessingErrors expected %d, the number that actually occurred, %d", expectedKnownGapsReprocessError, value))
+		}
+		log.Debug("&bc.Metrics.KnownGapsReprocessError: ", &bc.Metrics.KnownGapsReprocessError)
+		value = atomic.LoadUint64(&bc.Metrics.KnownGapsReprocessError)
+	}
+	curRetry = 0
+	value = atomic.LoadUint64(&bc.Metrics.ReorgInserts)
+	for value != expectedReorgs {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Too many retries have occurred. The number of Reorgs expected %d, the number that actually occurred, %d", expectedReorgs, atomic.LoadUint64(&bc.Metrics.ReorgInserts)))
+		}
+		value = atomic.LoadUint64(&bc.Metrics.ReorgInserts)
+	}
+}
+
+// A wrapper function to validate a few popular blocks
+func validatePopularBatchBlocks(bc *beaconclient.BeaconClient) {
+	validateSlot(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, 3, "proposed")
+	validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectParentRoot, BeaconNodeTester.TestEvents["100"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["100"].CorrectSignedBeaconBlockMhKey)
+	validateBeaconState(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectBeaconStateMhKey)
+
+	validateSlot(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, 3, "proposed")
+	validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, BeaconNodeTester.TestEvents["100"].HeadMessage.Block, BeaconNodeTester.TestEvents["101"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["101"].CorrectSignedBeaconBlockMhKey)
+	validateBeaconState(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, BeaconNodeTester.TestEvents["101"].CorrectBeaconStateMhKey)
+
+	validateSlot(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, "proposed")
+	validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectParentRoot, BeaconNodeTester.TestEvents["2375703"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["2375703"].CorrectSignedBeaconBlockMhKey)
+	validateBeaconState(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectBeaconStateMhKey)
+}
+
+// Make sure all rows have checked_out as false.
+func validateAllRowsCheckedOut(db sql.Database, checkStmt string) {
+	res, err := db.Exec(context.Background(), checkStmt)
+	Expect(err).ToNot(HaveOccurred())
+	rows, err := res.RowsAffected()
+	Expect(err).ToNot(HaveOccurred())
+	Expect(rows).To(Equal(int64(0)))
+}

pkg/beaconclient/checkbeaconserverstatus.go

@@ -0,0 +1,207 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package beaconclient
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync/atomic"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+var (
+	MissingBeaconServerType error = fmt.Errorf("The beacon server type provided is not handled.")
+	LighthouseMissingSlots  error = fmt.Errorf("Anchor is not nil. This means lighthouse has not backfilled all the slots from Genesis to head.")
+)
+
+// The sync response when checking if the node is synced.
+type Sync struct {
+	Data SyncData `json:"data"`
+}
+
+// The sync data
+type SyncData struct {
+	IsSync       bool   `json:"is_syncing"`
+	HeadSlot     string `json:"head_slot"`
+	SyncDistance string `json:"sync_distance"`
+}
+
+// This function will check to see if we are synced up with the head of chain.
+//{"data":{"is_syncing":true,"head_slot":"62528","sync_distance":"3734299"}}
+func (bc BeaconClient) CheckHeadSync() (bool, error) {
+	syncStatus, err := bc.QueryHeadSync()
+	if err != nil {
+		return true, nil
+	}
+	return syncStatus.Data.IsSync, nil
+}
+
+func (bc BeaconClient) QueryHeadSync() (Sync, error) {
+	var syncStatus Sync
+	bcSync := bc.ServerEndpoint + BcSyncStatusEndpoint
+	resp, err := http.Get(bcSync)
+
+	if err != nil {
+		loghelper.LogEndpoint(bcSync).Error("Unable to check the sync status")
+		return syncStatus, err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		loghelper.LogEndpoint(bcSync).WithFields(log.Fields{"returnCode": resp.StatusCode}).Error("Error when getting the sync status")
+		return syncStatus, fmt.Errorf("Querying the sync status returned a non 2xx status code, code provided: %d", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return syncStatus, err
+	}
+
+	if err := json.Unmarshal(body, &syncStatus); err != nil {
+		loghelper.LogEndpoint(bcSync).WithFields(log.Fields{
+			"rawMessage": string(body),
+			"err":        err,
+		}).Error("Unable to unmarshal sync status")
+		return syncStatus, err
+	}
+	return syncStatus, nil
+}
+
+// The response when checking the lighthouse nodes DB info: /lighthouse/database/info
+type LighthouseDatabaseInfo struct {
+	SchemaVersion int        `json:"schema_version"`
+	Config        LhDbConfig `json:"config"`
+	Split         LhDbSplit  `json:"split"`
+	Anchor        LhDbAnchor `json:"anchor"`
+}
+
+// The config field within the DatabaseInfo response.
+type LhDbConfig struct {
+	SlotsPerRestorePoint              int  `json:"slots_per_restore_point"`
+	SlotsPerRestorePointSetExplicitly bool `json:"slots_per_restore_point_set_explicitly"`
+	BlockCacheSize                    int  `json:"block_cache_size"`
+	CompactOnInit                     bool `json:"compact_on_init"`
+	CompactOnPrune                    bool `json:"compact_on_prune"`
+}
+
+// The split field within the DatabaseInfo response.
+type LhDbSplit struct {
+	Slot      string `json:"slot"`
+	StateRoot string `json:"state_root"`
+}
+
+// The anchor field within the DatabaseInfo response.
+type LhDbAnchor struct {
+	AnchorSlot        string `json:"anchor_slot"`
+	OldestBlockSlot   string `json:"oldest_block_slot"`
+	OldestBlockParent string `json:"oldest_block_parent"`
+	StateUpperLimit   string `json:"state_upper_limit"`
+	StateLowerLimit   string `json:"state_lower_limit"`
+}
+
+// This function will notify us what the head slot is.
+func (bc BeaconClient) queryHeadSlotInBeaconServer() (int, error) {
+	syncStatus, err := bc.QueryHeadSync()
+	if err != nil {
+		return 0, nil
+	}
+	headSlot, err := strconv.Atoi(syncStatus.Data.HeadSlot)
+	if err != nil {
+		return 0, nil
+	}
+	return headSlot, nil
+}
+
+// return the lighthouse Database Info
+func (bc BeaconClient) queryLighthouseDbInfo() (LighthouseDatabaseInfo, error) {
+	var dbInfo LighthouseDatabaseInfo
+
+	lhDbInfo := bc.ServerEndpoint + LhDbInfoEndpoint
+	resp, err := http.Get(lhDbInfo)
+
+	if err != nil {
+		loghelper.LogEndpoint(lhDbInfo).Error("Unable to get the lighthouse database information")
+		return dbInfo, err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		loghelper.LogEndpoint(lhDbInfo).WithFields(log.Fields{"returnCode": resp.StatusCode}).Error("Error when getting the lighthouse database information")
+		return dbInfo, fmt.Errorf("Querying the lighthouse database information returned a non 2xx status code, code provided: %d", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return dbInfo, err
+	}
+
+	if err := json.Unmarshal(body, &dbInfo); err != nil {
+		loghelper.LogEndpoint(lhDbInfo).WithFields(log.Fields{
+			"rawMessage": string(body),
+			"err":        err,
+		}).Error("Unable to unmarshal the lighthouse database information")
+		return dbInfo, err
+	}
+	return dbInfo, nil
+}
+
+// This function will tell us what the latest slot is that the beacon server has available. This is important as
+// it will ensure us that we have all slots prior to the given slot.
+func (bc BeaconClient) GetLatestSlotInBeaconServer(beaconServerType string) (int, error) {
+	switch strings.ToLower(beaconServerType) {
+	case "lighthouse":
+		headSlot, err := bc.queryHeadSlotInBeaconServer()
+		if err != nil {
+			return 0, err
+		}
+		lhDb, err := bc.queryLighthouseDbInfo()
+		if err != nil {
+			return 0, err
+		}
+		if lhDb.Anchor == (LhDbAnchor{}) {
+			//atomic.StoreInt64(&bc.LatestSlotInBeaconServer, int64(headSlot))
+			log.WithFields(log.Fields{
+				"headSlot": headSlot,
+			}).Info("Anchor is nil, the lighthouse client has all the nodes from genesis to head.")
+			return headSlot, nil
+		} else {
+			log.WithFields(log.Fields{
+				"lhDb.Anchor": lhDb.Anchor,
+			}).Info(LighthouseMissingSlots.Error())
+			log.Info("We will add a feature down the road to wait for anchor to be null, if its needed.")
+			return 0, LighthouseMissingSlots
+		}
+	default:
+		log.WithFields(log.Fields{"BeaconServerType": beaconServerType}).Error(MissingBeaconServerType.Error())
+		return 0, MissingBeaconServerType
+	}
+}
+
+// A wrapper function for updating the latest slot.
+func (bc BeaconClient) UpdateLatestSlotInBeaconServer(headSlot int64) {
+	curr := atomic.LoadInt64(&bc.LatestSlotInBeaconServer)
+	log.WithFields(log.Fields{
+		"Previous Latest Slot": curr,
+		"New Latest Slot":      headSlot,
+	}).Debug("Swapping Head Slot")
+	atomic.SwapInt64(&bc.LatestSlotInBeaconServer, int64(headSlot))
+}

pkg/beaconclient/databasewrite.go

@@ -0,0 +1,601 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package beaconclient
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/jackc/pgx/v4"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	// Statement to upsert to the eth_beacon.slots table.
+	UpsertSlotsStmt string = `
+INSERT INTO eth_beacon.slots (epoch, slot, block_root, state_root, status)
+VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
+	// Statement to upsert to the eth_beacon.signed_blocks table.
+	UpsertSignedBeaconBlockStmt string = `
+INSERT INTO eth_beacon.signed_block (slot, block_root, parent_block_root, eth1_block_hash, mh_key)
+VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
+	// Statement to upsert to the eth_beacon.state table.
+	UpsertBeaconState string = `
+INSERT INTO eth_beacon.state (slot, state_root, mh_key)
+VALUES ($1, $2, $3) ON CONFLICT (slot, state_root) DO NOTHING`
+	// Statement to upsert to the public.blocks table.
+	UpsertBlocksStmt string = `
+INSERT INTO public.blocks (key, data)
+VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`
+	UpdateForkedStmt string = `UPDATE eth_beacon.slots
+	SET status='forked'
+	WHERE slot=$1 AND block_root<>$2
+	RETURNING block_root;`
+	UpdateProposedStmt string = `UPDATE eth_beacon.slots
+	SET status='proposed'
+	WHERE slot=$1 AND block_root=$2
+	RETURNING block_root;`
+	CheckProposedStmt string = `SELECT slot, block_root
+	FROM eth_beacon.slots
+	WHERE slot=$1 AND block_root=$2;`
+	// Check to see if the slot and block_root exist in eth_beacon.signed_block
+	CheckSignedBeaconBlockStmt string = `SELECT slot, block_root
+	FROM eth_beacon.signed_block
+	WHERE slot=$1 AND block_root=$2`
+	// Check to see if the slot and state_root exist in eth_beacon.state
+	CheckBeaconStateStmt string = `SELECT slot, state_root
+	FROM eth_beacon.state
+	WHERE slot=$1 AND state_root=$2`
+	// Used to get a single slot from the table if it exists
+	QueryBySlotStmt string = `SELECT slot
+	FROM eth_beacon.slots
+	WHERE slot=$1`
+	// Statement to insert known_gaps. We don't pass in timestamp, we let the server take care of that one.
+	UpsertKnownGapsStmt string = `
+INSERT INTO eth_beacon.known_gaps (start_slot, end_slot, checked_out, reprocessing_error, entry_error, entry_process)
+VALUES ($1, $2, $3, $4, $5, $6) on CONFLICT (start_slot, end_slot) DO NOTHING`
+	UpsertKnownGapsErrorStmt string = `
+	UPDATE eth_beacon.known_gaps
+	SET reprocessing_error=$3, priority=priority+1
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Get the highest slot if one exists
+	QueryHighestSlotStmt string = "SELECT COALESCE(MAX(slot), 0) FROM eth_beacon.slots"
+)
+
+// Put all functionality to prepare the write object
+// And write it in this file.
+// Remove any of it from the processslot file.
+type DatabaseWriter struct {
+	Db                   sql.Database
+	Tx                   sql.Tx
+	Ctx                  context.Context
+	Metrics              *BeaconClientMetrics
+	DbSlots              *DbSlots
+	DbSignedBeaconBlock  *DbSignedBeaconBlock
+	DbBeaconState        *DbBeaconState
+	rawBeaconState       *[]byte
+	rawSignedBeaconBlock *[]byte
+}
+
+func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot string, parentBlockRoot string,
+	eth1BlockHash string, status string, rawSignedBeaconBlock *[]byte, rawBeaconState *[]byte, metrics *BeaconClientMetrics) (*DatabaseWriter, error) {
+	ctx := context.Background()
+	tx, err := db.Begin(ctx)
+	if err != nil {
+		loghelper.LogError(err).Error("We are unable to Begin a SQL transaction")
+	}
+	dw := &DatabaseWriter{
+		Db:                   db,
+		Tx:                   tx,
+		Ctx:                  ctx,
+		rawBeaconState:       rawBeaconState,
+		rawSignedBeaconBlock: rawSignedBeaconBlock,
+		Metrics:              metrics,
+	}
+	dw.prepareSlotsModel(slot, stateRoot, blockRoot, status)
+	err = dw.prepareSignedBeaconBlockModel(slot, blockRoot, parentBlockRoot, eth1BlockHash)
+	if err != nil {
+		return nil, err
+	}
+	err = dw.prepareBeaconStateModel(slot, stateRoot)
+	if err != nil {
+		return nil, err
+	}
+	return dw, err
+}
+
+// Write functions to write each all together...
+// Should I do one atomic write?
+// Create the model for the eth_beacon.slots table
+func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoot string, status string) {
+	dw.DbSlots = &DbSlots{
+		Epoch:     calculateEpoch(slot, bcSlotsPerEpoch),
+		Slot:      strconv.Itoa(slot),
+		StateRoot: stateRoot,
+		BlockRoot: blockRoot,
+		Status:    status,
+	}
+	log.Debug("dw.DbSlots: ", dw.DbSlots)
+
+}
+
+// Create the model for the eth_beacon.signed_block table.
+func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot int, blockRoot string, parentBlockRoot string, eth1BlockHash string) error {
+	mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.BlockRoot))
+	if err != nil {
+		return err
+	}
+	dw.DbSignedBeaconBlock = &DbSignedBeaconBlock{
+		Slot:          strconv.Itoa(slot),
+		BlockRoot:     blockRoot,
+		ParentBlock:   parentBlockRoot,
+		Eth1BlockHash: eth1BlockHash,
+		MhKey:         mhKey,
+	}
+	log.Debug("dw.DbSignedBeaconBlock: ", dw.DbSignedBeaconBlock)
+	return nil
+}
+
+// Create the model for the eth_beacon.state table.
+func (dw *DatabaseWriter) prepareBeaconStateModel(slot int, stateRoot string) error {
+	mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.StateRoot))
+	if err != nil {
+		return err
+	}
+	dw.DbBeaconState = &DbBeaconState{
+		Slot:      strconv.Itoa(slot),
+		StateRoot: stateRoot,
+		MhKey:     mhKey,
+	}
+	log.Debug("dw.DbBeaconState: ", dw.DbBeaconState)
+	return nil
+}
+
+// Add all the data for a given slot to a SQL transaction.
+// Originally it wrote to each table individually.
+func (dw *DatabaseWriter) transactFullSlot() error {
+	// If an error occurs, write to knownGaps table.
+	log.WithFields(log.Fields{
+		"slot": dw.DbSlots.Slot,
+	}).Debug("Starting to write to the DB.")
+	err := dw.transactSlots()
+	if err != nil {
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon.slots table...")
+		return err
+	}
+	log.Debug("We finished writing to the eth_beacon.slots table.")
+	if dw.DbSlots.Status != "skipped" {
+		//errG, _ := errgroup.WithContext(context.Background())
+		//errG.Go(func() error {
+		//	return dw.transactSignedBeaconBlocks()
+		//})
+		//errG.Go(func() error {
+		//	return dw.transactBeaconState()
+		//})
+		//if err := errG.Wait(); err != nil {
+		//	loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon block or state table...")
+		//	return err
+		//}
+		// Might want to seperate writing to public.blocks so we can do this concurrently...
+		// Cant concurrently write because we are using a transaction.
+		err := dw.transactSignedBeaconBlocks()
+		if err != nil {
+			loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon block table...")
+			return err
+		}
+		err = dw.transactBeaconState()
+		if err != nil {
+			loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon state table...")
+			return err
+		}
+	}
+	dw.Metrics.IncrementSlotInserts(1)
+	return nil
+}
+
+// Add data for the eth_beacon.slots table to a transaction. For now this is only one function.
+// But in the future if we need to incorporate any FK's or perform any actions to write to the
+// slots table we can do it all here.
+func (dw *DatabaseWriter) transactSlots() error {
+	return dw.upsertSlots()
+}
+
+// Upsert to the eth_beacon.slots table.
+func (dw *DatabaseWriter) upsertSlots() error {
+	_, err := dw.Tx.Exec(dw.Ctx, UpsertSlotsStmt, dw.DbSlots.Epoch, dw.DbSlots.Slot, dw.DbSlots.BlockRoot, dw.DbSlots.StateRoot, dw.DbSlots.Status)
+	if err != nil {
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the eth_beacon.slots table")
+		return err
+	}
+	return nil
+}
+
+// Add the information for the signed_block to a transaction.
+func (dw *DatabaseWriter) transactSignedBeaconBlocks() error {
+	err := dw.upsertPublicBlocks(dw.DbSignedBeaconBlock.MhKey, dw.rawSignedBeaconBlock)
+	if err != nil {
+		return err
+	}
+	err = dw.upsertSignedBeaconBlock()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Upsert to public.blocks.
+func (dw *DatabaseWriter) upsertPublicBlocks(key string, data *[]byte) error {
+	_, err := dw.Tx.Exec(dw.Ctx, UpsertBlocksStmt, key, *data)
+	if err != nil {
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the public.blocks table")
+		return err
+	}
+	return nil
+}
+
+// Upsert to the eth_beacon.signed_block table.
+func (dw *DatabaseWriter) upsertSignedBeaconBlock() error {
+	_, err := dw.Tx.Exec(dw.Ctx, UpsertSignedBeaconBlockStmt, dw.DbSignedBeaconBlock.Slot, dw.DbSignedBeaconBlock.BlockRoot, dw.DbSignedBeaconBlock.ParentBlock, dw.DbSignedBeaconBlock.Eth1BlockHash, dw.DbSignedBeaconBlock.MhKey)
+	if err != nil {
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).WithFields(log.Fields{"block_root": dw.DbSignedBeaconBlock.BlockRoot}).Error("Unable to write to the slot to the eth_beacon.signed_block table")
+		return err
+	}
+	return nil
+}
+
+// Add the information for the state to a transaction.
+func (dw *DatabaseWriter) transactBeaconState() error {
+	err := dw.upsertPublicBlocks(dw.DbBeaconState.MhKey, dw.rawBeaconState)
+	if err != nil {
+		return err
+	}
+	err = dw.upsertBeaconState()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Upsert to the eth_beacon.state table.
+func (dw *DatabaseWriter) upsertBeaconState() error {
+	_, err := dw.Tx.Exec(dw.Ctx, UpsertBeaconState, dw.DbBeaconState.Slot, dw.DbBeaconState.StateRoot, dw.DbBeaconState.MhKey)
+	if err != nil {
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the eth_beacon.state table")
+		return err
+	}
+	return nil
+}
+
+// Update a given slot to be marked as forked within a transaction. Provide the slot and the latest latestBlockRoot.
+// We will mark all entries for the given slot that don't match the provided latestBlockRoot as forked.
+func transactReorgs(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) {
+	slotNum, strErr := strconv.Atoi(slot)
+	if strErr != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, strErr).Error("We can't convert the slot to an int...")
+	}
+
+	forkCount, err := updateForked(tx, ctx, slot, latestBlockRoot)
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while updating all forks.")
+		transactKnownGaps(tx, ctx, 1, slotNum, slotNum, err, "reorg", metrics)
+	}
+	proposedCount, err := updateProposed(tx, ctx, slot, latestBlockRoot)
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while trying to update the proposed slot.")
+		transactKnownGaps(tx, ctx, 1, slotNum, slotNum, err, "reorg", metrics)
+	}
+
+	if forkCount > 0 {
+		loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
+			"forkCount": forkCount,
+		}).Info("Updated rows that were forked.")
+	} else {
+		loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
+			"forkCount": forkCount,
+		}).Warn("There were no forked rows to update.")
+	}
+
+	if proposedCount == 1 {
+		loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
+			"proposedCount": proposedCount,
+		}).Info("Updated the row that should have been marked as proposed.")
+	} else if proposedCount > 1 {
+		loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
+			"proposedCount": proposedCount,
+		}).Error("Too many rows were marked as proposed!")
+		transactKnownGaps(tx, ctx, 1, slotNum, slotNum, fmt.Errorf("Too many rows were marked as unproposed."), "reorg", metrics)
+	} else if proposedCount == 0 {
+		transactKnownGaps(tx, ctx, 1, slotNum, slotNum, fmt.Errorf("Unable to find properly proposed row in DB"), "reorg", metrics)
+		loghelper.LogReorg(slot, latestBlockRoot).Info("Updated the row that should have been marked as proposed.")
+	}
+
+	metrics.IncrementReorgsInsert(1)
+}
+
+// Wrapper function that will create a transaction and execute the function.
+func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) {
+	ctx := context.Background()
+	tx, err := db.Begin(ctx)
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Fatal("Unable to create a new transaction for reorgs")
+	}
+	defer func() {
+		err := tx.Rollback(ctx)
+		if err != nil && err != pgx.ErrTxClosed {
+			loghelper.LogError(err).Error("We were unable to Rollback a transaction for reorgs")
+		}
+	}()
+	transactReorgs(tx, ctx, slot, latestBlockRoot, metrics)
+	if err = tx.Commit(ctx); err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Fatal("Unable to execute the transaction for reorgs")
+	}
+}
+
+// Update the slots table by marking the old slot's as forked.
+func updateForked(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) {
+	res, err := tx.Exec(ctx, UpdateForkedStmt, slot, latestBlockRoot)
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the forked slots")
+		return 0, err
+	}
+	count, err := res.RowsAffected()
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("Unable to figure out how many entries were marked as forked.")
+		return 0, err
+	}
+	return count, err
+}
+
+// Mark a slot as proposed.
+func updateProposed(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) {
+	res, err := tx.Exec(ctx, UpdateProposedStmt, slot, latestBlockRoot)
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the proposed slot.")
+		return 0, err
+	}
+	count, err := res.RowsAffected()
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("Unable to figure out how many entries were marked as proposed")
+		return 0, err
+	}
+
+	return count, err
+}
+
+// A wrapper function to call upsertKnownGaps. This function will break down the range of known_gaps into
+// smaller chunks. For example, instead of having an entry of 1-101, if we increment the entries by 10 slots, we would
+// have 10 entries as follows: 1-10, 11-20, etc...
+func transactKnownGaps(tx sql.Tx, ctx context.Context, tableIncrement int, startSlot int, endSlot int, entryError error, entryProcess string, metric *BeaconClientMetrics) {
+	var entryErrorMsg string
+	if entryError == nil {
+		entryErrorMsg = ""
+	} else {
+		entryErrorMsg = entryError.Error()
+	}
+	if endSlot-startSlot <= tableIncrement {
+		kgModel := DbKnownGaps{
+			StartSlot:         strconv.Itoa(startSlot),
+			EndSlot:           strconv.Itoa(endSlot),
+			CheckedOut:        false,
+			ReprocessingError: "",
+			EntryError:        entryErrorMsg,
+			EntryProcess:      entryProcess,
+		}
+		upsertKnownGaps(tx, ctx, kgModel, metric)
+	} else {
+		totalSlots := endSlot - startSlot
+		var chunks int
+		chunks = totalSlots / tableIncrement
+		if totalSlots%tableIncrement != 0 {
+			chunks = chunks + 1
+		}
+
+		for i := 0; i < chunks; i++ {
+			var tempStart, tempEnd int
+			tempStart = startSlot + (i * tableIncrement)
+			if i+1 == chunks {
+				tempEnd = endSlot
+			} else {
+				tempEnd = startSlot + ((i + 1) * tableIncrement)
+			}
+			kgModel := DbKnownGaps{
+				StartSlot:         strconv.Itoa(tempStart),
+				EndSlot:           strconv.Itoa(tempEnd),
+				CheckedOut:        false,
+				ReprocessingError: "",
+				EntryError:        entryErrorMsg,
+				EntryProcess:      entryProcess,
+			}
+			upsertKnownGaps(tx, ctx, kgModel, metric)
+		}
+	}
+}
+
+// Wrapper function, instead of adding the knownGaps entries to a transaction, it will
+// create the transaction and write it.
+func writeKnownGaps(db sql.Database, tableIncrement int, startSlot int, endSlot int, entryError error, entryProcess string, metric *BeaconClientMetrics) {
+	ctx := context.Background()
+	tx, err := db.Begin(ctx)
+	if err != nil {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Fatal("Unable to create a new transaction for knownGaps")
+	}
+	defer func() {
+		err := tx.Rollback(ctx)
+		if err != nil && err != pgx.ErrTxClosed {
+			loghelper.LogError(err).Error("We were unable to Rollback a transaction for reorgs")
+		}
+	}()
+	transactKnownGaps(tx, ctx, tableIncrement, startSlot, endSlot, entryError, entryProcess, metric)
+	if err = tx.Commit(ctx); err != nil {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Fatal("Unable to execute the transaction for knownGaps")
+	}
+}
+
+// A function to upsert a single entry to the eth_beacon.known_gaps table.
+func upsertKnownGaps(tx sql.Tx, ctx context.Context, knModel DbKnownGaps, metric *BeaconClientMetrics) {
+	_, err := tx.Exec(ctx, UpsertKnownGapsStmt, knModel.StartSlot, knModel.EndSlot,
+		knModel.CheckedOut, knModel.ReprocessingError, knModel.EntryError, knModel.EntryProcess)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"err":       err,
+			"startSlot": knModel.StartSlot,
+			"endSlot":   knModel.EndSlot,
+		}).Fatal("We are unable to write to the eth_beacon.known_gaps table!!! We will stop the application because of that.")
+	}
+	log.WithFields(log.Fields{
+		"startSlot": knModel.StartSlot,
+		"endSlot":   knModel.EndSlot,
+	}).Warn("A new gap has been added to the eth_beacon.known_gaps table.")
+	metric.IncrementKnownGapsInserts(1)
+}
+
+// A function to write the gap between the highest slot in the DB and the first processed slot.
+func writeStartUpGaps(db sql.Database, tableIncrement int, firstSlot int, metric *BeaconClientMetrics) {
+	var maxSlot int
+	err := db.QueryRow(context.Background(), QueryHighestSlotStmt).Scan(&maxSlot)
+	if err != nil {
+		loghelper.LogError(err).Fatal("Unable to get the max block from the DB. We must close the application or we might have undetected gaps.")
+	}
+
+	if err != nil {
+		loghelper.LogError(err).WithFields(log.Fields{
+			"maxSlot": maxSlot,
+		}).Fatal("Unable to get convert max block from DB to int. We must close the application or we might have undetected gaps.")
+	}
+	if maxSlot != firstSlot-1 {
+		if maxSlot < firstSlot-1 {
+			if maxSlot == 0 {
+				writeKnownGaps(db, tableIncrement, maxSlot, firstSlot-1, fmt.Errorf(""), "startup", metric)
+			} else {
+				writeKnownGaps(db, tableIncrement, maxSlot+1, firstSlot-1, fmt.Errorf(""), "startup", metric)
+			}
+		} else {
+			log.WithFields(log.Fields{
+				"maxSlot":   maxSlot,
+				"firstSlot": firstSlot,
+			}).Warn("The maxSlot in the DB is greater than or equal to the first Slot we are processing.")
+		}
+	}
+}
+
+// A function to update a knownGap range with a reprocessing error.
+func updateKnownGapErrors(db sql.Database, startSlot int, endSlot int, reprocessingErr error, metric *BeaconClientMetrics) error {
+	res, err := db.Exec(context.Background(), UpsertKnownGapsErrorStmt, startSlot, endSlot, reprocessingErr.Error())
+	if err != nil {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Error("Unable to update reprocessing_error")
+		return err
+	}
+	row, err := res.RowsAffected()
+	if err != nil {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Error("Unable to count rows affected when trying to update reprocessing_error.")
+		return err
+	}
+	if row != 1 {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).WithFields(log.Fields{
+			"rowCount": row,
+		}).Error("The rows affected by the upsert for reprocessing_error is not 1.")
+		metric.IncrementKnownGapsReprocessError(1)
+		return err
+	}
+	metric.IncrementKnownGapsReprocessError(1)
+	return nil
+}
+
+// A quick helper function to calculate the epoch.
+func calculateEpoch(slot int, slotPerEpoch int) string {
+	epoch := slot / slotPerEpoch
+	return strconv.Itoa(epoch)
+}
+
+// A helper function to check to see if the slot is processed.
+func isSlotProcessed(db sql.Database, checkProcessStmt string, slot string) (bool, error) {
+	processRow, err := db.Exec(context.Background(), checkProcessStmt, slot)
+	if err != nil {
+		return false, err
+	}
+	row, err := processRow.RowsAffected()
+	if err != nil {
+		return false, err
+	}
+	if row > 0 {
+		return true, nil
+	}
+	return false, nil
+}
+
+// Check to see if this slot is in the DB. Check eth_beacon.slots, eth_beacon.signed_block
+// and eth_beacon.state. If the slot exists, return true
+func IsSlotInDb(ctx context.Context, db sql.Database, slot string, blockRoot string, stateRoot string) (bool, error) {
+	var (
+		isInBeaconState       bool
+		isInSignedBeaconBlock bool
+	)
+	errG, _ := errgroup.WithContext(context.Background())
+	errG.Go(func() error {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			var err error
+			isInBeaconState, err = checkSlotAndRoot(db, CheckBeaconStateStmt, slot, stateRoot)
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to check if the slot and stateroot exist in eth_beacon.state")
+			}
+			return err
+		}
+	})
+	errG.Go(func() error {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			var err error
+			isInSignedBeaconBlock, err = checkSlotAndRoot(db, CheckSignedBeaconBlockStmt, slot, blockRoot)
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to check if the slot and block_root exist in eth_beacon.signed_block")
+			}
+			return err
+		}
+	})
+	if err := errG.Wait(); err != nil {
+		return false, err
+	}
+	if isInBeaconState && isInSignedBeaconBlock {
+		return true, nil
+	}
+	return false, nil
+}
+
+// Provide a statement, slot, and root, and this function will check to see
+// if the slot and root exist in the table.
+func checkSlotAndRoot(db sql.Database, statement, slot, root string) (bool, error) {
+	processRow, err := db.Exec(context.Background(), statement, slot, root)
+	if err != nil {
+		return false, err
+	}
+	row, err := processRow.RowsAffected()
+	if err != nil {
+		return false, err
+	}
+	if row > 0 {
+		return true, nil
+	}
+	return false, nil
+}

pkg/beaconclient/healthcheck.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
 import (
@@ -5,7 +20,7 @@ import (
 	"net/http"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // This function will ensure that we can connect to the beacon client.
@@ -22,8 +37,8 @@ func (bc BeaconClient) CheckBeaconClient() error {
 	}
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		log.Error("We recieved a non 2xx status code when checking the health of the beacon node.")
+		loghelper.LogEndpoint(bcEndpoint).Error("We recieved a non 2xx status code when checking the health of the beacon node.")
-		log.Error("Health Endpoint Status Code: ", resp.StatusCode)
+		loghelper.LogEndpoint(bcEndpoint).Error("Health Endpoint Status Code: ", resp.StatusCode)
 		return fmt.Errorf("beacon Node Provided a non 2xx status code, code provided: %d", resp.StatusCode)
 	}
 

pkg/beaconclient/healthcheck_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient_test
 
 import (
@@ -5,18 +20,27 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	beaconclient "github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient"
+	beaconclient "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
 )
 
 var _ = Describe("Healthcheck", func() {
 	var (
-		BC    = beaconclient.CreateBeaconClient(context.Background(), "http", "localhost", 5052)
+		Bc    *beaconclient.BeaconClient
-		errBc = beaconclient.CreateBeaconClient(context.Background(), "http", "blah-blah", 1010)
+		errBc *beaconclient.BeaconClient
 	)
+
+	BeforeEach(func() {
+		var err error
+		Bc, err = beaconclient.CreateBeaconClient(context.Background(), "http", "localhost", 5052, 10, bcUniqueIdentifier, false)
+		Expect(err).ToNot(HaveOccurred())
+		errBc, err = beaconclient.CreateBeaconClient(context.Background(), "http", "blah-blah", 1010, 10, bcUniqueIdentifier, false)
+		Expect(err).ToNot(HaveOccurred())
+
+	})
 	Describe("Connecting to the lighthouse client", Label("integration"), func() {
 		Context("When the client is running", func() {
 			It("We should connect successfully", func() {
-				err := BC.CheckBeaconClient()
+				err := Bc.CheckBeaconClient()
 				Expect(err).To(BeNil())
 			})
 		})

pkg/beaconclient/incomingsse.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This package will handle all event subscriptions that utilize SSE.
 
 package beaconclient
@@ -7,7 +22,8 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
 )
 
 var (
@@ -17,14 +33,33 @@ var (
 // This function will capture all the SSE events for a given SseEvents object.
 // When new messages come in, it will ensure that they are decoded into JSON.
 // If any errors occur, it log the error information.
-func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P]) {
+func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P], errMetricInc func(uint64)) {
-	loghelper.LogEndpoint(eventHandler.Endpoint).Info("Subscribing to Messages")
+	go func() {
-	go eventHandler.SseClient.SubscribeChanRaw(eventHandler.MessagesCh)
+		errG := new(errgroup.Group)
+		errG.Go(func() error {
+			err := eventHandler.SseClient.SubscribeChanRaw(eventHandler.MessagesCh)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+		if err := errG.Wait(); err != nil {
+			log.WithFields(log.Fields{
+				"err":      err,
+				"endpoint": eventHandler.Endpoint,
+			}).Error("Unable to subscribe to the SSE endpoint.")
+			return
+		} else {
+			loghelper.LogEndpoint(eventHandler.Endpoint).Info("Successfully subscribed to the event stream.")
+		}
+
+	}()
 	for {
 		select {
 		case message := <-eventHandler.MessagesCh:
 			// Message can be nil if its a keep-alive message
 			if len(message.Data) != 0 {
+				log.WithFields(log.Fields{"msg": string(message.Data)}).Debug("We are going to send the following message to be processed.")
 				go processMsg(message.Data, eventHandler.ProcessCh, eventHandler.ErrorCh)
 			}
 
@@ -35,9 +70,7 @@ func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P]) {
 				"msg":      headErr.msg,
 			},
 			).Error("Unable to handle event.")
-
+			errMetricInc(1)
-		case process := <-eventHandler.ProcessCh:
-			log.WithFields(log.Fields{"processed": process}).Debug("Processesing a Message")
 		}
 	}
 }
@@ -60,7 +93,6 @@ func processMsg[P ProcessedEvents](msg []byte, processCh chan<- *P, errorCh chan
 // Capture all of the event topics.
 func (bc *BeaconClient) captureEventTopic() {
 	log.Info("We are capturing all SSE events")
-	go handleIncomingSseEvent(bc.HeadTracking)
+	go handleIncomingSseEvent(bc.HeadTracking, bc.Metrics.IncrementHeadError)
-	go handleIncomingSseEvent(bc.ReOrgTracking)
+	go handleIncomingSseEvent(bc.ReOrgTracking, bc.Metrics.IncrementReorgError)
-	go handleIncomingSseEvent(bc.FinalizationTracking)
 }

pkg/beaconclient/metrics.go

@@ -0,0 +1,150 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package beaconclient
+
+import (
+	"sync/atomic"
+
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+//Create a metric struct and register each channel with prometheus
+func CreateBeaconClientMetrics() (*BeaconClientMetrics, error) {
+	metrics := &BeaconClientMetrics{
+		SlotInserts:             0,
+		ReorgInserts:            0,
+		KnownGapsInserts:        0,
+		KnownGapsProcessed:      0,
+		KnownGapsReprocessError: 0,
+		HeadError:               0,
+		HeadReorgError:          0,
+	}
+	err := prometheusRegisterHelper("slot_inserts", "Keeps track of the number of slots we have inserted.", &metrics.SlotInserts)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("reorg_inserts", "Keeps track of the number of reorgs we have inserted.", &metrics.ReorgInserts)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("known_gaps_inserts", "Keeps track of the number of known gaps we have inserted.", &metrics.KnownGapsInserts)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("known_gaps_reprocess_error", "Keeps track of the number of known gaps that had errors when reprocessing, but the error was updated successfully.", &metrics.KnownGapsReprocessError)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("known_gaps_processed", "Keeps track of the number of known gaps we successfully processed.", &metrics.KnownGapsProcessed)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("historic_slots_processed", "Keeps track of the number of historic slots we successfully processed.", &metrics.HistoricSlotProcessed)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("head_error", "Keeps track of the number of errors we had processing head messages.", &metrics.HeadError)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("head_reorg_error", "Keeps track of the number of errors we had processing reorg messages.", &metrics.HeadReorgError)
+	if err != nil {
+		return nil, err
+	}
+	return metrics, nil
+}
+
+func prometheusRegisterHelper(name string, help string, varPointer *uint64) error {
+	err := prometheus.Register(prometheus.NewCounterFunc(
+		prometheus.CounterOpts{
+			Namespace:   "beacon_client",
+			Subsystem:   "",
+			Name:        name,
+			Help:        help,
+			ConstLabels: map[string]string{},
+		},
+		func() float64 {
+			return float64(atomic.LoadUint64(varPointer))
+		}))
+	if err != nil && err.Error() != "duplicate metrics collector registration attempted" {
+		loghelper.LogError(err).WithField("name", name).Error("Unable to register counter.")
+		return err
+	}
+	return nil
+}
+
+// A structure utilized for keeping track of various metrics. Currently, mostly used in testing.
+type BeaconClientMetrics struct {
+	SlotInserts             uint64 // Number of head events we successfully wrote to the DB.
+	ReorgInserts            uint64 // Number of reorg events we successfully wrote to the DB.
+	KnownGapsInserts        uint64 // Number of known_gaps we successfully wrote to the DB.
+	KnownGapsProcessed      uint64 // Number of knownGaps processed.
+	KnownGapsReprocessError uint64 // Number of knownGaps that were updated with an error.
+	HistoricSlotProcessed   uint64 // Number of historic slots successfully processed.
+	HeadError               uint64 // Number of errors that occurred when decoding the head message.
+	HeadReorgError          uint64 // Number of errors that occurred when decoding the reorg message.
+}
+
+// Wrapper function to increment inserts. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementSlotInserts(inc uint64) {
+	log.Debug("Incrementing Slot Insert")
+	atomic.AddUint64(&m.SlotInserts, inc)
+}
+
+// Wrapper function to increment reorgs. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementReorgsInsert(inc uint64) {
+	atomic.AddUint64(&m.ReorgInserts, inc)
+}
+
+// Wrapper function to increment known gaps. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementKnownGapsInserts(inc uint64) {
+	atomic.AddUint64(&m.KnownGapsInserts, inc)
+}
+
+// Wrapper function to increment known gaps processed. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementKnownGapsProcessed(inc uint64) {
+	atomic.AddUint64(&m.KnownGapsProcessed, inc)
+}
+
+// Wrapper function to increment head errors. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementHeadError(inc uint64) {
+	atomic.AddUint64(&m.HeadError, inc)
+}
+
+// Wrapper function to increment reorg errors. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementReorgError(inc uint64) {
+	atomic.AddUint64(&m.HeadReorgError, inc)
+}
+
+// Wrapper function to increment the number of knownGaps that were updated with reprocessing errors.
+//If we want to use mutexes later we can easily update all occurrences here.
+func (m *BeaconClientMetrics) IncrementKnownGapsReprocessError(inc uint64) {
+	atomic.AddUint64(&m.KnownGapsReprocessError, inc)
+}
+
+// Wrapper function to increment the number of historicSlots that were processed successfully.
+// If we want to use mutexes later we can easily update all occurrences here.
+func (m *BeaconClientMetrics) IncrementHistoricSlotProcessed(inc uint64) {
+	atomic.AddUint64(&m.HistoricSlotProcessed, inc)
+}

pkg/beaconclient/models.go

@@ -1,8 +1,23 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
 // This interface captured what the events can be for processed event streams.
 type ProcessedEvents interface {
-	Head | FinalizedCheckpoint | ChainReorg
+	Head | ChainReorg
 }
 
 // This struct captures the JSON representation of the head topic
@@ -35,3 +50,40 @@ type ChainReorg struct {
 	Epoch               string `json:"epoch"`
 	ExecutionOptimistic bool   `json:"execution_optimistic"`
 }
+
+// A struct to capture whats being written to the eth-beacon.slots table.
+type DbSlots struct {
+	Epoch     string // The epoch.
+	Slot      string // The slot.
+	BlockRoot string // The block root
+	StateRoot string // The state root
+	Status    string // The status, it can be proposed | forked | skipped.
+}
+
+// A struct to capture whats being written to eth-beacon.signed_block table.
+type DbSignedBeaconBlock struct {
+	Slot          string // The slot.
+	BlockRoot     string // The block root
+	ParentBlock   string // The parent block root.
+	Eth1BlockHash string // The eth1 block_hash
+	MhKey         string // The ipld multihash key.
+
+}
+
+// A struct to capture whats being written to eth-beacon.state table.
+type DbBeaconState struct {
+	Slot      string // The slot.
+	StateRoot string // The state root
+	MhKey     string // The ipld multihash key.
+}
+
+// A structure to capture whats being written to the eth-beacon.known_gaps table.
+type DbKnownGaps struct {
+	StartSlot         string // The start slot for known_gaps, inclusive.
+	EndSlot           string // The end slot for known_gaps, inclusive.
+	CheckedOut        bool   // Indicates if any process is currently processing this entry.
+	ReprocessingError string // The error that occurred when attempting to reprocess these entries.
+	EntryError        string // The error that caused this entry to be added to the table. Could be null.
+	EntryTime         string // The time this range was added to the DB. This can help us catch ranges that have not been processed for a long time due to some error.
+	EntryProcess      string // The entry process that added this process. Potential options are StartUp, Error, ManualEntry, HeadGap.
+}

pkg/beaconclient/multihash.go

@@ -0,0 +1,39 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package beaconclient
+
+import (
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	dshelp "github.com/ipfs/go-ipfs-ds-help"
+	"github.com/multiformats/go-multihash"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+const SSZ_SHA2_256_PREFIX uint64 = 0xb502
+
+// MultihashKeyFromSSZRoot converts a SSZ-SHA2-256 root hash into a blockstore prefixed multihash key
+func MultihashKeyFromSSZRoot(root []byte) (string, error) {
+	mh, err := multihash.Encode(root, SSZ_SHA2_256_PREFIX)
+	if err != nil {
+		loghelper.LogError(err).Error("Unable to create a multihash Key")
+		return "", err
+	}
+	dbKey := dshelp.MultihashToDsKey(mh)
+	mhKey := blockstore.BlockPrefix.String() + dbKey.String()
+	log.WithFields(log.Fields{"mhKey": mhKey, "len": len(root)}).Debug("The MHKEY")
+	return mhKey, nil
+}

pkg/beaconclient/processevents.go

@@ -1,38 +1,77 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This file contains all the functions to handle SSE events after they have been turned
 // to the structs.
 
 package beaconclient
 
-import log "github.com/sirupsen/logrus"
+import (
+	"fmt"
+	"strconv"
+
+	log "github.com/sirupsen/logrus"
+)
 
 // This function will perform the necessary steps to handle a reorg.
-func (bc *BeaconClient) handleReorgs() {
+func (bc *BeaconClient) handleReorg() {
 	log.Info("Starting to process reorgs.")
 	for {
-		// We will add real functionality later
 		reorg := <-bc.ReOrgTracking.ProcessCh
 		log.WithFields(log.Fields{"reorg": reorg}).Debug("Received a new reorg message.")
+		writeReorgs(bc.Db, reorg.Slot, reorg.NewHeadBlock, bc.Metrics)
 	}
 }
 
-// This function will perform the necessary steps to handle a reorg.
-func (bc *BeaconClient) handleFinalizedCheckpoint() {
-	log.Info("Starting to process finalized checkpoints.")
-	for {
-		// We will add real functionality later
-		finalized := <-bc.ReOrgTracking.ProcessCh
-		log.WithFields(log.Fields{"finalized": finalized}).Debug("Received a new finalized checkpoint.")
-	}
-
-}
-
 // This function will handle the latest head event.
 func (bc *BeaconClient) handleHead() {
 	log.Info("Starting to process head.")
+	errorSlots := 0
 	for {
-		// We will add real functionality later
+		head := <-bc.HeadTracking.ProcessCh
-		head := <-bc.ReOrgTracking.ProcessCh
+		// Process all the work here.
-		log.WithFields(log.Fields{"head": head}).Debug("Received a new head event.")
+		slot, err := strconv.Atoi(head.Slot)
-	}
+		if err != nil {
+			bc.HeadTracking.ErrorCh <- &SseError{
+				err: fmt.Errorf("Unable to turn the slot from string to int: %s", head.Slot),
+			}
+			errorSlots = errorSlots + 1
+			continue
+		}
+		if errorSlots != 0 && bc.PreviousSlot != 0 {
+			log.WithFields(log.Fields{
+				"lastProcessedSlot": bc.PreviousSlot,
+				"errorSlots":        errorSlots,
+			}).Warn("We added slots to the knownGaps table because we got bad head messages.")
+			writeKnownGaps(bc.Db, bc.KnownGapTableIncrement, bc.PreviousSlot+1, slot, fmt.Errorf("Bad Head Messages"), "headProcessing", bc.Metrics)
+			errorSlots = 0
+		}
 
+		log.WithFields(log.Fields{"head": head}).Debug("We are going to start processing the slot.")
+
+		// Not used anywhere yet but might be useful to have.
+		if bc.PreviousSlot == 0 && bc.PreviousBlockRoot == "" {
+			bc.StartingSlot = slot
+		}
+
+		go processHeadSlot(bc.Db, bc.ServerEndpoint, slot, head.Block, head.State, bc.PreviousSlot, bc.PreviousBlockRoot, bc.Metrics, bc.KnownGapTableIncrement, bc.CheckDb)
+
+		log.WithFields(log.Fields{"head": head.Slot}).Debug("We finished calling processHeadSlot.")
+
+		// Update the previous block
+		bc.PreviousSlot = slot
+		bc.PreviousBlockRoot = head.Block
+	}
 }

pkg/beaconclient/processhistoric.go

@@ -0,0 +1,269 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains all the code to process historic slots.
+
+package beaconclient
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/jackc/pgx/v4"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+var (
+	// Get a single highest priority and non-checked out row row from eth_beacon.historical_process
+	getHpEntryStmt string = `SELECT start_slot, end_slot FROM eth_beacon.historic_process
+	WHERE checked_out=false
+	ORDER BY priority ASC
+	LIMIT 1;`
+	// Used to periodically check to see if there is a new entry in the eth_beacon.historic_process table.
+	checkHpEntryStmt string = `SELECT * FROM eth_beacon.historic_process WHERE checked_out=false;`
+	// Used to checkout a row from the eth_beacon.historic_process table
+	lockHpEntryStmt string = `UPDATE eth_beacon.historic_process
+	SET checked_out=true, checked_out_by=$3
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to delete an entry from the eth_beacon.historic_process table
+	deleteHpEntryStmt string = `DELETE FROM eth_beacon.historic_process
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to update every single row that this node has checked out.
+	releaseHpLockStmt string = `UPDATE eth_beacon.historic_process
+	SET checked_out=false, checked_out_by=null
+	WHERE checked_out_by=$1`
+)
+
+type HistoricProcessing struct {
+	db                   sql.Database         //db connection
+	metrics              *BeaconClientMetrics // metrics for beaconclient
+	uniqueNodeIdentifier int                  // node unique identifier.
+}
+
+// Get a single row of historical slots from the table.
+func (hp HistoricProcessing) getSlotRange(ctx context.Context, slotCh chan<- slotsToProcess) []error {
+	return getBatchProcessRow(ctx, hp.db, getHpEntryStmt, checkHpEntryStmt, lockHpEntryStmt, slotCh, strconv.Itoa(hp.uniqueNodeIdentifier))
+}
+
+// Remove the table entry.
+func (hp HistoricProcessing) removeTableEntry(ctx context.Context, processCh <-chan slotsToProcess) error {
+	return removeRowPostProcess(ctx, hp.db, processCh, QueryBySlotStmt, deleteHpEntryStmt)
+}
+
+// Remove the table entry.
+func (hp HistoricProcessing) handleProcessingErrors(ctx context.Context, errMessages <-chan batchHistoricError) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case errMs := <-errMessages:
+			loghelper.LogSlotError(strconv.Itoa(errMs.slot), errMs.err)
+			writeKnownGaps(hp.db, 1, errMs.slot, errMs.slot, errMs.err, errMs.errProcess, hp.metrics)
+		}
+	}
+}
+
+// "un"-checkout the rows held by this DB in the eth_beacon.historical_process table.
+func (hp HistoricProcessing) releaseDbLocks() error {
+	log.Debug("Updating all the entries to eth_beacon.historical processing")
+	res, err := hp.db.Exec(context.Background(), releaseHpLockStmt, hp.uniqueNodeIdentifier)
+	if err != nil {
+		return fmt.Errorf("Unable to remove lock from eth_beacon.historical_processing table for node %d, error is %e", hp.uniqueNodeIdentifier, err)
+	}
+	log.Debug("Update all the entries to eth_beacon.historical processing")
+	rows, err := res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("Unable to calculated number of rows affected by releasing locks from eth_beacon.historical_processing table for node %d, error is %e", hp.uniqueNodeIdentifier, err)
+	}
+	log.WithField("rowCount", rows).Info("Released historicalProcess locks for specified rows.")
+	return nil
+}
+
+// Process the slot range.
+func processSlotRangeWorker(ctx context.Context, workCh <-chan int, errCh chan<- batchHistoricError, db sql.Database, serverAddress string, metrics *BeaconClientMetrics, checkDb bool, incrementTracker func(uint64)) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case slot := <-workCh:
+			log.Debug("Handling slot: ", slot)
+			err, errProcess := handleHistoricSlot(ctx, db, serverAddress, slot, metrics, checkDb)
+			if err != nil {
+				errMs := batchHistoricError{
+					err:        err,
+					errProcess: errProcess,
+					slot:       slot,
+				}
+				errCh <- errMs
+			} else {
+				incrementTracker(1)
+			}
+		}
+	}
+}
+
+// A wrapper function that insert the start_slot and end_slot from a single row into a channel.
+// It also locks the row by updating the checked_out column.
+// The statement for getting the start_slot and end_slot must be provided.
+// The statement for "locking" the row must also be provided.
+func getBatchProcessRow(ctx context.Context, db sql.Database, getStartEndSlotStmt string, checkNewRowsStmt string, checkOutRowStmt string, slotCh chan<- slotsToProcess, uniqueNodeIdentifier string) []error {
+	errCount := make([]error, 0)
+
+	// 5 is an arbitrary number. It allows us to retry a few times before
+	// ending the application.
+	prevErrCount := 0
+	for len(errCount) < 5 {
+		select {
+		case <-ctx.Done():
+			return errCount
+		default:
+			if len(errCount) != prevErrCount {
+				log.WithFields(log.Fields{
+					"errCount": errCount,
+				}).Error("New error entry added")
+			}
+			processRow, err := db.Exec(context.Background(), checkNewRowsStmt)
+			if err != nil {
+				errCount = append(errCount, err)
+			}
+			row, err := processRow.RowsAffected()
+			if err != nil {
+				errCount = append(errCount, err)
+			}
+			if row < 1 {
+				time.Sleep(3 * time.Second)
+				log.Debug("We are checking rows, be patient")
+				break
+			}
+			log.Debug("We found a new row")
+			dbCtx := context.Background()
+
+			// Setup TX
+			tx, err := db.Begin(dbCtx)
+			if err != nil {
+				loghelper.LogError(err).Error("We are unable to Begin a SQL transaction")
+				errCount = append(errCount, err)
+				break
+			}
+			defer func() {
+				err := tx.Rollback(dbCtx)
+				if err != nil && err != pgx.ErrTxClosed {
+					loghelper.LogError(err).Error("We were unable to Rollback a transaction")
+					errCount = append(errCount, err)
+				}
+			}()
+
+			// Query the DB for slots.
+			sp := slotsToProcess{}
+			err = tx.QueryRow(dbCtx, getStartEndSlotStmt).Scan(&sp.startSlot, &sp.endSlot)
+			if err != nil {
+				if err == pgx.ErrNoRows {
+					time.Sleep(1 * time.Second)
+					break
+				}
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), getStartEndSlotStmt, err).Error("Unable to get a row")
+				errCount = append(errCount, err)
+				break
+			}
+
+			// Checkout the Row
+			res, err := tx.Exec(dbCtx, checkOutRowStmt, sp.startSlot, sp.endSlot, uniqueNodeIdentifier)
+			if err != nil {
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).Error("Unable to checkout the row")
+				errCount = append(errCount, err)
+				break
+			}
+			rows, err := res.RowsAffected()
+			if err != nil {
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, fmt.Errorf("Unable to determine the rows affected when trying to checkout a row."))
+				errCount = append(errCount, err)
+				break
+			}
+			if rows > 1 {
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).WithFields(log.Fields{
+					"rowsReturn": rows,
+				}).Error("We locked too many rows.....")
+				errCount = append(errCount, err)
+				break
+			}
+			if rows == 0 {
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).WithFields(log.Fields{
+					"rowsReturn": rows,
+				}).Error("We did not lock a single row.")
+				errCount = append(errCount, err)
+				break
+			}
+			err = tx.Commit(dbCtx)
+			if err != nil {
+				loghelper.LogSlotRangeError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), err).Error("Unable commit transactions.")
+				errCount = append(errCount, err)
+				break
+			}
+			log.WithField("slots", sp).Debug("Added a new slots to be processed")
+			slotCh <- sp
+		}
+	}
+	log.WithFields(log.Fields{
+		"ErrCount": errCount,
+	}).Error("The ErrCounter")
+	return errCount
+}
+
+// After a row has been processed it should be removed from its appropriate table.
+func removeRowPostProcess(ctx context.Context, db sql.Database, processCh <-chan slotsToProcess, checkProcessedStmt, removeStmt string) error {
+	errCh := make(chan error)
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case slots := <-processCh:
+			// Make sure the start and end slot exist in the slots table.
+			go func() {
+				log.WithFields(log.Fields{
+					"startSlot": slots.startSlot,
+					"endSlot":   slots.endSlot,
+				}).Debug("Starting to check to see if the following slots have been processed")
+				for {
+					isStartProcess, err := isSlotProcessed(db, checkProcessedStmt, strconv.Itoa(slots.startSlot))
+					if err != nil {
+						errCh <- err
+					}
+					isEndProcess, err := isSlotProcessed(db, checkProcessedStmt, strconv.Itoa(slots.endSlot))
+					if err != nil {
+						errCh <- err
+					}
+					if isStartProcess && isEndProcess {
+						break
+					}
+					time.Sleep(3 * time.Second)
+				}
+
+				_, err := db.Exec(context.Background(), removeStmt, strconv.Itoa(slots.startSlot), strconv.Itoa(slots.endSlot))
+				if err != nil {
+					errCh <- err
+				}
+
+			}()
+			if len(errCh) != 0 {
+				return <-errCh
+			}
+		}
+	}
+}

pkg/beaconclient/processknowngaps.go

@@ -0,0 +1,137 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains all the code to process historic slots.
+
+package beaconclient
+
+import (
+	"context"
+	"strconv"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+var (
+	// Get a single non-checked out row row from eth_beacon.known_gaps.
+	getKgEntryStmt string = `SELECT start_slot, end_slot FROM eth_beacon.known_gaps
+	WHERE checked_out=false
+	ORDER BY priority ASC
+	LIMIT 1;`
+	// Used to periodically check to see if there is a new entry in the eth_beacon.known_gaps table.
+	checkKgEntryStmt string = `SELECT * FROM eth_beacon.known_gaps WHERE checked_out=false;`
+	// Used to checkout a row from the eth_beacon.known_gaps table
+	lockKgEntryStmt string = `UPDATE eth_beacon.known_gaps
+	SET checked_out=true, checked_out_by=$3
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to delete an entry from the knownGaps table
+	deleteKgEntryStmt string = `DELETE FROM eth_beacon.known_gaps
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to check to see if a single slot exists in the known_gaps table.
+	checkKgSingleSlotStmt string = `SELECT start_slot, end_slot FROM eth_beacon.known_gaps
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to update every single row that this node has checked out.
+	releaseKgLockStmt string = `UPDATE eth_beacon.known_gaps
+	SET checked_out=false, checked_out_by=null
+	WHERE checked_out_by=$1`
+)
+
+type KnownGapsProcessing struct {
+	db                   sql.Database         //db connection
+	metrics              *BeaconClientMetrics // metrics for beaconclient
+	uniqueNodeIdentifier int                  // node unique identifier.
+}
+
+// This function will perform all the heavy lifting for tracking the head of the chain.
+func (bc *BeaconClient) ProcessKnownGaps(ctx context.Context, maxWorkers int) []error {
+	log.Info("We are starting the known gaps processing service.")
+	bc.KnownGapsProcess = KnownGapsProcessing{db: bc.Db, uniqueNodeIdentifier: bc.UniqueNodeIdentifier, metrics: bc.Metrics}
+	errs := handleBatchProcess(ctx, maxWorkers, bc.KnownGapsProcess, bc.KnownGapsProcess.db, bc.ServerEndpoint, bc.Metrics, bc.CheckDb, bc.Metrics.IncrementKnownGapsProcessed)
+	log.Debug("Exiting known gaps processing service")
+	return errs
+}
+
+// This function will perform all the necessary clean up tasks for stopping historical processing.
+func (bc *BeaconClient) StopKnownGapsProcessing(cancel context.CancelFunc) error {
+	log.Info("We are stopping the known gaps processing service.")
+	cancel()
+	err := bc.KnownGapsProcess.releaseDbLocks()
+	if err != nil {
+		loghelper.LogError(err).WithField("uniqueIdentifier", bc.UniqueNodeIdentifier).Error("We were unable to remove the locks from the eth_beacon.known_gaps table. Manual Intervention is needed!")
+	}
+	return nil
+}
+
+// Get a single row of historical slots from the table.
+func (kgp KnownGapsProcessing) getSlotRange(ctx context.Context, slotCh chan<- slotsToProcess) []error {
+	return getBatchProcessRow(ctx, kgp.db, getKgEntryStmt, checkKgEntryStmt, lockKgEntryStmt, slotCh, strconv.Itoa(kgp.uniqueNodeIdentifier))
+}
+
+// Remove the table entry.
+func (kgp KnownGapsProcessing) removeTableEntry(ctx context.Context, processCh <-chan slotsToProcess) error {
+	return removeRowPostProcess(ctx, kgp.db, processCh, QueryBySlotStmt, deleteKgEntryStmt)
+}
+
+// Remove the table entry.
+func (kgp KnownGapsProcessing) handleProcessingErrors(ctx context.Context, errMessages <-chan batchHistoricError) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case errMs := <-errMessages:
+			// Check to see if this if this entry already exists.
+			res, err := kgp.db.Exec(context.Background(), checkKgSingleSlotStmt, errMs.slot, errMs.slot)
+			if err != nil {
+				loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).Error("Unable to see if this slot is in the eth_beacon.known_gaps table")
+			}
+
+			rows, err := res.RowsAffected()
+			if err != nil {
+				loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).WithFields(log.Fields{
+					"queryStatement": checkKgSingleSlotStmt,
+				}).Error("Unable to get the number of rows affected by this statement.")
+			}
+
+			if rows > 0 {
+				loghelper.LogSlotError(strconv.Itoa(errMs.slot), errMs.err).Error("We received an error when processing a knownGap")
+				err = updateKnownGapErrors(kgp.db, errMs.slot, errMs.slot, errMs.err, kgp.metrics)
+				if err != nil {
+					loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).Error("Error processing known gap")
+				}
+			} else {
+				writeKnownGaps(kgp.db, 1, errMs.slot, errMs.slot, errMs.err, errMs.errProcess, kgp.metrics)
+			}
+		}
+	}
+
+}
+
+// Updated checked_out column for the uniqueNodeIdentifier.
+func (kgp KnownGapsProcessing) releaseDbLocks() error {
+	log.Debug("Updating all the entries to eth_beacon.known_gaps")
+	res, err := kgp.db.Exec(context.Background(), releaseKgLockStmt, kgp.uniqueNodeIdentifier)
+	if err != nil {
+		return err
+	}
+	rows, err := res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	log.WithField("rowCount", rows).Info("Released knownGaps locks for specified rows.")
+	return nil
+}

pkg/beaconclient/processslot.go

@@ -0,0 +1,402 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// This file will keep track of all the code needed to process a slot.
+// To process a slot, it should have all the necessary data needed to write it to the DB.
+// But not actually write it.
+
+package beaconclient
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v4"
+	si "github.com/prysmaticlabs/prysm/consensus-types/interfaces"
+	"github.com/prysmaticlabs/prysm/consensus-types/wrapper"
+	dt "github.com/prysmaticlabs/prysm/encoding/ssz/detect"
+
+	// The below is temporary, once https://github.com/prysmaticlabs/prysm/issues/10006 has been resolved we wont need it.
+	// pb "github.com/prysmaticlabs/prysm/proto/prysm/v2"
+
+	state "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	ParentRootUnmarshalError  = "Unable to properly unmarshal the ParentRoot field in the SignedBeaconBlock."
+	MissingEth1Data           = "Can't get the Eth1 block_hash"
+	VersionedUnmarshalerError = "Unable to create a versioned unmarshaler"
+)
+
+type ProcessSlot struct {
+	// Generic
+
+	Slot               int                  // The slot number.
+	Epoch              int                  // The epoch number.
+	BlockRoot          string               // The hex encoded string of the BlockRoot.
+	StateRoot          string               // The hex encoded string of the StateRoot.
+	ParentBlockRoot    string               // The hex encoded string of the parent block.
+	Status             string               // The status of the block
+	HeadOrHistoric     string               // Is this the head or a historic slot. This is critical when trying to analyze errors and skipped slots.
+	Db                 sql.Database         // The DB object used to write to the DB.
+	Metrics            *BeaconClientMetrics // An object to keep track of the beaconclient metrics
+	PerformanceMetrics PerformanceMetrics   // An object to keep track of performance metrics.
+	// BeaconBlock
+
+	SszSignedBeaconBlock  []byte               // The entire SSZ encoded SignedBeaconBlock
+	FullSignedBeaconBlock si.SignedBeaconBlock // The unmarshaled BeaconState object, the unmarshalling could have errors.
+
+	// BeaconState
+	FullBeaconState state.BeaconState // The unmarshaled BeaconState object, the unmarshalling could have errors.
+	SszBeaconState  []byte            // The entire SSZ encoded BeaconState
+
+	// DB Write objects
+	DbSlotsModel             *DbSlots             // The model being written to the slots table.
+	DbSignedBeaconBlockModel *DbSignedBeaconBlock // The model being written to the signed_block table.
+	DbBeaconState            *DbBeaconState       // The model being written to the state table.
+}
+
+type PerformanceMetrics struct {
+	BeaconNodeBlockRetrievalTime time.Duration // How long it took to get the BeaconBlock from the Beacon Node.
+	BeaconNodeStateRetrievalTime time.Duration // How long it took to get the BeaconState from the Beacon Node.
+	ParseBeaconObjectForHash     time.Duration // How long it took to get some information from the beacon objects.
+	CheckDbPreProcessing         time.Duration // How long it takes to check the DB before processing a block.
+	CreateDbWriteObject          time.Duration // How long it takes to create a DB write object.
+	TransactSlotOnly             time.Duration // How long it takes to transact the slot information only.
+	CheckReorg                   time.Duration // How long it takes to check for Reorgs
+	CommitTransaction            time.Duration // How long it takes to commit the final transaction.
+	TotalDbTransaction           time.Duration // How long it takes from start to committing the entire DB transaction.
+	TotalProcessing              time.Duration // How long it took to process the entire slot.
+}
+
+// This function will do all the work to process the slot and write it to the DB.
+// It will return the error and error process. The error process is used for providing reach detail to the
+// known_gaps table.
+func processFullSlot(ctx context.Context, db sql.Database, serverAddress string, slot int, blockRoot string, stateRoot string, previousSlot int, previousBlockRoot string, headOrHistoric string, metrics *BeaconClientMetrics, knownGapsTableIncrement int, checkDb bool) (error, string) {
+	select {
+	case <-ctx.Done():
+		return nil, ""
+	default:
+		totalStart := time.Now()
+		ps := &ProcessSlot{
+			Slot:           slot,
+			BlockRoot:      blockRoot,
+			StateRoot:      stateRoot,
+			HeadOrHistoric: headOrHistoric,
+			Db:             db,
+			Metrics:        metrics,
+			PerformanceMetrics: PerformanceMetrics{
+				BeaconNodeBlockRetrievalTime: 0,
+				BeaconNodeStateRetrievalTime: 0,
+				ParseBeaconObjectForHash:     0,
+				CheckDbPreProcessing:         0,
+				CreateDbWriteObject:          0,
+				TransactSlotOnly:             0,
+				CheckReorg:                   0,
+				CommitTransaction:            0,
+				TotalDbTransaction:           0,
+				TotalProcessing:              0,
+			},
+		}
+
+		g, _ := errgroup.WithContext(context.Background())
+		vUnmarshalerCh := make(chan *dt.VersionedUnmarshaler, 1)
+
+		// Get the BeaconState.
+		g.Go(func() error {
+			select {
+			case <-ctx.Done():
+				return nil
+			default:
+				start := time.Now()
+				err := ps.getBeaconState(serverAddress, vUnmarshalerCh)
+				if err != nil {
+					return err
+				}
+				ps.PerformanceMetrics.BeaconNodeStateRetrievalTime = time.Since(start)
+				return nil
+			}
+		})
+
+		// Get the SignedBeaconBlock.
+		g.Go(func() error {
+			select {
+			case <-ctx.Done():
+				return nil
+			default:
+				start := time.Now()
+				err := ps.getSignedBeaconBlock(serverAddress, vUnmarshalerCh)
+				if err != nil {
+					return err
+				}
+				ps.PerformanceMetrics.BeaconNodeBlockRetrievalTime = time.Since(start)
+				return nil
+			}
+		})
+
+		if err := g.Wait(); err != nil {
+			return err, "processSlot"
+		}
+
+		parseBeaconTime := time.Now()
+		finalBlockRoot, finalStateRoot, finalEth1BlockHash, err := ps.provideFinalHash()
+		if err != nil {
+			return err, "CalculateBlockRoot"
+		}
+		ps.PerformanceMetrics.ParseBeaconObjectForHash = time.Since(parseBeaconTime)
+
+		if checkDb {
+			checkDbTime := time.Now()
+			inDb, err := IsSlotInDb(ctx, ps.Db, strconv.Itoa(ps.Slot), finalBlockRoot, finalStateRoot)
+			if err != nil {
+				return err, "checkDb"
+			}
+			if inDb {
+				log.WithField("slot", slot).Info("Slot already in the DB.")
+				return nil, ""
+			}
+			ps.PerformanceMetrics.CheckDbPreProcessing = time.Since(checkDbTime)
+		}
+
+		// Get this object ready to write
+		createDbWriteTime := time.Now()
+		dw, err := ps.createWriteObjects(finalBlockRoot, finalStateRoot, finalEth1BlockHash)
+		if err != nil {
+			return err, "blockRoot"
+		}
+		ps.PerformanceMetrics.CreateDbWriteObject = time.Since(createDbWriteTime)
+
+		// Write the object to the DB.
+		dbFullTransactionTime := time.Now()
+		defer func() {
+			err := dw.Tx.Rollback(dw.Ctx)
+			if err != nil && err != pgx.ErrTxClosed {
+				loghelper.LogError(err).Error("We were unable to Rollback a transaction")
+			}
+		}()
+
+		transactionTime := time.Now()
+		err = dw.transactFullSlot()
+		if err != nil {
+			return err, "processSlot"
+		}
+		ps.PerformanceMetrics.TransactSlotOnly = time.Since(transactionTime)
+
+		// Handle any reorgs or skipped slots.
+		reorgTime := time.Now()
+		headOrHistoric = strings.ToLower(headOrHistoric)
+		if headOrHistoric != "head" && headOrHistoric != "historic" {
+			return fmt.Errorf("headOrHistoric must be either historic or head!"), ""
+		}
+		if ps.HeadOrHistoric == "head" && previousSlot != 0 && previousBlockRoot != "" && ps.Status != "skipped" {
+			ps.checkPreviousSlot(dw.Tx, dw.Ctx, previousSlot, previousBlockRoot, knownGapsTableIncrement)
+		}
+		ps.PerformanceMetrics.CheckReorg = time.Since(reorgTime)
+
+		// Commit the transaction
+		commitTime := time.Now()
+		if err = dw.Tx.Commit(dw.Ctx); err != nil {
+			return err, "transactionCommit"
+		}
+		ps.PerformanceMetrics.CommitTransaction = time.Since(commitTime)
+
+		// Total metric capture time.
+		ps.PerformanceMetrics.TotalDbTransaction = time.Since(dbFullTransactionTime)
+		ps.PerformanceMetrics.TotalProcessing = time.Since(totalStart)
+
+		log.WithFields(log.Fields{
+			"slot":               slot,
+			"performanceMetrics": fmt.Sprintf("%+v\n", ps.PerformanceMetrics),
+		}).Debug("Performance Metric output!")
+
+		return nil, ""
+	}
+}
+
+// Handle a slot that is at head. A wrapper function for calling `handleFullSlot`.
+func processHeadSlot(db sql.Database, serverAddress string, slot int, blockRoot string, stateRoot string, previousSlot int, previousBlockRoot string, metrics *BeaconClientMetrics, knownGapsTableIncrement int, checkDb bool) {
+	// Get the knownGaps at startUp.
+	if previousSlot == 0 && previousBlockRoot == "" {
+		writeStartUpGaps(db, knownGapsTableIncrement, slot, metrics)
+	}
+	err, errReason := processFullSlot(context.Background(), db, serverAddress, slot, blockRoot, stateRoot, previousSlot, previousBlockRoot, "head", metrics, knownGapsTableIncrement, checkDb)
+	if err != nil {
+		writeKnownGaps(db, knownGapsTableIncrement, slot, slot, err, errReason, metrics)
+	}
+}
+
+// Handle a historic slot. A wrapper function for calling `handleFullSlot`.
+func handleHistoricSlot(ctx context.Context, db sql.Database, serverAddress string, slot int, metrics *BeaconClientMetrics, checkDb bool) (error, string) {
+	return processFullSlot(ctx, db, serverAddress, slot, "", "", 0, "", "historic", metrics, 1, checkDb)
+}
+
+// Update the SszSignedBeaconBlock and FullSignedBeaconBlock object with their respective values.
+func (ps *ProcessSlot) getSignedBeaconBlock(serverAddress string, vmCh <-chan *dt.VersionedUnmarshaler) error {
+	var blockIdentifier string // Used to query the block
+	if ps.BlockRoot != "" {
+		blockIdentifier = ps.BlockRoot
+	} else {
+		blockIdentifier = strconv.Itoa(ps.Slot)
+	}
+	blockEndpoint := serverAddress + BcBlockQueryEndpoint + blockIdentifier
+	var err error
+	var rc int
+	ps.SszSignedBeaconBlock, rc, err = querySsz(blockEndpoint, strconv.Itoa(ps.Slot))
+	if err != nil {
+		loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error("Unable to properly query the slot.")
+		return err
+	}
+
+	vm := <-vmCh
+	if rc != 200 {
+		ps.FullSignedBeaconBlock = &wrapper.Phase0SignedBeaconBlock{}
+		ps.SszSignedBeaconBlock = []byte{}
+		ps.ParentBlockRoot = ""
+		ps.Status = "skipped"
+		return nil
+	}
+
+	if vm == nil {
+		return fmt.Errorf(VersionedUnmarshalerError)
+	}
+
+	ps.FullSignedBeaconBlock, err = vm.UnmarshalBeaconBlock(ps.SszSignedBeaconBlock)
+	if err != nil {
+		loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Warn("Unable to process the slots SignedBeaconBlock")
+		return nil
+	}
+	ps.ParentBlockRoot = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().ParentRoot())
+	return nil
+}
+
+// Update the SszBeaconState and FullBeaconState object with their respective values.
+func (ps *ProcessSlot) getBeaconState(serverEndpoint string, vmCh chan<- *dt.VersionedUnmarshaler) error {
+	var stateIdentifier string // Used to query the state
+	if ps.StateRoot != "" {
+		stateIdentifier = ps.StateRoot
+	} else {
+		stateIdentifier = strconv.Itoa(ps.Slot)
+	}
+	stateEndpoint := serverEndpoint + BcStateQueryEndpoint + stateIdentifier
+	ps.SszBeaconState, _, _ = querySsz(stateEndpoint, strconv.Itoa(ps.Slot))
+
+	versionedUnmarshaler, err := dt.FromState(ps.SszBeaconState)
+	if err != nil {
+		loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error(VersionedUnmarshalerError)
+		vmCh <- nil
+		return fmt.Errorf(VersionedUnmarshalerError)
+	}
+	vmCh <- versionedUnmarshaler
+	ps.FullBeaconState, err = versionedUnmarshaler.UnmarshalBeaconState(ps.SszBeaconState)
+	if err != nil {
+		loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error("Unable to process the slots BeaconState")
+		return err
+	}
+	return nil
+}
+
+// Check to make sure that the previous block we processed is the parent of the current block.
+func (ps *ProcessSlot) checkPreviousSlot(tx sql.Tx, ctx context.Context, previousSlot int, previousBlockRoot string, knownGapsTableIncrement int) {
+	parentRoot := "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().ParentRoot())
+	slot := int(ps.FullBeaconState.Slot())
+	if previousSlot == slot {
+		log.WithFields(log.Fields{
+			"slot": slot,
+			"fork": true,
+		}).Warn("A fork occurred! The previous slot and current slot match.")
+		transactReorgs(tx, ctx, strconv.Itoa(ps.Slot), ps.BlockRoot, ps.Metrics)
+	} else if previousSlot > slot {
+		log.WithFields(log.Fields{
+			"previousSlot": previousSlot,
+			"curSlot":      slot,
+		}).Warn("We noticed the previous slot is greater than the current slot.")
+	} else if previousSlot+1 != slot {
+		log.WithFields(log.Fields{
+			"previousSlot": previousSlot,
+			"currentSlot":  slot,
+		}).Error("We skipped a few slots.")
+		transactKnownGaps(tx, ctx, knownGapsTableIncrement, previousSlot+1, slot-1, fmt.Errorf("Gaps during head processing"), "headGaps", ps.Metrics)
+	} else if previousBlockRoot != parentRoot {
+		log.WithFields(log.Fields{
+			"previousBlockRoot":  previousBlockRoot,
+			"currentBlockParent": parentRoot,
+		}).Error("The previousBlockRoot does not match the current blocks parent, an unprocessed fork might have occurred.")
+		transactReorgs(tx, ctx, strconv.Itoa(previousSlot), parentRoot, ps.Metrics)
+	} else {
+		log.Debug("Previous Slot and Current Slot are one distance from each other.")
+	}
+}
+
+// Transforms all the raw data into DB models that can be written to the DB.
+func (ps *ProcessSlot) createWriteObjects(blockRoot, stateRoot, eth1BlockHash string) (*DatabaseWriter, error) {
+	var status string
+	if ps.Status != "" {
+		status = ps.Status
+	} else {
+		status = "proposed"
+	}
+
+	dw, err := CreateDatabaseWrite(ps.Db, ps.Slot, stateRoot, blockRoot, ps.ParentBlockRoot, eth1BlockHash, status, &ps.SszSignedBeaconBlock, &ps.SszBeaconState, ps.Metrics)
+	if err != nil {
+		return dw, err
+	}
+
+	return dw, nil
+}
+
+// This function will return the final blockRoot, stateRoot, and eth1BlockHash that will be
+// used to write to a DB
+func (ps *ProcessSlot) provideFinalHash() (string, string, string, error) {
+	var (
+		stateRoot     string
+		blockRoot     string
+		eth1BlockHash string
+	)
+	if ps.Status == "skipped" {
+		stateRoot = ""
+		blockRoot = ""
+		eth1BlockHash = ""
+	} else {
+		if ps.StateRoot != "" {
+			stateRoot = ps.StateRoot
+		} else {
+			stateRoot = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().StateRoot())
+			log.Debug("StateRoot: ", stateRoot)
+		}
+
+		if ps.BlockRoot != "" {
+			blockRoot = ps.BlockRoot
+		} else {
+			var err error
+			rawBlockRoot, err := ps.FullSignedBeaconBlock.Block().HashTreeRoot()
+			//blockRoot, err = queryBlockRoot(blockRootEndpoint, strconv.Itoa(ps.Slot))
+			if err != nil {
+				return "", "", "", err
+			}
+			blockRoot = "0x" + hex.EncodeToString(rawBlockRoot[:])
+			log.WithFields(log.Fields{"blockRoot": blockRoot}).Debug("Block Root from ssz")
+		}
+		eth1BlockHash = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().Body().Eth1Data().BlockHash)
+	}
+	return blockRoot, stateRoot, eth1BlockHash, nil
+}

pkg/beaconclient/queryserver.go

@@ -1,101 +1,96 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This file will contain functions to query the Beacon Chain Server.
 
 package beaconclient
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
-// Attempt to use generics..
-// // These are types that append slot at the end of the URL to handle a request.
-// type SlotBasedRequests interface {
-// 	*specs.BeaconState | *specs.SignedBeaconBlock
-// 	UnmarshalSSZ([]byte) error
-// }
-//
-// func queryState[R SlotBasedRequests](endpoint string, slot string) (R, error) {
-// 	obj := new(R)
-// 	rawState, err := querySlot(endpoint, slot)
-// 	if err != nil {
-// 		return *obj, err
-// 	}
-//
-// 	err = &obj.UnmarshalSSZ(rawState)
-// 	err = (*obj).UnmarshalSSZ(rawState)
-// 	if err != nil {
-// 		loghelper.LogSlotError(slot, err).Error("Unable to unmarshal the SSZ response from the Beacon Node Successfully!")
-// 		return *obj, fmt.Errorf("Unable to unmarshal the SSZ response from the Beacon Node Successfully!: %s", err.Error())
-// 	}
-// 	return *obj, nil
-// }
-
-// This function will query a state object based on the slot provided.
-// The object is SSZ encoded.
-
-//type BeaconBlockResponse struct {
-//	version string `json: `
-//}
-
-// func queryState(endpoint string, slot string) (spectests.BeaconState, error) {
-// 	obj := new(spectests.BeaconState)
-// 	fullEndpoint := endpoint + slot
-// 	rawState, err := querySsz(fullEndpoint, slot)
-// 	if err != nil {
-// 		return *obj, err
-// 	}
-//
-// 	err = obj.UnmarshalSSZ(rawState)
-// 	if err != nil {
-// 		loghelper.LogSlotError(slot, err).Error("Unable to unmarshal the SSZ response from the Beacon Node")
-// 		return *obj, fmt.Errorf("Unable to unmarshal the SSZ response from the Beacon Node: %s", err.Error())
-// 	}
-// 	return *obj, nil
-// }
-//
-// // This function will query a state object based on the slot provided.
-// // The object is SSZ encoded.
-// func queryBlock(endpoint string, slot string) (spectests.SignedBeaconBlock, error) {
-// 	obj := new(spectests.SignedBeaconBlock)
-// 	fullEndpoint := endpoint + slot
-// 	rawBlock, err := querySsz(fullEndpoint, slot)
-// 	if err != nil {
-// 		return *obj, err
-// 	}
-//
-// 	err = obj.UnmarshalSSZ(rawBlock)
-// 	if err != nil {
-// 		loghelper.LogSlotError(slot, err).Error("Unable to unmarshal the SSZ response from the Beacon Node Successfully!")
-// 		return *obj, fmt.Errorf("Unable to unmarshal the SSZ response from the Beacon Node Successfully!: %s", err.Error())
-// 	}
-// 	return *obj, nil
-// }
-
 // A helper function to query endpoints that utilize slots.
-func querySsz(endpoint string, slot string) ([]byte, error) {
+func querySsz(endpoint string, slot string) ([]byte, int, error) {
-	log.WithFields(log.Fields{"endpoint": endpoint}).Info("Querying endpoint")
+	log.WithFields(log.Fields{"endpoint": endpoint}).Debug("Querying endpoint")
 	client := &http.Client{}
 	req, err := http.NewRequest("GET", endpoint, nil)
 	if err != nil {
 		loghelper.LogSlotError(slot, err).Error("Unable to create a request!")
-		return nil, fmt.Errorf("Unable to create a request!: %s", err.Error())
+		return nil, 0, fmt.Errorf("Unable to create a request!: %s", err.Error())
 	}
-	// Not set correctly
 	req.Header.Set("Accept", "application/octet-stream")
 	response, err := client.Do(req)
 	if err != nil {
 		loghelper.LogSlotError(slot, err).Error("Unable to query Beacon Node!")
-		return nil, fmt.Errorf("Unable to query Beacon Node: %s", err.Error())
+		return nil, 0, fmt.Errorf("Unable to query Beacon Node: %s", err.Error())
+	}
+	defer response.Body.Close()
+	rc := response.StatusCode
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		loghelper.LogSlotError(slot, err).Error("Unable to turn response into a []bytes array!")
+		return nil, rc, fmt.Errorf("Unable to turn response into a []bytes array!: %s", err.Error())
+	}
+	return body, rc, nil
+}
+
+// Object to unmarshal the BlockRootResponse
+type BlockRootResponse struct {
+	Data BlockRootMessage `json:"data"`
+}
+
+// Object to unmarshal the BlockRoot Message
+type BlockRootMessage struct {
+	Root string `json:"root"`
+}
+
+// A function to query the blockroot for a given slot.
+func queryBlockRoot(endpoint string, slot string) (string, error) {
+	log.WithFields(log.Fields{"endpoint": endpoint}).Debug("Querying endpoint")
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		loghelper.LogSlotError(slot, err).Error("Unable to create a request!")
+		return "", fmt.Errorf("Unable to create a request!: %s", err.Error())
+	}
+	req.Header.Set("Accept", "application/json")
+	response, err := client.Do(req)
+	if err != nil {
+		loghelper.LogSlotError(slot, err).Error("Unable to query Beacon Node!")
+		return "", fmt.Errorf("Unable to query Beacon Node: %s", err.Error())
 	}
 	defer response.Body.Close()
 	body, err := ioutil.ReadAll(response.Body)
 	if err != nil {
 		loghelper.LogSlotError(slot, err).Error("Unable to turn response into a []bytes array!")
-		return nil, fmt.Errorf("Unable to turn response into a []bytes array!: %s", err.Error())
+		return "", fmt.Errorf("Unable to turn response into a []bytes array!: %s", err.Error())
 	}
-	return body, nil
+
+	resp := BlockRootResponse{}
+	if err := json.Unmarshal(body, &resp); err != nil {
+		loghelper.LogEndpoint(endpoint).WithFields(log.Fields{
+			"rawMessage": string(body),
+			"err":        err,
+		}).Error("Unable to unmarshal the block root")
+		return "", err
+	}
+	return resp.Data.Root, nil
 }

pkg/beaconclient/systemvalidation_test.go

@@ -0,0 +1,69 @@
+package beaconclient_test
+
+import (
+	"os"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	//. "github.com/onsi/gomega"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+)
+
+var (
+	prodConfig = Config{
+		protocol:                os.Getenv("bc_protocol"),
+		address:                 os.Getenv("bc_address"),
+		port:                    getEnvInt(os.Getenv("bc_port")),
+		dbHost:                  os.Getenv("db_host"),
+		dbPort:                  getEnvInt(os.Getenv("db_port")),
+		dbName:                  os.Getenv("db_name"),
+		dbUser:                  os.Getenv("db_user"),
+		dbPassword:              os.Getenv("db_password"),
+		dbDriver:                os.Getenv("db_driver"),
+		knownGapsTableIncrement: 100000000,
+		bcUniqueIdentifier:      100,
+		checkDb:                 false,
+	}
+)
+var _ = Describe("Systemvalidation", Label("system"), func() {
+	Describe("Run the application against a running lighthouse node", func() {
+		Context("When we receive head messages", func() {
+			It("We should process the messages successfully", func() {
+				bc := setUpTest(prodConfig, "10000000000")
+				processProdHeadBlocks(bc, 3, 0, 0, 0)
+			})
+		})
+		Context("When we have historical and knownGaps slots to process", Label("system-batch"), func() {
+			It("Should process them successfully", func() {
+				bc := setUpTest(prodConfig, "10000000000")
+				//known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 100, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 2, 0, 0, 0)
+
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 2375703, 2375703, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+	})
+})
+
+// Wrapper function to get int env variables.
+func getEnvInt(envVar string) int {
+	val, err := strconv.Atoi(envVar)
+	if err != nil {
+		return 0
+	}
+	return val
+}
+
+// Start head tracking and wait for the expected results.
+func processProdHeadBlocks(bc *beaconclient.BeaconClient, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	go bc.CaptureHead()
+	time.Sleep(1 * time.Second)
+	validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
+}

pkg/database/sql/error.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package sql
 
 import (

pkg/database/sql/interfaces.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package sql
 
 import (

pkg/database/sql/postgres/config.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres
 
 import (
@@ -18,7 +33,7 @@ const (
 // DefaultConfig are default parameters for connecting to a Postgres sql
 var DefaultConfig = Config{
 	Hostname:     "localhost",
-	Port:         8077,
+	Port:         8076,
 	DatabaseName: "vulcanize_testing",
 	Username:     "vdbm",
 	Password:     "password",

pkg/database/sql/postgres/database.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres
 
 import (
@@ -5,7 +20,8 @@ import (
 	"fmt"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 var _ sql.Database = &DB{}
@@ -23,6 +39,35 @@ func NewPostgresDB(c Config) (*DB, error) {
 	return &DB{driver}, nil
 }
 
+// A simple wrapper to create a DB object to use.
+func SetupPostgresDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (sql.Database, error) {
+	log.Debug("Resolving Driver Type")
+	DbDriver, err := ResolveDriverType(driverName)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"err":                  err,
+			"driver_name_provided": driverName,
+		}).Error("Can't resolve driver type")
+	}
+	log.Info("Using Driver: ", DbDriver)
+
+	postgresConfig := Config{
+		Hostname:     dbHostname,
+		Port:         dbPort,
+		DatabaseName: dbName,
+		Username:     dbUsername,
+		Password:     dbPassword,
+		Driver:       DbDriver,
+	}
+	DB, err := NewPostgresDB(postgresConfig)
+
+	if err != nil {
+		loghelper.LogError(err).Error("Unable to connect to the DB")
+		return nil, err
+	}
+	return DB, err
+}
+
 // Create a driver based on the config
 func createDriver(c Config) (*pgxDriver, error) {
 	switch c.Driver {

pkg/database/sql/postgres/pgx.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres
 
 import (
@@ -8,7 +23,7 @@ import (
 	"github.com/jackc/pgconn"
 	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
 )
 
 // pgxDriver driver, implements sql.Driver

pkg/database/sql/postgres/pgx_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres_test
 
 import (
@@ -8,9 +23,9 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql/postgres"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/testhelpers"
 )
 
 var _ = Describe("Pgx", func() {
@@ -29,17 +44,18 @@ var _ = Describe("Pgx", func() {
 				_, err := postgres.NewPostgresDB(postgres.Config{
 					Driver: "PGX",
 				})
-				Expect(err).NotTo(BeNil())
+				Expect(err).To(HaveOccurred())
 
 				present, err := doesContainsSubstring(err.Error(), sql.DbConnectionFailedMsg)
 				Expect(present).To(BeTrue())
+				Expect(err).NotTo(HaveOccurred())
 			})
 		})
 		Context("The connection is successful", func() {
 			It("Should create a DB object", func() {
 				db, err := postgres.NewPostgresDB(postgres.DefaultConfig)
-				defer db.Close()
 				Expect(err).To(BeNil())
+				defer db.Close()
 			})
 		})
 	})

pkg/database/sql/postgres/postgres_suite_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres_test
 
 import (

pkg/gracefulshutdown/gracefulshutdown.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package gracefulshutdown
 
 import (
@@ -10,30 +25,34 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // operation is a clean up function on shutting down
 type Operation func(ctx context.Context) error
 
+var (
+	TimeoutErr = func(timeout string) error {
+		return fmt.Errorf("The Timeout %s, has been elapsed, the application will forcefully exit", timeout)
+	}
+)
+
 // gracefulShutdown waits for termination syscalls and doing clean up operations after received it
-func Shutdown(ctx context.Context, timeout time.Duration, ops map[string]Operation) (<-chan struct{}, <-chan error) {
+func Shutdown(ctx context.Context, notifierCh chan os.Signal, timeout time.Duration, ops map[string]Operation) (<-chan struct{}, <-chan error) {
 	waitCh := make(chan struct{})
 	errCh := make(chan error)
 	go func() {
-		s := make(chan os.Signal, 1)
 
 		// add any other syscalls that you want to be notified with
-		signal.Notify(s, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
+		signal.Notify(notifierCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
-		<-s
+		<-notifierCh
 
 		log.Info("Shutting Down your application")
 
 		// set timeout for the ops to be done to prevent system hang
 		timeoutFunc := time.AfterFunc(timeout, func() {
-			log.Warnf("timeout %d ms has been elapsed, force exit", timeout.Milliseconds())
+			log.Warnf(TimeoutErr(timeout.String()).Error())
-			errCh <- fmt.Errorf("Application shutdown took too long.")
+			errCh <- TimeoutErr(timeout.String())
-			return
 		})
 
 		defer timeoutFunc.Stop()

pkg/loghelper/log_endpoint.go

@@ -1,12 +0,0 @@
-package loghelper
-
-import (
-	log "github.com/sirupsen/logrus"
-)
-
-// A simple helper function that will help wrap the error message.
-func LogEndpoint(endpoint string) *log.Entry {
-	return log.WithFields(log.Fields{
-		"endpoint": endpoint,
-	})
-}

pkg/loghelper/log_error.go

@@ -1,20 +0,0 @@
-// A simple function to help with logging errors.
-package loghelper
-
-import (
-	log "github.com/sirupsen/logrus"
-)
-
-// A simple helper function that will help wrap the error message.
-func LogError(err error) *log.Entry {
-	return log.WithFields(log.Fields{
-		"err": err,
-	})
-}
-
-func LogSlotError(slot string, err error) *log.Entry {
-	return log.WithFields(log.Fields{
-		"err":  err,
-		"slot": slot,
-	})
-}

pkg/loghelper/logendpoint.go

@@ -0,0 +1,27 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package loghelper
+
+import (
+	log "github.com/sirupsen/logrus"
+)
+
+// A simple helper function that will help wrap the error message.
+func LogEndpoint(endpoint string) *log.Entry {
+	return log.WithFields(log.Fields{
+		"endpoint": endpoint,
+	})
+}

pkg/loghelper/logerror.go

@@ -0,0 +1,52 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// A simple function to help with logging errors.
+package loghelper
+
+import (
+	log "github.com/sirupsen/logrus"
+)
+
+// A simple helper function that will help wrap the error message.
+func LogError(err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err": err,
+	})
+}
+
+// A simple herlper function to log slot and error.
+func LogSlotError(slot string, err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err":  err,
+		"slot": slot,
+	})
+}
+
+func LogSlotRangeError(startSlot string, endSlot string, err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err":       err,
+		"startSlot": startSlot,
+		"endSlot":   endSlot,
+	})
+}
+func LogSlotRangeStatementError(startSlot string, endSlot string, statement string, err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err":          err,
+		"startSlot":    startSlot,
+		"endSlot":      endSlot,
+		"SqlStatement": statement,
+	})
+}

pkg/loghelper/logreorg.go

@@ -0,0 +1,37 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package loghelper
+
+import (
+	log "github.com/sirupsen/logrus"
+)
+
+// A simple helper function that will help wrap the reorg error messages.
+func LogReorgError(slot string, latestBlockRoot string, err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err":             err,
+		"slot":            slot,
+		"latestBlockRoot": latestBlockRoot,
+	})
+}
+
+// A simple helper function that will help wrap regular reorg messages.
+func LogReorg(slot string, latestBlockRoot string) *log.Entry {
+	return log.WithFields(log.Fields{
+		"slot":            slot,
+		"latestBlockRoot": latestBlockRoot,
+	})
+}

pkg/testhelpers/test_helper.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package testhelpers
 
 import (

pkg/version/version.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package version
 
 import "fmt"