Slight Performance improvements with Logging (#67)

* Slight Performance improvements with Logging

* Use exec in docker container

* Update packages for dockerfile

* Add flakey test and boot command specific logic

* Remove excess logging and add config val

.github/workflows/generic-testing.yml

@@ -0,0 +1,199 @@
+name: Test the stack.
+on:
+  workflow_call:
+    inputs:
+      stack-orchestrator-ref:
+        required: false
+        type: string
+      ipld-eth-beacon-db-ref:
+        required: false
+        type: string
+      ssz-data-ref:
+        required: false
+        type: string
+    secrets:
+      GHA_KEY:
+        required: true
+
+env:
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '7fb664270a0ba09e2caa3095e8c91f3fdb5b38af' }}
+  ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref  || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
+  ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }}
+  GOPATH: /tmp/go
+jobs:
+  build:
+    name: Run Docker Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-beacon-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-beacon-db-ref }}
+          repository: vulcanize/ipld-eth-beacon-db
+          path: "./ipld-eth-beacon-db/"
+          ssh-key: ${{secrets.GHA_KEY}}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
+          echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
+          echo eth_beacon_config_file=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer/config/cicd/boot.ipld-eth-beacon-indexer.json >> ./config.sh
+          echo eth_beacon_capture_mode=boot >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - name: Check to make sure HEALTH file is present
+        shell: bash
+        run: |
+          until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" --env-file ./config.sh cp ipld-eth-beacon-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
+          cat ./HEALTH
+          if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" cp ipld-eth-beacon-indexer:/root/ipld-eth-beacon-indexer.log . && cat ipld-eth-beacon-indexer.log && (exit 1); fi
+
+  unit-test:
+    name: Run Unit Tests
+    runs-on: ubuntu-latest
+    ## IF you want to update the default branch for `pull_request runs, do it after the ||`
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-beacon-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-beacon-db-ref }}
+          repository: vulcanize/ipld-eth-beacon-db
+          path: "./ipld-eth-beacon-db/"
+          ssh-key: ${{ secrets.GHA_KEY }}
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ssz-data-ref }}
+          repository: vulcanize/ssz-data
+          path: "./ipld-eth-beacon-indexer/pkg/beaconclient/ssz-data"
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
+          echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          cd ipld-eth-beacon-indexer
+          make unit-test-ci
+
+  integration-test:
+    name: Run Integration Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-beacon-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-beacon-db-ref }}
+          repository: vulcanize/ipld-eth-beacon-db
+          path: "./ipld-eth-beacon-db/"
+          ssh-key: ${{secrets.GHA_KEY}}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
+          echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          cd ipld-eth-beacon-indexer
+          make integration-test-ci
+
+  golangci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          args: --timeout 90s --disable deadcode,unused
+#          args: --timeout 90s --disable deadcode,

.github/workflows/on-pr.yml

@@ -7,8 +7,8 @@ on:
         description: "The branch, commit or sha from stack-orchestrator to checkout"
         required: false
         default: "main"
-      ipld-eth-db-ref:
-        description: "The branch, commit or sha from ipld-eth-db to checkout"
+      ipld-eth-beacon-db-ref:
+        description: "The branch, commit or sha from ipld-eth-beacon-db to checkout"
         required: false
         default: "main"
       ssz-data-ref:
@@ -22,185 +22,26 @@ on:
       - "!LICENSE"
       - "!.github/workflows/**"
       - ".github/workflows/on-pr.yml"
+      - ".github/workflows/tests.yml"
       - "**"
+  schedule:
+    - cron: '0 13 * * *' # Must be single quotes!!
 
-env:
-  stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref  || 'feature/client-build'}}
-  ipld-eth-db-ref: ${{ github.event.inputs.ipld-eth-db-ref  || 'minimal-beacon-chain-schema' }}
-  ssz-data-ref: ${{ github.event.inputs.ssz-data-ref  || 'main' }}
-  GOPATH: /tmp/go
 jobs:
-  build:
-    name: Run Docker Build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-ethcl-indexer"
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.stack-orchestrator-ref }}
-          path: "./stack-orchestrator/"
-          repository: vulcanize/stack-orchestrator
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ipld-eth-db-ref }}
-          repository: vulcanize/ipld-eth-db
-          path: "./ipld-eth-db/"
-          fetch-depth: 0
-
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh
-          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
-          echo ethcl_capture_mode=boot >> ./config.sh
-          echo ethcl_skip_sync=true >> ./config.sh
-          echo ethcl_known_gap_increment=1000000 >> ./config.sh
-          cat ./config.sh
-
-      - name: Run docker compose
-        run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" \
-          --env-file ./config.sh \
-          up -d --build
-
-      - name: Check to make sure HEALTH file is present
-        shell: bash
-        run: |
-          until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
-          cat ./HEALTH
-          if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else (exit 1); fi
-
-  unit-test:
-    name: Run Unit Tests
-    runs-on: ubuntu-latest
-    ## IF you want to update the default branch for `pull_request runs, do it after the ||`
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-ethcl-indexer"
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.stack-orchestrator-ref }}
-          path: "./stack-orchestrator/"
-          repository: vulcanize/stack-orchestrator
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ipld-eth-db-ref }}
-          repository: vulcanize/ipld-eth-db
-          path: "./ipld-eth-db/"
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ssz-data-ref }}
-          repository: vulcanize/ssz-data
-          path: "./ipld-ethcl-indexer/pkg/beaconclient/ssz-data"
-          fetch-depth: 0
-
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh
-          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
-          cat ./config.sh
-
-      - name: Run docker compose
-        run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db.yml" \
-          --env-file ./config.sh \
-          up -d --build
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-          check-latest: true
-
-      - name: Install packages
-        run: |
-          go install github.com/onsi/ginkgo/v2/ginkgo@latest
-          which ginkgo
-
-      - name: Run the tests using Make
-        run: |
-          cd ipld-ethcl-indexer
-          make unit-test-ci
-
-  integration-test:
-    name: Run Integration Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-ethcl-indexer"
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.stack-orchestrator-ref }}
-          path: "./stack-orchestrator/"
-          repository: vulcanize/stack-orchestrator
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ipld-eth-db-ref }}
-          repository: vulcanize/ipld-eth-db
-          path: "./ipld-eth-db/"
-          fetch-depth: 0
-
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh
-          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
-          echo ethcl_capture_mode=boot >> ./config.sh
-          cat ./config.sh
-
-      - name: Run docker compose
-        run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
-          --env-file ./config.sh \
-          up -d --build
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-          check-latest: true
-
-      - name: Install packages
-        run: |
-          go install github.com/onsi/ginkgo/v2/ginkgo@latest
-          which ginkgo
-
-      - name: Run the tests using Make
-        run: |
-          cd ipld-ethcl-indexer
-          make integration-test-ci
-
-  golangci:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-      - uses: actions/checkout@v3
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          args: --disable errcheck
+  trigger-tests:
+    if: github.event_name != 'schedule'
+    uses: ./.github/workflows/generic-testing.yml
+    with:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
+      ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
+      ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }}
+    secrets:
+      GHA_KEY: ${{secrets.GHA_KEY}}
+  system-testing:
+    uses: ./.github/workflows/system-tests.yml
+    with:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
+      ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
+    secrets:
+      GHA_KEY: ${{secrets.GHA_KEY}}
+      BC_ADDRESS: ${{secrets.BC_ADDRESS}}

.github/workflows/on-publish.yml

@@ -3,9 +3,28 @@ on:
   release:
     types: [published, edited]
 jobs:
+  trigger-tests:
+    uses: ./.github/workflows/generic-testing.yml
+    with:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
+      ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
+      ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }}
+    secrets:
+      GHA_KEY: ${{secrets.GHA_KEY}}
+  system-testing:
+    uses: ./.github/workflows/system-tests.yml
+    with:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
+      ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
+    secrets:
+      GHA_KEY: ${{secrets.GHA_KEY}}
+      BC_ADDRESS: ${{secrets.BC_ADDRESS}}
   build:
     name: Run docker build
     runs-on: ubuntu-latest
+    needs:
+      - trigger-tests
+      - system-testing
     steps:
       - uses: actions/checkout@v2
       - name: Get the version
@@ -14,11 +33,11 @@ jobs:
       - name: Run docker build
         run: make docker-build
       - name: Tag docker image
-        run: docker tag vulcanize/ipld-ethcl-indexer docker.pkg.github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer:${{steps.vars.outputs.sha}}
+        run: docker tag vulcanize/ipld-eth-beacon-indexer docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
       - name: Docker Login
         run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
       - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer:${{steps.vars.outputs.sha}}
+        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
   push_to_registries:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
@@ -32,10 +51,10 @@ jobs:
       - name: Docker Login to Github Registry
         run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
       - name: Docker Pull
-        run: docker pull docker.pkg.github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer:${{steps.vars.outputs.sha}}
+        run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
       - name: Docker Login to Docker Registry
         run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
       - name: Tag docker image
-        run: docker tag docker.pkg.github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer:${{steps.vars.outputs.sha}} vulcanize/ipld-ethcl-indexer:${{steps.vars.outputs.tag}}
+        run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-beacon-indexer:${{steps.vars.outputs.tag}}
       - name: Docker Push to Docker Hub
-        run: docker push vulcanize/ipld-ethcl-indexer:${{steps.vars.outputs.tag}}
+        run: docker push vulcanize/ipld-eth-beacon-indexer:${{steps.vars.outputs.tag}}

.github/workflows/system-tests.yml

@@ -0,0 +1,95 @@
+name: System Testing for the stack.
+on:
+  workflow_call:
+    inputs:
+      stack-orchestrator-ref:
+        required: false
+        type: string
+      ipld-eth-beacon-db-ref:
+        required: false
+        type: string
+    secrets:
+      GHA_KEY:
+        required: true
+      BC_ADDRESS:
+        required: true
+env:
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '7fb664270a0ba09e2caa3095e8c91f3fdb5b38af' }}
+  ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref  || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
+  GOPATH: /tmp/go
+  bc_protocol: "http"
+  bc_address: ${{secrets.BC_ADDRESS}}
+  bc_port: 5052
+  db_host: localhost
+  db_port: 8076
+  db_name: vulcanize_testing
+  db_user: vdbm
+  db_password: password
+  db_driver: "pgx"
+
+jobs:
+  system-testing:
+    runs-on: self-hosted
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-beacon-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-eth-beacon-db-ref }}
+          repository: vulcanize/ipld-eth-beacon-db
+          path: "./ipld-eth-beacon-db/"
+          ssh-key: ${{secrets.GHA_KEY}}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_beacon_db=$(pwd)/ipld-eth-beacon-db > ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        id: compose
+        shell: bash
+        run: |
+          ls "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml"
+          whoami
+          /usr/local/bin/docker-compose  \
+          -f "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          sleep 20
+          cd ipld-eth-beacon-indexer
+          make system-test-ci
+
+      - name: Clean up the docker containers
+        if: always() && steps.compose.outcome == 'success'
+        shell: bash
+        run: |
+          /usr/local/bin/docker-compose \
+          -f "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
+          --env-file ./config.sh \
+          down -v

.gitignore

@@ -4,5 +4,10 @@ ipld-ethcl-indexer.log
 report.json
 cover.profile
 temp/*
+.vscode/*
 pkg/beaconclient/ssz-data/
-*.test
+*.test
+ipld-eth-beacon-indexer.log
+ipld-eth-beacon-indexer
+config/local.ipld-eth-beacon-indexer-config.json
+config/docker.ipld-eth-beacon-indexer-config.json

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.18-alpine as builder
 
-WORKDIR /go/src/github.com/vulcanize/ipld-ethcl-indexer
-RUN apk --no-cache add ca-certificates make git g++ linux-headers
+WORKDIR /go/src/github.com/vulcanize/ipld-eth-beacon-indexer
+RUN apk --no-cache add ca-certificates make git g++ linux-headers libstdc++
 
 ENV GO111MODULE=on
 COPY go.mod .
@@ -9,12 +9,12 @@ COPY go.sum .
 RUN go mod tidy; go mod download
 COPY . .
 
-RUN GCO_ENABLED=0 GOOS=linux go build -race -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-ethcl-indexer .
-RUN chmod +x ipld-ethcl-indexer
+RUN GCO_ENABLED=0 GOOS=linux go build -race -ldflags="-s -w" -o ipld-eth-beacon-indexer .
+RUN chmod +x ipld-eth-beacon-indexer
 
 FROM frolvlad/alpine-bash:latest
-RUN apk --no-cache add ca-certificates
+RUN apk --no-cache add ca-certificates libstdc++
 WORKDIR /root/
-COPY --from=builder /go/src/github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer /root/ipld-ethcl-indexer
+COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer /root/ipld-eth-beacon-indexer
 ADD entrypoint.sh .
 ENTRYPOINT ["./entrypoint.sh"]

Makefile

@@ -67,6 +67,7 @@ unit-test-local:
 	go fmt ./...
 	$(GINKGO) -r --label-filter unit \
 	--randomize-all --randomize-suites \
+	--flake-attempts=3 \
 	--fail-on-pending --keep-going \
 	--trace
 
@@ -75,11 +76,30 @@ unit-test-ci:
 	go vet ./...
 	go fmt ./...
 	$(GINKGO) -r --label-filter unit \
+	--randomize-all --randomize-suites
+	--flake-attempts=3 \
+	--fail-on-pending --keep-going \
+	--cover --coverprofile=cover.profile \
+	--trace --json-report=report.json
+
+.PHONY: system-test-ci
+system-test-ci:
+	go vet ./...
+	go fmt ./...
+	$(GINKGO) -r --label-filter system \
 	--randomize-all --randomize-suites \
 	--fail-on-pending --keep-going \
 	--cover --coverprofile=cover.profile \
 	--trace --json-report=report.json
 
+.PHONY: system-test-local
+system-test-local:
+	go vet ./...
+	go fmt ./...
+	$(GINKGO) -r --label-filter system \
+	--randomize-all --randomize-suites \
+	--fail-on-pending --keep-going \
+	--trace
 
 .PHONY: build
 build:
@@ -89,4 +109,4 @@ build:
 ## Build docker image
 .PHONY: docker-build
 docker-build:
-	docker build -t vulcanize/ipld-ethcl-indexer .
+	docker build -t vulcanize/ipld-eth-beacon-indexer .

README.md

@@ -1,4 +1,4 @@
-- [ipld-ethcl-indexer](#ipld-ethcl-indexer)
+- [ipld-eth-beacon-indexer](#ipld-eth-beacon-indexer)
 - [Running the Application](#running-the-application)
 - [Development Patterns](#development-patterns)
   - [Logging](#logging)
@@ -8,7 +8,7 @@
 
 <small><i><a href='http://ecotrust-canada.github.io/markdown-toc/'>Table of contents generated with markdown-toc</a></i></small>
 
-# ipld-ethcl-indexer
+# ipld-eth-beacon-indexer
 
 This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node.
 
@@ -22,12 +22,12 @@ To run the application, do as follows:
 
 1. Setup the prerequisite applications.
    a. Run a beacon client (such as lighthouse).
-   b. Run a postgres DB.
+   b. Run a postgres DB for eth-beacon.
    c. You can utilize the `stack-orchestrator` [repository](https://github.com/vulcanize/stack-orchestrato).
 
    ```
    ./wrapper.sh -e skip \
-   -d ../docker/local/docker-compose-db.yml \
+   -d ../docker/local/docker-compose-ipld-eth-beacon-db.yml \
    -d ../docker/latest/docker-compose-lighthouse.yml \
    -v remove \
    -p ../local-config.sh
@@ -37,19 +37,7 @@ To run the application, do as follows:
 2. Run the start up command.
 
 ```
-go run -race main.go capture head --db.address localhost \
-  --db.password password \
-  --db.port 8077 \
-  --db.username vdbm \
-  --db.name vulcanize_testing \
-  --db.driver PGX \
-  --bc.address localhost \
-  --bc.port 5052 \
-  --bc.connectionProtocol http \
-  --t.skipSync=true \
-  --log.level info \
-  --log.output=true \
-  --kg.increment 100
+go run -race main.go capture full --config ./example.ipld-eth-beacon-indexer-config.json
 ```
 
 ## Running Tests

application_component.md

@@ -51,4 +51,4 @@ This package contains useful functions for logging.
 
 ## `internal/shutdown`
 
-This package is used to shutdown the `ipld-ethcl-indexer`. It calls the `pkg/gracefulshutdown` package.
+This package is used to shutdown the `ipld-eth-beacon-indexer`. It calls the `pkg/gracefulshutdown` package.

cmd/boot.go

@@ -1,19 +1,19 @@
-/*
-Copyright © 2022 Abdul Rabbani <abdulrabbani00@gmail.com>
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
 
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
 package cmd
 
 import (
@@ -23,9 +23,10 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // bootCmd represents the boot command
@@ -44,9 +45,11 @@ func bootApp() {
 	log.Info("Starting the application in boot mode.")
 	ctx := context.Background()
 
-	BC, DB, err := boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, testDisregardSync)
+	Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
+		viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
+		viper.GetInt("kg.increment"), "boot", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
 	if err != nil {
-		loghelper.LogError(err).Error("Unable to Start application")
+		StopApplicationPreBoot(err, Db)
 	}
 
 	log.Info("Boot complete, we are going to shutdown.")
@@ -57,11 +60,11 @@ func bootApp() {
 		notifierCh <- syscall.SIGTERM
 	}()
 
-	err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC)
+	err = shutdown.ShutdownBoot(ctx, notifierCh, maxWaitSecondsShutdown, Db, Bc)
 	if err != nil {
-		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!")
+		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
 	} else {
-		log.Info("Gracefully shutdown ipld-ethcl-indexer")
+		log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
 	}
 }
 

cmd/capture.go

@@ -1,10 +1,23 @@
-/*
-Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-*/
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"time"
 
@@ -13,19 +26,30 @@ import (
 )
 
 var (
-	dbUsername             string
-	dbPassword             string
-	dbName                 string
-	dbAddress              string
-	dbDriver               string
-	dbPort                 int
-	bcAddress              string
-	bcPort                 int
-	bcConnectionProtocol   string
-	bcType                 string
-	maxWaitSecondsShutdown time.Duration  = time.Duration(5) * time.Second
-	notifierCh             chan os.Signal = make(chan os.Signal, 1)
-	testDisregardSync      bool
+	dbUsername                 string
+	dbPassword                 string
+	dbName                     string
+	dbAddress                  string
+	dbDriver                   string
+	dbPort                     int
+	bcAddress                  string
+	bcPort                     int
+	bcBootRetryInterval        int
+	bcBootMaxRetry             int
+	bcConnectionProtocol       string
+	bcType                     string
+	bcMaxHistoricProcessWorker int
+	bcUniqueNodeIdentifier     int
+	bcCheckDb                  bool
+	kgMaxWorker                int
+	kgTableIncrement           int
+	kgProcessGaps              bool
+	pmMetrics                  bool
+	pmAddress                  string
+	pmPort                     int
+	maxWaitSecondsShutdown     time.Duration  = time.Duration(20) * time.Second
+	notifierCh                 chan os.Signal = make(chan os.Signal, 1)
+	testDisregardSync          bool
 )
 
 // captureCmd represents the capture command
@@ -50,35 +74,50 @@ func init() {
 	captureCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)")
 	captureCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)")
 	captureCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)")
-	err := captureCmd.MarkPersistentFlagRequired("db.username")
-	exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.password")
-	exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.address")
-	exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.port")
-	exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.name")
-	exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("db.driver")
-	exitErr(err)
+	//err := captureCmd.MarkPersistentFlagRequired("db.username")
+	// exitErr(err)
+	// err = captureCmd.MarkPersistentFlagRequired("db.password")
+	// exitErr(err)
+	// err = captureCmd.MarkPersistentFlagRequired("db.address")
+	// exitErr(err)
+	// err = captureCmd.MarkPersistentFlagRequired("db.port")
+	// exitErr(err)
+	// err = captureCmd.MarkPersistentFlagRequired("db.name")
+	// exitErr(err)
+	// err = captureCmd.MarkPersistentFlagRequired("db.driver")
+	// exitErr(err)
 
 	//// Beacon Client Specific
 	captureCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required)")
 	captureCmd.PersistentFlags().StringVarP(&bcType, "bc.type", "", "lighthouse", "The beacon client we are using, options are prysm and lighthouse.")
 	captureCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required )")
 	captureCmd.PersistentFlags().StringVarP(&bcConnectionProtocol, "bc.connectionProtocol", "", "http", "protocol for connecting to the beacon node.")
-	err = captureCmd.MarkPersistentFlagRequired("bc.address")
-	exitErr(err)
-	err = captureCmd.MarkPersistentFlagRequired("bc.port")
-	exitErr(err)
+	captureCmd.PersistentFlags().IntVarP(&bcBootRetryInterval, "bc.bootRetryInterval", "", 30, "The amount of time to wait between retries while booting the application")
+	captureCmd.PersistentFlags().IntVarP(&bcBootMaxRetry, "bc.bootMaxRetry", "", 5, "The amount of time to wait between retries while booting the application")
+	captureCmd.PersistentFlags().IntVarP(&bcMaxHistoricProcessWorker, "bc.maxHistoricProcessWorker", "", 30, "The number of workers that should be actively processing slots from the eth-beacon.historic_process table. Be careful of system memory.")
+	captureCmd.PersistentFlags().IntVarP(&bcUniqueNodeIdentifier, "bc.uniqueNodeIdentifier", "", 0, "The unique identifier of this application. Each application connecting to the DB should have a unique identifier.")
+	captureCmd.PersistentFlags().BoolVarP(&bcCheckDb, "bc.checkDb", "", true, "Should we check to see if the slot exists in the DB before writing it?")
+	// err = captureCmd.MarkPersistentFlagRequired("bc.address")
+	// exitErr(err)
+	// err = captureCmd.MarkPersistentFlagRequired("bc.port")
+	// exitErr(err)
+
+	//// Known Gaps specific
+	captureCmd.PersistentFlags().BoolVarP(&kgProcessGaps, "kg.processKnownGaps", "", true, "Should we process the slots within the eth-beacon.known_gaps table.")
+	captureCmd.PersistentFlags().IntVarP(&kgTableIncrement, "kg.increment", "", 10000, "The max slots within a single entry to the known_gaps table.")
+	captureCmd.PersistentFlags().IntVarP(&kgMaxWorker, "kg.maxKnownGapsWorker", "", 30, "The number of workers that should be actively processing slots from the eth-beacon.known_gaps table. Be careful of system memory.")
+
+	// Prometheus Specific
+	captureCmd.PersistentFlags().BoolVarP(&pmMetrics, "pm.metrics", "", true, "Should we capture prometheus metrics.")
+	captureCmd.PersistentFlags().StringVarP(&pmAddress, "pm.address", "", "localhost", "Address to send the prometheus metrics.")
+	captureCmd.PersistentFlags().IntVarP(&pmPort, "pm.port", "", 9000, "The port to send prometheus metrics.")
 
 	//// Testing Specific
 	captureCmd.PersistentFlags().BoolVar(&testDisregardSync, "t.skipSync", false, "Should we disregard the head sync?")
 
 	// Bind Flags with Viper
 	//// DB Flags
-	err = viper.BindPFlag("db.username", captureCmd.PersistentFlags().Lookup("db.username"))
+	err := viper.BindPFlag("db.username", captureCmd.PersistentFlags().Lookup("db.username"))
 	exitErr(err)
 	err = viper.BindPFlag("db.password", captureCmd.PersistentFlags().Lookup("db.password"))
 	exitErr(err)
@@ -88,14 +127,14 @@ func init() {
 	exitErr(err)
 	err = viper.BindPFlag("db.name", captureCmd.PersistentFlags().Lookup("db.name"))
 	exitErr(err)
+	err = viper.BindPFlag("db.driver", captureCmd.PersistentFlags().Lookup("db.driver"))
+	exitErr(err)
+
+	//// Testing Specific
 	err = viper.BindPFlag("t.skipSync", captureCmd.PersistentFlags().Lookup("t.skipSync"))
 	exitErr(err)
 
-	// Testing Specific
-	err = viper.BindPFlag("t.driver", captureCmd.PersistentFlags().Lookup("db.driver"))
-	exitErr(err)
-
-	// LH specific
+	//// LH specific
 	err = viper.BindPFlag("bc.address", captureCmd.PersistentFlags().Lookup("bc.address"))
 	exitErr(err)
 	err = viper.BindPFlag("bc.type", captureCmd.PersistentFlags().Lookup("bc.type"))
@@ -104,14 +143,40 @@ func init() {
 	exitErr(err)
 	err = viper.BindPFlag("bc.connectionProtocol", captureCmd.PersistentFlags().Lookup("bc.connectionProtocol"))
 	exitErr(err)
+	err = viper.BindPFlag("bc.bootRetryInterval", captureCmd.PersistentFlags().Lookup("bc.bootRetryInterval"))
+	exitErr(err)
+	err = viper.BindPFlag("bc.bootMaxRetry", captureCmd.PersistentFlags().Lookup("bc.bootMaxRetry"))
+	exitErr(err)
+	err = viper.BindPFlag("bc.maxHistoricProcessWorker", captureCmd.PersistentFlags().Lookup("bc.maxHistoricProcessWorker"))
+	exitErr(err)
+	err = viper.BindPFlag("bc.uniqueNodeIdentifier", captureCmd.PersistentFlags().Lookup("bc.uniqueNodeIdentifier"))
+	exitErr(err)
+	err = viper.BindPFlag("bc.checkDb", captureCmd.PersistentFlags().Lookup("bc.checkDb"))
+	exitErr(err)
 	// Here you will define your flags and configuration settings.
 
+	//// Known Gap Specific
+	err = viper.BindPFlag("kg.processKnownGaps", captureCmd.PersistentFlags().Lookup("kg.processKnownGaps"))
+	exitErr(err)
+	err = viper.BindPFlag("kg.increment", captureCmd.PersistentFlags().Lookup("kg.increment"))
+	exitErr(err)
+	err = viper.BindPFlag("kg.processKnownGaps", captureCmd.PersistentFlags().Lookup("kg.maxKnownGapsWorker"))
+	exitErr(err)
+
+	// Prometheus Specific
+	err = viper.BindPFlag("pm.metrics", captureCmd.PersistentFlags().Lookup("pm.metrics"))
+	exitErr(err)
+	err = viper.BindPFlag("pm.address", captureCmd.PersistentFlags().Lookup("pm.address"))
+	exitErr(err)
+	err = viper.BindPFlag("pm.port", captureCmd.PersistentFlags().Lookup("pm.port"))
+	exitErr(err)
 }
 
 // Helper function to catch any errors.
 // We need to capture these errors for the linter.
 func exitErr(err error) {
 	if err != nil {
+		fmt.Println("Error: ", err)
 		os.Exit(1)
 	}
 }

cmd/full.go

@@ -0,0 +1,118 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
+)
+
+// fullCmd represents the full command
+var fullCmd = &cobra.Command{
+	Use:   "full",
+	Short: "Capture all components of the application (head and historical)",
+	Long:  `Capture all components of the application (head and historical`,
+	Run: func(cmd *cobra.Command, args []string) {
+		startFullProcessing()
+	},
+}
+
+func init() {
+	captureCmd.AddCommand(fullCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// fullCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// fullCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}
+
+// Start the application to track at head and historical processing.
+func startFullProcessing() {
+	// Boot the application
+	log.Info("Starting the application in head tracking mode.")
+	ctx := context.Background()
+
+	Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
+		viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
+		viper.GetInt("kg.increment"), "head", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
+	if err != nil {
+		StopApplicationPreBoot(err, Db)
+	}
+
+	if viper.GetBool("pm.metrics") {
+		addr := viper.GetString("pm.address") + ":" + strconv.Itoa(viper.GetInt("pm.port"))
+		serveProm(addr)
+	}
+
+	log.Info("The Beacon Client has booted successfully!")
+	// Capture head blocks
+	go Bc.CaptureHead()
+
+	hpContext, hpCancel := context.WithCancel(context.Background())
+
+	errG, _ := errgroup.WithContext(context.Background())
+	errG.Go(func() error {
+		errs := Bc.CaptureHistoric(hpContext, viper.GetInt("bc.maxHistoricProcessWorker"))
+		if len(errs) != 0 {
+			if len(errs) != 0 {
+				log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing historic events")
+				return fmt.Errorf("Application ended because there were too many error when attempting to process historic")
+			}
+		}
+		return nil
+	})
+	kgCtx, KgCancel := context.WithCancel(context.Background())
+	if viper.GetBool("kg.processKnownGaps") {
+		go func() {
+			errG := new(errgroup.Group)
+			errG.Go(func() error {
+				errs := Bc.ProcessKnownGaps(kgCtx, viper.GetInt("kg.maxKnownGapsWorker"))
+				if len(errs) != 0 {
+					log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
+					return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
+				}
+				return nil
+			})
+			if err := errG.Wait(); err != nil {
+				loghelper.LogError(err).Error("Error with knownGaps processing")
+			}
+		}()
+	}
+
+	// Shutdown when the time is right.
+	err = shutdown.ShutdownFull(ctx, KgCancel, hpCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
+	if err != nil {
+		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
+	} else {
+		log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
+	}
+
+}

cmd/head.go

@@ -1,23 +1,35 @@
-/*
-Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-*/
 package cmd
 
 import (
 	"context"
-	"os"
+	"fmt"
+	"net/http"
+	"strconv"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
-)
-
-var (
-	kgTableIncrement int
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
 )
 
 // headCmd represents the head command
@@ -36,34 +48,65 @@ func startHeadTracking() {
 	log.Info("Starting the application in head tracking mode.")
 	ctx := context.Background()
 
-	BC, DB, err := boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, testDisregardSync)
+	Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
+		viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
+		viper.GetInt("kg.increment"), "head", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
 	if err != nil {
-		loghelper.LogError(err).Error("Unable to Start application")
-		if DB != nil {
-			DB.Close()
-		}
-		os.Exit(1)
+		StopApplicationPreBoot(err, Db)
+	}
+
+	if viper.GetBool("pm.metrics") {
+		addr := viper.GetString("pm.address") + ":" + strconv.Itoa(viper.GetInt("pm.port"))
+		serveProm(addr)
 	}
 
 	log.Info("The Beacon Client has booted successfully!")
 	// Capture head blocks
-	go BC.CaptureHead(kgTableIncrement)
+	go Bc.CaptureHead()
+	kgCtx, KgCancel := context.WithCancel(context.Background())
+	if viper.GetBool("kg.processKnownGaps") {
+		go func() {
+			errG := new(errgroup.Group)
+			errG.Go(func() error {
+				errs := Bc.ProcessKnownGaps(kgCtx, viper.GetInt("kg.maxKnownGapsWorker"))
+				if len(errs) != 0 {
+					log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
+					return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
+				}
+				return nil
+			})
+			if err := errG.Wait(); err != nil {
+				loghelper.LogError(err).Error("Error with knownGaps processing")
+			}
+		}()
+	}
 
 	// Shutdown when the time is right.
-	err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC)
+	err = shutdown.ShutdownHeadTracking(ctx, KgCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
 	if err != nil {
-		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!")
+		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
 	} else {
-		log.Info("Gracefully shutdown ipld-ethcl-indexer")
+		log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
 	}
 
 }
 
 func init() {
 	captureCmd.AddCommand(headCmd)
-
-	// Known Gaps specific
-	captureCmd.PersistentFlags().IntVarP(&kgTableIncrement, "kg.increment", "", 10000, "The max slots within a single entry to the known_gaps table.")
-	err := viper.BindPFlag("kg.increment", captureCmd.PersistentFlags().Lookup("kg.increment"))
-	exitErr(err)
+}
+
+// Start prometheus server
+func serveProm(addr string) {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+
+	srv := http.Server{
+		Addr:    addr,
+		Handler: mux,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			loghelper.LogError(err).WithField("endpoint", addr).Error("Error with prometheus")
+		}
+	}()
 }

cmd/historic.go

@@ -1,13 +1,35 @@
-/*
-Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-*/
 package cmd
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"strconv"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
 )
 
 // historicCmd represents the historic command
@@ -16,10 +38,69 @@ var historicCmd = &cobra.Command{
 	Short: "Capture the historic blocks and states.",
 	Long:  `Capture the historic blocks and states.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("historic called")
+		startHistoricProcessing()
 	},
 }
 
+// Start the application to process historical slots.
+func startHistoricProcessing() {
+	// Boot the application
+	log.Info("Starting the application in head tracking mode.")
+	ctx := context.Background()
+
+	Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
+		viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
+		viper.GetInt("kg.increment"), "historic", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
+	if err != nil {
+		StopApplicationPreBoot(err, Db)
+	}
+
+	if viper.GetBool("pm.metrics") {
+		addr := viper.GetString("pm.address") + ":" + strconv.Itoa(viper.GetInt("pm.port"))
+		serveProm(addr)
+	}
+
+	hpContext, hpCancel := context.WithCancel(context.Background())
+
+	errG, _ := errgroup.WithContext(context.Background())
+	errG.Go(func() error {
+		errs := Bc.CaptureHistoric(hpContext, viper.GetInt("bc.maxHistoricProcessWorker"))
+		if len(errs) != 0 {
+			if len(errs) != 0 {
+				log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing historic events")
+				return fmt.Errorf("Application ended because there were too many error when attempting to process historic")
+			}
+		}
+		return nil
+	})
+
+	kgContext, kgCancel := context.WithCancel(context.Background())
+	if viper.GetBool("kg.processKnownGaps") {
+		go func() {
+			errG := new(errgroup.Group)
+			errG.Go(func() error {
+				errs := Bc.ProcessKnownGaps(kgContext, viper.GetInt("kg.maxKnownGapsWorker"))
+				if len(errs) != 0 {
+					log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
+					return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
+				}
+				return nil
+			})
+			if err := errG.Wait(); err != nil {
+				loghelper.LogError(err).Error("Error with knownGaps processing")
+			}
+		}()
+	}
+
+	// Shutdown when the time is right.
+	err = shutdown.ShutdownHistoricProcessing(ctx, kgCancel, hpCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
+	if err != nil {
+		loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
+	} else {
+		log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
+	}
+}
+
 func init() {
 	captureCmd.AddCommand(historicCmd)
 
@@ -33,3 +114,12 @@ func init() {
 	// is called directly, e.g.:
 	// historicCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 }
+
+// Stop the application during its initial boot phases.
+func StopApplicationPreBoot(startErr error, db sql.Database) {
+	loghelper.LogError(startErr).Error("Unable to Start application")
+	if db != nil {
+		db.Close()
+	}
+	os.Exit(1)
+}

cmd/root.go

@@ -1,7 +1,19 @@
-/*
-Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-*/
 package cmd
 
 import (
@@ -20,7 +32,7 @@ var (
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:   "ipld-ethcl-indexer",
+	Use:   "ipld-eth-beacon-indexer",
 	Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.",
 	Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain.
 It can either do this will keeping track of head, or backfilling historic data.`,
@@ -35,6 +47,7 @@ It can either do this will keeping track of head, or backfilling historic data.`
 func Execute() {
 	err := rootCmd.Execute()
 	if err != nil {
+		fmt.Println("Err when executing rootCmd", err)
 		os.Exit(1)
 	}
 }
@@ -114,9 +127,9 @@ func init() {
 	// will be global for your application.
 
 	// Optional Flags
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)")
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-eth-beacon-indexer.yaml)")
 	rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
-	rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging")
+	rootCmd.PersistentFlags().String("log.file", "ipld-eth-beacon-indexer.log", "file path for logging")
 	rootCmd.PersistentFlags().Bool("log.output", true, "Should we log to STDOUT")
 	rootCmd.PersistentFlags().String("log.format", "json", "json or text")
 
@@ -147,10 +160,10 @@ func initConfig() {
 		home, err := os.UserHomeDir()
 		cobra.CheckErr(err)
 
-		// Search config in home directory with name ".ipld-ethcl-indexer" (without extension).
+		// Search config in home directory with name ".ipld-eth-beacon-indexer" (without extension).
 		viper.AddConfigPath(home)
 		viper.SetConfigType("yaml")
-		viper.SetConfigName(".ipld-ethcl-indexer")
+		viper.SetConfigName(".ipld-eth-beacon-indexer")
 	}
 
 	viper.AutomaticEnv() // read in environment variables that match

cmd/version.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package cmd
 
 import (
@@ -5,7 +20,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	v "github.com/vulcanize/ipld-ethcl-indexer/pkg/version"
+	v "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/version"
 )
 
 var (
@@ -32,7 +47,7 @@ to quickly create a Cobra application.`,
 			Patch: Patch,
 			Meta:  Meta,
 		}
-		log.Infof("ipld-ethcl-indexer version: %s", version.GetVersionWithMeta())
+		log.Infof("ipld-eth-beacon-indexer version: %s", version.GetVersionWithMeta())
 		fmt.Println(version.GetVersionWithMeta())
 	},
 }

config/cicd/boot.ipld-eth-beacon-indexer.json

@@ -0,0 +1,40 @@
+{
+  "db": {
+    "address": "ipld-eth-beacon-db",
+    "password": "password",
+    "port": 5432,
+    "username": "vdbm",
+    "name": "vulcanize_testing",
+    "driver": "PGX"
+  },
+  "bc": {
+    "address": "lighthouse",
+    "port": 5052,
+    "type": "lighthouse",
+    "bootRetryInterval": 30,
+    "bootMaxRetry": 5,
+    "maxHistoricProcessWorker": 2,
+    "connectionProtocol": "http",
+    "uniqueNodeIdentifier": 100,
+    "checkDb": true
+  },
+  "t": {
+    "skipSync": true
+  },
+  "log": {
+    "level": "debug",
+    "output": true,
+    "file": "./ipld-eth-beacon-indexer.log",
+    "format": "json"
+  },
+  "kg": {
+    "increment": 10000,
+    "processKnownGaps": true,
+    "maxKnownGapsWorker": 2
+  },
+  "pm": {
+    "address": "localhost",
+    "port": 9000,
+    "metrics": true
+  }
+}

config/example.ipld-eth-beacon-indexer-config.json

@@ -0,0 +1,40 @@
+{
+  "db": {
+    "address": "localhost",
+    "password": "password",
+    "port": 8076,
+    "username": "vdbm",
+    "name": "vulcanize_testing",
+    "driver": "PGX"
+  },
+  "bc": {
+    "address": "localhost",
+    "port": 5052,
+    "type": "lighthouse",
+    "bootRetryInterval": 30,
+    "bootMaxRetry": 5,
+    "maxHistoricProcessWorker": 2,
+    "connectionProtocol": "http",
+    "uniqueNodeIdentifier": 100,
+    "checkDb": true
+  },
+  "t": {
+    "skipSync": true
+  },
+  "log": {
+    "level": "debug",
+    "output": true,
+    "file": "./ipld-eth-beacon-indexer.log",
+    "format": "json"
+  },
+  "kg": {
+    "increment": 10000,
+    "processKnownGaps": true,
+    "maxKnownGapsWorker": 2
+  },
+  "pm": {
+    "address": "localhost",
+    "port": 9000,
+    "metrics": true
+  }
+}

entrypoint.sh

@@ -1,40 +1,24 @@
 #!/bin/bash
 
 sleep 10
-echo "Starting ipld-ethcl-indexer"
+echo "Starting ipld-eth-beacon-indexer"
 
-echo /root/ipld-ethcl-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \
-  --db.password $DB_PASSWORD \
-  --db.port $DB_PORT \
-  --db.username $DB_USER \
-  --db.name $DB_NAME \
-  --db.driver $DB_DRIVER \
-  --bc.address $BC_ADDRESS \
-  --bc.port $BC_PORT \
-  --log.level $LOG_LEVEL\
-  --t.skipSync=$SKIP_SYNC \
-  --kg.increment $KNOWN_GAP_INCREMENT
+echo /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
 
-/root/ipld-ethcl-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \
-  --db.password $DB_PASSWORD \
-  --db.port $DB_PORT \
-  --db.username $DB_USER \
-  --db.name $DB_NAME \
-  --db.driver $DB_DRIVER \
-  --bc.address $BC_ADDRESS \
-  --bc.port $BC_PORT \
-  --log.level $LOG_LEVEL \
-  --t.skipSync=$SKIP_SYNC \
-  --kg.increment $KNOWN_GAP_INCREMENT
+if [ ${CAPTURE_MODE} == "boot" ]; then
+    /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
+    rv=$?
 
-rv=$?
+    if [ $rv != 0 ]; then
+      echo "ipld-eth-beacon-indexer boot failed"
+    else
+      echo "ipld-eth-beacon-indexer boot succeeded"
+    fi
+      echo $rv > /root/HEALTH
+      echo $rv
+      cat /root/ipld-eth-beacon-indexer.output
 
-if [ $rv != 0 ]; then
-  echo "ipld-ethcl-indexer startup failed"
-  echo 1 > /root/HEALTH
+    tail -f /dev/null
 else
-  echo "ipld-ethcl-indexer startup succeeded"
-  echo 0 > /root/HEALTH
-fi
-
-tail -f /dev/null
+    exec /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
+fi

go.mod

@@ -1,4 +1,4 @@
-module github.com/vulcanize/ipld-ethcl-indexer
+module github.com/vulcanize/ipld-eth-beacon-indexer
 
 go 1.18
 
@@ -9,17 +9,23 @@ require (
 	github.com/multiformats/go-multihash v0.1.0
 	github.com/onsi/ginkgo/v2 v2.1.4
 	github.com/onsi/gomega v1.19.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc
 	github.com/sirupsen/logrus v1.8.1
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/ethereum/go-ethereum v1.10.17 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-block-format v0.0.3 // indirect
 	github.com/ipfs/go-cid v0.1.0 // indirect
@@ -40,8 +46,11 @@ require (
 	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
 	github.com/lib/pq v1.10.5 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/minio/highwayhash v1.0.1 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.0.4 // indirect
 	github.com/multiformats/go-base36 v0.1.0 // indirect
@@ -49,9 +58,17 @@ require (
 	github.com/multiformats/go-varint v0.0.6 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect
+	github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220303211031-f753e083138c // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/supranational/blst v0.3.5 // indirect
 	github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect
+	github.com/urfave/cli/v2 v2.3.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect

go.sum

@@ -39,6 +39,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
+contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
@@ -49,10 +50,15 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY=
+github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o=
 github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -66,12 +72,17 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7
 github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0=
 github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM=
 github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw=
+github.com/bazelbuild/rules_go v0.23.2 h1:Wxu7JjqnF78cKZbsBsARLSXx/jlGaSLCnUV3mTlyHvM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo=
+github.com/btcsuite/btcd/btcec/v2 v2.1.2 h1:YoYoC9J0jwfukodSBMzZYUVQ8PTiYg4BnOWiJVzTmLs=
 github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
@@ -79,6 +90,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -100,6 +113,7 @@ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -109,8 +123,10 @@ github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhr
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4=
 github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
 github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
 github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
 github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
@@ -124,6 +140,7 @@ github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r
 github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
 github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
 github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
+github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -156,16 +173,20 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v0.2.1 h1:fV3MLmabKIZ383XifUjFSwcoGee0v9qgPp8wy5svibE=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
@@ -188,6 +209,8 @@ github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -214,6 +237,7 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -227,11 +251,14 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -257,6 +284,7 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
@@ -284,7 +312,11 @@ github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuW
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e h1:wCMygKUQhmcQAjlk2Gquzq6dLmyMv2kF+llRspoRgrk=
+github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U=
+github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
 github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
+github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM=
 github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y=
@@ -441,12 +473,18 @@ github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
@@ -478,6 +516,7 @@ github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPR
 github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -521,6 +560,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@@ -528,6 +568,7 @@ github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/maxatome/go-testdeep v1.11.0 h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk=
@@ -535,6 +576,8 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0=
+github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -547,10 +590,16 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
 github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
@@ -593,10 +642,12 @@ github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2
 github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
@@ -616,6 +667,7 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -635,22 +687,41 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic=
 github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw=
 github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4=
+github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220303211031-f753e083138c h1:920mmUNkTZ9zIWXjArGv6iU45b85gjUvq/4L1ZHWVYQ=
+github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220303211031-f753e083138c/go.mod h1:4pWaT30XoEx1j8KNJf3TV+E3mQkaufn7mf+jRNb/Fuk=
+github.com/prysmaticlabs/protoc-gen-go-cast v0.0.0-20211014160335-757fae4f38c6 h1:+jhXLjEYVW4qU2z5SOxlxN+Hv/A9FDf0HpfDurfMEz0=
 github.com/prysmaticlabs/prysm v1.4.2-0.20220504145118-df695346a53c h1:OBWrQQlyAdjB/CelI3/Q4QVrZ0qdw2IXAhk8u4dXBVk=
 github.com/prysmaticlabs/prysm v1.4.2-0.20220504145118-df695346a53c/go.mod h1:z8vCgEaKYKD+6R5MRZvIOEw8aAsFydBidRbxvqtMpcU=
 github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o=
 github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNrWd0GQ/OoQfMtlg2uPRSuTzcSGrzwK8=
 github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc=
+github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
 github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -660,11 +731,13 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
 github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -674,6 +747,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -708,18 +782,26 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/supranational/blst v0.3.5 h1:/pey7U712GgJBSD1XTiJ5iBqjYIH3QNdrjRoGXlJJ60=
+github.com/supranational/blst v0.3.5/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
+github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
 github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo=
 github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4=
 github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
+github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
 github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
+github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
@@ -740,6 +822,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -868,12 +952,14 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
@@ -887,6 +973,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -904,6 +992,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -925,6 +1014,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -940,6 +1030,8 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -959,15 +1051,18 @@ golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -982,6 +1077,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1045,6 +1141,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1077,12 +1174,14 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513
 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
 google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1171,8 +1270,10 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4=
 gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
 gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
@@ -1181,6 +1282,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
@@ -1200,6 +1302,11 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
+k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk=
+k8s.io/client-go v0.18.3 h1:QaJzz92tsN67oorwzmoB0a9r9ZVHuD5ryjbCKP0U22k=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog/v2 v2.3.0 h1:WmkrnW7fdrm0/DMClc+HIxtftvxVIPAhlVwMQo5yLco=
+k8s.io/utils v0.0.0-20200520001619-278ece378a50 h1:ZtTUW5+ZWaoqjR3zOpRa7oFJ5d4aA22l4me/xArfOIc=
 lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
 lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
 lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
@@ -1207,3 +1314,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=

internal/boot/boot.go

@@ -1,21 +1,35 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package boot
 
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql/postgres"
 )
 
 var (
-	maxRetry                                 = 5  // Max times to try to connect to the DB or BC at boot.
-	retryInterval                            = 30 // The time to wait between each try.
-	DB            sql.Database               = &postgres.DB{}
-	BC            *beaconclient.BeaconClient = &beaconclient.BeaconClient{}
+	DB sql.Database               = &postgres.DB{}
+	BC *beaconclient.BeaconClient = &beaconclient.BeaconClient{}
 )
 
 // This function will perform some boot operations. If any steps fail, the application will fail to start.
@@ -27,14 +41,18 @@ var (
 // 2. Connect to the database.
 //
 // 3. Make sure the node is synced, unless disregardSync is true.
-func BootApplication(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int, bcConnectionProtocol string, disregardSync bool) (*beaconclient.BeaconClient, sql.Database, error) {
+func BootApplication(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string,
+	bcAddress string, bcPort int, bcConnectionProtocol string, bcKgTableIncrement int, disregardSync bool, uniqueNodeIdentifier int, checkDb bool) (*beaconclient.BeaconClient, sql.Database, error) {
 	log.Info("Booting the Application")
 
 	log.Debug("Creating the Beacon Client")
-	BC = beaconclient.CreateBeaconClient(ctx, bcConnectionProtocol, bcAddress, bcPort)
+	Bc, err := beaconclient.CreateBeaconClient(ctx, bcConnectionProtocol, bcAddress, bcPort, bcKgTableIncrement, uniqueNodeIdentifier, checkDb)
+	if err != nil {
+		return Bc, nil, err
+	}
 
 	log.Debug("Checking Beacon Client")
-	err := BC.CheckBeaconClient()
+	err = Bc.CheckBeaconClient()
 	if err != nil {
 		return nil, nil, err
 	}
@@ -45,40 +63,90 @@ func BootApplication(ctx context.Context, dbHostname string, dbPort int, dbName
 		return nil, nil, err
 	}
 
-	BC.Db = DB
+	Bc.Db = DB
 
 	var status bool
 	if !disregardSync {
-		status, err = BC.CheckHeadSync()
+		status, err = Bc.CheckHeadSync()
 		if err != nil {
 			log.Error("Unable to get the nodes sync status")
-			return BC, DB, err
+			return Bc, DB, err
 		}
 		if status {
 			log.Error("The node is still syncing..")
 			err = fmt.Errorf("The node is still syncing.")
-			return BC, DB, err
+			return Bc, DB, err
 		}
 	} else {
 		log.Warn("We are not checking to see if the node has synced to head.")
 	}
-	return BC, DB, nil
+	return Bc, DB, nil
 }
 
 // Add retry logic to ensure that we are give the Beacon Client and the DB time to start.
-func BootApplicationWithRetry(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int, bcConnectionProtocol string, disregardSync bool) (*beaconclient.BeaconClient, sql.Database, error) {
+func BootApplicationWithRetry(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string,
+	bcAddress string, bcPort int, bcConnectionProtocol string, bcType string, bcRetryInterval int, bcMaxRetry int, bcKgTableIncrement int,
+	startUpMode string, disregardSync bool, uniqueNodeIdentifier int, checkDb bool) (*beaconclient.BeaconClient, sql.Database, error) {
 	var err error
-	for i := 0; i < maxRetry; i++ {
-		BC, DB, err = BootApplication(ctx, dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName, bcAddress, bcPort, bcConnectionProtocol, disregardSync)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"retryNumber": i,
-				"err":         err,
-			}).Warn("Unable to boot application. Going to try again")
-			time.Sleep(time.Duration(retryInterval) * time.Second)
-			continue
+
+	if bcMaxRetry < 0 {
+		i := 0
+		for {
+			BC, DB, err = BootApplication(ctx, dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName,
+				bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, disregardSync, uniqueNodeIdentifier, checkDb)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"retryNumber": i,
+					"err":         err,
+				}).Warn("Unable to boot application. Going to try again")
+				time.Sleep(time.Duration(bcRetryInterval) * time.Second)
+				i = i + 1
+				continue
+			}
+			break
+		}
+	} else {
+		for i := 0; i < bcMaxRetry; i++ {
+			BC, DB, err = BootApplication(ctx, dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName,
+				bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, disregardSync, uniqueNodeIdentifier, checkDb)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"retryNumber": i,
+					"err":         err,
+				}).Warn("Unable to boot application. Going to try again")
+				time.Sleep(time.Duration(bcRetryInterval) * time.Second)
+				continue
+			}
+			break
 		}
-		break
 	}
+
+	switch strings.ToLower(startUpMode) {
+	case "head":
+		BC.PerformHeadTracking = true
+	case "historic":
+		log.Debug("Performing additional boot steps for historical processing")
+		BC.PerformHistoricalProcessing = true
+		// This field is not currently used.
+		// The idea is, that if we are doing historially processing and we get a slot
+		// greater than this slot, then we would rerun this function.
+		// this would ensure that we have the slots necessary for processing
+		// within the beacon server.
+
+		// We can implement this feature if we notice any errors.
+		headSlot, err := BC.GetLatestSlotInBeaconServer(bcType)
+		if err != nil {
+			return BC, DB, err
+		}
+		BC.UpdateLatestSlotInBeaconServer(int64(headSlot))
+		// Add another switch case for bcType if its ever needed.
+	case "boot":
+		log.Debug("Running application in boot mode.")
+	default:
+		log.WithFields(log.Fields{
+			"startUpMode": startUpMode,
+		}).Error("The startUpMode provided is not handled.")
+	}
+
 	return BC, DB, err
 }

internal/boot/boot_suite_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package boot_test
 
 import (

internal/boot/boot_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package boot_test
 
 import (
@@ -5,13 +20,13 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
 )
 
 var _ = Describe("Boot", func() {
 	var (
 		dbAddress            string = "localhost"
-		dbPort               int    = 8077
+		dbPort               int    = 8076
 		dbName               string = "vulcanize_testing"
 		dbUsername           string = "vdbm"
 		dbPassword           string = "password"
@@ -19,37 +34,57 @@ var _ = Describe("Boot", func() {
 		bcAddress            string = "localhost"
 		bcPort               int    = 5052
 		bcConnectionProtocol string = "http"
+		bcType               string = "lighthouse"
+		bcBootRetryInterval  int    = 1
+		bcBootMaxRetry       int    = 5
+		bcKgTableIncrement   int    = 10
+		bcUniqueIdentifier   int    = 100
+		bcCheckDb            bool   = false
 	)
 	Describe("Booting the application", Label("integration"), func() {
-		Context("When the DB and BC are both up and running, and we skip checking for a synced head", func() {
+		Context("When the DB and BC are both up and running, we skip checking for a synced head, and we are processing head", func() {
 			It("Should connect successfully", func() {
-				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, true)
+				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", true, bcUniqueIdentifier, bcCheckDb)
+				defer db.Close()
+				Expect(err).ToNot(HaveOccurred())
+			})
+		})
+		Context("When the DB and BC are both up and running, we skip checking for a synced head, and we are processing historic ", func() {
+			It("Should connect successfully", func() {
+				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "historic", true, bcUniqueIdentifier, bcCheckDb)
 				defer db.Close()
 				Expect(err).ToNot(HaveOccurred())
 			})
 		})
 		Context("When the DB and BC are both up and running, and we check for a synced head", func() {
 			It("Should not connect successfully", func() {
-				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, false)
+				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", false, bcUniqueIdentifier, bcCheckDb)
+				defer db.Close()
+				Expect(err).To(HaveOccurred())
+			})
+		})
+		Context("When the DB and BC are both up and running, we skip checking for a synced head, but the unique identifier is 0", func() {
+			It("Should not connect successfully", func() {
+				_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", false, 0, bcCheckDb)
 				defer db.Close()
 				Expect(err).To(HaveOccurred())
 			})
 		})
 		Context("When the DB is running but not the BC", func() {
 			It("Should not connect successfully", func() {
-				_, _, err := boot.BootApplication(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, true)
+				_, _, err := boot.BootApplication(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
 				Expect(err).To(HaveOccurred())
 			})
 		})
 		Context("When the BC is running but not the DB", func() {
 			It("Should not connect successfully", func() {
-				_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, true)
+				_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
 				Expect(err).To(HaveOccurred())
 			})
 		})
 		Context("When neither the BC or DB are running", func() {
 			It("Should not connect successfully", func() {
-				_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, true)
+				_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
 				Expect(err).To(HaveOccurred())
 			})
 		})

internal/shutdown/shutdown.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package shutdown
 
 import (
@@ -5,25 +20,16 @@ import (
 	"os"
 	"time"
 
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/gracefulshutdown"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/gracefulshutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // Shutdown all the internal services for the application.
-func ShutdownServices(ctx context.Context, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
-	successCh, errCh := gracefulshutdown.Shutdown(ctx, notifierCh, waitTime, map[string]gracefulshutdown.Operation{
-		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
-		"beaconClient": func(ctx context.Context) error {
-			defer DB.Close()
-			err := BC.StopHeadTracking()
-			if err != nil {
-				loghelper.LogError(err).Error("Unable to trigger shutdown of head tracking")
-			}
-			return err
-		},
-	})
+func ShutdownServices(ctx context.Context, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient, shutdownOperations map[string]gracefulshutdown.Operation) error {
+	//successCh, errCh := gracefulshutdown.Shutdown(ctx, notifierCh, waitTime, )
+	successCh, errCh := gracefulshutdown.Shutdown(ctx, notifierCh, waitTime, shutdownOperations)
 
 	select {
 	case <-successCh:
@@ -32,3 +38,82 @@ func ShutdownServices(ctx context.Context, notifierCh chan os.Signal, waitTime t
 		return err
 	}
 }
+
+// Wrapper function for shutting down the head tracking process.
+func ShutdownHeadTracking(ctx context.Context, kgCancel context.CancelFunc, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
+	return ShutdownServices(ctx, notifierCh, waitTime, DB, BC, map[string]gracefulshutdown.Operation{
+		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
+		"beaconClient": func(ctx context.Context) error {
+			defer DB.Close()
+			err := BC.StopHeadTracking()
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to trigger shutdown of head tracking")
+			}
+			if BC.KnownGapsProcess != (beaconclient.KnownGapsProcessing{}) {
+				err = BC.StopKnownGapsProcessing(kgCancel)
+				if err != nil {
+					loghelper.LogError(err).Error("Unable to stop processing known gaps")
+				}
+			}
+			return err
+		},
+	})
+}
+
+// Wrapper function for shutting down the head tracking process.
+func ShutdownHistoricProcessing(ctx context.Context, kgCancel, hpCancel context.CancelFunc, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
+	return ShutdownServices(ctx, notifierCh, waitTime, DB, BC, map[string]gracefulshutdown.Operation{
+		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
+		"beaconClient": func(ctx context.Context) error {
+			defer DB.Close()
+			err := BC.StopHistoric(hpCancel)
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to stop processing historic")
+			}
+			if BC.KnownGapsProcess != (beaconclient.KnownGapsProcessing{}) {
+				err = BC.StopKnownGapsProcessing(kgCancel)
+				if err != nil {
+					loghelper.LogError(err).Error("Unable to stop processing known gaps")
+				}
+			}
+			return err
+		},
+	})
+}
+
+// Shutdown the head and historical processing
+func ShutdownFull(ctx context.Context, kgCancel, hpCancel context.CancelFunc, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
+	return ShutdownServices(ctx, notifierCh, waitTime, DB, BC, map[string]gracefulshutdown.Operation{
+		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
+		"beaconClient": func(ctx context.Context) error {
+			defer DB.Close()
+			err := BC.StopHistoric(hpCancel)
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to stop processing historic")
+			}
+			if BC.KnownGapsProcess != (beaconclient.KnownGapsProcessing{}) {
+				err = BC.StopKnownGapsProcessing(kgCancel)
+				if err != nil {
+					loghelper.LogError(err).Error("Unable to stop processing known gaps")
+				}
+			}
+			err = BC.StopHeadTracking()
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to trigger shutdown of head tracking")
+			}
+
+			return err
+		},
+	})
+
+}
+
+// Wrapper function for shutting down the application in boot mode.
+func ShutdownBoot(ctx context.Context, notifierCh chan os.Signal, waitTime time.Duration, DB sql.Database, BC *beaconclient.BeaconClient) error {
+	return ShutdownServices(ctx, notifierCh, waitTime, DB, BC, map[string]gracefulshutdown.Operation{
+		// Combining DB shutdown with BC because BC needs DB open to cleanly shutdown.
+		"Database": func(ctx context.Context) error {
+			return DB.Close()
+		},
+	})
+}

internal/shutdown/shutdown_suite_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package shutdown_test
 
 import (

internal/shutdown/shutdown_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 //go:build !race
 // +build !race
 
@@ -13,44 +28,53 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/r3labs/sse"
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/boot"
-	"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/gracefulshutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/gracefulshutdown"
+)
+
+var (
+	dbAddress              string        = "localhost"
+	dbPort                 int           = 8076
+	dbName                 string        = "vulcanize_testing"
+	dbUsername             string        = "vdbm"
+	dbPassword             string        = "password"
+	dbDriver               string        = "PGX"
+	bcAddress              string        = "localhost"
+	bcPort                 int           = 5052
+	bcConnectionProtocol   string        = "http"
+	bcType                 string        = "lighthouse"
+	bcBootRetryInterval    int           = 1
+	bcBootMaxRetry         int           = 5
+	bcKgTableIncrement     int           = 10
+	bcUniqueIdentifier     int           = 100
+	bcCheckDb              bool          = false
+	maxWaitSecondsShutdown time.Duration = time.Duration(1) * time.Second
+	DB                     sql.Database
+	BC                     *beaconclient.BeaconClient
+	err                    error
+	ctx                    context.Context
+	notifierCh             chan os.Signal
 )
 
 var _ = Describe("Shutdown", func() {
-	var (
-		dbAddress              string        = "localhost"
-		dbPort                 int           = 8077
-		dbName                 string        = "vulcanize_testing"
-		dbUsername             string        = "vdbm"
-		dbPassword             string        = "password"
-		dbDriver               string        = "PGX"
-		bcAddress              string        = "localhost"
-		bcPort                 int           = 5052
-		bcConnectionProtocol   string        = "http"
-		maxWaitSecondsShutdown time.Duration = time.Duration(1) * time.Second
-		DB                     sql.Database
-		BC                     *beaconclient.BeaconClient
-		err                    error
-		ctx                    context.Context
-		notifierCh             chan os.Signal
-	)
 	BeforeEach(func() {
 		ctx = context.Background()
-		BC, DB, err = boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, true)
+		BC, DB, err = boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress,
+			bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", true, bcUniqueIdentifier, bcCheckDb)
 		notifierCh = make(chan os.Signal, 1)
 		Expect(err).To(BeNil())
 	})
 
-	Describe("Run Shutdown Function,", Label("integration"), func() {
+	Describe("Run Shutdown Function for head tracking,", Label("integration"), func() {
 		Context("When Channels are empty,", func() {
 			It("Should Shutdown Successfully.", func() {
 				go func() {
+					_, cancel := context.WithCancel(context.Background())
 					log.Debug("Starting shutdown chan")
-					err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC)
+					err = shutdown.ShutdownHeadTracking(ctx, cancel, notifierCh, maxWaitSecondsShutdown, DB, BC)
 					log.Debug("We have completed the shutdown...")
 					Expect(err).ToNot(HaveOccurred())
 				}()
@@ -61,8 +85,9 @@ var _ = Describe("Shutdown", func() {
 				shutdownCh := make(chan bool)
 				//log.SetLevel(log.DebugLevel)
 				go func() {
+					_, cancel := context.WithCancel(context.Background())
 					log.Debug("Starting shutdown chan")
-					err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC)
+					err = shutdown.ShutdownHeadTracking(ctx, cancel, notifierCh, maxWaitSecondsShutdown, DB, BC)
 					log.Debug("We have completed the shutdown...")
 					Expect(err).ToNot(HaveOccurred())
 					shutdownCh <- true
@@ -95,7 +120,8 @@ var _ = Describe("Shutdown", func() {
 				//log.SetLevel(log.DebugLevel)
 				go func() {
 					log.Debug("Starting shutdown chan")
-					err = shutdown.ShutdownServices(ctx, notifierCh, maxWaitSecondsShutdown, DB, BC)
+					_, cancel := context.WithCancel(context.Background())
+					err = shutdown.ShutdownHeadTracking(ctx, cancel, notifierCh, maxWaitSecondsShutdown, DB, BC)
 					log.Debug("We have completed the shutdown...")
 					Expect(err).To(MatchError(gracefulshutdown.TimeoutErr(maxWaitSecondsShutdown.String())))
 					shutdownCh <- true

main.go

@@ -1,10 +1,25 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 /*
 Copyright © 2022 NAME HERE <EMAIL ADDRESS>
 
 */
 package main
 
-import "github.com/vulcanize/ipld-ethcl-indexer/cmd"
+import "github.com/vulcanize/ipld-eth-beacon-indexer/cmd"
 
 func main() {
 	cmd.Execute()

pkg/beaconclient/beaconclient.go

@@ -1,12 +1,28 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
 import (
 	"context"
 	"fmt"
+	"math/rand"
 
 	"github.com/r3labs/sse"
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
 )
 
 // TODO: Use prysms config values instead of hardcoding them here.
@@ -16,7 +32,8 @@ var (
 	bcReorgTopicEndpoint = "/eth/v1/events?topics=chain_reorg" // Endpoint used to subscribe to the head of the chain
 	BcBlockQueryEndpoint = "/eth/v2/beacon/blocks/"            // Endpoint to query individual Blocks
 	BcStateQueryEndpoint = "/eth/v2/debug/beacon/states/"      // Endpoint to query individual States
-	BcSyncStatusEndpoint = "/eth/v1/node/syncing"
+	BcSyncStatusEndpoint = "/eth/v1/node/syncing"              // The endpoint to check to see if the beacon server is still trying to sync to head.
+	LhDbInfoEndpoint     = "/lighthouse/database/info"         // The endpoint for the LIGHTHOUSE server to get the database information.
 	BcBlockRootEndpoint  = func(slot string) string {
 		return "/eth/v1/beacon/blocks/" + slot + "/root"
 	}
@@ -25,30 +42,34 @@ var (
 	//bcFinalizedTopicEndpoint  = "/eth/v1/events?topics=finalized_checkpoint" // Endpoint used to subscribe to the head of the chain
 )
 
-// A structure utilized for keeping track of various metrics. Currently, mostly used in testing.
-type BeaconClientMetrics struct {
-	HeadTrackingInserts uint64 // Number of head events we wrote to the DB.
-	HeadTrackingReorgs  uint64 // The number of reorg events written to the DB.
-}
-
 // A struct that capture the Beacon Server that the Beacon Client will be interacting with and querying.
 type BeaconClient struct {
-	Context                     context.Context      // A context generic context with multiple uses.
-	ServerEndpoint              string               // What is the endpoint of the beacon server.
-	PerformHistoricalProcessing bool                 // Should we perform historical processing?
-	Db                          sql.Database         // Database object used for reads and writes.
-	Metrics                     *BeaconClientMetrics // An object used to keep track of certain BeaconClient Metrics.
-	KnownGapTableIncrement      int                  // The max number of slots within a single known_gaps table entry.
+	Context                context.Context      // A context generic context with multiple uses.
+	ServerEndpoint         string               // What is the endpoint of the beacon server.
+	Db                     sql.Database         // Database object used for reads and writes.
+	Metrics                *BeaconClientMetrics // An object used to keep track of certain BeaconClient Metrics.
+	KnownGapTableIncrement int                  // The max number of slots within a single known_gaps table entry.
+	UniqueNodeIdentifier   int                  // The unique identifier within the cluster of this individual node.
+	KnownGapsProcess       KnownGapsProcessing  // object keeping track of knowngaps processing
+	CheckDb                bool                 // Should we check the DB to see if the slot exists before processing it?
 
 	// Used for Head Tracking
+
 	PerformHeadTracking bool                   // Should we track head?
 	StartingSlot        int                    // If we're performing head tracking. What is the first slot we processed.
 	PreviousSlot        int                    // Whats the previous slot we processed
 	PreviousBlockRoot   string                 // Whats the previous block root, used to check the next blocks parent.
-	CheckKnownGaps      bool                   // Should we check for gaps at start up.
 	HeadTracking        *SseEvents[Head]       // Track the head block
 	ReOrgTracking       *SseEvents[ChainReorg] // Track all Reorgs
 	//FinalizationTracking        *SseEvents[FinalizedCheckpoint] // Track all finalization checkpoints
+
+	// Used for Historical Processing
+
+	// The latest available slot within the Beacon Server. We can't query any slot greater than this.
+	// This value is lazily updated. Therefore at times it will be outdated.
+	LatestSlotInBeaconServer    int64
+	PerformHistoricalProcessing bool               // Should we perform historical processing?
+	HistoricalProcess           HistoricProcessing // object keeping track of historical processing
 }
 
 // A struct to keep track of relevant the head event topic.
@@ -67,20 +88,30 @@ type SseError struct {
 }
 
 // A Function to create the BeaconClient.
-func CreateBeaconClient(ctx context.Context, connectionProtocol string, bcAddress string, bcPort int) *BeaconClient {
+func CreateBeaconClient(ctx context.Context, connectionProtocol string, bcAddress string, bcPort int, bcKgTableIncrement int, uniqueNodeIdentifier int, checkDb bool) (*BeaconClient, error) {
+	if uniqueNodeIdentifier == 0 {
+		uniqueNodeIdentifier := rand.Int()
+		log.WithField("randomUniqueNodeIdentifier", uniqueNodeIdentifier).Warn("No uniqueNodeIdentifier provided, we are going to use a randomly generated one.")
+	}
+
+	metrics, err := CreateBeaconClientMetrics()
+	if err != nil {
+		return nil, err
+	}
+
 	endpoint := fmt.Sprintf("%s://%s:%d", connectionProtocol, bcAddress, bcPort)
 	log.Info("Creating the BeaconClient")
 	return &BeaconClient{
-		Context:        ctx,
-		ServerEndpoint: endpoint,
-		HeadTracking:   createSseEvent[Head](endpoint, BcHeadTopicEndpoint),
-		ReOrgTracking:  createSseEvent[ChainReorg](endpoint, bcReorgTopicEndpoint),
-		Metrics: &BeaconClientMetrics{
-			HeadTrackingInserts: 0,
-			HeadTrackingReorgs:  0,
-		},
+		Context:                ctx,
+		ServerEndpoint:         endpoint,
+		KnownGapTableIncrement: bcKgTableIncrement,
+		HeadTracking:           createSseEvent[Head](endpoint, BcHeadTopicEndpoint),
+		ReOrgTracking:          createSseEvent[ChainReorg](endpoint, bcReorgTopicEndpoint),
+		Metrics:                metrics,
+		UniqueNodeIdentifier:   uniqueNodeIdentifier,
+		CheckDb:                checkDb,
 		//FinalizationTracking: createSseEvent[FinalizedCheckpoint](endpoint, bcFinalizedTopicEndpoint),
-	}
+	}, nil
 }
 
 // Create all the channels to handle a SSE events

pkg/beaconclient/beaconclient_suite_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient_test
 
 import (

pkg/beaconclient/capturehead.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This file will call all the functions to start and stop capturing the head of the beacon chain.
 
 package beaconclient
@@ -6,16 +21,13 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // This function will perform all the heavy lifting for tracking the head of the chain.
-func (bc *BeaconClient) CaptureHead(knownGapsTableIncrement int) {
-	bc.KnownGapTableIncrement = knownGapsTableIncrement
+func (bc *BeaconClient) CaptureHead() {
 	log.Info("We are tracking the head of the chain.")
-	//bc.tempHelper()
 	go bc.handleHead()
-	//go bc.handleFinalizedCheckpoint()
 	go bc.handleReorg()
 	bc.captureEventTopic()
 }

pkg/beaconclient/capturehistoric.go

@@ -0,0 +1,180 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// This file will call all the functions to start and stop capturing the head of the beacon chain.
+
+package beaconclient
+
+import (
+	"context"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
+)
+
+// This function will perform all the heavy lifting for tracking the head of the chain.
+func (bc *BeaconClient) CaptureHistoric(ctx context.Context, maxWorkers int) []error {
+	log.Info("We are starting the historical processing service.")
+	bc.HistoricalProcess = HistoricProcessing{db: bc.Db, metrics: bc.Metrics, uniqueNodeIdentifier: bc.UniqueNodeIdentifier}
+	errs := handleBatchProcess(ctx, maxWorkers, bc.HistoricalProcess, bc.HistoricalProcess.db, bc.ServerEndpoint, bc.Metrics, bc.CheckDb, bc.Metrics.IncrementHistoricSlotProcessed)
+	log.Debug("Exiting Historical")
+	return errs
+}
+
+// This function will perform all the necessary clean up tasks for stopping historical processing.
+func (bc *BeaconClient) StopHistoric(cancel context.CancelFunc) error {
+	log.Info("We are stopping the historical processing service.")
+	cancel()
+	err := bc.HistoricalProcess.releaseDbLocks()
+	if err != nil {
+		loghelper.LogError(err).WithField("uniqueIdentifier", bc.UniqueNodeIdentifier).Error("We were unable to remove the locks from the eth_beacon.historic_processing table. Manual Intervention is needed!")
+	}
+	return nil
+}
+
+// An interface to enforce any batch processing. Currently there are two use cases for this.
+//
+// 1. Historic Processing
+//
+// 2. Known Gaps Processing
+type BatchProcessing interface {
+	getSlotRange(context.Context, chan<- slotsToProcess) []error       // Write the slots to process in a channel, return an error if you cant get the next slots to write.
+	handleProcessingErrors(context.Context, <-chan batchHistoricError) // Custom logic to handle errors.
+	removeTableEntry(context.Context, <-chan slotsToProcess) error     // With the provided start and end slot, remove the entry from the database.
+	releaseDbLocks() error                                             // Update the checked_out column to false for whatever table is being updated.
+}
+
+/// ^^^
+// Might be better to remove the interface and create a single struct that historicalProcessing
+// and knownGapsProcessing can use. The struct would contain all the SQL strings that they need.
+// And the only difference in logic for processing would be within the error handling.
+// Which can be a function we pass into handleBatchProcess()
+
+// A struct to pass around indicating a table entry for slots to process.
+type slotsToProcess struct {
+	startSlot int // The start slot
+	endSlot   int // The end slot
+}
+
+type batchHistoricError struct {
+	err        error  // The error that occurred when attempting to a slot
+	errProcess string // The process that caused the error.
+	slot       int    // The slot which the error is for.
+}
+
+// Wrapper function for the BatchProcessing interface.
+// This function will take the structure that needs batch processing.
+// It follows a generic format.
+// Get new entries from any given table.
+// 1. Add it to the slotsCh.
+//
+// 2. Run the maximum specified workers to handle individual slots. We need a maximum because we don't want
+// To store too many SSZ objects in memory.
+//
+// 3. Process the slots and send the err to the ErrCh. Each structure can define how it wants its own errors handled.
+//
+// 4. Remove the slot entry from the DB.
+//
+// 5. Handle any errors.
+func handleBatchProcess(ctx context.Context, maxWorkers int, bp BatchProcessing, db sql.Database, serverEndpoint string, metrics *BeaconClientMetrics, checkDb bool, incrementTracker func(uint64)) []error {
+	slotsCh := make(chan slotsToProcess)
+	workCh := make(chan int)
+	processedCh := make(chan slotsToProcess)
+	errCh := make(chan batchHistoricError)
+	finalErrCh := make(chan []error, 1)
+
+	// Checkout Rows with same node Identifier.
+	err := bp.releaseDbLocks()
+	if err != nil {
+		loghelper.LogError(err).Error(("We are unable to un-checkout entries at the start!"))
+	}
+
+	// Start workers
+	for w := 1; w <= maxWorkers; w++ {
+		log.WithFields(log.Fields{"maxWorkers": maxWorkers}).Debug("Starting batch processing workers")
+
+		go processSlotRangeWorker(ctx, workCh, errCh, db, serverEndpoint, metrics, checkDb, incrementTracker)
+	}
+
+	// Process all ranges and send each individual slot to the worker.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case slots := <-slotsCh:
+				if slots.startSlot > slots.endSlot {
+					log.Error("We received a batch process request where the startSlot is greater than the end slot.")
+					errCh <- batchHistoricError{
+						err:        fmt.Errorf("We received a startSlot where the start was greater than the end."),
+						errProcess: "RangeOrder",
+						slot:       slots.startSlot,
+					}
+					errCh <- batchHistoricError{
+						err:        fmt.Errorf("We received a endSlot where the start was greater than the end."),
+						errProcess: "RangeOrder",
+						slot:       slots.endSlot,
+					}
+				} else if slots.startSlot == slots.endSlot {
+					log.WithField("slot", slots.startSlot).Debug("Added new slot to workCh")
+					workCh <- slots.startSlot
+					processedCh <- slots
+				} else {
+					for i := slots.startSlot; i <= slots.endSlot; i++ {
+						workCh <- i
+						log.WithField("slot", i).Debug("Added new slot to workCh")
+					}
+					processedCh <- slots
+				}
+			}
+
+		}
+	}()
+
+	// Remove entries, end the application if a row cannot be removed..
+	go func() {
+		errG := new(errgroup.Group)
+		errG.Go(func() error {
+			return bp.removeTableEntry(ctx, processedCh)
+		})
+		if err := errG.Wait(); err != nil {
+			finalErrCh <- []error{err}
+		}
+	}()
+	// Process errors from slot processing.
+	go bp.handleProcessingErrors(ctx, errCh)
+
+	// Get slots from the DB.
+	go func() {
+		errs := bp.getSlotRange(ctx, slotsCh) // Periodically adds new entries....
+		if errs != nil {
+			finalErrCh <- errs
+		}
+		finalErrCh <- nil
+		log.Debug("We are stopping the processing of adding new entries")
+	}()
+	log.Debug("Waiting for shutdown signal from channel")
+	select {
+	case <-ctx.Done():
+		log.Debug("Received shutdown signal from channel")
+		return nil
+	case errs := <-finalErrCh:
+		log.Debug("Finishing the batchProcess")
+		return errs
+	}
+}

pkg/beaconclient/capturehistoric_test.go

@@ -0,0 +1,290 @@
+package beaconclient_test
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/jarcoal/httpmock"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+)
+
+var (
+	kgCheckCheckedOutStmt = `SELECT * FROM eth_beacon.known_gaps WHERE checked_out=true `
+	hpCheckCheckedOutStmt = `SELECT * FROM eth_beacon.historic_process WHERE checked_out=true `
+)
+
+var _ = Describe("Capturehistoric", func() {
+
+	Describe("Run the application in historic mode", Label("unit", "behavioral", "historical"), func() {
+		Context("Phase0 + Altairs: When we need to process a multiple blocks in a multiple entries in the eth_beacon.historic_process table.", Label("deb"), func() {
+			It("Successfully Process the Blocks", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 100, 101, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 2, 0, 0, 0)
+				// Run Two seperate processes
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 2375703, 2375703, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+		Context("When the start block is greater than the endBlock", func() {
+			It("Should Add two entries to the knownGaps table", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 101, 100, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 0, 0, 2, 0)
+			})
+		})
+		Context("Processing the Genesis block", Label("genesis"), func() {
+			It("Should Process properly", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "100")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 0, 0, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 1, 0, 0, 0)
+				validateSlot(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, 0, "proposed")
+				validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, BeaconNodeTester.TestEvents["0"].CorrectParentRoot, BeaconNodeTester.TestEvents["0"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["0"].CorrectSignedBeaconBlockMhKey)
+				validateBeaconState(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, BeaconNodeTester.TestEvents["0"].CorrectBeaconStateMhKey)
+			})
+		})
+		Context("When there is a skipped slot", func() {
+			It("Should process the slot properly.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "3797055")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 3797056, 3797056, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 1, 0, 0, 0)
+				validateSlot(bc, BeaconNodeTester.TestEvents["3797056"].HeadMessage, 118658, "skipped")
+			})
+		})
+	})
+	Describe("Running the Application to process Known Gaps", Label("unit", "behavioral", "knownGaps"), func() {
+		Context("Phase0 + Altairs: When we need to process a multiple blocks in a multiple entries in the eth_beacon.known_gaps table.", func() {
+			It("Successfully Process the Blocks", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToKnownGaps(bc, 100, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 2, 0, 0, 0)
+				// Run Two seperate processes
+				BeaconNodeTester.writeEventToKnownGaps(bc, 2375703, 2375703)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+		Context("When the start block is greater than the endBlock", func() {
+			It("Should Add two entries to the knownGaps table", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "104")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				BeaconNodeTester.writeEventToKnownGaps(bc, 101, 100)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 2, 0, 2, 0)
+			})
+		})
+		Context("When theres a reprocessing error", Label("reprocessingError"), func() {
+			It("Should update the reprocessing error.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "99")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				// We dont have an entry in the BeaconNodeTester for this slot
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 105, 105, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 0, 0, 1, 0)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 0, 0, 1, 1)
+			})
+		})
+	})
+	Describe("Running the application in Historic, Head, and KnownGaps mode", Label("unit", "historical", "full"), func() {
+		Context("When it recieves a head, historic and known Gaps message (in order)", func() {
+			It("Should process them all successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				// Head
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 100, 100, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 2, 0, 0, 0)
+
+				// Known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 101, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+		Context("When it recieves a historic, head and known Gaps message (in order)", func() {
+			It("Should process them all successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 100, 100, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 1, 0, 0, 0)
+
+				// Head
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+
+				// Known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 101, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+		Context("When it recieves a known Gaps, historic and head  message (in order)", func() {
+			It("Should process them all successfully.", func() {
+				bc := setUpTest(BeaconNodeTester.TestConfig, "2375702")
+				BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
+				defer httpmock.DeactivateAndReset()
+				// Known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 101, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 1, 0, 0, 0)
+
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 100, 100, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 2, 0, 0, 0)
+
+				// Head
+				BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, maxRetry, 1, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+	})
+})
+
+// This function will write an even to the eth_beacon.known_gaps table
+func (tbc TestBeaconNode) writeEventToKnownGaps(bc *beaconclient.BeaconClient, startSlot, endSlot int) {
+	log.Debug("We are writing the necessary events to batch process")
+	insertKnownGapsStmt := `INSERT INTO eth_beacon.known_gaps (start_slot, end_slot)
+	VALUES ($1, $2);`
+	res, err := bc.Db.Exec(context.Background(), insertKnownGapsStmt, startSlot, endSlot)
+	Expect(err).ToNot(HaveOccurred())
+	rows, err := res.RowsAffected()
+	if rows != 1 {
+		Fail("We didnt write...")
+	}
+	Expect(err).ToNot(HaveOccurred())
+}
+
+// This function will write an even to the eth_beacon.known_gaps table
+func (tbc TestBeaconNode) writeEventToHistoricProcess(bc *beaconclient.BeaconClient, startSlot, endSlot, priority int) {
+	log.Debug("We are writing the necessary events to batch process")
+	insertHistoricProcessingStmt := `INSERT INTO eth_beacon.historic_process (start_slot, end_slot, priority)
+	VALUES ($1, $2, $3);`
+	res, err := bc.Db.Exec(context.Background(), insertHistoricProcessingStmt, startSlot, endSlot, priority)
+	Expect(err).ToNot(HaveOccurred())
+	rows, err := res.RowsAffected()
+	if rows != 1 {
+		Fail("We didnt write...")
+	}
+	Expect(err).ToNot(HaveOccurred())
+}
+
+// Start the CaptureHistoric function, and check for the correct inserted slots.
+func (tbc TestBeaconNode) runHistoricalProcess(bc *beaconclient.BeaconClient, maxWorkers int, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	ctx, cancel := context.WithCancel(context.Background())
+	go bc.CaptureHistoric(ctx, maxWorkers)
+	validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
+	log.Debug("Calling the stop function for historical processing..")
+	err := bc.StopHistoric(cancel)
+	time.Sleep(5 * time.Second)
+	Expect(err).ToNot(HaveOccurred())
+	validateAllRowsCheckedOut(bc.Db, hpCheckCheckedOutStmt)
+}
+
+// Wrapper function that processes knownGaps
+func (tbc TestBeaconNode) runKnownGapsProcess(bc *beaconclient.BeaconClient, maxWorkers int, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	ctx, cancel := context.WithCancel(context.Background())
+	go bc.ProcessKnownGaps(ctx, maxWorkers)
+	validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
+	err := bc.StopKnownGapsProcessing(cancel)
+	time.Sleep(5 * time.Second)
+	Expect(err).ToNot(HaveOccurred())
+	validateAllRowsCheckedOut(bc.Db, kgCheckCheckedOutStmt)
+}
+
+func validateMetrics(bc *beaconclient.BeaconClient, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	curRetry := 0
+	value := atomic.LoadUint64(&bc.Metrics.SlotInserts)
+	for value != expectedInserts {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Too many retries have occurred. The number of inserts expected %d, the number that actually occurred, %d", expectedInserts, atomic.LoadUint64(&bc.Metrics.SlotInserts)))
+		}
+		value = atomic.LoadUint64(&bc.Metrics.SlotInserts)
+	}
+	curRetry = 0
+	value = atomic.LoadUint64(&bc.Metrics.KnownGapsInserts)
+	for value != expectedKnownGaps {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Too many retries have occurred. The number of knownGaps expected %d, the number that actually occurred, %d", expectedKnownGaps, atomic.LoadUint64(&bc.Metrics.KnownGapsInserts)))
+		}
+		value = atomic.LoadUint64(&bc.Metrics.KnownGapsInserts)
+	}
+	curRetry = 0
+	value = atomic.LoadUint64(&bc.Metrics.KnownGapsReprocessError)
+	for value != expectedKnownGapsReprocessError {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Too many retries have occurred. The number of knownGapsReprocessingErrors expected %d, the number that actually occurred, %d", expectedKnownGapsReprocessError, value))
+		}
+		log.Debug("&bc.Metrics.KnownGapsReprocessError: ", &bc.Metrics.KnownGapsReprocessError)
+		value = atomic.LoadUint64(&bc.Metrics.KnownGapsReprocessError)
+	}
+	curRetry = 0
+	value = atomic.LoadUint64(&bc.Metrics.ReorgInserts)
+	for value != expectedReorgs {
+		time.Sleep(1 * time.Second)
+		curRetry = curRetry + 1
+		if curRetry == maxRetry {
+			Fail(fmt.Sprintf("Too many retries have occurred. The number of Reorgs expected %d, the number that actually occurred, %d", expectedReorgs, atomic.LoadUint64(&bc.Metrics.ReorgInserts)))
+		}
+		value = atomic.LoadUint64(&bc.Metrics.ReorgInserts)
+	}
+}
+
+// A wrapper function to validate a few popular blocks
+func validatePopularBatchBlocks(bc *beaconclient.BeaconClient) {
+	validateSlot(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, 3, "proposed")
+	validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectParentRoot, BeaconNodeTester.TestEvents["100"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["100"].CorrectSignedBeaconBlockMhKey)
+	validateBeaconState(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectBeaconStateMhKey)
+
+	validateSlot(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, 3, "proposed")
+	validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, BeaconNodeTester.TestEvents["100"].HeadMessage.Block, BeaconNodeTester.TestEvents["101"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["101"].CorrectSignedBeaconBlockMhKey)
+	validateBeaconState(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, BeaconNodeTester.TestEvents["101"].CorrectBeaconStateMhKey)
+
+	validateSlot(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, "proposed")
+	validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectParentRoot, BeaconNodeTester.TestEvents["2375703"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["2375703"].CorrectSignedBeaconBlockMhKey)
+	validateBeaconState(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectBeaconStateMhKey)
+}
+
+// Make sure all rows have checked_out as false.
+func validateAllRowsCheckedOut(db sql.Database, checkStmt string) {
+	res, err := db.Exec(context.Background(), checkStmt)
+	Expect(err).ToNot(HaveOccurred())
+	rows, err := res.RowsAffected()
+	Expect(err).ToNot(HaveOccurred())
+	Expect(rows).To(Equal(int64(0)))
+}

pkg/beaconclient/checkbeaconserverstatus.go

@@ -0,0 +1,207 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package beaconclient
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync/atomic"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+var (
+	MissingBeaconServerType error = fmt.Errorf("The beacon server type provided is not handled.")
+	LighthouseMissingSlots  error = fmt.Errorf("Anchor is not nil. This means lighthouse has not backfilled all the slots from Genesis to head.")
+)
+
+// The sync response when checking if the node is synced.
+type Sync struct {
+	Data SyncData `json:"data"`
+}
+
+// The sync data
+type SyncData struct {
+	IsSync       bool   `json:"is_syncing"`
+	HeadSlot     string `json:"head_slot"`
+	SyncDistance string `json:"sync_distance"`
+}
+
+// This function will check to see if we are synced up with the head of chain.
+//{"data":{"is_syncing":true,"head_slot":"62528","sync_distance":"3734299"}}
+func (bc BeaconClient) CheckHeadSync() (bool, error) {
+	syncStatus, err := bc.QueryHeadSync()
+	if err != nil {
+		return true, nil
+	}
+	return syncStatus.Data.IsSync, nil
+}
+
+func (bc BeaconClient) QueryHeadSync() (Sync, error) {
+	var syncStatus Sync
+	bcSync := bc.ServerEndpoint + BcSyncStatusEndpoint
+	resp, err := http.Get(bcSync)
+
+	if err != nil {
+		loghelper.LogEndpoint(bcSync).Error("Unable to check the sync status")
+		return syncStatus, err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		loghelper.LogEndpoint(bcSync).WithFields(log.Fields{"returnCode": resp.StatusCode}).Error("Error when getting the sync status")
+		return syncStatus, fmt.Errorf("Querying the sync status returned a non 2xx status code, code provided: %d", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return syncStatus, err
+	}
+
+	if err := json.Unmarshal(body, &syncStatus); err != nil {
+		loghelper.LogEndpoint(bcSync).WithFields(log.Fields{
+			"rawMessage": string(body),
+			"err":        err,
+		}).Error("Unable to unmarshal sync status")
+		return syncStatus, err
+	}
+	return syncStatus, nil
+}
+
+// The response when checking the lighthouse nodes DB info: /lighthouse/database/info
+type LighthouseDatabaseInfo struct {
+	SchemaVersion int        `json:"schema_version"`
+	Config        LhDbConfig `json:"config"`
+	Split         LhDbSplit  `json:"split"`
+	Anchor        LhDbAnchor `json:"anchor"`
+}
+
+// The config field within the DatabaseInfo response.
+type LhDbConfig struct {
+	SlotsPerRestorePoint              int  `json:"slots_per_restore_point"`
+	SlotsPerRestorePointSetExplicitly bool `json:"slots_per_restore_point_set_explicitly"`
+	BlockCacheSize                    int  `json:"block_cache_size"`
+	CompactOnInit                     bool `json:"compact_on_init"`
+	CompactOnPrune                    bool `json:"compact_on_prune"`
+}
+
+// The split field within the DatabaseInfo response.
+type LhDbSplit struct {
+	Slot      string `json:"slot"`
+	StateRoot string `json:"state_root"`
+}
+
+// The anchor field within the DatabaseInfo response.
+type LhDbAnchor struct {
+	AnchorSlot        string `json:"anchor_slot"`
+	OldestBlockSlot   string `json:"oldest_block_slot"`
+	OldestBlockParent string `json:"oldest_block_parent"`
+	StateUpperLimit   string `json:"state_upper_limit"`
+	StateLowerLimit   string `json:"state_lower_limit"`
+}
+
+// This function will notify us what the head slot is.
+func (bc BeaconClient) queryHeadSlotInBeaconServer() (int, error) {
+	syncStatus, err := bc.QueryHeadSync()
+	if err != nil {
+		return 0, nil
+	}
+	headSlot, err := strconv.Atoi(syncStatus.Data.HeadSlot)
+	if err != nil {
+		return 0, nil
+	}
+	return headSlot, nil
+}
+
+// return the lighthouse Database Info
+func (bc BeaconClient) queryLighthouseDbInfo() (LighthouseDatabaseInfo, error) {
+	var dbInfo LighthouseDatabaseInfo
+
+	lhDbInfo := bc.ServerEndpoint + LhDbInfoEndpoint
+	resp, err := http.Get(lhDbInfo)
+
+	if err != nil {
+		loghelper.LogEndpoint(lhDbInfo).Error("Unable to get the lighthouse database information")
+		return dbInfo, err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		loghelper.LogEndpoint(lhDbInfo).WithFields(log.Fields{"returnCode": resp.StatusCode}).Error("Error when getting the lighthouse database information")
+		return dbInfo, fmt.Errorf("Querying the lighthouse database information returned a non 2xx status code, code provided: %d", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return dbInfo, err
+	}
+
+	if err := json.Unmarshal(body, &dbInfo); err != nil {
+		loghelper.LogEndpoint(lhDbInfo).WithFields(log.Fields{
+			"rawMessage": string(body),
+			"err":        err,
+		}).Error("Unable to unmarshal the lighthouse database information")
+		return dbInfo, err
+	}
+	return dbInfo, nil
+}
+
+// This function will tell us what the latest slot is that the beacon server has available. This is important as
+// it will ensure us that we have all slots prior to the given slot.
+func (bc BeaconClient) GetLatestSlotInBeaconServer(beaconServerType string) (int, error) {
+	switch strings.ToLower(beaconServerType) {
+	case "lighthouse":
+		headSlot, err := bc.queryHeadSlotInBeaconServer()
+		if err != nil {
+			return 0, err
+		}
+		lhDb, err := bc.queryLighthouseDbInfo()
+		if err != nil {
+			return 0, err
+		}
+		if lhDb.Anchor == (LhDbAnchor{}) {
+			//atomic.StoreInt64(&bc.LatestSlotInBeaconServer, int64(headSlot))
+			log.WithFields(log.Fields{
+				"headSlot": headSlot,
+			}).Info("Anchor is nil, the lighthouse client has all the nodes from genesis to head.")
+			return headSlot, nil
+		} else {
+			log.WithFields(log.Fields{
+				"lhDb.Anchor": lhDb.Anchor,
+			}).Info(LighthouseMissingSlots.Error())
+			log.Info("We will add a feature down the road to wait for anchor to be null, if its needed.")
+			return 0, LighthouseMissingSlots
+		}
+	default:
+		log.WithFields(log.Fields{"BeaconServerType": beaconServerType}).Error(MissingBeaconServerType.Error())
+		return 0, MissingBeaconServerType
+	}
+}
+
+// A wrapper function for updating the latest slot.
+func (bc BeaconClient) UpdateLatestSlotInBeaconServer(headSlot int64) {
+	curr := atomic.LoadInt64(&bc.LatestSlotInBeaconServer)
+	log.WithFields(log.Fields{
+		"Previous Latest Slot": curr,
+		"New Latest Slot":      headSlot,
+	}).Debug("Swapping Head Slot")
+	atomic.SwapInt64(&bc.LatestSlotInBeaconServer, int64(headSlot))
+}

pkg/beaconclient/checksyncstatus.go

@@ -1,57 +0,0 @@
-package beaconclient
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
-)
-
-// The sync response
-type Sync struct {
-	Data SyncData `json:"data"`
-}
-
-// The sync data
-type SyncData struct {
-	IsSync       bool   `json:"is_syncing"`
-	HeadSlot     string `json:"head_slot"`
-	SyncDistance string `json:"sync_distance"`
-}
-
-// This function will check to see if we are synced up with the head of chain.
-//{"data":{"is_syncing":true,"head_slot":"62528","sync_distance":"3734299"}}
-func (bc BeaconClient) CheckHeadSync() (bool, error) {
-	bcSync := bc.ServerEndpoint + BcSyncStatusEndpoint
-	resp, err := http.Get(bcSync)
-
-	if err != nil {
-		loghelper.LogEndpoint(bcSync).Error("Unable to check the sync status")
-		return true, err
-	}
-
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		loghelper.LogEndpoint(bcSync).WithFields(log.Fields{"returnCode": resp.StatusCode}).Error("Error when getting the sync status")
-		return true, fmt.Errorf("Querying the sync status returned a non 2xx status code, code provided: %d", resp.StatusCode)
-	}
-
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return true, err
-	}
-
-	var syncStatus Sync
-	if err := json.Unmarshal(body, &syncStatus); err != nil {
-		loghelper.LogEndpoint(bcSync).WithFields(log.Fields{
-			"rawMessage": string(body),
-			"err":        err,
-		}).Error("Unable to unmarshal sync status")
-		return true, err
-	}
-
-	return syncStatus.Data.IsSync, nil
-}

pkg/beaconclient/databasewrite.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
 import (
@@ -5,44 +20,63 @@ import (
 	"fmt"
 	"strconv"
 
+	"github.com/jackc/pgx/v4"
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
 )
 
 var (
-	// Statement to upsert to the ethcl.slots table.
+	// Statement to upsert to the eth_beacon.slots table.
 	UpsertSlotsStmt string = `
-INSERT INTO ethcl.slots (epoch, slot, block_root, state_root, status)
+INSERT INTO eth_beacon.slots (epoch, slot, block_root, state_root, status)
 VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
-	// Statement to upsert to the ethcl.signed_beacon_blocks table.
+	// Statement to upsert to the eth_beacon.signed_blocks table.
 	UpsertSignedBeaconBlockStmt string = `
-INSERT INTO ethcl.signed_beacon_block (slot, block_root, parent_block_root, eth1_block_hash, mh_key)
+INSERT INTO eth_beacon.signed_block (slot, block_root, parent_block_root, eth1_block_hash, mh_key)
 VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
-	// Statement to upsert to the ethcl.beacon_state table.
+	// Statement to upsert to the eth_beacon.state table.
 	UpsertBeaconState string = `
-INSERT INTO ethcl.beacon_state (slot, state_root, mh_key)
+INSERT INTO eth_beacon.state (slot, state_root, mh_key)
 VALUES ($1, $2, $3) ON CONFLICT (slot, state_root) DO NOTHING`
 	// Statement to upsert to the public.blocks table.
 	UpsertBlocksStmt string = `
 INSERT INTO public.blocks (key, data)
 VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`
-	UpdateForkedStmt string = `UPDATE ethcl.slots
+	UpdateForkedStmt string = `UPDATE eth_beacon.slots
 	SET status='forked'
 	WHERE slot=$1 AND block_root<>$2
 	RETURNING block_root;`
-	UpdateProposedStmt string = `UPDATE ethcl.slots
+	UpdateProposedStmt string = `UPDATE eth_beacon.slots
 	SET status='proposed'
 	WHERE slot=$1 AND block_root=$2
 	RETURNING block_root;`
 	CheckProposedStmt string = `SELECT slot, block_root
-	FROM ethcl.slots
+	FROM eth_beacon.slots
 	WHERE slot=$1 AND block_root=$2;`
+	// Check to see if the slot and block_root exist in eth_beacon.signed_block
+	CheckSignedBeaconBlockStmt string = `SELECT slot, block_root
+	FROM eth_beacon.signed_block
+	WHERE slot=$1 AND block_root=$2`
+	// Check to see if the slot and state_root exist in eth_beacon.state
+	CheckBeaconStateStmt string = `SELECT slot, state_root
+	FROM eth_beacon.state
+	WHERE slot=$1 AND state_root=$2`
+	// Used to get a single slot from the table if it exists
+	QueryBySlotStmt string = `SELECT slot
+	FROM eth_beacon.slots
+	WHERE slot=$1`
 	// Statement to insert known_gaps. We don't pass in timestamp, we let the server take care of that one.
 	UpsertKnownGapsStmt string = `
-INSERT INTO ethcl.known_gaps (start_slot, end_slot, checked_out, reprocessing_error, entry_error, entry_process)
+INSERT INTO eth_beacon.known_gaps (start_slot, end_slot, checked_out, reprocessing_error, entry_error, entry_process)
 VALUES ($1, $2, $3, $4, $5, $6) on CONFLICT (start_slot, end_slot) DO NOTHING`
-	QueryHighestSlotStmt string = "SELECT COALESCE(MAX(slot), 0) FROM ethcl.slots"
+	UpsertKnownGapsErrorStmt string = `
+	UPDATE eth_beacon.known_gaps
+	SET reprocessing_error=$3, priority=priority+1
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Get the highest slot if one exists
+	QueryHighestSlotStmt string = "SELECT COALESCE(MAX(slot), 0) FROM eth_beacon.slots"
 )
 
 // Put all functionality to prepare the write object
@@ -50,24 +84,33 @@ VALUES ($1, $2, $3, $4, $5, $6) on CONFLICT (start_slot, end_slot) DO NOTHING`
 // Remove any of it from the processslot file.
 type DatabaseWriter struct {
 	Db                   sql.Database
+	Tx                   sql.Tx
+	Ctx                  context.Context
 	Metrics              *BeaconClientMetrics
 	DbSlots              *DbSlots
 	DbSignedBeaconBlock  *DbSignedBeaconBlock
 	DbBeaconState        *DbBeaconState
-	rawBeaconState       []byte
-	rawSignedBeaconBlock []byte
+	rawBeaconState       *[]byte
+	rawSignedBeaconBlock *[]byte
 }
 
 func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot string, parentBlockRoot string,
-	eth1BlockHash string, status string, rawSignedBeaconBlock []byte, rawBeaconState []byte, metrics *BeaconClientMetrics) (*DatabaseWriter, error) {
+	eth1BlockHash string, status string, rawSignedBeaconBlock *[]byte, rawBeaconState *[]byte, metrics *BeaconClientMetrics) (*DatabaseWriter, error) {
+	ctx := context.Background()
+	tx, err := db.Begin(ctx)
+	if err != nil {
+		loghelper.LogError(err).Error("We are unable to Begin a SQL transaction")
+	}
 	dw := &DatabaseWriter{
 		Db:                   db,
+		Tx:                   tx,
+		Ctx:                  ctx,
 		rawBeaconState:       rawBeaconState,
 		rawSignedBeaconBlock: rawSignedBeaconBlock,
 		Metrics:              metrics,
 	}
 	dw.prepareSlotsModel(slot, stateRoot, blockRoot, status)
-	err := dw.prepareSignedBeaconBlockModel(slot, blockRoot, parentBlockRoot, eth1BlockHash)
+	err = dw.prepareSignedBeaconBlockModel(slot, blockRoot, parentBlockRoot, eth1BlockHash)
 	if err != nil {
 		return nil, err
 	}
@@ -80,7 +123,7 @@ func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot
 
 // Write functions to write each all together...
 // Should I do one atomic write?
-// Create the model for the ethcl.slots table
+// Create the model for the eth_beacon.slots table
 func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoot string, status string) {
 	dw.DbSlots = &DbSlots{
 		Epoch:     calculateEpoch(slot, bcSlotsPerEpoch),
@@ -93,7 +136,7 @@ func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoo
 
 }
 
-// Create the model for the ethcl.signed_beacon_block table.
+// Create the model for the eth_beacon.signed_block table.
 func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot int, blockRoot string, parentBlockRoot string, eth1BlockHash string) error {
 	mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.BlockRoot))
 	if err != nil {
@@ -110,7 +153,7 @@ func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot int, blockRoot stri
 	return nil
 }
 
-// Create the model for the ethcl.beacon_state table.
+// Create the model for the eth_beacon.state table.
 func (dw *DatabaseWriter) prepareBeaconStateModel(slot int, stateRoot string) error {
 	mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.StateRoot))
 	if err != nil {
@@ -125,47 +168,67 @@ func (dw *DatabaseWriter) prepareBeaconStateModel(slot int, stateRoot string) er
 	return nil
 }
 
-// Write all the data for a given slot.
-func (dw *DatabaseWriter) writeFullSlot() error {
-	// Add errors for each function call
+// Add all the data for a given slot to a SQL transaction.
+// Originally it wrote to each table individually.
+func (dw *DatabaseWriter) transactFullSlot() error {
 	// If an error occurs, write to knownGaps table.
-	err := dw.writeSlots()
+	log.WithFields(log.Fields{
+		"slot": dw.DbSlots.Slot,
+	}).Debug("Starting to write to the DB.")
+	err := dw.transactSlots()
 	if err != nil {
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon.slots table...")
 		return err
 	}
+	log.Debug("We finished writing to the eth_beacon.slots table.")
 	if dw.DbSlots.Status != "skipped" {
-		err = dw.writeSignedBeaconBlocks()
+		//errG, _ := errgroup.WithContext(context.Background())
+		//errG.Go(func() error {
+		//	return dw.transactSignedBeaconBlocks()
+		//})
+		//errG.Go(func() error {
+		//	return dw.transactBeaconState()
+		//})
+		//if err := errG.Wait(); err != nil {
+		//	loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon block or state table...")
+		//	return err
+		//}
+		// Might want to seperate writing to public.blocks so we can do this concurrently...
+		// Cant concurrently write because we are using a transaction.
+		err := dw.transactSignedBeaconBlocks()
 		if err != nil {
+			loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon block table...")
 			return err
 		}
-		err = dw.writeBeaconState()
+		err = dw.transactBeaconState()
 		if err != nil {
+			loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon state table...")
 			return err
 		}
 	}
-	dw.Metrics.IncrementHeadTrackingInserts(1)
+	dw.Metrics.IncrementSlotInserts(1)
 	return nil
 }
 
-// Write the information for the generic slots table. For now this is only one function.
+// Add data for the eth_beacon.slots table to a transaction. For now this is only one function.
 // But in the future if we need to incorporate any FK's or perform any actions to write to the
 // slots table we can do it all here.
-func (dw *DatabaseWriter) writeSlots() error {
+func (dw *DatabaseWriter) transactSlots() error {
 	return dw.upsertSlots()
 }
 
-// Upsert to the ethcl.slots table.
+// Upsert to the eth_beacon.slots table.
 func (dw *DatabaseWriter) upsertSlots() error {
-	_, err := dw.Db.Exec(context.Background(), UpsertSlotsStmt, dw.DbSlots.Epoch, dw.DbSlots.Slot, dw.DbSlots.BlockRoot, dw.DbSlots.StateRoot, dw.DbSlots.Status)
+	_, err := dw.Tx.Exec(dw.Ctx, UpsertSlotsStmt, dw.DbSlots.Epoch, dw.DbSlots.Slot, dw.DbSlots.BlockRoot, dw.DbSlots.StateRoot, dw.DbSlots.Status)
 	if err != nil {
-		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the ethcl.slots table")
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the eth_beacon.slots table")
 		return err
 	}
 	return nil
 }
 
-// Write the information for the signed_beacon_block.
-func (dw *DatabaseWriter) writeSignedBeaconBlocks() error {
+// Add the information for the signed_block to a transaction.
+func (dw *DatabaseWriter) transactSignedBeaconBlocks() error {
 	err := dw.upsertPublicBlocks(dw.DbSignedBeaconBlock.MhKey, dw.rawSignedBeaconBlock)
 	if err != nil {
 		return err
@@ -178,8 +241,8 @@ func (dw *DatabaseWriter) writeSignedBeaconBlocks() error {
 }
 
 // Upsert to public.blocks.
-func (dw *DatabaseWriter) upsertPublicBlocks(key string, data []byte) error {
-	_, err := dw.Db.Exec(context.Background(), UpsertBlocksStmt, key, data)
+func (dw *DatabaseWriter) upsertPublicBlocks(key string, data *[]byte) error {
+	_, err := dw.Tx.Exec(dw.Ctx, UpsertBlocksStmt, key, *data)
 	if err != nil {
 		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the public.blocks table")
 		return err
@@ -187,18 +250,18 @@ func (dw *DatabaseWriter) upsertPublicBlocks(key string, data []byte) error {
 	return nil
 }
 
-// Upsert to the ethcl.signed_beacon_block table.
+// Upsert to the eth_beacon.signed_block table.
 func (dw *DatabaseWriter) upsertSignedBeaconBlock() error {
-	_, err := dw.Db.Exec(context.Background(), UpsertSignedBeaconBlockStmt, dw.DbSignedBeaconBlock.Slot, dw.DbSignedBeaconBlock.BlockRoot, dw.DbSignedBeaconBlock.ParentBlock, dw.DbSignedBeaconBlock.Eth1BlockHash, dw.DbSignedBeaconBlock.MhKey)
+	_, err := dw.Tx.Exec(dw.Ctx, UpsertSignedBeaconBlockStmt, dw.DbSignedBeaconBlock.Slot, dw.DbSignedBeaconBlock.BlockRoot, dw.DbSignedBeaconBlock.ParentBlock, dw.DbSignedBeaconBlock.Eth1BlockHash, dw.DbSignedBeaconBlock.MhKey)
 	if err != nil {
-		loghelper.LogSlotError(dw.DbSlots.Slot, err).WithFields(log.Fields{"block_root": dw.DbSignedBeaconBlock.BlockRoot}).Error("Unable to write to the slot to the ethcl.signed_beacon_block table")
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).WithFields(log.Fields{"block_root": dw.DbSignedBeaconBlock.BlockRoot}).Error("Unable to write to the slot to the eth_beacon.signed_block table")
 		return err
 	}
 	return nil
 }
 
-// Write the information for the beacon_state.
-func (dw *DatabaseWriter) writeBeaconState() error {
+// Add the information for the state to a transaction.
+func (dw *DatabaseWriter) transactBeaconState() error {
 	err := dw.upsertPublicBlocks(dw.DbBeaconState.MhKey, dw.rawBeaconState)
 	if err != nil {
 		return err
@@ -210,33 +273,33 @@ func (dw *DatabaseWriter) writeBeaconState() error {
 	return nil
 }
 
-// Upsert to the ethcl.beacon_state table.
+// Upsert to the eth_beacon.state table.
 func (dw *DatabaseWriter) upsertBeaconState() error {
-	_, err := dw.Db.Exec(context.Background(), UpsertBeaconState, dw.DbBeaconState.Slot, dw.DbBeaconState.StateRoot, dw.DbBeaconState.MhKey)
+	_, err := dw.Tx.Exec(dw.Ctx, UpsertBeaconState, dw.DbBeaconState.Slot, dw.DbBeaconState.StateRoot, dw.DbBeaconState.MhKey)
 	if err != nil {
-		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the ethcl.beacon_state table")
+		loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the eth_beacon.state table")
 		return err
 	}
 	return nil
 }
 
-// Update a given slot to be marked as forked. Provide the slot and the latest latestBlockRoot.
+// Update a given slot to be marked as forked within a transaction. Provide the slot and the latest latestBlockRoot.
 // We will mark all entries for the given slot that don't match the provided latestBlockRoot as forked.
-func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) {
+func transactReorgs(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) {
 	slotNum, strErr := strconv.Atoi(slot)
 	if strErr != nil {
 		loghelper.LogReorgError(slot, latestBlockRoot, strErr).Error("We can't convert the slot to an int...")
 	}
 
-	forkCount, err := updateForked(db, slot, latestBlockRoot)
+	forkCount, err := updateForked(tx, ctx, slot, latestBlockRoot)
 	if err != nil {
 		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while updating all forks.")
-		writeKnownGaps(db, 1, slotNum, slotNum, err, "reorg")
+		transactKnownGaps(tx, ctx, 1, slotNum, slotNum, err, "reorg", metrics)
 	}
-	proposedCount, err := updateProposed(db, slot, latestBlockRoot)
+	proposedCount, err := updateProposed(tx, ctx, slot, latestBlockRoot)
 	if err != nil {
 		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while trying to update the proposed slot.")
-		writeKnownGaps(db, 1, slotNum, slotNum, err, "reorg")
+		transactKnownGaps(tx, ctx, 1, slotNum, slotNum, err, "reorg", metrics)
 	}
 
 	if forkCount > 0 {
@@ -257,32 +320,39 @@ func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *
 		loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
 			"proposedCount": proposedCount,
 		}).Error("Too many rows were marked as proposed!")
-		writeKnownGaps(db, 1, slotNum, slotNum, err, "reorg")
+		transactKnownGaps(tx, ctx, 1, slotNum, slotNum, fmt.Errorf("Too many rows were marked as unproposed."), "reorg", metrics)
 	} else if proposedCount == 0 {
-		var count int
-		err := db.QueryRow(context.Background(), CheckProposedStmt, slot, latestBlockRoot).Scan(count)
-		if err != nil {
-			loghelper.LogReorgError(slot, latestBlockRoot, err).Error("Unable to query proposed rows after reorg.")
-			writeKnownGaps(db, 1, slotNum, slotNum, err, "reorg")
-		}
-		if count != 1 {
-			loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
-				"proposedCount": count,
-			}).Warn("The proposed block was not marked as proposed...")
-			writeKnownGaps(db, 1, slotNum, slotNum, err, "reorg")
-		} else {
-			loghelper.LogReorg(slot, latestBlockRoot).Info("Updated the row that should have been marked as proposed.")
-		}
+		transactKnownGaps(tx, ctx, 1, slotNum, slotNum, fmt.Errorf("Unable to find properly proposed row in DB"), "reorg", metrics)
+		loghelper.LogReorg(slot, latestBlockRoot).Info("Updated the row that should have been marked as proposed.")
 	}
 
-	metrics.IncrementHeadTrackingReorgs(1)
+	metrics.IncrementReorgsInsert(1)
+}
+
+// Wrapper function that will create a transaction and execute the function.
+func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) {
+	ctx := context.Background()
+	tx, err := db.Begin(ctx)
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Fatal("Unable to create a new transaction for reorgs")
+	}
+	defer func() {
+		err := tx.Rollback(ctx)
+		if err != nil && err != pgx.ErrTxClosed {
+			loghelper.LogError(err).Error("We were unable to Rollback a transaction for reorgs")
+		}
+	}()
+	transactReorgs(tx, ctx, slot, latestBlockRoot, metrics)
+	if err = tx.Commit(ctx); err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Fatal("Unable to execute the transaction for reorgs")
+	}
 }
 
 // Update the slots table by marking the old slot's as forked.
-func updateForked(db sql.Database, slot string, latestBlockRoot string) (int64, error) {
-	res, err := db.Exec(context.Background(), UpdateForkedStmt, slot, latestBlockRoot)
+func updateForked(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) {
+	res, err := tx.Exec(ctx, UpdateForkedStmt, slot, latestBlockRoot)
 	if err != nil {
-		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the ethcl.slots table with the forked slots")
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the forked slots")
 		return 0, err
 	}
 	count, err := res.RowsAffected()
@@ -293,10 +363,11 @@ func updateForked(db sql.Database, slot string, latestBlockRoot string) (int64,
 	return count, err
 }
 
-func updateProposed(db sql.Database, slot string, latestBlockRoot string) (int64, error) {
-	res, err := db.Exec(context.Background(), UpdateProposedStmt, slot, latestBlockRoot)
+// Mark a slot as proposed.
+func updateProposed(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) {
+	res, err := tx.Exec(ctx, UpdateProposedStmt, slot, latestBlockRoot)
 	if err != nil {
-		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the ethcl.slots table with the proposed slot.")
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the proposed slot.")
 		return 0, err
 	}
 	count, err := res.RowsAffected()
@@ -308,68 +379,95 @@ func updateProposed(db sql.Database, slot string, latestBlockRoot string) (int64
 	return count, err
 }
 
-// A wrapper function to call upsertKnownGaps. This function will break down the range of known_gaos into
+// A wrapper function to call upsertKnownGaps. This function will break down the range of known_gaps into
 // smaller chunks. For example, instead of having an entry of 1-101, if we increment the entries by 10 slots, we would
 // have 10 entries as follows: 1-10, 11-20, etc...
-func writeKnownGaps(db sql.Database, tableIncrement int, startSlot int, endSlot int, entryError error, entryProcess string) {
+func transactKnownGaps(tx sql.Tx, ctx context.Context, tableIncrement int, startSlot int, endSlot int, entryError error, entryProcess string, metric *BeaconClientMetrics) {
+	var entryErrorMsg string
+	if entryError == nil {
+		entryErrorMsg = ""
+	} else {
+		entryErrorMsg = entryError.Error()
+	}
 	if endSlot-startSlot <= tableIncrement {
 		kgModel := DbKnownGaps{
 			StartSlot:         strconv.Itoa(startSlot),
 			EndSlot:           strconv.Itoa(endSlot),
 			CheckedOut:        false,
 			ReprocessingError: "",
-			EntryError:        entryError.Error(),
+			EntryError:        entryErrorMsg,
 			EntryProcess:      entryProcess,
 		}
-		upsertKnownGaps(db, kgModel)
-	}
-	totalSlots := endSlot - startSlot
-	var chunks int
-	chunks = totalSlots / tableIncrement
-	if totalSlots%tableIncrement != 0 {
-		chunks = chunks + 1
-	}
-
-	for i := 0; i < chunks; i++ {
-		var tempStart, tempEnd int
-		tempStart = startSlot + (i * tableIncrement)
-		if i+1 == chunks {
-			tempEnd = endSlot
-		} else {
-			tempEnd = startSlot + ((i + 1) * tableIncrement)
+		upsertKnownGaps(tx, ctx, kgModel, metric)
+	} else {
+		totalSlots := endSlot - startSlot
+		var chunks int
+		chunks = totalSlots / tableIncrement
+		if totalSlots%tableIncrement != 0 {
+			chunks = chunks + 1
 		}
-		kgModel := DbKnownGaps{
-			StartSlot:         strconv.Itoa(tempStart),
-			EndSlot:           strconv.Itoa(tempEnd),
-			CheckedOut:        false,
-			ReprocessingError: "",
-			EntryError:        entryError.Error(),
-			EntryProcess:      entryProcess,
-		}
-		upsertKnownGaps(db, kgModel)
-	}
 
+		for i := 0; i < chunks; i++ {
+			var tempStart, tempEnd int
+			tempStart = startSlot + (i * tableIncrement)
+			if i+1 == chunks {
+				tempEnd = endSlot
+			} else {
+				tempEnd = startSlot + ((i + 1) * tableIncrement)
+			}
+			kgModel := DbKnownGaps{
+				StartSlot:         strconv.Itoa(tempStart),
+				EndSlot:           strconv.Itoa(tempEnd),
+				CheckedOut:        false,
+				ReprocessingError: "",
+				EntryError:        entryErrorMsg,
+				EntryProcess:      entryProcess,
+			}
+			upsertKnownGaps(tx, ctx, kgModel, metric)
+		}
+	}
 }
 
-// A function to upsert a single entry to the ethcl.known_gaps table.
-func upsertKnownGaps(db sql.Database, knModel DbKnownGaps) {
-	_, err := db.Exec(context.Background(), UpsertKnownGapsStmt, knModel.StartSlot, knModel.EndSlot,
+// Wrapper function, instead of adding the knownGaps entries to a transaction, it will
+// create the transaction and write it.
+func writeKnownGaps(db sql.Database, tableIncrement int, startSlot int, endSlot int, entryError error, entryProcess string, metric *BeaconClientMetrics) {
+	ctx := context.Background()
+	tx, err := db.Begin(ctx)
+	if err != nil {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Fatal("Unable to create a new transaction for knownGaps")
+	}
+	defer func() {
+		err := tx.Rollback(ctx)
+		if err != nil && err != pgx.ErrTxClosed {
+			loghelper.LogError(err).Error("We were unable to Rollback a transaction for reorgs")
+		}
+	}()
+	transactKnownGaps(tx, ctx, tableIncrement, startSlot, endSlot, entryError, entryProcess, metric)
+	if err = tx.Commit(ctx); err != nil {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Fatal("Unable to execute the transaction for knownGaps")
+	}
+}
+
+// A function to upsert a single entry to the eth_beacon.known_gaps table.
+func upsertKnownGaps(tx sql.Tx, ctx context.Context, knModel DbKnownGaps, metric *BeaconClientMetrics) {
+	_, err := tx.Exec(ctx, UpsertKnownGapsStmt, knModel.StartSlot, knModel.EndSlot,
 		knModel.CheckedOut, knModel.ReprocessingError, knModel.EntryError, knModel.EntryProcess)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"err":       err,
 			"startSlot": knModel.StartSlot,
 			"endSlot":   knModel.EndSlot,
-		}).Fatal("We are unable to write to the ethcl.known_gaps table!!! We will stop the application because of that.")
+		}).Fatal("We are unable to write to the eth_beacon.known_gaps table!!! We will stop the application because of that.")
 	}
 	log.WithFields(log.Fields{
 		"startSlot": knModel.StartSlot,
 		"endSlot":   knModel.EndSlot,
-	}).Warn("A new gap has been added to the ethcl.known_gaps table.")
+	}).Warn("A new gap has been added to the eth_beacon.known_gaps table.")
+	metric.IncrementKnownGapsInserts(1)
 }
 
 // A function to write the gap between the highest slot in the DB and the first processed slot.
-func writeStartUpGaps(db sql.Database, tableIncrement int, firstSlot int) {
+func writeStartUpGaps(db sql.Database, tableIncrement int, firstSlot int, metric *BeaconClientMetrics) {
 	var maxSlot int
 	err := db.QueryRow(context.Background(), QueryHighestSlotStmt).Scan(&maxSlot)
 	if err != nil {
@@ -381,7 +479,43 @@ func writeStartUpGaps(db sql.Database, tableIncrement int, firstSlot int) {
 			"maxSlot": maxSlot,
 		}).Fatal("Unable to get convert max block from DB to int. We must close the application or we might have undetected gaps.")
 	}
-	writeKnownGaps(db, tableIncrement, maxSlot, firstSlot, fmt.Errorf(""), "startup")
+	if maxSlot != firstSlot-1 {
+		if maxSlot < firstSlot-1 {
+			if maxSlot == 0 {
+				writeKnownGaps(db, tableIncrement, maxSlot, firstSlot-1, fmt.Errorf(""), "startup", metric)
+			} else {
+				writeKnownGaps(db, tableIncrement, maxSlot+1, firstSlot-1, fmt.Errorf(""), "startup", metric)
+			}
+		} else {
+			log.WithFields(log.Fields{
+				"maxSlot":   maxSlot,
+				"firstSlot": firstSlot,
+			}).Warn("The maxSlot in the DB is greater than or equal to the first Slot we are processing.")
+		}
+	}
+}
+
+// A function to update a knownGap range with a reprocessing error.
+func updateKnownGapErrors(db sql.Database, startSlot int, endSlot int, reprocessingErr error, metric *BeaconClientMetrics) error {
+	res, err := db.Exec(context.Background(), UpsertKnownGapsErrorStmt, startSlot, endSlot, reprocessingErr.Error())
+	if err != nil {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Error("Unable to update reprocessing_error")
+		return err
+	}
+	row, err := res.RowsAffected()
+	if err != nil {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Error("Unable to count rows affected when trying to update reprocessing_error.")
+		return err
+	}
+	if row != 1 {
+		loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).WithFields(log.Fields{
+			"rowCount": row,
+		}).Error("The rows affected by the upsert for reprocessing_error is not 1.")
+		metric.IncrementKnownGapsReprocessError(1)
+		return err
+	}
+	metric.IncrementKnownGapsReprocessError(1)
+	return nil
 }
 
 // A quick helper function to calculate the epoch.
@@ -389,3 +523,79 @@ func calculateEpoch(slot int, slotPerEpoch int) string {
 	epoch := slot / slotPerEpoch
 	return strconv.Itoa(epoch)
 }
+
+// A helper function to check to see if the slot is processed.
+func isSlotProcessed(db sql.Database, checkProcessStmt string, slot string) (bool, error) {
+	processRow, err := db.Exec(context.Background(), checkProcessStmt, slot)
+	if err != nil {
+		return false, err
+	}
+	row, err := processRow.RowsAffected()
+	if err != nil {
+		return false, err
+	}
+	if row > 0 {
+		return true, nil
+	}
+	return false, nil
+}
+
+// Check to see if this slot is in the DB. Check eth_beacon.slots, eth_beacon.signed_block
+// and eth_beacon.state. If the slot exists, return true
+func IsSlotInDb(ctx context.Context, db sql.Database, slot string, blockRoot string, stateRoot string) (bool, error) {
+	var (
+		isInBeaconState       bool
+		isInSignedBeaconBlock bool
+	)
+	errG, _ := errgroup.WithContext(context.Background())
+	errG.Go(func() error {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			var err error
+			isInBeaconState, err = checkSlotAndRoot(db, CheckBeaconStateStmt, slot, stateRoot)
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to check if the slot and stateroot exist in eth_beacon.state")
+			}
+			return err
+		}
+	})
+	errG.Go(func() error {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			var err error
+			isInSignedBeaconBlock, err = checkSlotAndRoot(db, CheckSignedBeaconBlockStmt, slot, blockRoot)
+			if err != nil {
+				loghelper.LogError(err).Error("Unable to check if the slot and block_root exist in eth_beacon.signed_block")
+			}
+			return err
+		}
+	})
+	if err := errG.Wait(); err != nil {
+		return false, err
+	}
+	if isInBeaconState && isInSignedBeaconBlock {
+		return true, nil
+	}
+	return false, nil
+}
+
+// Provide a statement, slot, and root, and this function will check to see
+// if the slot and root exist in the table.
+func checkSlotAndRoot(db sql.Database, statement, slot, root string) (bool, error) {
+	processRow, err := db.Exec(context.Background(), statement, slot, root)
+	if err != nil {
+		return false, err
+	}
+	row, err := processRow.RowsAffected()
+	if err != nil {
+		return false, err
+	}
+	if row > 0 {
+		return true, nil
+	}
+	return false, nil
+}

pkg/beaconclient/healthcheck.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
 import (
@@ -5,7 +20,7 @@ import (
 	"net/http"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // This function will ensure that we can connect to the beacon client.

pkg/beaconclient/healthcheck_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient_test
 
 import (
@@ -5,18 +20,27 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	beaconclient "github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient"
+	beaconclient "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
 )
 
 var _ = Describe("Healthcheck", func() {
 	var (
-		BC    = beaconclient.CreateBeaconClient(context.Background(), "http", "localhost", 5052)
-		errBc = beaconclient.CreateBeaconClient(context.Background(), "http", "blah-blah", 1010)
+		Bc    *beaconclient.BeaconClient
+		errBc *beaconclient.BeaconClient
 	)
+
+	BeforeEach(func() {
+		var err error
+		Bc, err = beaconclient.CreateBeaconClient(context.Background(), "http", "localhost", 5052, 10, bcUniqueIdentifier, false)
+		Expect(err).ToNot(HaveOccurred())
+		errBc, err = beaconclient.CreateBeaconClient(context.Background(), "http", "blah-blah", 1010, 10, bcUniqueIdentifier, false)
+		Expect(err).ToNot(HaveOccurred())
+
+	})
 	Describe("Connecting to the lighthouse client", Label("integration"), func() {
 		Context("When the client is running", func() {
 			It("We should connect successfully", func() {
-				err := BC.CheckBeaconClient()
+				err := Bc.CheckBeaconClient()
 				Expect(err).To(BeNil())
 			})
 		})

pkg/beaconclient/incomingsse.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This package will handle all event subscriptions that utilize SSE.
 
 package beaconclient
@@ -7,7 +22,7 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -18,7 +33,7 @@ var (
 // This function will capture all the SSE events for a given SseEvents object.
 // When new messages come in, it will ensure that they are decoded into JSON.
 // If any errors occur, it log the error information.
-func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P]) {
+func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P], errMetricInc func(uint64)) {
 	go func() {
 		errG := new(errgroup.Group)
 		errG.Go(func() error {
@@ -44,6 +59,7 @@ func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P]) {
 		case message := <-eventHandler.MessagesCh:
 			// Message can be nil if its a keep-alive message
 			if len(message.Data) != 0 {
+				log.WithFields(log.Fields{"msg": string(message.Data)}).Debug("We are going to send the following message to be processed.")
 				go processMsg(message.Data, eventHandler.ProcessCh, eventHandler.ErrorCh)
 			}
 
@@ -54,9 +70,7 @@ func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P]) {
 				"msg":      headErr.msg,
 			},
 			).Error("Unable to handle event.")
-
-		case process := <-eventHandler.ProcessCh:
-			log.WithFields(log.Fields{"processed": process}).Debug("Processesing a Message")
+			errMetricInc(1)
 		}
 	}
 }
@@ -79,6 +93,6 @@ func processMsg[P ProcessedEvents](msg []byte, processCh chan<- *P, errorCh chan
 // Capture all of the event topics.
 func (bc *BeaconClient) captureEventTopic() {
 	log.Info("We are capturing all SSE events")
-	go handleIncomingSseEvent(bc.HeadTracking)
-	go handleIncomingSseEvent(bc.ReOrgTracking)
+	go handleIncomingSseEvent(bc.HeadTracking, bc.Metrics.IncrementHeadError)
+	go handleIncomingSseEvent(bc.ReOrgTracking, bc.Metrics.IncrementReorgError)
 }

pkg/beaconclient/metrics.go

@@ -1,15 +1,150 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+//Create a metric struct and register each channel with prometheus
+func CreateBeaconClientMetrics() (*BeaconClientMetrics, error) {
+	metrics := &BeaconClientMetrics{
+		SlotInserts:             0,
+		ReorgInserts:            0,
+		KnownGapsInserts:        0,
+		KnownGapsProcessed:      0,
+		KnownGapsReprocessError: 0,
+		HeadError:               0,
+		HeadReorgError:          0,
+	}
+	err := prometheusRegisterHelper("slot_inserts", "Keeps track of the number of slots we have inserted.", &metrics.SlotInserts)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("reorg_inserts", "Keeps track of the number of reorgs we have inserted.", &metrics.ReorgInserts)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("known_gaps_inserts", "Keeps track of the number of known gaps we have inserted.", &metrics.KnownGapsInserts)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("known_gaps_reprocess_error", "Keeps track of the number of known gaps that had errors when reprocessing, but the error was updated successfully.", &metrics.KnownGapsReprocessError)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("known_gaps_processed", "Keeps track of the number of known gaps we successfully processed.", &metrics.KnownGapsProcessed)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("historic_slots_processed", "Keeps track of the number of historic slots we successfully processed.", &metrics.HistoricSlotProcessed)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("head_error", "Keeps track of the number of errors we had processing head messages.", &metrics.HeadError)
+	if err != nil {
+		return nil, err
+	}
+	err = prometheusRegisterHelper("head_reorg_error", "Keeps track of the number of errors we had processing reorg messages.", &metrics.HeadReorgError)
+	if err != nil {
+		return nil, err
+	}
+	return metrics, nil
+}
+
+func prometheusRegisterHelper(name string, help string, varPointer *uint64) error {
+	err := prometheus.Register(prometheus.NewCounterFunc(
+		prometheus.CounterOpts{
+			Namespace:   "beacon_client",
+			Subsystem:   "",
+			Name:        name,
+			Help:        help,
+			ConstLabels: map[string]string{},
+		},
+		func() float64 {
+			return float64(atomic.LoadUint64(varPointer))
+		}))
+	if err != nil && err.Error() != "duplicate metrics collector registration attempted" {
+		loghelper.LogError(err).WithField("name", name).Error("Unable to register counter.")
+		return err
+	}
+	return nil
+}
+
+// A structure utilized for keeping track of various metrics. Currently, mostly used in testing.
+type BeaconClientMetrics struct {
+	SlotInserts             uint64 // Number of head events we successfully wrote to the DB.
+	ReorgInserts            uint64 // Number of reorg events we successfully wrote to the DB.
+	KnownGapsInserts        uint64 // Number of known_gaps we successfully wrote to the DB.
+	KnownGapsProcessed      uint64 // Number of knownGaps processed.
+	KnownGapsReprocessError uint64 // Number of knownGaps that were updated with an error.
+	HistoricSlotProcessed   uint64 // Number of historic slots successfully processed.
+	HeadError               uint64 // Number of errors that occurred when decoding the head message.
+	HeadReorgError          uint64 // Number of errors that occurred when decoding the reorg message.
+}
 
 // Wrapper function to increment inserts. If we want to use mutexes later we can easily update all
 // occurrences here.
-func (m *BeaconClientMetrics) IncrementHeadTrackingInserts(inc uint64) {
-	atomic.AddUint64(&m.HeadTrackingInserts, inc)
+func (m *BeaconClientMetrics) IncrementSlotInserts(inc uint64) {
+	log.Debug("Incrementing Slot Insert")
+	atomic.AddUint64(&m.SlotInserts, inc)
 }
 
 // Wrapper function to increment reorgs. If we want to use mutexes later we can easily update all
 // occurrences here.
-func (m *BeaconClientMetrics) IncrementHeadTrackingReorgs(inc uint64) {
-	atomic.AddUint64(&m.HeadTrackingReorgs, inc)
+func (m *BeaconClientMetrics) IncrementReorgsInsert(inc uint64) {
+	atomic.AddUint64(&m.ReorgInserts, inc)
+}
+
+// Wrapper function to increment known gaps. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementKnownGapsInserts(inc uint64) {
+	atomic.AddUint64(&m.KnownGapsInserts, inc)
+}
+
+// Wrapper function to increment known gaps processed. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementKnownGapsProcessed(inc uint64) {
+	atomic.AddUint64(&m.KnownGapsProcessed, inc)
+}
+
+// Wrapper function to increment head errors. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementHeadError(inc uint64) {
+	atomic.AddUint64(&m.HeadError, inc)
+}
+
+// Wrapper function to increment reorg errors. If we want to use mutexes later we can easily update all
+// occurrences here.
+func (m *BeaconClientMetrics) IncrementReorgError(inc uint64) {
+	atomic.AddUint64(&m.HeadReorgError, inc)
+}
+
+// Wrapper function to increment the number of knownGaps that were updated with reprocessing errors.
+//If we want to use mutexes later we can easily update all occurrences here.
+func (m *BeaconClientMetrics) IncrementKnownGapsReprocessError(inc uint64) {
+	atomic.AddUint64(&m.KnownGapsReprocessError, inc)
+}
+
+// Wrapper function to increment the number of historicSlots that were processed successfully.
+// If we want to use mutexes later we can easily update all occurrences here.
+func (m *BeaconClientMetrics) IncrementHistoricSlotProcessed(inc uint64) {
+	atomic.AddUint64(&m.HistoricSlotProcessed, inc)
 }

pkg/beaconclient/models.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
 // This interface captured what the events can be for processed event streams.
@@ -36,7 +51,7 @@ type ChainReorg struct {
 	ExecutionOptimistic bool   `json:"execution_optimistic"`
 }
 
-// A struct to capture whats being written to the ethcl.slots table.
+// A struct to capture whats being written to the eth-beacon.slots table.
 type DbSlots struct {
 	Epoch     string // The epoch.
 	Slot      string // The slot.
@@ -45,7 +60,7 @@ type DbSlots struct {
 	Status    string // The status, it can be proposed | forked | skipped.
 }
 
-// A struct to capture whats being written to ethcl.signed_beacon_block table.
+// A struct to capture whats being written to eth-beacon.signed_block table.
 type DbSignedBeaconBlock struct {
 	Slot          string // The slot.
 	BlockRoot     string // The block root
@@ -55,14 +70,14 @@ type DbSignedBeaconBlock struct {
 
 }
 
-// A struct to capture whats being written to ethcl.beacon_state table.
+// A struct to capture whats being written to eth-beacon.state table.
 type DbBeaconState struct {
 	Slot      string // The slot.
 	StateRoot string // The state root
 	MhKey     string // The ipld multihash key.
 }
 
-// A structure to capture whats being written to the ethcl.known_gaps table.
+// A structure to capture whats being written to the eth-beacon.known_gaps table.
 type DbKnownGaps struct {
 	StartSlot         string // The start slot for known_gaps, inclusive.
 	EndSlot           string // The end slot for known_gaps, inclusive.

pkg/beaconclient/multihash.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package beaconclient
 
 import (
@@ -5,10 +20,10 @@ import (
 	dshelp "github.com/ipfs/go-ipfs-ds-help"
 	"github.com/multiformats/go-multihash"
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
-const SSZ_SHA2_256_PREFIX uint64 = 0xb501
+const SSZ_SHA2_256_PREFIX uint64 = 0xb502
 
 // MultihashKeyFromSSZRoot converts a SSZ-SHA2-256 root hash into a blockstore prefixed multihash key
 func MultihashKeyFromSSZRoot(root []byte) (string, error) {

pkg/beaconclient/processevents.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This file contains all the functions to handle SSE events after they have been turned
 // to the structs.
 
@@ -8,7 +23,6 @@ import (
 	"strconv"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
 )
 
 // This function will perform the necessary steps to handle a reorg.
@@ -39,20 +53,25 @@ func (bc *BeaconClient) handleHead() {
 		if errorSlots != 0 && bc.PreviousSlot != 0 {
 			log.WithFields(log.Fields{
 				"lastProcessedSlot": bc.PreviousSlot,
-				"errorMessages":     errorSlots,
+				"errorSlots":        errorSlots,
 			}).Warn("We added slots to the knownGaps table because we got bad head messages.")
-			writeKnownGaps(bc.Db, bc.KnownGapTableIncrement, bc.PreviousSlot, bcSlotsPerEpoch+errorSlots, fmt.Errorf("Bad Head Messages"), "headProcessing")
+			writeKnownGaps(bc.Db, bc.KnownGapTableIncrement, bc.PreviousSlot+1, slot, fmt.Errorf("Bad Head Messages"), "headProcessing", bc.Metrics)
+			errorSlots = 0
 		}
 
-		err = processHeadSlot(bc.Db, bc.ServerEndpoint, slot, head.Block, head.State, bc.PreviousSlot, bc.PreviousBlockRoot, bc.Metrics, bc.KnownGapTableIncrement)
-		if err != nil {
-			loghelper.LogSlotError(head.Slot, err).Error("Unable to process a slot")
+		log.WithFields(log.Fields{"head": head}).Debug("We are going to start processing the slot.")
+
+		// Not used anywhere yet but might be useful to have.
+		if bc.PreviousSlot == 0 && bc.PreviousBlockRoot == "" {
+			bc.StartingSlot = slot
 		}
-		log.WithFields(log.Fields{"head": head}).Debug("Received a new head event.")
+
+		go processHeadSlot(bc.Db, bc.ServerEndpoint, slot, head.Block, head.State, bc.PreviousSlot, bc.PreviousBlockRoot, bc.Metrics, bc.KnownGapTableIncrement, bc.CheckDb)
+
+		log.WithFields(log.Fields{"head": head.Slot}).Debug("We finished calling processHeadSlot.")
 
 		// Update the previous block
 		bc.PreviousSlot = slot
 		bc.PreviousBlockRoot = head.Block
 	}
-
 }

pkg/beaconclient/processhistoric.go

@@ -0,0 +1,269 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains all the code to process historic slots.
+
+package beaconclient
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/jackc/pgx/v4"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+var (
+	// Get a single highest priority and non-checked out row row from eth_beacon.historical_process
+	getHpEntryStmt string = `SELECT start_slot, end_slot FROM eth_beacon.historic_process
+	WHERE checked_out=false
+	ORDER BY priority ASC
+	LIMIT 1;`
+	// Used to periodically check to see if there is a new entry in the eth_beacon.historic_process table.
+	checkHpEntryStmt string = `SELECT * FROM eth_beacon.historic_process WHERE checked_out=false;`
+	// Used to checkout a row from the eth_beacon.historic_process table
+	lockHpEntryStmt string = `UPDATE eth_beacon.historic_process
+	SET checked_out=true, checked_out_by=$3
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to delete an entry from the eth_beacon.historic_process table
+	deleteHpEntryStmt string = `DELETE FROM eth_beacon.historic_process
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to update every single row that this node has checked out.
+	releaseHpLockStmt string = `UPDATE eth_beacon.historic_process
+	SET checked_out=false, checked_out_by=null
+	WHERE checked_out_by=$1`
+)
+
+type HistoricProcessing struct {
+	db                   sql.Database         //db connection
+	metrics              *BeaconClientMetrics // metrics for beaconclient
+	uniqueNodeIdentifier int                  // node unique identifier.
+}
+
+// Get a single row of historical slots from the table.
+func (hp HistoricProcessing) getSlotRange(ctx context.Context, slotCh chan<- slotsToProcess) []error {
+	return getBatchProcessRow(ctx, hp.db, getHpEntryStmt, checkHpEntryStmt, lockHpEntryStmt, slotCh, strconv.Itoa(hp.uniqueNodeIdentifier))
+}
+
+// Remove the table entry.
+func (hp HistoricProcessing) removeTableEntry(ctx context.Context, processCh <-chan slotsToProcess) error {
+	return removeRowPostProcess(ctx, hp.db, processCh, QueryBySlotStmt, deleteHpEntryStmt)
+}
+
+// Remove the table entry.
+func (hp HistoricProcessing) handleProcessingErrors(ctx context.Context, errMessages <-chan batchHistoricError) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case errMs := <-errMessages:
+			loghelper.LogSlotError(strconv.Itoa(errMs.slot), errMs.err)
+			writeKnownGaps(hp.db, 1, errMs.slot, errMs.slot, errMs.err, errMs.errProcess, hp.metrics)
+		}
+	}
+}
+
+// "un"-checkout the rows held by this DB in the eth_beacon.historical_process table.
+func (hp HistoricProcessing) releaseDbLocks() error {
+	log.Debug("Updating all the entries to eth_beacon.historical processing")
+	res, err := hp.db.Exec(context.Background(), releaseHpLockStmt, hp.uniqueNodeIdentifier)
+	if err != nil {
+		return fmt.Errorf("Unable to remove lock from eth_beacon.historical_processing table for node %d, error is %e", hp.uniqueNodeIdentifier, err)
+	}
+	log.Debug("Update all the entries to eth_beacon.historical processing")
+	rows, err := res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("Unable to calculated number of rows affected by releasing locks from eth_beacon.historical_processing table for node %d, error is %e", hp.uniqueNodeIdentifier, err)
+	}
+	log.WithField("rowCount", rows).Info("Released historicalProcess locks for specified rows.")
+	return nil
+}
+
+// Process the slot range.
+func processSlotRangeWorker(ctx context.Context, workCh <-chan int, errCh chan<- batchHistoricError, db sql.Database, serverAddress string, metrics *BeaconClientMetrics, checkDb bool, incrementTracker func(uint64)) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case slot := <-workCh:
+			log.Debug("Handling slot: ", slot)
+			err, errProcess := handleHistoricSlot(ctx, db, serverAddress, slot, metrics, checkDb)
+			if err != nil {
+				errMs := batchHistoricError{
+					err:        err,
+					errProcess: errProcess,
+					slot:       slot,
+				}
+				errCh <- errMs
+			} else {
+				incrementTracker(1)
+			}
+		}
+	}
+}
+
+// A wrapper function that insert the start_slot and end_slot from a single row into a channel.
+// It also locks the row by updating the checked_out column.
+// The statement for getting the start_slot and end_slot must be provided.
+// The statement for "locking" the row must also be provided.
+func getBatchProcessRow(ctx context.Context, db sql.Database, getStartEndSlotStmt string, checkNewRowsStmt string, checkOutRowStmt string, slotCh chan<- slotsToProcess, uniqueNodeIdentifier string) []error {
+	errCount := make([]error, 0)
+
+	// 5 is an arbitrary number. It allows us to retry a few times before
+	// ending the application.
+	prevErrCount := 0
+	for len(errCount) < 5 {
+		select {
+		case <-ctx.Done():
+			return errCount
+		default:
+			if len(errCount) != prevErrCount {
+				log.WithFields(log.Fields{
+					"errCount": errCount,
+				}).Error("New error entry added")
+			}
+			processRow, err := db.Exec(context.Background(), checkNewRowsStmt)
+			if err != nil {
+				errCount = append(errCount, err)
+			}
+			row, err := processRow.RowsAffected()
+			if err != nil {
+				errCount = append(errCount, err)
+			}
+			if row < 1 {
+				time.Sleep(3 * time.Second)
+				log.Debug("We are checking rows, be patient")
+				break
+			}
+			log.Debug("We found a new row")
+			dbCtx := context.Background()
+
+			// Setup TX
+			tx, err := db.Begin(dbCtx)
+			if err != nil {
+				loghelper.LogError(err).Error("We are unable to Begin a SQL transaction")
+				errCount = append(errCount, err)
+				break
+			}
+			defer func() {
+				err := tx.Rollback(dbCtx)
+				if err != nil && err != pgx.ErrTxClosed {
+					loghelper.LogError(err).Error("We were unable to Rollback a transaction")
+					errCount = append(errCount, err)
+				}
+			}()
+
+			// Query the DB for slots.
+			sp := slotsToProcess{}
+			err = tx.QueryRow(dbCtx, getStartEndSlotStmt).Scan(&sp.startSlot, &sp.endSlot)
+			if err != nil {
+				if err == pgx.ErrNoRows {
+					time.Sleep(1 * time.Second)
+					break
+				}
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), getStartEndSlotStmt, err).Error("Unable to get a row")
+				errCount = append(errCount, err)
+				break
+			}
+
+			// Checkout the Row
+			res, err := tx.Exec(dbCtx, checkOutRowStmt, sp.startSlot, sp.endSlot, uniqueNodeIdentifier)
+			if err != nil {
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).Error("Unable to checkout the row")
+				errCount = append(errCount, err)
+				break
+			}
+			rows, err := res.RowsAffected()
+			if err != nil {
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, fmt.Errorf("Unable to determine the rows affected when trying to checkout a row."))
+				errCount = append(errCount, err)
+				break
+			}
+			if rows > 1 {
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).WithFields(log.Fields{
+					"rowsReturn": rows,
+				}).Error("We locked too many rows.....")
+				errCount = append(errCount, err)
+				break
+			}
+			if rows == 0 {
+				loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).WithFields(log.Fields{
+					"rowsReturn": rows,
+				}).Error("We did not lock a single row.")
+				errCount = append(errCount, err)
+				break
+			}
+			err = tx.Commit(dbCtx)
+			if err != nil {
+				loghelper.LogSlotRangeError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), err).Error("Unable commit transactions.")
+				errCount = append(errCount, err)
+				break
+			}
+			log.WithField("slots", sp).Debug("Added a new slots to be processed")
+			slotCh <- sp
+		}
+	}
+	log.WithFields(log.Fields{
+		"ErrCount": errCount,
+	}).Error("The ErrCounter")
+	return errCount
+}
+
+// After a row has been processed it should be removed from its appropriate table.
+func removeRowPostProcess(ctx context.Context, db sql.Database, processCh <-chan slotsToProcess, checkProcessedStmt, removeStmt string) error {
+	errCh := make(chan error)
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case slots := <-processCh:
+			// Make sure the start and end slot exist in the slots table.
+			go func() {
+				log.WithFields(log.Fields{
+					"startSlot": slots.startSlot,
+					"endSlot":   slots.endSlot,
+				}).Debug("Starting to check to see if the following slots have been processed")
+				for {
+					isStartProcess, err := isSlotProcessed(db, checkProcessedStmt, strconv.Itoa(slots.startSlot))
+					if err != nil {
+						errCh <- err
+					}
+					isEndProcess, err := isSlotProcessed(db, checkProcessedStmt, strconv.Itoa(slots.endSlot))
+					if err != nil {
+						errCh <- err
+					}
+					if isStartProcess && isEndProcess {
+						break
+					}
+					time.Sleep(3 * time.Second)
+				}
+
+				_, err := db.Exec(context.Background(), removeStmt, strconv.Itoa(slots.startSlot), strconv.Itoa(slots.endSlot))
+				if err != nil {
+					errCh <- err
+				}
+
+			}()
+			if len(errCh) != 0 {
+				return <-errCh
+			}
+		}
+	}
+}

pkg/beaconclient/processknowngaps.go

@@ -0,0 +1,137 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains all the code to process historic slots.
+
+package beaconclient
+
+import (
+	"context"
+	"strconv"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+)
+
+var (
+	// Get a single non-checked out row row from eth_beacon.known_gaps.
+	getKgEntryStmt string = `SELECT start_slot, end_slot FROM eth_beacon.known_gaps
+	WHERE checked_out=false
+	ORDER BY priority ASC
+	LIMIT 1;`
+	// Used to periodically check to see if there is a new entry in the eth_beacon.known_gaps table.
+	checkKgEntryStmt string = `SELECT * FROM eth_beacon.known_gaps WHERE checked_out=false;`
+	// Used to checkout a row from the eth_beacon.known_gaps table
+	lockKgEntryStmt string = `UPDATE eth_beacon.known_gaps
+	SET checked_out=true, checked_out_by=$3
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to delete an entry from the knownGaps table
+	deleteKgEntryStmt string = `DELETE FROM eth_beacon.known_gaps
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to check to see if a single slot exists in the known_gaps table.
+	checkKgSingleSlotStmt string = `SELECT start_slot, end_slot FROM eth_beacon.known_gaps
+	WHERE start_slot=$1 AND end_slot=$2;`
+	// Used to update every single row that this node has checked out.
+	releaseKgLockStmt string = `UPDATE eth_beacon.known_gaps
+	SET checked_out=false, checked_out_by=null
+	WHERE checked_out_by=$1`
+)
+
+type KnownGapsProcessing struct {
+	db                   sql.Database         //db connection
+	metrics              *BeaconClientMetrics // metrics for beaconclient
+	uniqueNodeIdentifier int                  // node unique identifier.
+}
+
+// This function will perform all the heavy lifting for tracking the head of the chain.
+func (bc *BeaconClient) ProcessKnownGaps(ctx context.Context, maxWorkers int) []error {
+	log.Info("We are starting the known gaps processing service.")
+	bc.KnownGapsProcess = KnownGapsProcessing{db: bc.Db, uniqueNodeIdentifier: bc.UniqueNodeIdentifier, metrics: bc.Metrics}
+	errs := handleBatchProcess(ctx, maxWorkers, bc.KnownGapsProcess, bc.KnownGapsProcess.db, bc.ServerEndpoint, bc.Metrics, bc.CheckDb, bc.Metrics.IncrementKnownGapsProcessed)
+	log.Debug("Exiting known gaps processing service")
+	return errs
+}
+
+// This function will perform all the necessary clean up tasks for stopping historical processing.
+func (bc *BeaconClient) StopKnownGapsProcessing(cancel context.CancelFunc) error {
+	log.Info("We are stopping the known gaps processing service.")
+	cancel()
+	err := bc.KnownGapsProcess.releaseDbLocks()
+	if err != nil {
+		loghelper.LogError(err).WithField("uniqueIdentifier", bc.UniqueNodeIdentifier).Error("We were unable to remove the locks from the eth_beacon.known_gaps table. Manual Intervention is needed!")
+	}
+	return nil
+}
+
+// Get a single row of historical slots from the table.
+func (kgp KnownGapsProcessing) getSlotRange(ctx context.Context, slotCh chan<- slotsToProcess) []error {
+	return getBatchProcessRow(ctx, kgp.db, getKgEntryStmt, checkKgEntryStmt, lockKgEntryStmt, slotCh, strconv.Itoa(kgp.uniqueNodeIdentifier))
+}
+
+// Remove the table entry.
+func (kgp KnownGapsProcessing) removeTableEntry(ctx context.Context, processCh <-chan slotsToProcess) error {
+	return removeRowPostProcess(ctx, kgp.db, processCh, QueryBySlotStmt, deleteKgEntryStmt)
+}
+
+// Remove the table entry.
+func (kgp KnownGapsProcessing) handleProcessingErrors(ctx context.Context, errMessages <-chan batchHistoricError) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case errMs := <-errMessages:
+			// Check to see if this if this entry already exists.
+			res, err := kgp.db.Exec(context.Background(), checkKgSingleSlotStmt, errMs.slot, errMs.slot)
+			if err != nil {
+				loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).Error("Unable to see if this slot is in the eth_beacon.known_gaps table")
+			}
+
+			rows, err := res.RowsAffected()
+			if err != nil {
+				loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).WithFields(log.Fields{
+					"queryStatement": checkKgSingleSlotStmt,
+				}).Error("Unable to get the number of rows affected by this statement.")
+			}
+
+			if rows > 0 {
+				loghelper.LogSlotError(strconv.Itoa(errMs.slot), errMs.err).Error("We received an error when processing a knownGap")
+				err = updateKnownGapErrors(kgp.db, errMs.slot, errMs.slot, errMs.err, kgp.metrics)
+				if err != nil {
+					loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).Error("Error processing known gap")
+				}
+			} else {
+				writeKnownGaps(kgp.db, 1, errMs.slot, errMs.slot, errMs.err, errMs.errProcess, kgp.metrics)
+			}
+		}
+	}
+
+}
+
+// Updated checked_out column for the uniqueNodeIdentifier.
+func (kgp KnownGapsProcessing) releaseDbLocks() error {
+	log.Debug("Updating all the entries to eth_beacon.known_gaps")
+	res, err := kgp.db.Exec(context.Background(), releaseKgLockStmt, kgp.uniqueNodeIdentifier)
+	if err != nil {
+		return err
+	}
+	rows, err := res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	log.WithField("rowCount", rows).Info("Released knownGaps locks for specified rows.")
+	return nil
+}

pkg/beaconclient/processslot.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This file will keep track of all the code needed to process a slot.
 // To process a slot, it should have all the necessary data needed to write it to the DB.
 // But not actually write it.
@@ -5,130 +20,243 @@
 package beaconclient
 
 import (
+	"context"
 	"encoding/hex"
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v4"
+	si "github.com/prysmaticlabs/prysm/consensus-types/interfaces"
+	"github.com/prysmaticlabs/prysm/consensus-types/wrapper"
+	dt "github.com/prysmaticlabs/prysm/encoding/ssz/detect"
 
 	// The below is temporary, once https://github.com/prysmaticlabs/prysm/issues/10006 has been resolved we wont need it.
 	// pb "github.com/prysmaticlabs/prysm/proto/prysm/v2"
 
-	st "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	state "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
+	"golang.org/x/sync/errgroup"
 )
 
 var (
-	SlotUnmarshalError = func(obj string) string {
-		return fmt.Sprintf("Unable to properly unmarshal the Slot field in the %s.", obj)
-	}
-	ParentRootUnmarshalError = "Unable to properly unmarshal the ParentRoot field in the SignedBeaconBlock."
-	MissingIdentifiedError   = "Can't query state without a set slot or block_root"
-	MissingEth1Data          = "Can't get the Eth1 block_hash"
+	ParentRootUnmarshalError  = "Unable to properly unmarshal the ParentRoot field in the SignedBeaconBlock."
+	MissingEth1Data           = "Can't get the Eth1 block_hash"
+	VersionedUnmarshalerError = "Unable to create a versioned unmarshaler"
 )
 
 type ProcessSlot struct {
 	// Generic
 
-	Slot            int                  // The slot number.
-	Epoch           int                  // The epoch number.
-	BlockRoot       string               // The hex encoded string of the BlockRoot.
-	StateRoot       string               // The hex encoded string of the StateRoot.
-	ParentBlockRoot string               // The hex encoded string of the parent block.
-	Status          string               // The status of the block
-	HeadOrHistoric  string               // Is this the head or a historic slot. This is critical when trying to analyze errors and skipped slots.
-	Db              sql.Database         // The DB object used to write to the DB.
-	Metrics         *BeaconClientMetrics // An object to keep track of the beaconclient metrics
+	Slot               int                  // The slot number.
+	Epoch              int                  // The epoch number.
+	BlockRoot          string               // The hex encoded string of the BlockRoot.
+	StateRoot          string               // The hex encoded string of the StateRoot.
+	ParentBlockRoot    string               // The hex encoded string of the parent block.
+	Status             string               // The status of the block
+	HeadOrHistoric     string               // Is this the head or a historic slot. This is critical when trying to analyze errors and skipped slots.
+	Db                 sql.Database         // The DB object used to write to the DB.
+	Metrics            *BeaconClientMetrics // An object to keep track of the beaconclient metrics
+	PerformanceMetrics PerformanceMetrics   // An object to keep track of performance metrics.
 	// BeaconBlock
 
-	SszSignedBeaconBlock  []byte                // The entire SSZ encoded SignedBeaconBlock
-	FullSignedBeaconBlock *st.SignedBeaconBlock // The unmarshaled BeaconState object, the unmarshalling could have errors.
+	SszSignedBeaconBlock  []byte               // The entire SSZ encoded SignedBeaconBlock
+	FullSignedBeaconBlock si.SignedBeaconBlock // The unmarshaled BeaconState object, the unmarshalling could have errors.
 
 	// BeaconState
-	FullBeaconState *st.BeaconState // The unmarshaled BeaconState object, the unmarshalling could have errors.
-	SszBeaconState  []byte          // The entire SSZ encoded BeaconState
+	FullBeaconState state.BeaconState // The unmarshaled BeaconState object, the unmarshalling could have errors.
+	SszBeaconState  []byte            // The entire SSZ encoded BeaconState
 
 	// DB Write objects
 	DbSlotsModel             *DbSlots             // The model being written to the slots table.
-	DbSignedBeaconBlockModel *DbSignedBeaconBlock // The model being written to the signed_beacon_block table.
-	DbBeaconState            *DbBeaconState       // The model being written to the beacon_state table.
+	DbSignedBeaconBlockModel *DbSignedBeaconBlock // The model being written to the signed_block table.
+	DbBeaconState            *DbBeaconState       // The model being written to the state table.
+}
+
+type PerformanceMetrics struct {
+	BeaconNodeBlockRetrievalTime time.Duration // How long it took to get the BeaconBlock from the Beacon Node.
+	BeaconNodeStateRetrievalTime time.Duration // How long it took to get the BeaconState from the Beacon Node.
+	ParseBeaconObjectForHash     time.Duration // How long it took to get some information from the beacon objects.
+	CheckDbPreProcessing         time.Duration // How long it takes to check the DB before processing a block.
+	CreateDbWriteObject          time.Duration // How long it takes to create a DB write object.
+	TransactSlotOnly             time.Duration // How long it takes to transact the slot information only.
+	CheckReorg                   time.Duration // How long it takes to check for Reorgs
+	CommitTransaction            time.Duration // How long it takes to commit the final transaction.
+	TotalDbTransaction           time.Duration // How long it takes from start to committing the entire DB transaction.
+	TotalProcessing              time.Duration // How long it took to process the entire slot.
 }
 
 // This function will do all the work to process the slot and write it to the DB.
-func processFullSlot(db sql.Database, serverAddress string, slot int, blockRoot string, stateRoot string, previousSlot int, previousBlockRoot string, headOrHistoric string, metrics *BeaconClientMetrics, knownGapsTableIncrement int) error {
-	ps := &ProcessSlot{
-		Slot:           slot,
-		BlockRoot:      blockRoot,
-		StateRoot:      stateRoot,
-		HeadOrHistoric: headOrHistoric,
-		Db:             db,
-		Metrics:        metrics,
-	}
+// It will return the error and error process. The error process is used for providing reach detail to the
+// known_gaps table.
+func processFullSlot(ctx context.Context, db sql.Database, serverAddress string, slot int, blockRoot string, stateRoot string, previousSlot int, previousBlockRoot string, headOrHistoric string, metrics *BeaconClientMetrics, knownGapsTableIncrement int, checkDb bool) (error, string) {
+	select {
+	case <-ctx.Done():
+		return nil, ""
+	default:
+		totalStart := time.Now()
+		ps := &ProcessSlot{
+			Slot:           slot,
+			BlockRoot:      blockRoot,
+			StateRoot:      stateRoot,
+			HeadOrHistoric: headOrHistoric,
+			Db:             db,
+			Metrics:        metrics,
+			PerformanceMetrics: PerformanceMetrics{
+				BeaconNodeBlockRetrievalTime: 0,
+				BeaconNodeStateRetrievalTime: 0,
+				ParseBeaconObjectForHash:     0,
+				CheckDbPreProcessing:         0,
+				CreateDbWriteObject:          0,
+				TransactSlotOnly:             0,
+				CheckReorg:                   0,
+				CommitTransaction:            0,
+				TotalDbTransaction:           0,
+				TotalProcessing:              0,
+			},
+		}
 
-	// Get the BeaconState.
-	err := ps.getBeaconState(serverAddress)
-	if err != nil {
-		writeKnownGaps(ps.Db, 1, ps.Slot, ps.Slot, err, "processSlot")
-		return err
-	}
+		g, _ := errgroup.WithContext(context.Background())
+		vUnmarshalerCh := make(chan *dt.VersionedUnmarshaler, 1)
 
-	// Get the SignedBeaconBlock.
-	err = ps.getSignedBeaconBlock(serverAddress)
-	if err != nil {
-		writeKnownGaps(ps.Db, 1, ps.Slot, ps.Slot, err, "processSlot")
-		return err
-	}
+		// Get the BeaconState.
+		g.Go(func() error {
+			select {
+			case <-ctx.Done():
+				return nil
+			default:
+				start := time.Now()
+				err := ps.getBeaconState(serverAddress, vUnmarshalerCh)
+				if err != nil {
+					return err
+				}
+				ps.PerformanceMetrics.BeaconNodeStateRetrievalTime = time.Since(start)
+				return nil
+			}
+		})
 
-	if ps.HeadOrHistoric == "head" && previousSlot == 0 && previousBlockRoot == "" {
-		writeStartUpGaps(db, knownGapsTableIncrement, ps.Slot)
-	}
+		// Get the SignedBeaconBlock.
+		g.Go(func() error {
+			select {
+			case <-ctx.Done():
+				return nil
+			default:
+				start := time.Now()
+				err := ps.getSignedBeaconBlock(serverAddress, vUnmarshalerCh)
+				if err != nil {
+					return err
+				}
+				ps.PerformanceMetrics.BeaconNodeBlockRetrievalTime = time.Since(start)
+				return nil
+			}
+		})
 
-	// Get this object ready to write
-	blockRootEndpoint := serverAddress + BcBlockRootEndpoint(strconv.Itoa(ps.Slot))
-	dw, err := ps.createWriteObjects(blockRootEndpoint)
-	if err != nil {
-		writeKnownGaps(ps.Db, 1, ps.Slot, ps.Slot, err, "blockRoot")
-		return err
-	}
-	// Write the object to the DB.
-	err = dw.writeFullSlot()
-	if err != nil {
-		writeKnownGaps(ps.Db, 1, ps.Slot, ps.Slot, err, "processSlot")
-	}
+		if err := g.Wait(); err != nil {
+			return err, "processSlot"
+		}
 
-	// Handle any reorgs or skipped slots.
-	headOrHistoric = strings.ToLower(headOrHistoric)
-	if headOrHistoric != "head" && headOrHistoric != "historic" {
-		return fmt.Errorf("headOrHistoric must be either historic or head!")
+		parseBeaconTime := time.Now()
+		finalBlockRoot, finalStateRoot, finalEth1BlockHash, err := ps.provideFinalHash()
+		if err != nil {
+			return err, "CalculateBlockRoot"
+		}
+		ps.PerformanceMetrics.ParseBeaconObjectForHash = time.Since(parseBeaconTime)
+
+		if checkDb {
+			checkDbTime := time.Now()
+			inDb, err := IsSlotInDb(ctx, ps.Db, strconv.Itoa(ps.Slot), finalBlockRoot, finalStateRoot)
+			if err != nil {
+				return err, "checkDb"
+			}
+			if inDb {
+				log.WithField("slot", slot).Info("Slot already in the DB.")
+				return nil, ""
+			}
+			ps.PerformanceMetrics.CheckDbPreProcessing = time.Since(checkDbTime)
+		}
+
+		// Get this object ready to write
+		createDbWriteTime := time.Now()
+		dw, err := ps.createWriteObjects(finalBlockRoot, finalStateRoot, finalEth1BlockHash)
+		if err != nil {
+			return err, "blockRoot"
+		}
+		ps.PerformanceMetrics.CreateDbWriteObject = time.Since(createDbWriteTime)
+
+		// Write the object to the DB.
+		dbFullTransactionTime := time.Now()
+		defer func() {
+			err := dw.Tx.Rollback(dw.Ctx)
+			if err != nil && err != pgx.ErrTxClosed {
+				loghelper.LogError(err).Error("We were unable to Rollback a transaction")
+			}
+		}()
+
+		transactionTime := time.Now()
+		err = dw.transactFullSlot()
+		if err != nil {
+			return err, "processSlot"
+		}
+		ps.PerformanceMetrics.TransactSlotOnly = time.Since(transactionTime)
+
+		// Handle any reorgs or skipped slots.
+		reorgTime := time.Now()
+		headOrHistoric = strings.ToLower(headOrHistoric)
+		if headOrHistoric != "head" && headOrHistoric != "historic" {
+			return fmt.Errorf("headOrHistoric must be either historic or head!"), ""
+		}
+		if ps.HeadOrHistoric == "head" && previousSlot != 0 && previousBlockRoot != "" && ps.Status != "skipped" {
+			ps.checkPreviousSlot(dw.Tx, dw.Ctx, previousSlot, previousBlockRoot, knownGapsTableIncrement)
+		}
+		ps.PerformanceMetrics.CheckReorg = time.Since(reorgTime)
+
+		// Commit the transaction
+		commitTime := time.Now()
+		if err = dw.Tx.Commit(dw.Ctx); err != nil {
+			return err, "transactionCommit"
+		}
+		ps.PerformanceMetrics.CommitTransaction = time.Since(commitTime)
+
+		// Total metric capture time.
+		ps.PerformanceMetrics.TotalDbTransaction = time.Since(dbFullTransactionTime)
+		ps.PerformanceMetrics.TotalProcessing = time.Since(totalStart)
+
+		log.WithFields(log.Fields{
+			"slot":               slot,
+			"performanceMetrics": fmt.Sprintf("%+v\n", ps.PerformanceMetrics),
+		}).Debug("Performance Metric output!")
+
+		return nil, ""
 	}
-	if ps.HeadOrHistoric == "head" && previousSlot != 0 && previousBlockRoot != "" && ps.Status != "skipped" {
-		ps.checkPreviousSlot(previousSlot, previousBlockRoot, knownGapsTableIncrement)
-	}
-	return nil
 }
 
 // Handle a slot that is at head. A wrapper function for calling `handleFullSlot`.
-func processHeadSlot(db sql.Database, serverAddress string, slot int, blockRoot string, stateRoot string, previousSlot int, previousBlockRoot string, metrics *BeaconClientMetrics, knownGapsTableIncrement int) error {
-	return processFullSlot(db, serverAddress, slot, blockRoot, stateRoot, previousSlot, previousBlockRoot, "head", metrics, knownGapsTableIncrement)
+func processHeadSlot(db sql.Database, serverAddress string, slot int, blockRoot string, stateRoot string, previousSlot int, previousBlockRoot string, metrics *BeaconClientMetrics, knownGapsTableIncrement int, checkDb bool) {
+	// Get the knownGaps at startUp.
+	if previousSlot == 0 && previousBlockRoot == "" {
+		writeStartUpGaps(db, knownGapsTableIncrement, slot, metrics)
+	}
+	err, errReason := processFullSlot(context.Background(), db, serverAddress, slot, blockRoot, stateRoot, previousSlot, previousBlockRoot, "head", metrics, knownGapsTableIncrement, checkDb)
+	if err != nil {
+		writeKnownGaps(db, knownGapsTableIncrement, slot, slot, err, errReason, metrics)
+	}
 }
 
 // Handle a historic slot. A wrapper function for calling `handleFullSlot`.
-// Commented because of the linter...... LOL
-//func handleHistoricSlot(db sql.Database, serverAddress string, slot int) error {
-//	return handleFullSlot(db, serverAddress, slot, "", "", 0, "", "historic")
-//}
+func handleHistoricSlot(ctx context.Context, db sql.Database, serverAddress string, slot int, metrics *BeaconClientMetrics, checkDb bool) (error, string) {
+	return processFullSlot(ctx, db, serverAddress, slot, "", "", 0, "", "historic", metrics, 1, checkDb)
+}
 
 // Update the SszSignedBeaconBlock and FullSignedBeaconBlock object with their respective values.
-func (ps *ProcessSlot) getSignedBeaconBlock(serverAddress string) error {
+func (ps *ProcessSlot) getSignedBeaconBlock(serverAddress string, vmCh <-chan *dt.VersionedUnmarshaler) error {
 	var blockIdentifier string // Used to query the block
 	if ps.BlockRoot != "" {
 		blockIdentifier = ps.BlockRoot
-	} else if ps.Slot != 0 {
-		blockIdentifier = strconv.Itoa(ps.Slot)
 	} else {
-		log.Error(MissingIdentifiedError)
-		return fmt.Errorf(MissingIdentifiedError)
+		blockIdentifier = strconv.Itoa(ps.Slot)
 	}
 	blockEndpoint := serverAddress + BcBlockQueryEndpoint + blockIdentifier
 	var err error
@@ -139,96 +267,111 @@ func (ps *ProcessSlot) getSignedBeaconBlock(serverAddress string) error {
 		return err
 	}
 
+	vm := <-vmCh
 	if rc != 200 {
-		ps.FullSignedBeaconBlock = &st.SignedBeaconBlock{}
+		ps.FullSignedBeaconBlock = &wrapper.Phase0SignedBeaconBlock{}
 		ps.SszSignedBeaconBlock = []byte{}
 		ps.ParentBlockRoot = ""
 		ps.Status = "skipped"
 		return nil
 	}
 
-	ps.FullSignedBeaconBlock = &st.SignedBeaconBlock{}
-	err = ps.FullSignedBeaconBlock.UnmarshalSSZ(ps.SszSignedBeaconBlock)
-
-	if err != nil {
-		if ps.FullSignedBeaconBlock.Block.Slot == 0 {
-			loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error(SlotUnmarshalError("SignedBeaconBlock"))
-			return fmt.Errorf(SlotUnmarshalError("SignedBeaconBlock"))
-		} else if ps.FullSignedBeaconBlock.Block.ParentRoot == nil {
-			loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error(ParentRootUnmarshalError)
-			return fmt.Errorf(ParentRootUnmarshalError)
-		} else if hex.EncodeToString(ps.FullBeaconState.Eth1Data.BlockHash) == "" {
-			loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error(MissingEth1Data)
-			return fmt.Errorf(MissingEth1Data)
-		}
-		log.Warn("We received a processing error: ", err)
+	if vm == nil {
+		return fmt.Errorf(VersionedUnmarshalerError)
 	}
-	ps.ParentBlockRoot = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block.ParentRoot)
+
+	ps.FullSignedBeaconBlock, err = vm.UnmarshalBeaconBlock(ps.SszSignedBeaconBlock)
+	if err != nil {
+		loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Warn("Unable to process the slots SignedBeaconBlock")
+		return nil
+	}
+	ps.ParentBlockRoot = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().ParentRoot())
 	return nil
 }
 
 // Update the SszBeaconState and FullBeaconState object with their respective values.
-func (ps *ProcessSlot) getBeaconState(serverEndpoint string) error {
+func (ps *ProcessSlot) getBeaconState(serverEndpoint string, vmCh chan<- *dt.VersionedUnmarshaler) error {
 	var stateIdentifier string // Used to query the state
 	if ps.StateRoot != "" {
 		stateIdentifier = ps.StateRoot
-	} else if ps.Slot != 0 {
-		stateIdentifier = strconv.Itoa(ps.Slot)
 	} else {
-		log.Error(MissingIdentifiedError)
-		return fmt.Errorf(MissingIdentifiedError)
+		stateIdentifier = strconv.Itoa(ps.Slot)
 	}
 	stateEndpoint := serverEndpoint + BcStateQueryEndpoint + stateIdentifier
 	ps.SszBeaconState, _, _ = querySsz(stateEndpoint, strconv.Itoa(ps.Slot))
 
-	ps.FullBeaconState = new(st.BeaconState)
-	err := ps.FullBeaconState.UnmarshalSSZ(ps.SszBeaconState)
-
+	versionedUnmarshaler, err := dt.FromState(ps.SszBeaconState)
 	if err != nil {
-		if ps.FullBeaconState.Slot == 0 {
-			loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error(SlotUnmarshalError("BeaconState"))
-			return fmt.Errorf(SlotUnmarshalError("BeaconState"))
-		}
+		loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error(VersionedUnmarshalerError)
+		vmCh <- nil
+		return fmt.Errorf(VersionedUnmarshalerError)
+	}
+	vmCh <- versionedUnmarshaler
+	ps.FullBeaconState, err = versionedUnmarshaler.UnmarshalBeaconState(ps.SszBeaconState)
+	if err != nil {
+		loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error("Unable to process the slots BeaconState")
+		return err
 	}
 	return nil
 }
 
 // Check to make sure that the previous block we processed is the parent of the current block.
-func (ps *ProcessSlot) checkPreviousSlot(previousSlot int, previousBlockRoot string, knownGapsTableIncrement int) {
-	parentRoot := "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block.ParentRoot)
-	if previousSlot == int(ps.FullBeaconState.Slot) {
+func (ps *ProcessSlot) checkPreviousSlot(tx sql.Tx, ctx context.Context, previousSlot int, previousBlockRoot string, knownGapsTableIncrement int) {
+	parentRoot := "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().ParentRoot())
+	slot := int(ps.FullBeaconState.Slot())
+	if previousSlot == slot {
 		log.WithFields(log.Fields{
-			"slot": ps.FullBeaconState.Slot,
+			"slot": slot,
 			"fork": true,
 		}).Warn("A fork occurred! The previous slot and current slot match.")
-		writeReorgs(ps.Db, strconv.Itoa(ps.Slot), ps.BlockRoot, ps.Metrics)
-	} else if previousSlot+1 != int(ps.FullBeaconState.Slot) {
+		transactReorgs(tx, ctx, strconv.Itoa(ps.Slot), ps.BlockRoot, ps.Metrics)
+	} else if previousSlot > slot {
 		log.WithFields(log.Fields{
 			"previousSlot": previousSlot,
-			"currentSlot":  ps.FullBeaconState.Slot,
+			"curSlot":      slot,
+		}).Warn("We noticed the previous slot is greater than the current slot.")
+	} else if previousSlot+1 != slot {
+		log.WithFields(log.Fields{
+			"previousSlot": previousSlot,
+			"currentSlot":  slot,
 		}).Error("We skipped a few slots.")
-		writeKnownGaps(ps.Db, knownGapsTableIncrement, previousSlot+1, int(ps.FullBeaconState.Slot)-1, fmt.Errorf("Gaps during head processing"), "headGaps")
+		transactKnownGaps(tx, ctx, knownGapsTableIncrement, previousSlot+1, slot-1, fmt.Errorf("Gaps during head processing"), "headGaps", ps.Metrics)
 	} else if previousBlockRoot != parentRoot {
 		log.WithFields(log.Fields{
 			"previousBlockRoot":  previousBlockRoot,
 			"currentBlockParent": parentRoot,
 		}).Error("The previousBlockRoot does not match the current blocks parent, an unprocessed fork might have occurred.")
-		writeReorgs(ps.Db, strconv.Itoa(previousSlot), parentRoot, ps.Metrics)
-		writeKnownGaps(ps.Db, 1, ps.Slot-1, ps.Slot-1, fmt.Errorf("Incorrect Parent"), "processSlot")
+		transactReorgs(tx, ctx, strconv.Itoa(previousSlot), parentRoot, ps.Metrics)
 	} else {
 		log.Debug("Previous Slot and Current Slot are one distance from each other.")
 	}
 }
 
 // Transforms all the raw data into DB models that can be written to the DB.
-func (ps *ProcessSlot) createWriteObjects(blockRootEndpoint string) (*DatabaseWriter, error) {
+func (ps *ProcessSlot) createWriteObjects(blockRoot, stateRoot, eth1BlockHash string) (*DatabaseWriter, error) {
+	var status string
+	if ps.Status != "" {
+		status = ps.Status
+	} else {
+		status = "proposed"
+	}
+
+	dw, err := CreateDatabaseWrite(ps.Db, ps.Slot, stateRoot, blockRoot, ps.ParentBlockRoot, eth1BlockHash, status, &ps.SszSignedBeaconBlock, &ps.SszBeaconState, ps.Metrics)
+	if err != nil {
+		return dw, err
+	}
+
+	return dw, nil
+}
+
+// This function will return the final blockRoot, stateRoot, and eth1BlockHash that will be
+// used to write to a DB
+func (ps *ProcessSlot) provideFinalHash() (string, string, string, error) {
 	var (
 		stateRoot     string
 		blockRoot     string
-		status        string
 		eth1BlockHash string
 	)
-
 	if ps.Status == "skipped" {
 		stateRoot = ""
 		blockRoot = ""
@@ -237,7 +380,7 @@ func (ps *ProcessSlot) createWriteObjects(blockRootEndpoint string) (*DatabaseWr
 		if ps.StateRoot != "" {
 			stateRoot = ps.StateRoot
 		} else {
-			stateRoot = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block.StateRoot)
+			stateRoot = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().StateRoot())
 			log.Debug("StateRoot: ", stateRoot)
 		}
 
@@ -245,24 +388,15 @@ func (ps *ProcessSlot) createWriteObjects(blockRootEndpoint string) (*DatabaseWr
 			blockRoot = ps.BlockRoot
 		} else {
 			var err error
-			blockRoot, err = queryBlockRoot(blockRootEndpoint, strconv.Itoa(ps.Slot))
+			rawBlockRoot, err := ps.FullSignedBeaconBlock.Block().HashTreeRoot()
+			//blockRoot, err = queryBlockRoot(blockRootEndpoint, strconv.Itoa(ps.Slot))
 			if err != nil {
-				return nil, err
+				return "", "", "", err
 			}
+			blockRoot = "0x" + hex.EncodeToString(rawBlockRoot[:])
+			log.WithFields(log.Fields{"blockRoot": blockRoot}).Debug("Block Root from ssz")
 		}
-		eth1BlockHash = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block.Body.Eth1Data.BlockHash)
+		eth1BlockHash = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().Body().Eth1Data().BlockHash)
 	}
-
-	if ps.Status != "" {
-		status = ps.Status
-	} else {
-		status = "proposed"
-	}
-
-	dw, err := CreateDatabaseWrite(ps.Db, ps.Slot, stateRoot, blockRoot, ps.ParentBlockRoot, eth1BlockHash, status, ps.SszSignedBeaconBlock, ps.SszBeaconState, ps.Metrics)
-	if err != nil {
-		return dw, err
-	}
-
-	return dw, nil
+	return blockRoot, stateRoot, eth1BlockHash, nil
 }

pkg/beaconclient/queryserver.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // This file will contain functions to query the Beacon Chain Server.
 
 package beaconclient
@@ -9,7 +24,7 @@ import (
 	"net/http"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // A helper function to query endpoints that utilize slots.

pkg/beaconclient/systemvalidation_test.go

@@ -0,0 +1,69 @@
+package beaconclient_test
+
+import (
+	"os"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	//. "github.com/onsi/gomega"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
+)
+
+var (
+	prodConfig = Config{
+		protocol:                os.Getenv("bc_protocol"),
+		address:                 os.Getenv("bc_address"),
+		port:                    getEnvInt(os.Getenv("bc_port")),
+		dbHost:                  os.Getenv("db_host"),
+		dbPort:                  getEnvInt(os.Getenv("db_port")),
+		dbName:                  os.Getenv("db_name"),
+		dbUser:                  os.Getenv("db_user"),
+		dbPassword:              os.Getenv("db_password"),
+		dbDriver:                os.Getenv("db_driver"),
+		knownGapsTableIncrement: 100000000,
+		bcUniqueIdentifier:      100,
+		checkDb:                 false,
+	}
+)
+var _ = Describe("Systemvalidation", Label("system"), func() {
+	Describe("Run the application against a running lighthouse node", func() {
+		Context("When we receive head messages", func() {
+			It("We should process the messages successfully", func() {
+				bc := setUpTest(prodConfig, "10000000000")
+				processProdHeadBlocks(bc, 3, 0, 0, 0)
+			})
+		})
+		Context("When we have historical and knownGaps slots to process", Label("system-batch"), func() {
+			It("Should process them successfully", func() {
+				bc := setUpTest(prodConfig, "10000000000")
+				//known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 100, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 2, 0, 0, 0)
+
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 2375703, 2375703, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+	})
+})
+
+// Wrapper function to get int env variables.
+func getEnvInt(envVar string) int {
+	val, err := strconv.Atoi(envVar)
+	if err != nil {
+		return 0
+	}
+	return val
+}
+
+// Start head tracking and wait for the expected results.
+func processProdHeadBlocks(bc *beaconclient.BeaconClient, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	go bc.CaptureHead()
+	time.Sleep(1 * time.Second)
+	validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
+}

pkg/database/sql/error.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package sql
 
 import (

pkg/database/sql/interfaces.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package sql
 
 import (

pkg/database/sql/postgres/config.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres
 
 import (
@@ -18,7 +33,7 @@ const (
 // DefaultConfig are default parameters for connecting to a Postgres sql
 var DefaultConfig = Config{
 	Hostname:     "localhost",
-	Port:         8077,
+	Port:         8076,
 	DatabaseName: "vulcanize_testing",
 	Username:     "vdbm",
 	Password:     "password",

pkg/database/sql/postgres/database.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres
 
 import (
@@ -5,8 +20,8 @@ import (
 	"fmt"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 var _ sql.Database = &DB{}
@@ -34,7 +49,7 @@ func SetupPostgresDb(dbHostname string, dbPort int, dbName string, dbUsername st
 			"driver_name_provided": driverName,
 		}).Error("Can't resolve driver type")
 	}
-	log.Info("Using Driver:", DbDriver)
+	log.Info("Using Driver: ", DbDriver)
 
 	postgresConfig := Config{
 		Hostname:     dbHostname,

pkg/database/sql/postgres/pgx.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres
 
 import (
@@ -8,7 +23,7 @@ import (
 	"github.com/jackc/pgconn"
 	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
 )
 
 // pgxDriver driver, implements sql.Driver

pkg/database/sql/postgres/pgx_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres_test
 
 import (
@@ -8,9 +23,9 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql/postgres"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/testhelpers"
 )
 
 var _ = Describe("Pgx", func() {

pkg/database/sql/postgres/postgres_suite_test.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package postgres_test
 
 import (

pkg/gracefulshutdown/gracefulshutdown.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package gracefulshutdown
 
 import (
@@ -10,7 +25,7 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+	"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
 )
 
 // operation is a clean up function on shutting down

pkg/loghelper/logendpoint.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package loghelper
 
 import (

pkg/loghelper/logerror.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // A simple function to help with logging errors.
 package loghelper
 
@@ -12,9 +27,26 @@ func LogError(err error) *log.Entry {
 	})
 }
 
+// A simple herlper function to log slot and error.
 func LogSlotError(slot string, err error) *log.Entry {
 	return log.WithFields(log.Fields{
 		"err":  err,
 		"slot": slot,
 	})
 }
+
+func LogSlotRangeError(startSlot string, endSlot string, err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err":       err,
+		"startSlot": startSlot,
+		"endSlot":   endSlot,
+	})
+}
+func LogSlotRangeStatementError(startSlot string, endSlot string, statement string, err error) *log.Entry {
+	return log.WithFields(log.Fields{
+		"err":          err,
+		"startSlot":    startSlot,
+		"endSlot":      endSlot,
+		"SqlStatement": statement,
+	})
+}

pkg/loghelper/logreorg.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package loghelper
 
 import (

pkg/testhelpers/test_helper.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package testhelpers
 
 import (

pkg/version/version.go

@@ -1,3 +1,18 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package version
 
 import "fmt"