update cicd pipeline

.github/workflows/on-pr.yml

@@ -22,189 +22,13 @@ on:
       - "!LICENSE"
       - "!.github/workflows/**"
       - ".github/workflows/on-pr.yml"
+      - ".github/workflows/tests.yml"
       - "**"
 
-env:
-  stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref  || 'main'}}
-  ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref  || 'feature/historic-processing' }}
-  ssz-data-ref: ${{ github.event.inputs.ssz-data-ref  || 'main' }}
-  GOPATH: /tmp/go
 jobs:
-  build:
-    name: Run Docker Build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-ethcl-indexer"
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.stack-orchestrator-ref }}
-          path: "./stack-orchestrator/"
-          repository: vulcanize/stack-orchestrator
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ipld-ethcl-db-ref }}
-          repository: vulcanize/ipld-ethcl-db
-          path: "./ipld-ethcl-db/"
-          token: ${{ secrets.GH_PAT }}
-          fetch-depth: 0
-
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh
-          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
-          echo ethcl_capture_mode=boot >> ./config.sh
-          echo ethcl_skip_sync=true >> ./config.sh
-          echo ethcl_known_gap_increment=1000000 >> ./config.sh
-          cat ./config.sh
-
-      - name: Run docker compose
-        run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" \
-          --env-file ./config.sh \
-          up -d --build
-
-      - name: Check to make sure HEALTH file is present
-        shell: bash
-        run: |
-          until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
-          cat ./HEALTH
-          if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/ipld-ethcl-indexer.log . && cat ipld-ethcl-indexer.log && (exit 1); fi
-
-  unit-test:
-    name: Run Unit Tests
-    runs-on: ubuntu-latest
-    ## IF you want to update the default branch for `pull_request runs, do it after the ||`
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-ethcl-indexer"
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.stack-orchestrator-ref }}
-          path: "./stack-orchestrator/"
-          repository: vulcanize/stack-orchestrator
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ipld-ethcl-db-ref }}
-          repository: vulcanize/ipld-ethcl-db
-          path: "./ipld-ethcl-db/"
-          token: ${{ secrets.GH_PAT }}
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ssz-data-ref }}
-          repository: vulcanize/ssz-data
-          path: "./ipld-ethcl-indexer/pkg/beaconclient/ssz-data"
-          fetch-depth: 0
-
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh
-          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
-          cat ./config.sh
-
-      - name: Run docker compose
-        run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \
-          --env-file ./config.sh \
-          up -d --build
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-          check-latest: true
-
-      - name: Install packages
-        run: |
-          go install github.com/onsi/ginkgo/v2/ginkgo@latest
-          which ginkgo
-
-      - name: Run the tests using Make
-        run: |
-          cd ipld-ethcl-indexer
-          make unit-test-ci
-
-  integration-test:
-    name: Run Integration Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-ethcl-indexer"
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.stack-orchestrator-ref }}
-          path: "./stack-orchestrator/"
-          repository: vulcanize/stack-orchestrator
-          fetch-depth: 0
-
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.ipld-ethcl-db-ref }}
-          repository: vulcanize/ipld-ethcl-db
-          path: "./ipld-ethcl-db/"
-          token: ${{ secrets.GH_PAT }}
-          fetch-depth: 0
-
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh
-          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
-          echo ethcl_capture_mode=boot >> ./config.sh
-          cat ./config.sh
-
-      - name: Run docker compose
-        run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
-          --env-file ./config.sh \
-          up -d --build
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-          check-latest: true
-
-      - name: Install packages
-        run: |
-          go install github.com/onsi/ginkgo/v2/ginkgo@latest
-          which ginkgo
-
-      - name: Run the tests using Make
-        run: |
-          cd ipld-ethcl-indexer
-          make integration-test-ci
-
-  golangci:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.17.0"
-      - uses: actions/checkout@v3
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          args: --timeout 90s --disable deadcode,unused
-#          args: --timeout 90s --disable deadcode,
+  trigger-tests:
+    uses: ./.github/workflows/tests.yml
+    with:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
+      ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }}
+      ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }}

.github/workflows/on-publish.yml

@@ -3,9 +3,16 @@ on:
   release:
     types: [published, edited]
 jobs:
+  trigger-tests:
+    uses: ./.github/workflows/tests.yml
+    with:
+      stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
+      ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }}
+      ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }}
   build:
     name: Run docker build
     runs-on: ubuntu-latest
+    needs: trigger-tests
     steps:
       - uses: actions/checkout@v2
       - name: Get the version

.github/workflows/tests.yml

@@ -0,0 +1,198 @@
+name: Test the stack.
+on:
+  workflow_call:
+    inputs:
+      stack-orchestrator-ref:
+        required: false
+        type: string
+      ipld-ethcl-db-ref:
+        required: false
+        type: string
+      ssz-data-ref:
+        required: false
+        type: string
+
+env:
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || 'develop' }}
+  ipld-ethcl-db-ref: ${{ inputs.ipld-ethcl-db-ref  || 'feature/historic-processing' }}
+  ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }}
+  GOPATH: /tmp/go
+jobs:
+  build:
+    name: Run Docker Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-ethcl-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-ethcl-db-ref }}
+          repository: vulcanize/ipld-ethcl-db
+          path: "./ipld-ethcl-db/"
+          token: ${{ secrets.GH_PAT }}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh
+          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
+          echo ethcl_capture_mode=boot >> ./config.sh
+          echo ethcl_skip_sync=true >> ./config.sh
+          echo ethcl_known_gap_increment=1000000 >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - name: Check to make sure HEALTH file is present
+        shell: bash
+        run: |
+          until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
+          cat ./HEALTH
+          if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/ipld-ethcl-indexer.log . && cat ipld-ethcl-indexer.log && (exit 1); fi
+
+  unit-test:
+    name: Run Unit Tests
+    runs-on: ubuntu-latest
+    ## IF you want to update the default branch for `pull_request runs, do it after the ||`
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-ethcl-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-ethcl-db-ref }}
+          repository: vulcanize/ipld-ethcl-db
+          path: "./ipld-ethcl-db/"
+          token: ${{ secrets.GH_PAT }}
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ssz-data-ref }}
+          repository: vulcanize/ssz-data
+          path: "./ipld-ethcl-indexer/pkg/beaconclient/ssz-data"
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh
+          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.17.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          cd ipld-ethcl-indexer
+          make unit-test-ci
+
+  integration-test:
+    name: Run Integration Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-ethcl-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-ethcl-db-ref }}
+          repository: vulcanize/ipld-ethcl-db
+          path: "./ipld-ethcl-db/"
+          token: ${{ secrets.GH_PAT }}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh
+          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
+          echo ethcl_capture_mode=boot >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.17.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          cd ipld-ethcl-indexer
+          make integration-test-ci
+
+  golangci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.17.0"
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          args: --timeout 90s --disable deadcode,unused
+#          args: --timeout 90s --disable deadcode,

pkg/beaconclient/capturehistoric.go

@@ -57,6 +57,12 @@ type BatchProcessing interface {
 	releaseDbLocks() error                            // Update the checked_out column to false for whatever table is being updated.
 }
 
+/// ^^^
+// Might be better to remove the interface and create a single struct that historicalProcessing
+// and knownGapsProcessing can use. The struct would contain all the SQL strings that they need.
+// And the only difference in logic for processing would be within the error handling.
+// Which can be a function we pass into handleBatchProcess()
+
 // A struct to pass around indicating a table entry for slots to process.
 type slotsToProcess struct {
 	startSlot int // The start slot