.github/workflows/generic-testing.yml

@@ -16,7 +16,7 @@ on:
         required: true
 
 env:
-  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || 'develop' }}
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '2e0d4f4f3cba4dc526df96818351a109dc295efa' }}
   ipld-ethcl-db-ref: ${{ inputs.ipld-ethcl-db-ref  || 'feature/historic-processing' }}
   ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }}
   GOPATH: /tmp/go
@@ -118,7 +118,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: ">=1.17.0"
+          go-version: ">=1.18.0"
           check-latest: true
 
       - name: Install packages
@@ -174,7 +174,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: ">=1.17.0"
+          go-version: ">=1.18.0"
           check-latest: true
 
       - name: Install packages
@@ -187,12 +187,18 @@ jobs:
           cd ipld-ethcl-indexer
           make integration-test-ci
 
+  system-testing:
+    uses: ./.github/workflows/system-tests.yml
+    with:
+      stack-orchestrator-ref: ${{ env.stack-orchestrator-ref }}
+      ipld-ethcl-db-ref: ${{ env.ipld-ethcl-db-ref }}
+
   golangci:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: ">=1.17.0"
+          go-version: ">=1.18.0"
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/on-pr.yml

@@ -24,10 +24,12 @@ on:
       - ".github/workflows/on-pr.yml"
       - ".github/workflows/tests.yml"
       - "**"
+  schedule:
+    - cron: '0 13 * * *'
 
 jobs:
   trigger-tests:
-    uses: ./.github/workflows/tests.yml
+    uses: ./.github/workflows/generic-testing.yml
     with:
       stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
       ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }}

.github/workflows/on-publish.yml

@@ -4,7 +4,7 @@ on:
     types: [published, edited]
 jobs:
   trigger-tests:
-    uses: ./.github/workflows/tests.yml
+    uses: ./.github/workflows/generic-testing.yml
     with:
       stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
       ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }}

.github/workflows/system-tests.yml

@@ -0,0 +1,90 @@
+name: System Testing for the stack.
+on:
+  workflow_call:
+    inputs:
+      stack-orchestrator-ref:
+        required: false
+        type: string
+      ipld-ethcl-db-ref:
+        required: false
+        type: string
+    secrets:
+      GHA_KEY:
+        required: true
+env:
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '2e0d4f4f3cba4dc526df96818351a109dc295efa' }}
+  ipld-ethcl-db-ref: ${{ inputs.ipld-ethcl-db-ref  || 'feature/historic-processing' }}
+  GOPATH: /tmp/go
+  bc_protocol: "http"
+  bc_address: ${{secrets.BC_ADDRESS}}
+  bc_port: 5052
+  db_host: localhost
+  db_port: 8076
+  db_name: vulcanize_testing
+  db_user: vdbm
+  db_password: password
+  db_driver: "pgx"
+
+jobs:
+  system-testing:
+    runs-on: self-hosted
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-ethcl-indexer"
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.stack-orchestrator-ref }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+          fetch-depth: 0
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ipld-ethcl-db-ref }}
+          repository: vulcanize/ipld-ethcl-db
+          path: "./ipld-ethcl-db/"
+          ssh-key: ${{secrets.GHA_KEY}}
+          fetch-depth: 0
+
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh
+          echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh
+          echo ethcl_capture_mode=boot >> ./config.sh
+          cat ./config.sh
+
+      - name: Run docker compose
+        id: compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \
+          --env-file ./config.sh \
+          up -d --build
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Install packages
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@latest
+          which ginkgo
+
+      - name: Run the tests using Make
+        run: |
+          cd ipld-ethcl-indexer
+          make system-test-ci
+
+      - name: Clean up the docker containers
+        if: steps.compose.outcome == 'success'
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \
+          --env-file ./config.sh \
+          down

Makefile

@@ -80,6 +80,24 @@ unit-test-ci:
 	--cover --coverprofile=cover.profile \
 	--trace --json-report=report.json
 
+.PHONY: system-test-ci
+system-test-ci:
+	go vet ./...
+	go fmt ./...
+	$(GINKGO) -r --label-filter system \
+	--randomize-all --randomize-suites \
+	--fail-on-pending --keep-going \
+	--cover --coverprofile=cover.profile \
+	--trace --json-report=report.json
+
+.PHONY: system-test-local
+system-test-local:
+	go vet ./...
+	go fmt ./...
+	$(GINKGO) -r --label-filter system \
+	--randomize-all --randomize-suites \
+	--fail-on-pending --keep-going \
+	--trace
 
 .PHONY: build
 build:

pkg/beaconclient/databasewrite.go

@@ -479,7 +479,14 @@ func writeStartUpGaps(db sql.Database, tableIncrement int, firstSlot int, metric
 		}).Fatal("Unable to get convert max block from DB to int. We must close the application or we might have undetected gaps.")
 	}
 	if maxSlot != firstSlot-1 {
-		writeKnownGaps(db, tableIncrement, maxSlot+1, firstSlot-1, fmt.Errorf(""), "startup", metric)
+		if maxSlot < firstSlot-1 {
+			writeKnownGaps(db, tableIncrement, maxSlot+1, firstSlot-1, fmt.Errorf(""), "startup", metric)
+		} else {
+			log.WithFields(log.Fields{
+				"maxSlot":   maxSlot,
+				"firstSlot": firstSlot,
+			}).Warn("The maxSlot in the DB is greater than or equal to the first Slot we are processing.")
+		}
 	}
 }
 

pkg/beaconclient/systemvalidation_test.go

@@ -0,0 +1,70 @@
+package beaconclient_test
+
+import (
+	"os"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	//. "github.com/onsi/gomega"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient"
+	"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper"
+)
+
+var (
+	prodConfig = Config{
+		protocol:                os.Getenv("bc_protocol"),
+		address:                 os.Getenv("bc_address"),
+		port:                    getEnvInt(os.Getenv("bc_port")),
+		dbHost:                  os.Getenv("db_host"),
+		dbPort:                  getEnvInt(os.Getenv("db_port")),
+		dbName:                  os.Getenv("db_name"),
+		dbUser:                  os.Getenv("db_user"),
+		dbPassword:              os.Getenv("db_password"),
+		dbDriver:                os.Getenv("db_driver"),
+		knownGapsTableIncrement: 100000000,
+		bcUniqueIdentifier:      100,
+		checkDb:                 false,
+	}
+)
+var _ = Describe("Systemvalidation", Label("system"), func() {
+	Describe("Run the application against a running lighthouse node", func() {
+		Context("When we receive head messages", func() {
+			It("We should process the messages successfully", func() {
+				bc := setUpTest(prodConfig, "10000000000")
+				processProdHeadBlocks(bc, 3, 0, 0, 0)
+			})
+		})
+		Context("When we have historical and knownGaps slots to process", Label("system-batch"), func() {
+			It("Should process them successfully", func() {
+				bc := setUpTest(prodConfig, "10000000000")
+				//known Gaps
+				BeaconNodeTester.writeEventToKnownGaps(bc, 100, 101)
+				BeaconNodeTester.runKnownGapsProcess(bc, 2, 2, 0, 0, 0)
+
+				// Historical
+				BeaconNodeTester.writeEventToHistoricProcess(bc, 2375703, 2375703, 10)
+				BeaconNodeTester.runHistoricalProcess(bc, 2, 3, 0, 0, 0)
+
+				time.Sleep(2 * time.Second)
+				validatePopularBatchBlocks(bc)
+			})
+		})
+	})
+})
+
+// Wrapper function to get int env variables.
+func getEnvInt(envVar string) int {
+	val, err := strconv.Atoi(envVar)
+	if err != nil {
+		loghelper.LogError(err).WithField("envVar", envVar).Fatal("Unable to turn env string to int")
+	}
+	return val
+}
+
+// Start head tracking and wait for the expected results.
+func processProdHeadBlocks(bc *beaconclient.BeaconClient, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
+	go bc.CaptureHead()
+	time.Sleep(1 * time.Second)
+	validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
+}