Update code and CI/CD

.github/workflows/on-pr.yml

@@ -11,6 +11,10 @@ on:
         description: "The branch, commit or sha from ipld-eth-db to checkout"
         required: false
         default: "main"
+      ssz-data-ref:
+        description: "The branch, commit or sha from ssz-data to checkout"
+        required: false
+        default: "main"
   pull_request:
     paths:
       - "!**.md"
@@ -23,6 +27,7 @@ on:
 env:
   stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref  || '21d076268730e3f25fcec6371c1aca1bf48040d8'}}
   ipld-eth-db-ref: ${{ github.event.inputs.ipld-eth-db-ref  || 'minimal-beacon-chain-schema' }}
+  ssz-data-ref: ${{ github.event.inputs.ssz-data-ref  || 'main' }}
   GOPATH: /tmp/go
 jobs:
   build:
@@ -96,6 +101,13 @@ jobs:
           path: "./ipld-eth-db/"
           fetch-depth: 0
 
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.ssz-data-ref }}
+          repository: vulcanize/ipld-eth-db
+          path: "./ipld-ethcl-indexer/pkg/beaconclient/ssz-data"
+          fetch-depth: 0
+
       - name: Create config file
         run: |
           echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh

Makefile

@@ -68,7 +68,7 @@ unit-test-local:
 	$(GINKGO) -r --label-filter unit \
 	--randomize-all --randomize-suites \
 	--fail-on-pending --keep-going \
-	--race --trace
+	--trace
 
 .PHONY: unit-test-ci
 unit-test-ci:
@@ -78,7 +78,7 @@ unit-test-ci:
 	--randomize-all --randomize-suites \
 	--fail-on-pending --keep-going \
 	--cover --coverprofile=cover.profile \
-	--race --trace --json-report=report.json
+	--trace --json-report=report.json
 
 
 .PHONY: build

README.md

@@ -14,9 +14,27 @@ This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s
 
 To learn more about the applications individual components, please read the [application components](/application_component.md).
 
-# Running the Application
+# Quick Start
 
-To run the application, utilize the following command, and update the values as needed.
+## Running the Application
+
+To run the application, do as follows:
+
+1. Setup the prerequisite applications.
+   a. Run a beacon client (such as lighthouse).
+   b. Run a postgres DB.
+   c. You can utilize the `stack-orchestrator` [repository](https://github.com/vulcanize/stack-orchestrato).
+
+   ```
+   ./wrapper.sh -e skip \
+   -d ../docker/local/docker-compose-db.yml \
+   -d ../docker/latest/docker-compose-lighthouse.yml \
+   -v remove \
+   -p ../local-config.sh
+
+   ```
+
+2. Run the start up command.
 
 ```
 go run main.go capture head --db.address localhost \
@@ -27,9 +45,19 @@ go run main.go capture head --db.address localhost \
   --db.driver PGX \
   --bc.address localhost \
   --bc.port 5052 \
-  --log.level info
+  --bc.connectionProtocol http \
+  --log.level info \
+  --log.output=true
 ```
 
+## Running Tests
+
+To run tests, you will need to clone another repository which contains all the ssz files.
+
+1. `git clone git@github.com:vulcanize/ssz-data.git pkg/beaconclient/ssz-data`
+2. To run unit tests, make sure you have a DB running: `make unit-test-local`
+3. To run integration tests, make sure you have a lighthouse client and a DB running: `make integration-test-local-no-race` .
+
 # Development Patterns
 
 This section will cover some generic development patterns utilizes.
@@ -61,17 +89,6 @@ This project utilizes `ginkgo` for testing. A few notes on testing:
 - Unit tests should not rely on any running service (except for a postgres DB). If a running service is needed. Utilize an integration test.
 - Integration tests must contain the `Label("integration")`.
 
-### Testing the `pkg/beaconclient`
-
-To test the `pkg/beaconclient`, keep the following in mind.
-
-#### Get SSZ Files
-
-To test the `/pkg/beaconclient`, you will need to download data locally.
-
-1. [Install Minio](https://docs.min.io/minio/baremetal/quickstart/quickstart.html), you only need the client, `mc`.
-2. Run: `mc cp`
-
 #### Understanding Testing Components
 
 A few notes about the testing components.

pkg/beaconclient/capturehead_test.go

@@ -204,7 +204,7 @@ var _ = Describe("Capturehead", func() {
 		//	})
 	})
 
-	Describe("ReOrg Scenario", Label("dry"), func() {
+	Describe("ReOrg Scenario", Label("unit", "behavioral"), func() {
 		Context("Altair: Multiple head messages for the same slot.", func() {
 			It("The previous block should be marked as 'forked', the new block should be the only one marked as 'proposed'.", func() {
 				BeaconNodeTester.testMultipleHead(TestEvents["2375703-dummy"].HeadMessage, TestEvents["2375703"].HeadMessage, 74240)
@@ -266,10 +266,10 @@ func setUpTest(config Config) *beaconclient.BeaconClient {
 
 // A helper function to validate the expected output.
 func validateSlot(bc *beaconclient.BeaconClient, headMessage *beaconclient.Head, correctEpoch int, correctStatus string) {
-	epoch, slot, blockRoot, stateRoot, status := queryDbSlotAndBlock(bc.Db, headMessage.Slot, headMessage.Block)
+	epoch, dbSlot, blockRoot, stateRoot, status := queryDbSlotAndBlock(bc.Db, headMessage.Slot, headMessage.Block)
-	slot, err := strconv.Atoi(headMessage.Slot)
+	baseSlot, err := strconv.Atoi(headMessage.Slot)
 	Expect(err).ToNot(HaveOccurred())
-	Expect(slot).To(Equal(slot))
+	Expect(dbSlot).To(Equal(baseSlot))
 	Expect(epoch).To(Equal(correctEpoch))
 	Expect(blockRoot).To(Equal(headMessage.Block))
 	Expect(stateRoot).To(Equal(headMessage.State))
@@ -397,6 +397,7 @@ func (tbc TestBeaconNode) provideSsz(slotIdentifier string, sszIdentifier string
 				Expect(err).ToNot(HaveOccurred())
 			} else {
 				block.Block.ParentRoot, err = hex.DecodeString(Message.MimicConfig.ParentRoot)
+				Expect(err).ToNot(HaveOccurred())
 			}
 			return block.MarshalSSZ()
 		}
@@ -406,7 +407,8 @@ func (tbc TestBeaconNode) provideSsz(slotIdentifier string, sszIdentifier string
 				return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
 			}
 			state := st.BeaconState{}
-			state.UnmarshalSSZ(dat)
+			err = state.UnmarshalSSZ(dat)
+			Expect(err)
 			slot, err := strconv.ParseUint(Message.HeadMessage.Slot, 10, 64)
 			Expect(err).ToNot(HaveOccurred())
 			state.Slot = types.Slot(slot)

pkg/beaconclient/databasewrite.go

@@ -169,9 +169,13 @@ func (dw *DatabaseWriter) upsertBeaconState() {
 // We will mark all entries for the given slot that don't match the provided latestBlockRoot as forked.
 func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) {
 	forkCount, err := updateForked(db, slot, latestBlockRoot)
+	if err != nil {
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while updating all forks.")
+		// Add to knownGaps Table
+	}
 	proposedCount, err := updateProposed(db, slot, latestBlockRoot)
 	if err != nil {
-		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble processing a reorg.")
+		loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while trying to update the proposed slot.")
 		// Add to knownGaps Table
 	}