diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index bc3d878..6b50a8c 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -11,6 +11,10 @@ on: description: "The branch, commit or sha from ipld-eth-db to checkout" required: false default: "main" + ssz-data-ref: + description: "The branch, commit or sha from ssz-data to checkout" + required: false + default: "main" pull_request: paths: - "!**.md" @@ -23,6 +27,7 @@ on: env: stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || '21d076268730e3f25fcec6371c1aca1bf48040d8'}} ipld-eth-db-ref: ${{ github.event.inputs.ipld-eth-db-ref || 'minimal-beacon-chain-schema' }} + ssz-data-ref: ${{ github.event.inputs.ssz-data-ref || 'main' }} GOPATH: /tmp/go jobs: build: @@ -96,6 +101,13 @@ jobs: path: "./ipld-eth-db/" fetch-depth: 0 + - uses: actions/checkout@v3 + with: + ref: ${{ env.ssz-data-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-ethcl-indexer/pkg/beaconclient/ssz-data" + fetch-depth: 0 + - name: Create config file run: | echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh diff --git a/Makefile b/Makefile index b8a12b5..e9ed9b4 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ unit-test-local: $(GINKGO) -r --label-filter unit \ --randomize-all --randomize-suites \ --fail-on-pending --keep-going \ - --race --trace + --trace .PHONY: unit-test-ci unit-test-ci: @@ -78,7 +78,7 @@ unit-test-ci: --randomize-all --randomize-suites \ --fail-on-pending --keep-going \ --cover --coverprofile=cover.profile \ - --race --trace --json-report=report.json + --trace --json-report=report.json .PHONY: build diff --git a/README.md b/README.md index a074c52..52b433b 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,27 @@ This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s To learn more about the applications individual components, please read the [application components](/application_component.md). -# Running the Application +# Quick Start -To run the application, utilize the following command, and update the values as needed. +## Running the Application + +To run the application, do as follows: + +1. Setup the prerequisite applications. + a. Run a beacon client (such as lighthouse). + b. Run a postgres DB. + c. You can utilize the `stack-orchestrator` [repository](https://github.com/vulcanize/stack-orchestrato). + + ``` + ./wrapper.sh -e skip \ + -d ../docker/local/docker-compose-db.yml \ + -d ../docker/latest/docker-compose-lighthouse.yml \ + -v remove \ + -p ../local-config.sh + + ``` + +2. Run the start up command. ``` go run main.go capture head --db.address localhost \ @@ -27,9 +45,19 @@ go run main.go capture head --db.address localhost \ --db.driver PGX \ --bc.address localhost \ --bc.port 5052 \ - --log.level info + --bc.connectionProtocol http \ + --log.level info \ + --log.output=true ``` +## Running Tests + +To run tests, you will need to clone another repository which contains all the ssz files. + +1. `git clone git@github.com:vulcanize/ssz-data.git pkg/beaconclient/ssz-data` +2. To run unit tests, make sure you have a DB running: `make unit-test-local` +3. To run integration tests, make sure you have a lighthouse client and a DB running: `make integration-test-local-no-race` . + # Development Patterns This section will cover some generic development patterns utilizes. @@ -61,17 +89,6 @@ This project utilizes `ginkgo` for testing. A few notes on testing: - Unit tests should not rely on any running service (except for a postgres DB). If a running service is needed. Utilize an integration test. - Integration tests must contain the `Label("integration")`. -### Testing the `pkg/beaconclient` - -To test the `pkg/beaconclient`, keep the following in mind. - -#### Get SSZ Files - -To test the `/pkg/beaconclient`, you will need to download data locally. - -1. [Install Minio](https://docs.min.io/minio/baremetal/quickstart/quickstart.html), you only need the client, `mc`. -2. Run: `mc cp` - #### Understanding Testing Components A few notes about the testing components. diff --git a/pkg/beaconclient/capturehead_test.go b/pkg/beaconclient/capturehead_test.go index 0f6d324..ea2196a 100644 --- a/pkg/beaconclient/capturehead_test.go +++ b/pkg/beaconclient/capturehead_test.go @@ -204,7 +204,7 @@ var _ = Describe("Capturehead", func() { // }) }) - Describe("ReOrg Scenario", Label("dry"), func() { + Describe("ReOrg Scenario", Label("unit", "behavioral"), func() { Context("Altair: Multiple head messages for the same slot.", func() { It("The previous block should be marked as 'forked', the new block should be the only one marked as 'proposed'.", func() { BeaconNodeTester.testMultipleHead(TestEvents["2375703-dummy"].HeadMessage, TestEvents["2375703"].HeadMessage, 74240) @@ -266,10 +266,10 @@ func setUpTest(config Config) *beaconclient.BeaconClient { // A helper function to validate the expected output. func validateSlot(bc *beaconclient.BeaconClient, headMessage *beaconclient.Head, correctEpoch int, correctStatus string) { - epoch, slot, blockRoot, stateRoot, status := queryDbSlotAndBlock(bc.Db, headMessage.Slot, headMessage.Block) - slot, err := strconv.Atoi(headMessage.Slot) + epoch, dbSlot, blockRoot, stateRoot, status := queryDbSlotAndBlock(bc.Db, headMessage.Slot, headMessage.Block) + baseSlot, err := strconv.Atoi(headMessage.Slot) Expect(err).ToNot(HaveOccurred()) - Expect(slot).To(Equal(slot)) + Expect(dbSlot).To(Equal(baseSlot)) Expect(epoch).To(Equal(correctEpoch)) Expect(blockRoot).To(Equal(headMessage.Block)) Expect(stateRoot).To(Equal(headMessage.State)) @@ -397,6 +397,7 @@ func (tbc TestBeaconNode) provideSsz(slotIdentifier string, sszIdentifier string Expect(err).ToNot(HaveOccurred()) } else { block.Block.ParentRoot, err = hex.DecodeString(Message.MimicConfig.ParentRoot) + Expect(err).ToNot(HaveOccurred()) } return block.MarshalSSZ() } @@ -406,7 +407,8 @@ func (tbc TestBeaconNode) provideSsz(slotIdentifier string, sszIdentifier string return nil, fmt.Errorf("Can't find the slot file, %s", slotFile) } state := st.BeaconState{} - state.UnmarshalSSZ(dat) + err = state.UnmarshalSSZ(dat) + Expect(err) slot, err := strconv.ParseUint(Message.HeadMessage.Slot, 10, 64) Expect(err).ToNot(HaveOccurred()) state.Slot = types.Slot(slot) diff --git a/pkg/beaconclient/databasewrite.go b/pkg/beaconclient/databasewrite.go index ec06ceb..c6ae8a0 100644 --- a/pkg/beaconclient/databasewrite.go +++ b/pkg/beaconclient/databasewrite.go @@ -169,9 +169,13 @@ func (dw *DatabaseWriter) upsertBeaconState() { // We will mark all entries for the given slot that don't match the provided latestBlockRoot as forked. func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) { forkCount, err := updateForked(db, slot, latestBlockRoot) + if err != nil { + loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while updating all forks.") + // Add to knownGaps Table + } proposedCount, err := updateProposed(db, slot, latestBlockRoot) if err != nil { - loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble processing a reorg.") + loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while trying to update the proposed slot.") // Add to knownGaps Table }