diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 1e7f347..c570dd7 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -22,189 +22,13 @@ on: - "!LICENSE" - "!.github/workflows/**" - ".github/workflows/on-pr.yml" + - ".github/workflows/tests.yml" - "**" -env: - stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || 'main'}} - ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || 'feature/historic-processing' }} - ssz-data-ref: ${{ github.event.inputs.ssz-data-ref || 'main' }} - GOPATH: /tmp/go jobs: - build: - name: Run Docker Build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - path: "./ipld-ethcl-indexer" - - - uses: actions/checkout@v3 - with: - ref: ${{ env.stack-orchestrator-ref }} - path: "./stack-orchestrator/" - repository: vulcanize/stack-orchestrator - fetch-depth: 0 - - - uses: actions/checkout@v3 - with: - ref: ${{ env.ipld-ethcl-db-ref }} - repository: vulcanize/ipld-ethcl-db - path: "./ipld-ethcl-db/" - token: ${{ secrets.GH_PAT }} - fetch-depth: 0 - - - name: Create config file - run: | - echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh - echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh - echo ethcl_capture_mode=boot >> ./config.sh - echo ethcl_skip_sync=true >> ./config.sh - echo ethcl_known_gap_increment=1000000 >> ./config.sh - cat ./config.sh - - - name: Run docker compose - run: | - docker-compose \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" \ - --env-file ./config.sh \ - up -d --build - - - name: Check to make sure HEALTH file is present - shell: bash - run: | - until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done - cat ./HEALTH - if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/ipld-ethcl-indexer.log . && cat ipld-ethcl-indexer.log && (exit 1); fi - - unit-test: - name: Run Unit Tests - runs-on: ubuntu-latest - ## IF you want to update the default branch for `pull_request runs, do it after the ||` - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - uses: actions/checkout@v2 - with: - path: "./ipld-ethcl-indexer" - - - uses: actions/checkout@v3 - with: - ref: ${{ env.stack-orchestrator-ref }} - path: "./stack-orchestrator/" - repository: vulcanize/stack-orchestrator - fetch-depth: 0 - - - uses: actions/checkout@v3 - with: - ref: ${{ env.ipld-ethcl-db-ref }} - repository: vulcanize/ipld-ethcl-db - path: "./ipld-ethcl-db/" - token: ${{ secrets.GH_PAT }} - fetch-depth: 0 - - - uses: actions/checkout@v3 - with: - ref: ${{ env.ssz-data-ref }} - repository: vulcanize/ssz-data - path: "./ipld-ethcl-indexer/pkg/beaconclient/ssz-data" - fetch-depth: 0 - - - name: Create config file - run: | - echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh - echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh - cat ./config.sh - - - name: Run docker compose - run: | - docker-compose \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ - --env-file ./config.sh \ - up -d --build - - - uses: actions/setup-go@v3 - with: - go-version: ">=1.17.0" - check-latest: true - - - name: Install packages - run: | - go install github.com/onsi/ginkgo/v2/ginkgo@latest - which ginkgo - - - name: Run the tests using Make - run: | - cd ipld-ethcl-indexer - make unit-test-ci - - integration-test: - name: Run Integration Tests - runs-on: ubuntu-latest - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - uses: actions/checkout@v2 - with: - path: "./ipld-ethcl-indexer" - - - uses: actions/checkout@v3 - with: - ref: ${{ env.stack-orchestrator-ref }} - path: "./stack-orchestrator/" - repository: vulcanize/stack-orchestrator - fetch-depth: 0 - - - uses: actions/checkout@v3 - with: - ref: ${{ env.ipld-ethcl-db-ref }} - repository: vulcanize/ipld-ethcl-db - path: "./ipld-ethcl-db/" - token: ${{ secrets.GH_PAT }} - fetch-depth: 0 - - - name: Create config file - run: | - echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh - echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh - echo ethcl_capture_mode=boot >> ./config.sh - cat ./config.sh - - - name: Run docker compose - run: | - docker-compose \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \ - --env-file ./config.sh \ - up -d --build - - - uses: actions/setup-go@v3 - with: - go-version: ">=1.17.0" - check-latest: true - - - name: Install packages - run: | - go install github.com/onsi/ginkgo/v2/ginkgo@latest - which ginkgo - - - name: Run the tests using Make - run: | - cd ipld-ethcl-indexer - make integration-test-ci - - golangci: - runs-on: ubuntu-latest - steps: - - uses: actions/setup-go@v3 - with: - go-version: ">=1.17.0" - - uses: actions/checkout@v3 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - args: --timeout 90s --disable deadcode,unused -# args: --timeout 90s --disable deadcode, + trigger-tests: + uses: ./.github/workflows/tests.yml + with: + stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }} + ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }} + ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }} diff --git a/.github/workflows/on-publish.yml b/.github/workflows/on-publish.yml index 801a1c9..348b76b 100644 --- a/.github/workflows/on-publish.yml +++ b/.github/workflows/on-publish.yml @@ -3,9 +3,16 @@ on: release: types: [published, edited] jobs: + trigger-tests: + uses: ./.github/workflows/tests.yml + with: + stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }} + ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }} + ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }} build: name: Run docker build runs-on: ubuntu-latest + needs: trigger-tests steps: - uses: actions/checkout@v2 - name: Get the version diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..85af609 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,198 @@ +name: Test the stack. +on: + workflow_call: + inputs: + stack-orchestrator-ref: + required: false + type: string + ipld-ethcl-db-ref: + required: false + type: string + ssz-data-ref: + required: false + type: string + +env: + stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || 'develop' }} + ipld-ethcl-db-ref: ${{ inputs.ipld-ethcl-db-ref || 'feature/historic-processing' }} + ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }} + GOPATH: /tmp/go +jobs: + build: + name: Run Docker Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.stack-orchestrator-ref }} + path: "./stack-orchestrator/" + repository: vulcanize/stack-orchestrator + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-ethcl-db-ref }} + repository: vulcanize/ipld-ethcl-db + path: "./ipld-ethcl-db/" + token: ${{ secrets.GH_PAT }} + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + echo ethcl_capture_mode=boot >> ./config.sh + echo ethcl_skip_sync=true >> ./config.sh + echo ethcl_known_gap_increment=1000000 >> ./config.sh + cat ./config.sh + + - name: Run docker compose + run: | + docker-compose \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" \ + --env-file ./config.sh \ + up -d --build + + - name: Check to make sure HEALTH file is present + shell: bash + run: | + until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done + cat ./HEALTH + if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/ipld-ethcl-indexer.log . && cat ipld-ethcl-indexer.log && (exit 1); fi + + unit-test: + name: Run Unit Tests + runs-on: ubuntu-latest + ## IF you want to update the default branch for `pull_request runs, do it after the ||` + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.stack-orchestrator-ref }} + path: "./stack-orchestrator/" + repository: vulcanize/stack-orchestrator + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-ethcl-db-ref }} + repository: vulcanize/ipld-ethcl-db + path: "./ipld-ethcl-db/" + token: ${{ secrets.GH_PAT }} + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ssz-data-ref }} + repository: vulcanize/ssz-data + path: "./ipld-ethcl-indexer/pkg/beaconclient/ssz-data" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + cat ./config.sh + + - name: Run docker compose + run: | + docker-compose \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ + --env-file ./config.sh \ + up -d --build + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + check-latest: true + + - name: Install packages + run: | + go install github.com/onsi/ginkgo/v2/ginkgo@latest + which ginkgo + + - name: Run the tests using Make + run: | + cd ipld-ethcl-indexer + make unit-test-ci + + integration-test: + name: Run Integration Tests + runs-on: ubuntu-latest + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.stack-orchestrator-ref }} + path: "./stack-orchestrator/" + repository: vulcanize/stack-orchestrator + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-ethcl-db-ref }} + repository: vulcanize/ipld-ethcl-db + path: "./ipld-ethcl-db/" + token: ${{ secrets.GH_PAT }} + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + echo ethcl_capture_mode=boot >> ./config.sh + cat ./config.sh + + - name: Run docker compose + run: | + docker-compose \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \ + --env-file ./config.sh \ + up -d --build + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + check-latest: true + + - name: Install packages + run: | + go install github.com/onsi/ginkgo/v2/ginkgo@latest + which ginkgo + + - name: Run the tests using Make + run: | + cd ipld-ethcl-indexer + make integration-test-ci + + golangci: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + args: --timeout 90s --disable deadcode,unused +# args: --timeout 90s --disable deadcode, diff --git a/pkg/beaconclient/capturehistoric.go b/pkg/beaconclient/capturehistoric.go index 1b0631a..cd88c90 100644 --- a/pkg/beaconclient/capturehistoric.go +++ b/pkg/beaconclient/capturehistoric.go @@ -57,6 +57,12 @@ type BatchProcessing interface { releaseDbLocks() error // Update the checked_out column to false for whatever table is being updated. } +/// ^^^ +// Might be better to remove the interface and create a single struct that historicalProcessing +// and knownGapsProcessing can use. The struct would contain all the SQL strings that they need. +// And the only difference in logic for processing would be within the error handling. +// Which can be a function we pass into handleBatchProcess() + // A struct to pass around indicating a table entry for slots to process. type slotsToProcess struct { startSlot int // The start slot