cerc-io/ipld-eth-beacon-indexer
19
0
Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases 9 Wiki Activity

Testing for Batch Processing #56

Merged
abdulrabbani00 merged 30 commits from feature/48-test-historical into develop 2022-06-09 21:32:46 +00:00
Conversation 1 Commits 30 Files Changed 47 +1867 -256
46 changed files with 1867 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 4e694d4c1f - Show all commits

68
.github/workflows/generic-testing.yml vendored
View File

@ -5,7 +5,7 @@ on:
stack-orchestrator-ref: stack-orchestrator-ref:
required: false required: false
type: string type: string
ipld-ethcl-db-ref: ipld-eth-beacon-db-ref:
required: false required: false
type: string type: string
ssz-data-ref: ssz-data-ref:
@ -16,8 +16,8 @@ on:
required: true required: true
env: env:
stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '2e0d4f4f3cba4dc526df96818351a109dc295efa' }} stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '3048a224100ceb122d6da71328bf3803dff72a01' }}
ipld-ethcl-db-ref: ${{ inputs.ipld-ethcl-db-ref || 'feature/historic-processing' }} ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }} ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }}
GOPATH: /tmp/go GOPATH: /tmp/go
jobs: jobs:
@ -27,7 +27,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
path: "./ipld-ethcl-indexer" path: "./ipld-eth-beacon-indexer"
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
@ -38,36 +38,36 @@ jobs:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
ref: ${{ env.ipld-ethcl-db-ref }} ref: ${{ env.ipld-eth-beacon-db-ref }}
repository: vulcanize/ipld-ethcl-db repository: vulcanize/ipld-eth-beacon-db
path: "./ipld-ethcl-db/" path: "./ipld-eth-beacon-db/"
ssh-key: ${{secrets.GHA_KEY}} ssh-key: ${{secrets.GHA_KEY}}
fetch-depth: 0 fetch-depth: 0
- name: Create config file - name: Create config file
run: | run: |
echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
echo ethcl_capture_mode=boot >> ./config.sh echo eth_beacon_capture_mode=boot >> ./config.sh
echo ethcl_skip_sync=true >> ./config.sh echo eth_beacon_skip_sync=true >> ./config.sh
echo ethcl_known_gap_increment=1000000 >> ./config.sh echo eth_beacon_known_gap_increment=1000000 >> ./config.sh
cat ./config.sh cat ./config.sh
- name: Run docker compose - name: Run docker compose
run: | run: |
docker-compose \ docker-compose \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" \
--env-file ./config.sh \ --env-file ./config.sh \
up -d --build up -d --build
- name: Check to make sure HEALTH file is present - name: Check to make sure HEALTH file is present
shell: bash shell: bash
run: | run: |
until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" cp ipld-eth-beacon-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
cat ./HEALTH cat ./HEALTH
if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-ethcl-indexer.yml" cp ipld-ethcl-indexer:/root/ipld-ethcl-indexer.log . && cat ipld-ethcl-indexer.log && (exit 1); fi if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" cp ipld-eth-beacon-indexer:/root/ipld-eth-beacon-indexer.log . && cat ipld-eth-beacon-indexer.log && (exit 1); fi
unit-test: unit-test:
name: Run Unit Tests name: Run Unit Tests
@ -79,7 +79,7 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
path: "./ipld-ethcl-indexer" path: "./ipld-eth-beacon-indexer"
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
@ -90,9 +90,9 @@ jobs:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
ref: ${{ env.ipld-ethcl-db-ref }} ref: ${{ env.ipld-eth-beacon-db-ref }}
repository: vulcanize/ipld-ethcl-db repository: vulcanize/ipld-eth-beacon-db
path: "./ipld-ethcl-db/" path: "./ipld-eth-beacon-db/"
ssh-key: ${{ secrets.GHA_KEY }} ssh-key: ${{ secrets.GHA_KEY }}
fetch-depth: 0 fetch-depth: 0
@ -100,19 +100,19 @@ jobs:
with: with:
ref: ${{ env.ssz-data-ref }} ref: ${{ env.ssz-data-ref }}
repository: vulcanize/ssz-data repository: vulcanize/ssz-data
path: "./ipld-ethcl-indexer/pkg/beaconclient/ssz-data" path: "./ipld-eth-beacon-indexer/pkg/beaconclient/ssz-data"
fetch-depth: 0 fetch-depth: 0
- name: Create config file - name: Create config file
run: | run: |
echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
cat ./config.sh cat ./config.sh
- name: Run docker compose - name: Run docker compose
run: | run: |
docker-compose \ docker-compose \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
--env-file ./config.sh \ --env-file ./config.sh \
up -d --build up -d --build
@ -128,7 +128,7 @@ jobs:
- name: Run the tests using Make - name: Run the tests using Make
run: | run: |
cd ipld-ethcl-indexer cd ipld-eth-beacon-indexer
make unit-test-ci make unit-test-ci
integration-test: integration-test:
@ -140,7 +140,7 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
path: "./ipld-ethcl-indexer" path: "./ipld-eth-beacon-indexer"
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
@ -151,23 +151,23 @@ jobs:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
ref: ${{ env.ipld-ethcl-db-ref }} ref: ${{ env.ipld-eth-beacon-db-ref }}
repository: vulcanize/ipld-ethcl-db repository: vulcanize/ipld-eth-beacon-db
path: "./ipld-ethcl-db/" path: "./ipld-eth-beacon-db/"
ssh-key: ${{secrets.GHA_KEY}} ssh-key: ${{secrets.GHA_KEY}}
fetch-depth: 0 fetch-depth: 0
- name: Create config file - name: Create config file
run: | run: |
echo vulcanize_ipld_ethcl_db=$GITHUB_WORKSPACE/ipld-ethcl-db/ > ./config.sh echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
echo ethcl_capture_mode=boot >> ./config.sh echo eth_beacon_capture_mode=boot >> ./config.sh
cat ./config.sh cat ./config.sh
- name: Run docker compose - name: Run docker compose
run: | run: |
docker-compose \ docker-compose \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
--env-file ./config.sh \ --env-file ./config.sh \
up -d --build up -d --build
@ -184,7 +184,7 @@ jobs:
- name: Run the tests using Make - name: Run the tests using Make
run: | run: |
cd ipld-ethcl-indexer cd ipld-eth-beacon-indexer
make integration-test-ci make integration-test-ci
golangci: golangci:

10
.github/workflows/on-pr.yml vendored
View File

@ -7,8 +7,8 @@ on:
description: "The branch, commit or sha from stack-orchestrator to checkout" description: "The branch, commit or sha from stack-orchestrator to checkout"
required: false required: false
default: "main" default: "main"
ipld-ethcl-db-ref: ipld-eth-beacon-db-ref:
description: "The branch, commit or sha from ipld-ethcl-db to checkout" description: "The branch, commit or sha from ipld-eth-beacon-db to checkout"
required: false required: false
default: "main" default: "main"
ssz-data-ref: ssz-data-ref:
@ -25,14 +25,14 @@ on:
- ".github/workflows/tests.yml" - ".github/workflows/tests.yml"
- "**" - "**"
schedule: schedule:
- cron: '0 13 * * *' # Must be single quotes!! - cron: "0 13 * * *" # Must be single quotes!!
jobs: jobs:
trigger-tests: trigger-tests:
uses: ./.github/workflows/generic-testing.yml uses: ./.github/workflows/generic-testing.yml
with: with:
stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }} stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }} ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }} ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }}
secrets: secrets:
GHA_KEY: ${{secrets.GHA_KEY}} GHA_KEY: ${{secrets.GHA_KEY}}
@ -40,7 +40,7 @@ jobs:
uses: ./.github/workflows/system-tests.yml uses: ./.github/workflows/system-tests.yml
with: with:
stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }} stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }} ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
secrets: secrets:
GHA_KEY: ${{secrets.GHA_KEY}} GHA_KEY: ${{secrets.GHA_KEY}}
BC_ADDRESS: ${{secrets.BC_ADDRESS}} BC_ADDRESS: ${{secrets.BC_ADDRESS}}

14
.github/workflows/on-publish.yml vendored
View File

@ -7,7 +7,7 @@ jobs:
uses: ./.github/workflows/generic-testing.yml uses: ./.github/workflows/generic-testing.yml
with: with:
stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }} stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }} ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }} ssz-data-ref: ${{ github.event.inputs.ssz-data-ref }}
secrets: secrets:
GHA_KEY: ${{secrets.GHA_KEY}} GHA_KEY: ${{secrets.GHA_KEY}}
@ -15,7 +15,7 @@ jobs:
uses: ./.github/workflows/system-tests.yml uses: ./.github/workflows/system-tests.yml
with: with:
stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }} stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref }}
ipld-ethcl-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref }} ipld-eth-beacon-db-ref: ${{ github.event.inputs.ipld-eth-beacon-db-ref }}
secrets: secrets:
GHA_KEY: ${{secrets.GHA_KEY}} GHA_KEY: ${{secrets.GHA_KEY}}
BC_ADDRESS: ${{secrets.BC_ADDRESS}} BC_ADDRESS: ${{secrets.BC_ADDRESS}}
@ -33,11 +33,11 @@ jobs:
- name: Run docker build - name: Run docker build
run: make docker-build run: make docker-build
- name: Tag docker image - name: Tag docker image
run: docker tag vulcanize/ipld-ethcl-indexer docker.pkg.github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer:${{steps.vars.outputs.sha}} run: docker tag vulcanize/ipld-eth-beacon-indexer docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
- name: Docker Login - name: Docker Login
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
- name: Docker Push - name: Docker Push
run: docker push docker.pkg.github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer:${{steps.vars.outputs.sha}} run: docker push docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
push_to_registries: push_to_registries:
name: Push Docker image to Docker Hub name: Push Docker image to Docker Hub
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -51,10 +51,10 @@ jobs:
- name: Docker Login to Github Registry - name: Docker Login to Github Registry
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
- name: Docker Pull - name: Docker Pull
run: docker pull docker.pkg.github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer:${{steps.vars.outputs.sha}} run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}}
- name: Docker Login to Docker Registry - name: Docker Login to Docker Registry
run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
- name: Tag docker image - name: Tag docker image
run: docker tag docker.pkg.github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer:${{steps.vars.outputs.sha}} vulcanize/ipld-ethcl-indexer:${{steps.vars.outputs.tag}} run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-beacon-indexer:${{steps.vars.outputs.tag}}
- name: Docker Push to Docker Hub - name: Docker Push to Docker Hub
run: docker push vulcanize/ipld-ethcl-indexer:${{steps.vars.outputs.tag}} run: docker push vulcanize/ipld-eth-beacon-indexer:${{steps.vars.outputs.tag}}

24
.github/workflows/system-tests.yml vendored
View File

@ -5,7 +5,7 @@ on:
stack-orchestrator-ref: stack-orchestrator-ref:
required: false required: false
type: string type: string
ipld-ethcl-db-ref: ipld-eth-beacon-db-ref:
required: false required: false
type: string type: string
secrets: secrets:
@ -14,8 +14,8 @@ on:
BC_ADDRESS: BC_ADDRESS:
required: true required: true
env: env:
stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '2e0d4f4f3cba4dc526df96818351a109dc295efa' }} stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '3048a224100ceb122d6da71328bf3803dff72a01' }}
ipld-ethcl-db-ref: ${{ inputs.ipld-ethcl-db-ref || 'feature/historic-processing' }} ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
GOPATH: /tmp/go GOPATH: /tmp/go
bc_protocol: "http" bc_protocol: "http"
bc_address: ${{secrets.BC_ADDRESS}} bc_address: ${{secrets.BC_ADDRESS}}
@ -37,7 +37,7 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
path: "./ipld-ethcl-indexer" path: "./ipld-eth-beacon-indexer"
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
@ -48,23 +48,23 @@ jobs:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
ref: ${{ env.ipld-ethcl-db-ref }} ref: ${{ env.ipld-eth-beacon-db-ref }}
repository: vulcanize/ipld-ethcl-db repository: vulcanize/ipld-eth-beacon-db
path: "./ipld-ethcl-db/" path: "./ipld-eth-beacon-db/"
ssh-key: ${{secrets.GHA_KEY}} ssh-key: ${{secrets.GHA_KEY}}
fetch-depth: 0 fetch-depth: 0
- name: Create config file - name: Create config file
run: | run: |
echo vulcanize_ipld_ethcl_db=$(pwd)/ipld-ethcl-db > ./config.sh echo vulcanize_ipld_eth_beacon_db=$(pwd)/ipld-eth-beacon-db > ./config.sh
cat ./config.sh cat ./config.sh
- name: Run docker compose - name: Run docker compose
id: compose id: compose
run: | run: |
ls "./stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" ls "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml"
sudo docker-compose \ sudo docker-compose \
-f "./stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ -f "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
--env-file ./config.sh \ --env-file ./config.sh \
up -d --build up -d --build
@ -80,13 +80,13 @@ jobs:
- name: Run the tests using Make - name: Run the tests using Make
run: | run: |
cd ipld-ethcl-indexer cd ipld-eth-beacon-indexer
make system-test-ci make system-test-ci
- name: Clean up the docker containers - name: Clean up the docker containers
if: steps.compose.outcome == 'success' if: steps.compose.outcome == 'success'
run: | run: |
sudo docker-compose \ sudo docker-compose \
-f "./stack-orchestrator/docker/local/docker-compose-ethcl-db.yml" \ -f "./stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
--env-file ./config.sh \ --env-file ./config.sh \
down -v down -v

8
Dockerfile
View File

@ -1,6 +1,6 @@
FROM golang:1.18-alpine as builder FROM golang:1.18-alpine as builder
WORKDIR /go/src/github.com/vulcanize/ipld-ethcl-indexer WORKDIR /go/src/github.com/vulcanize/ipld-eth-beacon-indexer
RUN apk --no-cache add ca-certificates make git g++ linux-headers RUN apk --no-cache add ca-certificates make git g++ linux-headers
ENV GO111MODULE=on ENV GO111MODULE=on
@ -9,12 +9,12 @@ COPY go.sum .
RUN go mod tidy; go mod download RUN go mod tidy; go mod download
COPY . . COPY . .
RUN GCO_ENABLED=0 GOOS=linux go build -race -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-ethcl-indexer . RUN GCO_ENABLED=0 GOOS=linux go build -race -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-beacon-indexer .
RUN chmod +x ipld-ethcl-indexer RUN chmod +x ipld-eth-beacon-indexer
FROM frolvlad/alpine-bash:latest FROM frolvlad/alpine-bash:latest
RUN apk --no-cache add ca-certificates RUN apk --no-cache add ca-certificates
WORKDIR /root/ WORKDIR /root/
COPY --from=builder /go/src/github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer /root/ipld-ethcl-indexer COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer /root/ipld-eth-beacon-indexer
ADD entrypoint.sh . ADD entrypoint.sh .
ENTRYPOINT ["./entrypoint.sh"] ENTRYPOINT ["./entrypoint.sh"]

2
Makefile
View File

@ -107,4 +107,4 @@ build:
## Build docker image ## Build docker image
.PHONY: docker-build .PHONY: docker-build
docker-build: docker-build:
docker build -t vulcanize/ipld-ethcl-indexer . docker build -t vulcanize/ipld-eth-beacon-indexer .

10
README.md
View File

@ -1,4 +1,4 @@
- [ipld-ethcl-indexer](#ipld-ethcl-indexer) - [ipld-eth-beacon-indexer](#ipld-eth-beacon-indexer)
- [Running the Application](#running-the-application) - [Running the Application](#running-the-application)
- [Development Patterns](#development-patterns) - [Development Patterns](#development-patterns)
- [Logging](#logging) - [Logging](#logging)
@ -8,7 +8,7 @@
<small><i><a href='http://ecotrust-canada.github.io/markdown-toc/'>Table of contents generated with markdown-toc</a></i></small> <small><i><a href='http://ecotrust-canada.github.io/markdown-toc/'>Table of contents generated with markdown-toc</a></i></small>
# ipld-ethcl-indexer # ipld-eth-beacon-indexer
This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node. This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node.
@ -22,12 +22,12 @@ To run the application, do as follows:
1. Setup the prerequisite applications. 1. Setup the prerequisite applications.
a. Run a beacon client (such as lighthouse). a. Run a beacon client (such as lighthouse).
b. Run a postgres DB for ethcl. b. Run a postgres DB for eth-beacon.
c. You can utilize the `stack-orchestrator` [repository](https://github.com/vulcanize/stack-orchestrato). c. You can utilize the `stack-orchestrator` [repository](https://github.com/vulcanize/stack-orchestrato).
``` ```
./wrapper.sh -e skip \ ./wrapper.sh -e skip \
-d ../docker/local/docker-compose-ethcl-db.yml \ -d ../docker/local/docker-compose-eth-beacon-db.yml \
-d ../docker/latest/docker-compose-lighthouse.yml \ -d ../docker/latest/docker-compose-lighthouse.yml \
-v remove \ -v remove \
-p ../local-config.sh -p ../local-config.sh
@ -37,7 +37,7 @@ To run the application, do as follows:
2. Run the start up command. 2. Run the start up command.
``` ```
go run -race main.go capture historic --config ./example.ipld-ethcl-indexer-config.json go run -race main.go capture historic --config ./example.ipld-eth-beacon-indexer-config.json
``` ```
## Running Tests ## Running Tests

2
application_component.md
View File

@ -51,4 +51,4 @@ This package contains useful functions for logging.
## `internal/shutdown` ## `internal/shutdown`
This package is used to shutdown the `ipld-ethcl-indexer`. It calls the `pkg/gracefulshutdown` package. This package is used to shutdown the `ipld-eth-beacon-indexer`. It calls the `pkg/gracefulshutdown` package.

10
cmd/boot.go
View File

@ -24,9 +24,9 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipld-ethcl-indexer/internal/boot" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
// bootCmd represents the boot command // bootCmd represents the boot command
@ -62,9 +62,9 @@ func bootApp() {
err = shutdown.ShutdownBoot(ctx, notifierCh, maxWaitSecondsShutdown, Db, Bc) err = shutdown.ShutdownBoot(ctx, notifierCh, maxWaitSecondsShutdown, Db, Bc)
if err != nil { if err != nil {
loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!") loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
} else { } else {
log.Info("Gracefully shutdown ipld-ethcl-indexer") log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
} }
} }

6
cmd/capture.go
View File

@ -94,7 +94,7 @@ func init() {
captureCmd.PersistentFlags().StringVarP(&bcConnectionProtocol, "bc.connectionProtocol", "", "http", "protocol for connecting to the beacon node.") captureCmd.PersistentFlags().StringVarP(&bcConnectionProtocol, "bc.connectionProtocol", "", "http", "protocol for connecting to the beacon node.")
captureCmd.PersistentFlags().IntVarP(&bcBootRetryInterval, "bc.bootRetryInterval", "", 30, "The amount of time to wait between retries while booting the application") captureCmd.PersistentFlags().IntVarP(&bcBootRetryInterval, "bc.bootRetryInterval", "", 30, "The amount of time to wait between retries while booting the application")
captureCmd.PersistentFlags().IntVarP(&bcBootMaxRetry, "bc.bootMaxRetry", "", 5, "The amount of time to wait between retries while booting the application") captureCmd.PersistentFlags().IntVarP(&bcBootMaxRetry, "bc.bootMaxRetry", "", 5, "The amount of time to wait between retries while booting the application")
captureCmd.PersistentFlags().IntVarP(&bcMaxHistoricProcessWorker, "bc.maxHistoricProcessWorker", "", 30, "The number of workers that should be actively processing slots from the ethcl.historic_process table. Be careful of system memory.") captureCmd.PersistentFlags().IntVarP(&bcMaxHistoricProcessWorker, "bc.maxHistoricProcessWorker", "", 30, "The number of workers that should be actively processing slots from the eth-beacon.historic_process table. Be careful of system memory.")
captureCmd.PersistentFlags().IntVarP(&bcUniqueNodeIdentifier, "bc.uniqueNodeIdentifier", "", 0, "The unique identifier of this application. Each application connecting to the DB should have a unique identifier.") captureCmd.PersistentFlags().IntVarP(&bcUniqueNodeIdentifier, "bc.uniqueNodeIdentifier", "", 0, "The unique identifier of this application. Each application connecting to the DB should have a unique identifier.")
captureCmd.PersistentFlags().BoolVarP(&bcCheckDb, "bc.checkDb", "", true, "Should we check to see if the slot exists in the DB before writing it?") captureCmd.PersistentFlags().BoolVarP(&bcCheckDb, "bc.checkDb", "", true, "Should we check to see if the slot exists in the DB before writing it?")
// err = captureCmd.MarkPersistentFlagRequired("bc.address") // err = captureCmd.MarkPersistentFlagRequired("bc.address")
@ -103,9 +103,9 @@ func init() {
// exitErr(err) // exitErr(err)
//// Known Gaps specific //// Known Gaps specific
captureCmd.PersistentFlags().BoolVarP(&kgProcessGaps, "kg.processKnownGaps", "", true, "Should we process the slots within the ethcl.known_gaps table.") captureCmd.PersistentFlags().BoolVarP(&kgProcessGaps, "kg.processKnownGaps", "", true, "Should we process the slots within the eth-beacon.known_gaps table.")
captureCmd.PersistentFlags().IntVarP(&kgTableIncrement, "kg.increment", "", 10000, "The max slots within a single entry to the known_gaps table.") captureCmd.PersistentFlags().IntVarP(&kgTableIncrement, "kg.increment", "", 10000, "The max slots within a single entry to the known_gaps table.")
captureCmd.PersistentFlags().IntVarP(&kgMaxWorker, "kg.maxKnownGapsWorker", "", 30, "The number of workers that should be actively processing slots from the ethcl.known_gaps table. Be careful of system memory.") captureCmd.PersistentFlags().IntVarP(&kgMaxWorker, "kg.maxKnownGapsWorker", "", 30, "The number of workers that should be actively processing slots from the eth-beacon.known_gaps table. Be careful of system memory.")
// Prometheus Specific // Prometheus Specific
captureCmd.PersistentFlags().BoolVarP(&pmMetrics, "pm.metrics", "", true, "Should we capture prometheus metrics.") captureCmd.PersistentFlags().BoolVarP(&pmMetrics, "pm.metrics", "", true, "Should we capture prometheus metrics.")

10
cmd/full.go
View File

@ -24,9 +24,9 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipld-ethcl-indexer/internal/boot" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -110,9 +110,9 @@ func startFullProcessing() {
// Shutdown when the time is right. // Shutdown when the time is right.
err = shutdown.ShutdownFull(ctx, KgCancel, hpCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc) err = shutdown.ShutdownFull(ctx, KgCancel, hpCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
if err != nil { if err != nil {
loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!") loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
} else { } else {
log.Info("Gracefully shutdown ipld-ethcl-indexer") log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
} }
} }

10
cmd/head.go
View File

@ -26,9 +26,9 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipld-ethcl-indexer/internal/boot" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -84,9 +84,9 @@ func startHeadTracking() {
// Shutdown when the time is right. // Shutdown when the time is right.
err = shutdown.ShutdownHeadTracking(ctx, KgCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc) err = shutdown.ShutdownHeadTracking(ctx, KgCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
if err != nil { if err != nil {
loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!") loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
} else { } else {
log.Info("Gracefully shutdown ipld-ethcl-indexer") log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
} }
} }

12
cmd/historic.go
View File

@ -25,10 +25,10 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipld-ethcl-indexer/internal/boot" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -95,9 +95,9 @@ func startHistoricProcessing() {
// Shutdown when the time is right. // Shutdown when the time is right.
err = shutdown.ShutdownHistoricProcessing(ctx, kgCancel, hpCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc) err = shutdown.ShutdownHistoricProcessing(ctx, kgCancel, hpCancel, notifierCh, maxWaitSecondsShutdown, Db, Bc)
if err != nil { if err != nil {
loghelper.LogError(err).Error("Ungracefully Shutdown ipld-ethcl-indexer!") loghelper.LogError(err).Error("Ungracefully Shutdown ipld-eth-beacon-indexer!")
} else { } else {
log.Info("Gracefully shutdown ipld-ethcl-indexer") log.Info("Gracefully shutdown ipld-eth-beacon-indexer")
} }
} }

10
cmd/root.go
View File

@ -32,7 +32,7 @@ var (
// rootCmd represents the base command when called without any subcommands // rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{ var rootCmd = &cobra.Command{
Use: "ipld-ethcl-indexer", Use: "ipld-eth-beacon-indexer",
Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.", Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.",
Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain. Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain.
It can either do this will keeping track of head, or backfilling historic data.`, It can either do this will keeping track of head, or backfilling historic data.`,
@ -127,9 +127,9 @@ func init() {
// will be global for your application. // will be global for your application.
// Optional Flags // Optional Flags
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-eth-beacon-indexer.yaml)")
rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging") rootCmd.PersistentFlags().String("log.file", "ipld-eth-beacon-indexer.log", "file path for logging")
rootCmd.PersistentFlags().Bool("log.output", true, "Should we log to STDOUT") rootCmd.PersistentFlags().Bool("log.output", true, "Should we log to STDOUT")
rootCmd.PersistentFlags().String("log.format", "json", "json or text") rootCmd.PersistentFlags().String("log.format", "json", "json or text")
@ -160,10 +160,10 @@ func initConfig() {
home, err := os.UserHomeDir() home, err := os.UserHomeDir()
cobra.CheckErr(err) cobra.CheckErr(err)
// Search config in home directory with name ".ipld-ethcl-indexer" (without extension). // Search config in home directory with name ".ipld-eth-beacon-indexer" (without extension).
viper.AddConfigPath(home) viper.AddConfigPath(home)
viper.SetConfigType("yaml") viper.SetConfigType("yaml")
viper.SetConfigName(".ipld-ethcl-indexer") viper.SetConfigName(".ipld-eth-beacon-indexer")
} }
viper.AutomaticEnv() // read in environment variables that match viper.AutomaticEnv() // read in environment variables that match

4
cmd/version.go
View File

@ -20,7 +20,7 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
v "github.com/vulcanize/ipld-ethcl-indexer/pkg/version" v "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/version"
) )
var ( var (
@ -47,7 +47,7 @@ to quickly create a Cobra application.`,
Patch: Patch, Patch: Patch,
Meta: Meta, Meta: Meta,
} }
log.Infof("ipld-ethcl-indexer version: %s", version.GetVersionWithMeta()) log.Infof("ipld-eth-beacon-indexer version: %s", version.GetVersionWithMeta())
fmt.Println(version.GetVersionWithMeta()) fmt.Println(version.GetVersionWithMeta())
}, },
} }

10
entrypoint.sh
View File

@ -1,9 +1,9 @@
#!/bin/bash #!/bin/bash
sleep 10 sleep 10
echo "Starting ipld-ethcl-indexer" echo "Starting ipld-eth-beacon-indexer"
echo /root/ipld-ethcl-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \ echo /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \
--db.password $DB_PASSWORD \ --db.password $DB_PASSWORD \
--db.port $DB_PORT \ --db.port $DB_PORT \
--db.username $DB_USER \ --db.username $DB_USER \
@ -15,7 +15,7 @@ echo /root/ipld-ethcl-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \
--t.skipSync=$SKIP_SYNC \ --t.skipSync=$SKIP_SYNC \
--kg.increment $KNOWN_GAP_INCREMENT --kg.increment $KNOWN_GAP_INCREMENT
/root/ipld-ethcl-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \ /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \
--db.password $DB_PASSWORD \ --db.password $DB_PASSWORD \
--db.port $DB_PORT \ --db.port $DB_PORT \
--db.username $DB_USER \ --db.username $DB_USER \
@ -30,10 +30,10 @@ echo /root/ipld-ethcl-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \
rv=$? rv=$?
if [ $rv != 0 ]; then if [ $rv != 0 ]; then
echo "ipld-ethcl-indexer startup failed" echo "ipld-eth-beacon-indexer startup failed"
echo 1 > /root/HEALTH echo 1 > /root/HEALTH
else else
echo "ipld-ethcl-indexer startup succeeded" echo "ipld-eth-beacon-indexer startup succeeded"
echo 0 > /root/HEALTH echo 0 > /root/HEALTH
fi fi

2
example.ipld-ethcl-indexer-config.json → example.ipld-eth-beacon-indexer-config.json
View File

@ -24,7 +24,7 @@
"log": { "log": {
"level": "debug", "level": "debug",
"output": true, "output": true,
"file": "./ipld-ethcl-indexer.log", "file": "./ipld-eth-beacon-indexer.log",
"format": "json" "format": "json"
}, },
"kg": { "kg": {

2
go.mod
View File

@ -1,4 +1,4 @@
module github.com/vulcanize/ipld-ethcl-indexer module github.com/vulcanize/ipld-eth-beacon-indexer
go 1.18 go 1.18

6
internal/boot/boot.go
View File

@ -22,9 +22,9 @@ import (
"time" "time"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql/postgres"
) )
var ( var (

2
internal/boot/boot_test.go
View File

@ -20,7 +20,7 @@ import (
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/ipld-ethcl-indexer/internal/boot" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
) )
var _ = Describe("Boot", func() { var _ = Describe("Boot", func() {

8
internal/shutdown/shutdown.go
View File

@ -20,10 +20,10 @@ import (
"os" "os"
"time" "time"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/gracefulshutdown" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/gracefulshutdown"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
// Shutdown all the internal services for the application. // Shutdown all the internal services for the application.

10
internal/shutdown/shutdown_test.go
View File

@ -28,11 +28,11 @@ import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/r3labs/sse" "github.com/r3labs/sse"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/internal/boot" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
"github.com/vulcanize/ipld-ethcl-indexer/internal/shutdown" "github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/gracefulshutdown" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/gracefulshutdown"
) )
var ( var (

1611
ipld-eth-beacon-indexer.log Normal file
View File

File diff suppressed because it is too large Load Diff

2
main.go
View File

@ -19,7 +19,7 @@ Copyright © 2022 NAME HERE <EMAIL ADDRESS>
*/ */
package main package main
import "github.com/vulcanize/ipld-ethcl-indexer/cmd" import "github.com/vulcanize/ipld-eth-beacon-indexer/cmd"
func main() { func main() {
cmd.Execute() cmd.Execute()

2
pkg/beaconclient/beaconclient.go
View File

@ -22,7 +22,7 @@ import (
"github.com/r3labs/sse" "github.com/r3labs/sse"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
) )
// TODO: Use prysms config values instead of hardcoding them here. // TODO: Use prysms config values instead of hardcoding them here.

2
pkg/beaconclient/capturehead.go
View File

@ -21,7 +21,7 @@ import (
"time" "time"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
// This function will perform all the heavy lifting for tracking the head of the chain. // This function will perform all the heavy lifting for tracking the head of the chain.

44
pkg/beaconclient/capturehead_test.go
View File

@ -39,9 +39,9 @@ import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql/postgres"
) )
var ( var (
@ -474,16 +474,16 @@ func setUpTest(config Config, maxSlot string) *beaconclient.BeaconClient {
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
// Drop all records from the DB. // Drop all records from the DB.
clearEthclDbTables(db) clearEthBeaconDbTables(db)
// Add an slot to the ethcl.slots table so it we can control how known_gaps are handled. // Add an slot to the eth_beacon.slots table so it we can control how known_gaps are handled.
writeSlot(db, maxSlot) writeSlot(db, maxSlot)
bc.Db = db bc.Db = db
return bc return bc
} }
// A helper function to validate the expected output from the ethcl.slots table. // A helper function to validate the expected output from the eth_beacon.slots table.
func validateSlot(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctEpoch int, correctStatus string) { func validateSlot(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctEpoch int, correctStatus string) {
epoch, dbSlot, blockRoot, stateRoot, status := queryDbSlotAndBlock(bc.Db, headMessage.Slot, headMessage.Block) epoch, dbSlot, blockRoot, stateRoot, status := queryDbSlotAndBlock(bc.Db, headMessage.Slot, headMessage.Block)
log.Info("validateSlot: ", headMessage) log.Info("validateSlot: ", headMessage)
@ -496,7 +496,7 @@ func validateSlot(bc *beaconclient.BeaconClient, headMessage beaconclient.Head,
Expect(status).To(Equal(correctStatus)) Expect(status).To(Equal(correctStatus))
} }
// A helper function to validate the expected output from the ethcl.signed_beacon_block table. // A helper function to validate the expected output from the eth_beacon.signed_block table.
func validateSignedBeaconBlock(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctParentRoot string, correctEth1BlockHash string, correctMhKey string) { func validateSignedBeaconBlock(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctParentRoot string, correctEth1BlockHash string, correctMhKey string) {
dbSlot, blockRoot, parentRoot, eth1BlockHash, mhKey := queryDbSignedBeaconBlock(bc.Db, headMessage.Slot, headMessage.Block) dbSlot, blockRoot, parentRoot, eth1BlockHash, mhKey := queryDbSignedBeaconBlock(bc.Db, headMessage.Slot, headMessage.Block)
log.Info("validateSignedBeaconBlock: ", headMessage) log.Info("validateSignedBeaconBlock: ", headMessage)
@ -510,7 +510,7 @@ func validateSignedBeaconBlock(bc *beaconclient.BeaconClient, headMessage beacon
} }
// A helper function to validate the expected output from the ethcl.beacon_state table. // A helper function to validate the expected output from the eth_beacon.state table.
func validateBeaconState(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctMhKey string) { func validateBeaconState(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctMhKey string) {
dbSlot, stateRoot, mhKey := queryDbBeaconState(bc.Db, headMessage.Slot, headMessage.State) dbSlot, stateRoot, mhKey := queryDbBeaconState(bc.Db, headMessage.Slot, headMessage.State)
log.Info("validateBeaconState: ", headMessage) log.Info("validateBeaconState: ", headMessage)
@ -550,21 +550,21 @@ func sendHeadMessage(bc *beaconclient.BeaconClient, head beaconclient.Head, maxR
} }
} }
// A helper function to query the ethcl.slots table based on the slot and block_root // A helper function to query the eth_beacon.slots table based on the slot and block_root
func queryDbSlotAndBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, int, string, string, string) { func queryDbSlotAndBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, int, string, string, string) {
sqlStatement := `SELECT epoch, slot, block_root, state_root, status FROM ethcl.slots WHERE slot=$1 AND block_root=$2;` sqlStatement := `SELECT epoch, slot, block_root, state_root, status FROM eth_beacon.slots WHERE slot=$1 AND block_root=$2;`
var epoch, slot int var epoch, slot int
var blockRoot, stateRoot, status string var blockRoot, stateRoot, status string
log.Debug("Starting to query the ethcl.slots table, ", querySlot, " ", queryBlockRoot) log.Debug("Starting to query the eth_beacon.slots table, ", querySlot, " ", queryBlockRoot)
err := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot).Scan(&epoch, &slot, &blockRoot, &stateRoot, &status) err := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot).Scan(&epoch, &slot, &blockRoot, &stateRoot, &status)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
log.Debug("Querying the ethcl.slots table complete") log.Debug("Querying the eth_beacon.slots table complete")
return epoch, slot, blockRoot, stateRoot, status return epoch, slot, blockRoot, stateRoot, status
} }
// A helper function to query the ethcl.signed_beacon_block table based on the slot and block_root. // A helper function to query the eth_beacon.signed_block table based on the slot and block_root.
func queryDbSignedBeaconBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, string, string, string, string) { func queryDbSignedBeaconBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, string, string, string, string) {
sqlStatement := `SELECT slot, block_root, parent_block_root, eth1_block_hash, mh_key FROM ethcl.signed_beacon_block WHERE slot=$1 AND block_root=$2;` sqlStatement := `SELECT slot, block_root, parent_block_root, eth1_block_hash, mh_key FROM eth_beacon.signed_block WHERE slot=$1 AND block_root=$2;`
var slot int var slot int
var blockRoot, parent_block_root, eth1_block_hash, mh_key string var blockRoot, parent_block_root, eth1_block_hash, mh_key string
row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot) row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot)
@ -573,9 +573,9 @@ func queryDbSignedBeaconBlock(db sql.Database, querySlot string, queryBlockRoot
return slot, blockRoot, parent_block_root, eth1_block_hash, mh_key return slot, blockRoot, parent_block_root, eth1_block_hash, mh_key
} }
// A helper function to query the ethcl.signed_beacon_block table based on the slot and block_root. // A helper function to query the eth_beacon.signed_block table based on the slot and block_root.
func queryDbBeaconState(db sql.Database, querySlot string, queryStateRoot string) (int, string, string) { func queryDbBeaconState(db sql.Database, querySlot string, queryStateRoot string) (int, string, string) {
sqlStatement := `SELECT slot, state_root, mh_key FROM ethcl.beacon_state WHERE slot=$1 AND state_root=$2;` sqlStatement := `SELECT slot, state_root, mh_key FROM eth_beacon.state WHERE slot=$1 AND state_root=$2;`
var slot int var slot int
var stateRoot, mh_key string var stateRoot, mh_key string
row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryStateRoot) row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryStateRoot)
@ -587,7 +587,7 @@ func queryDbBeaconState(db sql.Database, querySlot string, queryStateRoot string
// Count the entries in the knownGaps table. // Count the entries in the knownGaps table.
func countKnownGapsTable(db sql.Database) int { func countKnownGapsTable(db sql.Database) int {
var count int var count int
sqlStatement := "SELECT COUNT(*) FROM ethcl.known_gaps" sqlStatement := "SELECT COUNT(*) FROM eth_beacon.known_gaps"
err := db.QueryRow(context.Background(), sqlStatement).Scan(&count) err := db.QueryRow(context.Background(), sqlStatement).Scan(&count)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
return count return count
@ -595,7 +595,7 @@ func countKnownGapsTable(db sql.Database) int {
// Return the start and end slot // Return the start and end slot
func queryKnownGaps(db sql.Database, queryStartGap string, QueryEndGap string) (int, int) { func queryKnownGaps(db sql.Database, queryStartGap string, QueryEndGap string) (int, int) {
sqlStatement := `SELECT start_slot, end_slot FROM ethcl.known_gaps WHERE start_slot=$1 AND end_slot=$2;` sqlStatement := `SELECT start_slot, end_slot FROM eth_beacon.known_gaps WHERE start_slot=$1 AND end_slot=$2;`
var startGap, endGap int var startGap, endGap int
row := db.QueryRow(context.Background(), sqlStatement, queryStartGap, QueryEndGap) row := db.QueryRow(context.Background(), sqlStatement, queryStartGap, QueryEndGap)
err := row.Scan(&startGap, &endGap) err := row.Scan(&startGap, &endGap)
@ -603,16 +603,16 @@ func queryKnownGaps(db sql.Database, queryStartGap string, QueryEndGap string) (
return startGap, endGap return startGap, endGap
} }
// A function that will remove all entries from the ethcl tables for you. // A function that will remove all entries from the eth_beacon tables for you.
func clearEthclDbTables(db sql.Database) { func clearEthBeaconDbTables(db sql.Database) {
deleteQueries := []string{"DELETE FROM ethcl.slots;", "DELETE FROM ethcl.signed_beacon_block;", "DELETE FROM ethcl.beacon_state;", "DELETE FROM ethcl.known_gaps;", "DELETE FROM ethcl.historic_process;", "DELETE FROM public.blocks;"} deleteQueries := []string{"DELETE FROM eth_beacon.slots;", "DELETE FROM eth_beacon.signed_block;", "DELETE FROM eth_beacon.state;", "DELETE FROM eth_beacon.known_gaps;", "DELETE FROM eth_beacon.historic_process;", "DELETE FROM public.blocks;"}
for _, queries := range deleteQueries { for _, queries := range deleteQueries {
_, err := db.Exec(context.Background(), queries) _, err := db.Exec(context.Background(), queries)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
} }
} }
// Write an entry to the ethcl.slots table with just a slot number // Write an entry to the eth_beacon.slots table with just a slot number
func writeSlot(db sql.Database, slot string) { func writeSlot(db sql.Database, slot string) {
_, err := db.Exec(context.Background(), beaconclient.UpsertSlotsStmt, "0", slot, "", "", "") _, err := db.Exec(context.Background(), beaconclient.UpsertSlotsStmt, "0", slot, "", "", "")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())

6
pkg/beaconclient/capturehistoric.go
View File

@ -22,8 +22,8 @@ import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -41,7 +41,7 @@ func (bc *BeaconClient) StopHistoric(cancel context.CancelFunc) error {
log.Info("We are stopping the historical processing service.") log.Info("We are stopping the historical processing service.")
err := bc.HistoricalProcess.releaseDbLocks(cancel) err := bc.HistoricalProcess.releaseDbLocks(cancel)
if err != nil { if err != nil {
loghelper.LogError(err).WithField("uniqueIdentifier", bc.UniqueNodeIdentifier).Error("We were unable to remove the locks from the ethcl.historic_processing table. Manual Intervention is needed!") loghelper.LogError(err).WithField("uniqueIdentifier", bc.UniqueNodeIdentifier).Error("We were unable to remove the locks from the eth_beacon.historic_processing table. Manual Intervention is needed!")
} }
return nil return nil
} }

20
pkg/beaconclient/capturehistoric_test.go
View File

@ -10,19 +10,19 @@ import (
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
) )
var ( var (
kgCheckCheckedOutStmt = `SELECT * FROM ethcl.known_gaps WHERE checked_out=true ` kgCheckCheckedOutStmt = `SELECT * FROM eth_beacon.known_gaps WHERE checked_out=true `
hpCheckCheckedOutStmt = `SELECT * FROM ethcl.historic_process WHERE checked_out=true ` hpCheckCheckedOutStmt = `SELECT * FROM eth_beacon.historic_process WHERE checked_out=true `
) )
var _ = Describe("Capturehistoric", func() { var _ = Describe("Capturehistoric", func() {
Describe("Run the application in historic mode", Label("unit", "behavioral", "historical"), func() { Describe("Run the application in historic mode", Label("unit", "behavioral", "historical"), func() {
Context("Phase0 + Altairs: When we need to process a multiple blocks in a multiple entries in the ethcl.historic_process table.", Label("deb"), func() { Context("Phase0 + Altairs: When we need to process a multiple blocks in a multiple entries in the eth_beacon.historic_process table.", Label("deb"), func() {
It("Successfully Process the Blocks", func() { It("Successfully Process the Blocks", func() {
bc := setUpTest(BeaconNodeTester.TestConfig, "99") bc := setUpTest(BeaconNodeTester.TestConfig, "99")
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot) BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
@ -70,7 +70,7 @@ var _ = Describe("Capturehistoric", func() {
}) })
}) })
Describe("Running the Application to process Known Gaps", Label("unit", "behavioral", "knownGaps"), func() { Describe("Running the Application to process Known Gaps", Label("unit", "behavioral", "knownGaps"), func() {
Context("Phase0 + Altairs: When we need to process a multiple blocks in a multiple entries in the ethcl.known_gaps table.", func() { Context("Phase0 + Altairs: When we need to process a multiple blocks in a multiple entries in the eth_beacon.known_gaps table.", func() {
It("Successfully Process the Blocks", func() { It("Successfully Process the Blocks", func() {
bc := setUpTest(BeaconNodeTester.TestConfig, "99") bc := setUpTest(BeaconNodeTester.TestConfig, "99")
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot) BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
@ -170,10 +170,10 @@ var _ = Describe("Capturehistoric", func() {
}) })
}) })
// This function will write an even to the ethcl.known_gaps table // This function will write an even to the eth_beacon.known_gaps table
func (tbc TestBeaconNode) writeEventToKnownGaps(bc *beaconclient.BeaconClient, startSlot, endSlot int) { func (tbc TestBeaconNode) writeEventToKnownGaps(bc *beaconclient.BeaconClient, startSlot, endSlot int) {
log.Debug("We are writing the necessary events to batch process") log.Debug("We are writing the necessary events to batch process")
insertKnownGapsStmt := `INSERT INTO ethcl.known_gaps (start_slot, end_slot) insertKnownGapsStmt := `INSERT INTO eth_beacon.known_gaps (start_slot, end_slot)
VALUES ($1, $2);` VALUES ($1, $2);`
res, err := bc.Db.Exec(context.Background(), insertKnownGapsStmt, startSlot, endSlot) res, err := bc.Db.Exec(context.Background(), insertKnownGapsStmt, startSlot, endSlot)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -184,10 +184,10 @@ func (tbc TestBeaconNode) writeEventToKnownGaps(bc *beaconclient.BeaconClient, s
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
} }
// This function will write an even to the ethcl.known_gaps table // This function will write an even to the eth_beacon.known_gaps table
func (tbc TestBeaconNode) writeEventToHistoricProcess(bc *beaconclient.BeaconClient, startSlot, endSlot, priority int) { func (tbc TestBeaconNode) writeEventToHistoricProcess(bc *beaconclient.BeaconClient, startSlot, endSlot, priority int) {
log.Debug("We are writing the necessary events to batch process") log.Debug("We are writing the necessary events to batch process")
insertHistoricProcessingStmt := `INSERT INTO ethcl.historic_process (start_slot, end_slot, priority) insertHistoricProcessingStmt := `INSERT INTO eth_beacon.historic_process (start_slot, end_slot, priority)
VALUES ($1, $2, $3);` VALUES ($1, $2, $3);`
res, err := bc.Db.Exec(context.Background(), insertHistoricProcessingStmt, startSlot, endSlot, priority) res, err := bc.Db.Exec(context.Background(), insertHistoricProcessingStmt, startSlot, endSlot, priority)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())

2
pkg/beaconclient/checkbeaconserverstatus.go
View File

@ -25,7 +25,7 @@ import (
"sync/atomic" "sync/atomic"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
var ( var (

90
pkg/beaconclient/databasewrite.go
View File

@ -22,61 +22,61 @@ import (
"github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
var ( var (
// Statement to upsert to the ethcl.slots table. // Statement to upsert to the eth_beacon.slots table.
UpsertSlotsStmt string = ` UpsertSlotsStmt string = `
INSERT INTO ethcl.slots (epoch, slot, block_root, state_root, status) INSERT INTO eth_beacon.slots (epoch, slot, block_root, state_root, status)
VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING` VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
// Statement to upsert to the ethcl.signed_beacon_blocks table. // Statement to upsert to the eth_beacon.signed_blocks table.
UpsertSignedBeaconBlockStmt string = ` UpsertSignedBeaconBlockStmt string = `
INSERT INTO ethcl.signed_beacon_block (slot, block_root, parent_block_root, eth1_block_hash, mh_key) INSERT INTO eth_beacon.signed_block (slot, block_root, parent_block_root, eth1_block_hash, mh_key)
VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING` VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
// Statement to upsert to the ethcl.beacon_state table. // Statement to upsert to the eth_beacon.state table.
UpsertBeaconState string = ` UpsertBeaconState string = `
INSERT INTO ethcl.beacon_state (slot, state_root, mh_key) INSERT INTO eth_beacon.state (slot, state_root, mh_key)
VALUES ($1, $2, $3) ON CONFLICT (slot, state_root) DO NOTHING` VALUES ($1, $2, $3) ON CONFLICT (slot, state_root) DO NOTHING`
// Statement to upsert to the public.blocks table. // Statement to upsert to the public.blocks table.
UpsertBlocksStmt string = ` UpsertBlocksStmt string = `
INSERT INTO public.blocks (key, data) INSERT INTO public.blocks (key, data)
VALUES ($1, $2) ON CONFLICT (key) DO NOTHING` VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`
UpdateForkedStmt string = `UPDATE ethcl.slots UpdateForkedStmt string = `UPDATE eth_beacon.slots
SET status='forked' SET status='forked'
WHERE slot=$1 AND block_root<>$2 WHERE slot=$1 AND block_root<>$2
RETURNING block_root;` RETURNING block_root;`
UpdateProposedStmt string = `UPDATE ethcl.slots UpdateProposedStmt string = `UPDATE eth_beacon.slots
SET status='proposed' SET status='proposed'
WHERE slot=$1 AND block_root=$2 WHERE slot=$1 AND block_root=$2
RETURNING block_root;` RETURNING block_root;`
CheckProposedStmt string = `SELECT slot, block_root CheckProposedStmt string = `SELECT slot, block_root
FROM ethcl.slots FROM eth_beacon.slots
WHERE slot=$1 AND block_root=$2;` WHERE slot=$1 AND block_root=$2;`
// Check to see if the slot and block_root exist in ethcl.signed_beacon_block // Check to see if the slot and block_root exist in eth_beacon.signed_block
CheckSignedBeaconBlockStmt string = `SELECT slot, block_root CheckSignedBeaconBlockStmt string = `SELECT slot, block_root
FROM ethcl.signed_beacon_block FROM eth_beacon.signed_block
WHERE slot=$1 AND block_root=$2` WHERE slot=$1 AND block_root=$2`
// Check to see if the slot and state_root exist in ethcl.beacon_state // Check to see if the slot and state_root exist in eth_beacon.state
CheckBeaconStateStmt string = `SELECT slot, state_root CheckBeaconStateStmt string = `SELECT slot, state_root
FROM ethcl.beacon_state FROM eth_beacon.state
WHERE slot=$1 AND state_root=$2` WHERE slot=$1 AND state_root=$2`
// Used to get a single slot from the table if it exists // Used to get a single slot from the table if it exists
QueryBySlotStmt string = `SELECT slot QueryBySlotStmt string = `SELECT slot
FROM ethcl.slots FROM eth_beacon.slots
WHERE slot=$1` WHERE slot=$1`
// Statement to insert known_gaps. We don't pass in timestamp, we let the server take care of that one. // Statement to insert known_gaps. We don't pass in timestamp, we let the server take care of that one.
UpsertKnownGapsStmt string = ` UpsertKnownGapsStmt string = `
INSERT INTO ethcl.known_gaps (start_slot, end_slot, checked_out, reprocessing_error, entry_error, entry_process) INSERT INTO eth_beacon.known_gaps (start_slot, end_slot, checked_out, reprocessing_error, entry_error, entry_process)
VALUES ($1, $2, $3, $4, $5, $6) on CONFLICT (start_slot, end_slot) DO NOTHING` VALUES ($1, $2, $3, $4, $5, $6) on CONFLICT (start_slot, end_slot) DO NOTHING`
UpsertKnownGapsErrorStmt string = ` UpsertKnownGapsErrorStmt string = `
UPDATE ethcl.known_gaps UPDATE eth_beacon.known_gaps
SET reprocessing_error=$3, priority=priority+1 SET reprocessing_error=$3, priority=priority+1
WHERE start_slot=$1 AND end_slot=$2;` WHERE start_slot=$1 AND end_slot=$2;`
// Get the highest slot if one exists // Get the highest slot if one exists
QueryHighestSlotStmt string = "SELECT COALESCE(MAX(slot), 0) FROM ethcl.slots" QueryHighestSlotStmt string = "SELECT COALESCE(MAX(slot), 0) FROM eth_beacon.slots"
) )
// Put all functionality to prepare the write object // Put all functionality to prepare the write object
@ -123,7 +123,7 @@ func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot
// Write functions to write each all together... // Write functions to write each all together...
// Should I do one atomic write? // Should I do one atomic write?
// Create the model for the ethcl.slots table // Create the model for the eth_beacon.slots table
func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoot string, status string) { func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoot string, status string) {
dw.DbSlots = &DbSlots{ dw.DbSlots = &DbSlots{
Epoch: calculateEpoch(slot, bcSlotsPerEpoch), Epoch: calculateEpoch(slot, bcSlotsPerEpoch),
@ -136,7 +136,7 @@ func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoo
} }
// Create the model for the ethcl.signed_beacon_block table. // Create the model for the eth_beacon.signed_block table.
func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot int, blockRoot string, parentBlockRoot string, eth1BlockHash string) error { func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot int, blockRoot string, parentBlockRoot string, eth1BlockHash string) error {
mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.BlockRoot)) mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.BlockRoot))
if err != nil { if err != nil {
@ -153,7 +153,7 @@ func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot int, blockRoot stri
return nil return nil
} }
// Create the model for the ethcl.beacon_state table. // Create the model for the eth_beacon.state table.
func (dw *DatabaseWriter) prepareBeaconStateModel(slot int, stateRoot string) error { func (dw *DatabaseWriter) prepareBeaconStateModel(slot int, stateRoot string) error {
mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.StateRoot)) mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.StateRoot))
if err != nil { if err != nil {
@ -177,10 +177,10 @@ func (dw *DatabaseWriter) transactFullSlot() error {
}).Debug("Starting to write to the DB.") }).Debug("Starting to write to the DB.")
err := dw.transactSlots() err := dw.transactSlots()
if err != nil { if err != nil {
loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the ethcl.slots table...") loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon.slots table...")
return err return err
} }
log.Debug("We finished writing to the ethcl.slots table.") log.Debug("We finished writing to the eth_beacon.slots table.")
if dw.DbSlots.Status != "skipped" { if dw.DbSlots.Status != "skipped" {
//errG, _ := errgroup.WithContext(context.Background()) //errG, _ := errgroup.WithContext(context.Background())
//errG.Go(func() error { //errG.Go(func() error {
@ -190,18 +190,18 @@ func (dw *DatabaseWriter) transactFullSlot() error {
// return dw.transactBeaconState() // return dw.transactBeaconState()
//}) //})
//if err := errG.Wait(); err != nil { //if err := errG.Wait(); err != nil {
// loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the ethcl block or state table...") // loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon block or state table...")
// return err // return err
//} //}
// Might want to seperate writing to public.blocks so we can do this concurrently... // Might want to seperate writing to public.blocks so we can do this concurrently...
err := dw.transactSignedBeaconBlocks() err := dw.transactSignedBeaconBlocks()
if err != nil { if err != nil {
loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the ethcl block table...") loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon block table...")
return err return err
} }
err = dw.transactBeaconState() err = dw.transactBeaconState()
if err != nil { if err != nil {
loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the ethcl state table...") loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("We couldn't write to the eth_beacon state table...")
return err return err
} }
} }
@ -209,24 +209,24 @@ func (dw *DatabaseWriter) transactFullSlot() error {
return nil return nil
} }
// Add data for the ethcl.slots table to a transaction. For now this is only one function. // Add data for the eth_beacon.slots table to a transaction. For now this is only one function.
// But in the future if we need to incorporate any FK's or perform any actions to write to the // But in the future if we need to incorporate any FK's or perform any actions to write to the
// slots table we can do it all here. // slots table we can do it all here.
func (dw *DatabaseWriter) transactSlots() error { func (dw *DatabaseWriter) transactSlots() error {
return dw.upsertSlots() return dw.upsertSlots()
} }
// Upsert to the ethcl.slots table. // Upsert to the eth_beacon.slots table.
func (dw *DatabaseWriter) upsertSlots() error { func (dw *DatabaseWriter) upsertSlots() error {
_, err := dw.Tx.Exec(dw.Ctx, UpsertSlotsStmt, dw.DbSlots.Epoch, dw.DbSlots.Slot, dw.DbSlots.BlockRoot, dw.DbSlots.StateRoot, dw.DbSlots.Status) _, err := dw.Tx.Exec(dw.Ctx, UpsertSlotsStmt, dw.DbSlots.Epoch, dw.DbSlots.Slot, dw.DbSlots.BlockRoot, dw.DbSlots.StateRoot, dw.DbSlots.Status)
if err != nil { if err != nil {
loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the ethcl.slots table") loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the eth_beacon.slots table")
return err return err
} }
return nil return nil
} }
// Add the information for the signed_beacon_block to a transaction. // Add the information for the signed_block to a transaction.
func (dw *DatabaseWriter) transactSignedBeaconBlocks() error { func (dw *DatabaseWriter) transactSignedBeaconBlocks() error {
err := dw.upsertPublicBlocks(dw.DbSignedBeaconBlock.MhKey, dw.rawSignedBeaconBlock) err := dw.upsertPublicBlocks(dw.DbSignedBeaconBlock.MhKey, dw.rawSignedBeaconBlock)
if err != nil { if err != nil {
@ -249,17 +249,17 @@ func (dw *DatabaseWriter) upsertPublicBlocks(key string, data []byte) error {
return nil return nil
} }
// Upsert to the ethcl.signed_beacon_block table. // Upsert to the eth_beacon.signed_block table.
func (dw *DatabaseWriter) upsertSignedBeaconBlock() error { func (dw *DatabaseWriter) upsertSignedBeaconBlock() error {
_, err := dw.Tx.Exec(dw.Ctx, UpsertSignedBeaconBlockStmt, dw.DbSignedBeaconBlock.Slot, dw.DbSignedBeaconBlock.BlockRoot, dw.DbSignedBeaconBlock.ParentBlock, dw.DbSignedBeaconBlock.Eth1BlockHash, dw.DbSignedBeaconBlock.MhKey) _, err := dw.Tx.Exec(dw.Ctx, UpsertSignedBeaconBlockStmt, dw.DbSignedBeaconBlock.Slot, dw.DbSignedBeaconBlock.BlockRoot, dw.DbSignedBeaconBlock.ParentBlock, dw.DbSignedBeaconBlock.Eth1BlockHash, dw.DbSignedBeaconBlock.MhKey)
if err != nil { if err != nil {
loghelper.LogSlotError(dw.DbSlots.Slot, err).WithFields(log.Fields{"block_root": dw.DbSignedBeaconBlock.BlockRoot}).Error("Unable to write to the slot to the ethcl.signed_beacon_block table") loghelper.LogSlotError(dw.DbSlots.Slot, err).WithFields(log.Fields{"block_root": dw.DbSignedBeaconBlock.BlockRoot}).Error("Unable to write to the slot to the eth_beacon.signed_block table")
return err return err
} }
return nil return nil
} }
// Add the information for the beacon_state to a transaction. // Add the information for the state to a transaction.
func (dw *DatabaseWriter) transactBeaconState() error { func (dw *DatabaseWriter) transactBeaconState() error {
err := dw.upsertPublicBlocks(dw.DbBeaconState.MhKey, dw.rawBeaconState) err := dw.upsertPublicBlocks(dw.DbBeaconState.MhKey, dw.rawBeaconState)
if err != nil { if err != nil {
@ -272,11 +272,11 @@ func (dw *DatabaseWriter) transactBeaconState() error {
return nil return nil
} }
// Upsert to the ethcl.beacon_state table. // Upsert to the eth_beacon.state table.
func (dw *DatabaseWriter) upsertBeaconState() error { func (dw *DatabaseWriter) upsertBeaconState() error {
_, err := dw.Tx.Exec(dw.Ctx, UpsertBeaconState, dw.DbBeaconState.Slot, dw.DbBeaconState.StateRoot, dw.DbBeaconState.MhKey) _, err := dw.Tx.Exec(dw.Ctx, UpsertBeaconState, dw.DbBeaconState.Slot, dw.DbBeaconState.StateRoot, dw.DbBeaconState.MhKey)
if err != nil { if err != nil {
loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the ethcl.beacon_state table") loghelper.LogSlotError(dw.DbSlots.Slot, err).Error("Unable to write to the slot to the eth_beacon.state table")
return err return err
} }
return nil return nil
@ -351,7 +351,7 @@ func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *
func updateForked(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) { func updateForked(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) {
res, err := tx.Exec(ctx, UpdateForkedStmt, slot, latestBlockRoot) res, err := tx.Exec(ctx, UpdateForkedStmt, slot, latestBlockRoot)
if err != nil { if err != nil {
loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the ethcl.slots table with the forked slots") loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the forked slots")
return 0, err return 0, err
} }
count, err := res.RowsAffected() count, err := res.RowsAffected()
@ -366,7 +366,7 @@ func updateForked(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot s
func updateProposed(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) { func updateProposed(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) {
res, err := tx.Exec(ctx, UpdateProposedStmt, slot, latestBlockRoot) res, err := tx.Exec(ctx, UpdateProposedStmt, slot, latestBlockRoot)
if err != nil { if err != nil {
loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the ethcl.slots table with the proposed slot.") loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the proposed slot.")
return 0, err return 0, err
} }
count, err := res.RowsAffected() count, err := res.RowsAffected()
@ -447,7 +447,7 @@ func writeKnownGaps(db sql.Database, tableIncrement int, startSlot int, endSlot
} }
} }
// A function to upsert a single entry to the ethcl.known_gaps table. // A function to upsert a single entry to the eth_beacon.known_gaps table.
func upsertKnownGaps(tx sql.Tx, ctx context.Context, knModel DbKnownGaps, metric *BeaconClientMetrics) { func upsertKnownGaps(tx sql.Tx, ctx context.Context, knModel DbKnownGaps, metric *BeaconClientMetrics) {
_, err := tx.Exec(ctx, UpsertKnownGapsStmt, knModel.StartSlot, knModel.EndSlot, _, err := tx.Exec(ctx, UpsertKnownGapsStmt, knModel.StartSlot, knModel.EndSlot,
knModel.CheckedOut, knModel.ReprocessingError, knModel.EntryError, knModel.EntryProcess) knModel.CheckedOut, knModel.ReprocessingError, knModel.EntryError, knModel.EntryProcess)
@ -456,12 +456,12 @@ func upsertKnownGaps(tx sql.Tx, ctx context.Context, knModel DbKnownGaps, metric
"err": err, "err": err,
"startSlot": knModel.StartSlot, "startSlot": knModel.StartSlot,
"endSlot": knModel.EndSlot, "endSlot": knModel.EndSlot,
}).Fatal("We are unable to write to the ethcl.known_gaps table!!! We will stop the application because of that.") }).Fatal("We are unable to write to the eth_beacon.known_gaps table!!! We will stop the application because of that.")
} }
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"startSlot": knModel.StartSlot, "startSlot": knModel.StartSlot,
"endSlot": knModel.EndSlot, "endSlot": knModel.EndSlot,
}).Warn("A new gap has been added to the ethcl.known_gaps table.") }).Warn("A new gap has been added to the eth_beacon.known_gaps table.")
metric.IncrementKnownGapsInserts(1) metric.IncrementKnownGapsInserts(1)
} }
@ -539,8 +539,8 @@ func isSlotProcessed(db sql.Database, checkProcessStmt string, slot string) (boo
return false, nil return false, nil
} }
// Check to see if this slot is in the DB. Check ethcl.slots, ethcl.signed_beacon_block // Check to see if this slot is in the DB. Check eth_beacon.slots, eth_beacon.signed_block
// and ethcl.beacon_state. If the slot exists, return true // and eth_beacon.state. If the slot exists, return true
func IsSlotInDb(ctx context.Context, db sql.Database, slot string, blockRoot string, stateRoot string) (bool, error) { func IsSlotInDb(ctx context.Context, db sql.Database, slot string, blockRoot string, stateRoot string) (bool, error) {
var ( var (
isInBeaconState bool isInBeaconState bool
@ -555,7 +555,7 @@ func IsSlotInDb(ctx context.Context, db sql.Database, slot string, blockRoot str
default: default:
isInBeaconState, err = checkSlotAndRoot(db, CheckBeaconStateStmt, slot, stateRoot) isInBeaconState, err = checkSlotAndRoot(db, CheckBeaconStateStmt, slot, stateRoot)
if err != nil { if err != nil {
loghelper.LogError(err).Error("Unable to check if the slot and stateroot exist in ethcl.beacon_state") loghelper.LogError(err).Error("Unable to check if the slot and stateroot exist in eth_beacon.state")
} }
return err return err
} }
@ -567,7 +567,7 @@ func IsSlotInDb(ctx context.Context, db sql.Database, slot string, blockRoot str
default: default:
isInSignedBeaconBlock, err = checkSlotAndRoot(db, CheckSignedBeaconBlockStmt, slot, blockRoot) isInSignedBeaconBlock, err = checkSlotAndRoot(db, CheckSignedBeaconBlockStmt, slot, blockRoot)
if err != nil { if err != nil {
loghelper.LogError(err).Error("Unable to check if the slot and block_root exist in ethcl.signed_beacon_block") loghelper.LogError(err).Error("Unable to check if the slot and block_root exist in eth_beacon.signed_block")
} }
return err return err
} }

2
pkg/beaconclient/healthcheck.go
View File

@ -20,7 +20,7 @@ import (
"net/http" "net/http"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
// This function will ensure that we can connect to the beacon client. // This function will ensure that we can connect to the beacon client.

2
pkg/beaconclient/healthcheck_test.go
View File

@ -20,7 +20,7 @@ import (
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
beaconclient "github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient" beaconclient "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
) )
var _ = Describe("Healthcheck", func() { var _ = Describe("Healthcheck", func() {

2
pkg/beaconclient/incomingsse.go
View File

@ -22,7 +22,7 @@ import (
"time" "time"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )

2
pkg/beaconclient/metrics.go
View File

@ -20,7 +20,7 @@ import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
//Create a metric struct and register each channel with prometheus //Create a metric struct and register each channel with prometheus

8
pkg/beaconclient/models.go
View File

@ -51,7 +51,7 @@ type ChainReorg struct {
ExecutionOptimistic bool `json:"execution_optimistic"` ExecutionOptimistic bool `json:"execution_optimistic"`
} }
// A struct to capture whats being written to the ethcl.slots table. // A struct to capture whats being written to the eth-beacon.slots table.
type DbSlots struct { type DbSlots struct {
Epoch string // The epoch. Epoch string // The epoch.
Slot string // The slot. Slot string // The slot.
@ -60,7 +60,7 @@ type DbSlots struct {
Status string // The status, it can be proposed | forked | skipped. Status string // The status, it can be proposed | forked | skipped.
} }
// A struct to capture whats being written to ethcl.signed_beacon_block table. // A struct to capture whats being written to eth-beacon.signed_block table.
type DbSignedBeaconBlock struct { type DbSignedBeaconBlock struct {
Slot string // The slot. Slot string // The slot.
BlockRoot string // The block root BlockRoot string // The block root
@ -70,14 +70,14 @@ type DbSignedBeaconBlock struct {
} }
// A struct to capture whats being written to ethcl.beacon_state table. // A struct to capture whats being written to eth-beacon.state table.
type DbBeaconState struct { type DbBeaconState struct {
Slot string // The slot. Slot string // The slot.
StateRoot string // The state root StateRoot string // The state root
MhKey string // The ipld multihash key. MhKey string // The ipld multihash key.
} }
// A structure to capture whats being written to the ethcl.known_gaps table. // A structure to capture whats being written to the eth-beacon.known_gaps table.
type DbKnownGaps struct { type DbKnownGaps struct {
StartSlot string // The start slot for known_gaps, inclusive. StartSlot string // The start slot for known_gaps, inclusive.
EndSlot string // The end slot for known_gaps, inclusive. EndSlot string // The end slot for known_gaps, inclusive.

2
pkg/beaconclient/multihash.go
View File

@ -20,7 +20,7 @@ import (
dshelp "github.com/ipfs/go-ipfs-ds-help" dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/multiformats/go-multihash" "github.com/multiformats/go-multihash"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
const SSZ_SHA2_256_PREFIX uint64 = 0xb502 const SSZ_SHA2_256_PREFIX uint64 = 0xb502

32
pkg/beaconclient/processhistoric.go
View File

@ -26,27 +26,27 @@ import (
"github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
var ( var (
// Get a single highest priority and non-checked out row row from ethcl.historical_process // Get a single highest priority and non-checked out row row from eth_beacon.historical_process
getHpEntryStmt string = `SELECT start_slot, end_slot FROM ethcl.historic_process getHpEntryStmt string = `SELECT start_slot, end_slot FROM eth_beacon.historic_process
WHERE checked_out=false WHERE checked_out=false
ORDER BY priority ASC ORDER BY priority ASC
LIMIT 1;` LIMIT 1;`
// Used to periodically check to see if there is a new entry in the ethcl.historic_process table. // Used to periodically check to see if there is a new entry in the eth_beacon.historic_process table.
checkHpEntryStmt string = `SELECT * FROM ethcl.historic_process WHERE checked_out=false;` checkHpEntryStmt string = `SELECT * FROM eth_beacon.historic_process WHERE checked_out=false;`
// Used to checkout a row from the ethcl.historic_process table // Used to checkout a row from the eth_beacon.historic_process table
lockHpEntryStmt string = `UPDATE ethcl.historic_process lockHpEntryStmt string = `UPDATE eth_beacon.historic_process
SET checked_out=true, checked_out_by=$3 SET checked_out=true, checked_out_by=$3
WHERE start_slot=$1 AND end_slot=$2;` WHERE start_slot=$1 AND end_slot=$2;`
// Used to delete an entry from the ethcl.historic_process table // Used to delete an entry from the eth_beacon.historic_process table
deleteHpEntryStmt string = `DELETE FROM ethcl.historic_process deleteHpEntryStmt string = `DELETE FROM eth_beacon.historic_process
WHERE start_slot=$1 AND end_slot=$2;` WHERE start_slot=$1 AND end_slot=$2;`
// Used to update every single row that this node has checked out. // Used to update every single row that this node has checked out.
releaseHpLockStmt string = `UPDATE ethcl.historic_process releaseHpLockStmt string = `UPDATE eth_beacon.historic_process
SET checked_out=false, checked_out_by=null SET checked_out=false, checked_out_by=null
WHERE checked_out_by=$1` WHERE checked_out_by=$1`
) )
@ -80,20 +80,20 @@ func (hp HistoricProcessing) handleProcessingErrors(ctx context.Context, errMess
} }
} }
// "un"-checkout the rows held by this DB in the ethcl.historical_process table. // "un"-checkout the rows held by this DB in the eth_beacon.historical_process table.
func (hp HistoricProcessing) releaseDbLocks(cancel context.CancelFunc) error { func (hp HistoricProcessing) releaseDbLocks(cancel context.CancelFunc) error {
cancel() cancel()
log.Debug("Updating all the entries to ethcl.historical processing") log.Debug("Updating all the entries to eth_beacon.historical processing")
log.Debug("Db: ", hp.db) log.Debug("Db: ", hp.db)
log.Debug("hp.uniqueNodeIdentifier ", hp.uniqueNodeIdentifier) log.Debug("hp.uniqueNodeIdentifier ", hp.uniqueNodeIdentifier)
res, err := hp.db.Exec(context.Background(), releaseHpLockStmt, hp.uniqueNodeIdentifier) res, err := hp.db.Exec(context.Background(), releaseHpLockStmt, hp.uniqueNodeIdentifier)
if err != nil { if err != nil {
return fmt.Errorf("Unable to remove lock from ethcl.historical_processing table for node %d, error is %e", hp.uniqueNodeIdentifier, err) return fmt.Errorf("Unable to remove lock from eth_beacon.historical_processing table for node %d, error is %e", hp.uniqueNodeIdentifier, err)
} }
log.Debug("Update all the entries to ethcl.historical processing") log.Debug("Update all the entries to eth_beacon.historical processing")
rows, err := res.RowsAffected() rows, err := res.RowsAffected()
if err != nil { if err != nil {
return fmt.Errorf("Unable to calculated number of rows affected by releasing locks from ethcl.historical_processing table for node %d, error is %e", hp.uniqueNodeIdentifier, err) return fmt.Errorf("Unable to calculated number of rows affected by releasing locks from eth_beacon.historical_processing table for node %d, error is %e", hp.uniqueNodeIdentifier, err)
} }
log.WithField("rowCount", rows).Info("Released historicalProcess locks for specified rows.") log.WithField("rowCount", rows).Info("Released historicalProcess locks for specified rows.")
return nil return nil

28
pkg/beaconclient/processknowngaps.go
View File

@ -23,30 +23,30 @@ import (
"strconv" "strconv"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
var ( var (
// Get a single non-checked out row row from ethcl.known_gaps. // Get a single non-checked out row row from eth_beacon.known_gaps.
getKgEntryStmt string = `SELECT start_slot, end_slot FROM ethcl.known_gaps getKgEntryStmt string = `SELECT start_slot, end_slot FROM eth_beacon.known_gaps
WHERE checked_out=false WHERE checked_out=false
ORDER BY priority ASC ORDER BY priority ASC
LIMIT 1;` LIMIT 1;`
// Used to periodically check to see if there is a new entry in the ethcl.known_gaps table. // Used to periodically check to see if there is a new entry in the eth_beacon.known_gaps table.
checkKgEntryStmt string = `SELECT * FROM ethcl.known_gaps WHERE checked_out=false;` checkKgEntryStmt string = `SELECT * FROM eth_beacon.known_gaps WHERE checked_out=false;`
// Used to checkout a row from the ethcl.known_gaps table // Used to checkout a row from the eth_beacon.known_gaps table
lockKgEntryStmt string = `UPDATE ethcl.known_gaps lockKgEntryStmt string = `UPDATE eth_beacon.known_gaps
SET checked_out=true, checked_out_by=$3 SET checked_out=true, checked_out_by=$3
WHERE start_slot=$1 AND end_slot=$2;` WHERE start_slot=$1 AND end_slot=$2;`
// Used to delete an entry from the knownGaps table // Used to delete an entry from the knownGaps table
deleteKgEntryStmt string = `DELETE FROM ethcl.known_gaps deleteKgEntryStmt string = `DELETE FROM eth_beacon.known_gaps
WHERE start_slot=$1 AND end_slot=$2;` WHERE start_slot=$1 AND end_slot=$2;`
// Used to check to see if a single slot exists in the known_gaps table. // Used to check to see if a single slot exists in the known_gaps table.
checkKgSingleSlotStmt string = `SELECT start_slot, end_slot FROM ethcl.known_gaps checkKgSingleSlotStmt string = `SELECT start_slot, end_slot FROM eth_beacon.known_gaps
WHERE start_slot=$1 AND end_slot=$2;` WHERE start_slot=$1 AND end_slot=$2;`
// Used to update every single row that this node has checked out. // Used to update every single row that this node has checked out.
releaseKgLockStmt string = `UPDATE ethcl.known_gaps releaseKgLockStmt string = `UPDATE eth_beacon.known_gaps
SET checked_out=false, checked_out_by=null SET checked_out=false, checked_out_by=null
WHERE checked_out_by=$1` WHERE checked_out_by=$1`
) )
@ -71,7 +71,7 @@ func (bc *BeaconClient) StopKnownGapsProcessing(cancel context.CancelFunc) error
log.Info("We are stopping the known gaps processing service.") log.Info("We are stopping the known gaps processing service.")
err := bc.KnownGapsProcess.releaseDbLocks(cancel) err := bc.KnownGapsProcess.releaseDbLocks(cancel)
if err != nil { if err != nil {
loghelper.LogError(err).WithField("uniqueIdentifier", bc.UniqueNodeIdentifier).Error("We were unable to remove the locks from the ethcl.known_gaps table. Manual Intervention is needed!") loghelper.LogError(err).WithField("uniqueIdentifier", bc.UniqueNodeIdentifier).Error("We were unable to remove the locks from the eth_beacon.known_gaps table. Manual Intervention is needed!")
} }
return nil return nil
} }
@ -96,7 +96,7 @@ func (kgp KnownGapsProcessing) handleProcessingErrors(ctx context.Context, errMe
// Check to see if this if this entry already exists. // Check to see if this if this entry already exists.
res, err := kgp.db.Exec(context.Background(), checkKgSingleSlotStmt, errMs.slot, errMs.slot) res, err := kgp.db.Exec(context.Background(), checkKgSingleSlotStmt, errMs.slot, errMs.slot)
if err != nil { if err != nil {
loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).Error("Unable to see if this slot is in the ethcl.known_gaps table") loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).Error("Unable to see if this slot is in the eth_beacon.known_gaps table")
} }
rows, err := res.RowsAffected() rows, err := res.RowsAffected()
@ -123,7 +123,7 @@ func (kgp KnownGapsProcessing) handleProcessingErrors(ctx context.Context, errMe
// Updated checked_out column for the uniqueNodeIdentifier. // Updated checked_out column for the uniqueNodeIdentifier.
func (kgp KnownGapsProcessing) releaseDbLocks(cancel context.CancelFunc) error { func (kgp KnownGapsProcessing) releaseDbLocks(cancel context.CancelFunc) error {
cancel() cancel()
log.Debug("Updating all the entries to ethcl.known_gaps") log.Debug("Updating all the entries to eth_beacon.known_gaps")
log.Debug("Db: ", kgp.db) log.Debug("Db: ", kgp.db)
log.Debug("kgp.uniqueNodeIdentifier ", kgp.uniqueNodeIdentifier) log.Debug("kgp.uniqueNodeIdentifier ", kgp.uniqueNodeIdentifier)
res, err := kgp.db.Exec(context.Background(), releaseKgLockStmt, kgp.uniqueNodeIdentifier) res, err := kgp.db.Exec(context.Background(), releaseKgLockStmt, kgp.uniqueNodeIdentifier)

8
pkg/beaconclient/processslot.go
View File

@ -36,8 +36,8 @@ import (
state "github.com/prysmaticlabs/prysm/beacon-chain/state" state "github.com/prysmaticlabs/prysm/beacon-chain/state"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -70,8 +70,8 @@ type ProcessSlot struct {
// DB Write objects // DB Write objects
DbSlotsModel *DbSlots // The model being written to the slots table. DbSlotsModel *DbSlots // The model being written to the slots table.
DbSignedBeaconBlockModel *DbSignedBeaconBlock // The model being written to the signed_beacon_block table. DbSignedBeaconBlockModel *DbSignedBeaconBlock // The model being written to the signed_block table.
DbBeaconState *DbBeaconState // The model being written to the beacon_state table. DbBeaconState *DbBeaconState // The model being written to the state table.
} }
// This function will do all the work to process the slot and write it to the DB. // This function will do all the work to process the slot and write it to the DB.

2
pkg/beaconclient/queryserver.go
View File

@ -24,7 +24,7 @@ import (
"net/http" "net/http"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
// A helper function to query endpoints that utilize slots. // A helper function to query endpoints that utilize slots.

2
pkg/beaconclient/systemvalidation_test.go
View File

@ -7,7 +7,7 @@ import (
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
//. "github.com/onsi/gomega" //. "github.com/onsi/gomega"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/beaconclient" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
) )
var ( var (

4
pkg/database/sql/postgres/database.go
View File

@ -20,8 +20,8 @@ import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
var _ sql.Database = &DB{} var _ sql.Database = &DB{}

2
pkg/database/sql/postgres/pgx.go
View File

@ -23,7 +23,7 @@ import (
"github.com/jackc/pgconn" "github.com/jackc/pgconn"
"github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool" "github.com/jackc/pgx/v4/pgxpool"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
) )
// pgxDriver driver, implements sql.Driver // pgxDriver driver, implements sql.Driver

6
pkg/database/sql/postgres/pgx_test.go
View File

@ -23,9 +23,9 @@ import (
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql/postgres"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/testhelpers"
) )
var _ = Describe("Pgx", func() { var _ = Describe("Pgx", func() {

2
pkg/gracefulshutdown/gracefulshutdown.go
View File

@ -25,7 +25,7 @@ import (
"time" "time"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" "github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
) )
// operation is a clean up function on shutting down // operation is a clean up function on shutting down