Compare commits
18 Commits
feature/st
...
main
Author | SHA1 | Date | |
---|---|---|---|
8ded6246dc | |||
|
4381c77a9b | ||
|
3c80a207fd | ||
8033a956d4 | |||
|
815a5654b2 | ||
|
1bd64266bc | ||
9e96e4225a | |||
a1658421a1 | |||
4fe07c45f1 | |||
56ae836b09 | |||
20804bd30c | |||
0c77e6fd1d | |||
27fa54c6dc | |||
|
fbde7cd175 | ||
|
b3c2d5915c | ||
|
bb8c861bfd | ||
|
802f80057b | ||
|
cfbad8092d |
11
.github/workflows/generic-testing.yml
vendored
11
.github/workflows/generic-testing.yml
vendored
@ -17,7 +17,7 @@ on:
|
||||
|
||||
env:
|
||||
stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '7fb664270a0ba09e2caa3095e8c91f3fdb5b38af' }}
|
||||
ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
|
||||
ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref || '6b38fe9b18f7b19a803c626b742cafdccc1a2365' }}
|
||||
ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }}
|
||||
GOPATH: /tmp/go
|
||||
jobs:
|
||||
@ -50,6 +50,7 @@ jobs:
|
||||
echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
|
||||
echo eth_beacon_config_file=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer/config/cicd/boot.ipld-eth-beacon-indexer.json >> ./config.sh
|
||||
echo eth_beacon_capture_mode=boot >> ./config.sh
|
||||
echo CAPTURE_MODE=boot >> config.sh
|
||||
cat ./config.sh
|
||||
|
||||
- name: Run docker compose
|
||||
@ -64,7 +65,11 @@ jobs:
|
||||
- name: Check to make sure HEALTH file is present
|
||||
shell: bash
|
||||
run: |
|
||||
until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" --env-file ./config.sh cp ipld-eth-beacon-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
|
||||
until $(docker compose \
|
||||
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-db.yml" \
|
||||
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" \
|
||||
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-lighthouse.yml" \
|
||||
--env-file ./config.sh cp ipld-eth-beacon-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
|
||||
cat ./HEALTH
|
||||
if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" cp ipld-eth-beacon-indexer:/root/ipld-eth-beacon-indexer.log . && cat ipld-eth-beacon-indexer.log && (exit 1); fi
|
||||
|
||||
@ -122,7 +127,7 @@ jobs:
|
||||
|
||||
- name: Install packages
|
||||
run: |
|
||||
go install github.com/onsi/ginkgo/v2/ginkgo@latest
|
||||
go install github.com/onsi/ginkgo/v2/ginkgo@2.1.4
|
||||
which ginkgo
|
||||
|
||||
- name: Run the tests using Make
|
||||
|
29
.github/workflows/issues-notion-sync.yml
vendored
Normal file
29
.github/workflows/issues-notion-sync.yml
vendored
Normal file
@ -0,0 +1,29 @@
|
||||
name: Notion Sync
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
issues:
|
||||
types:
|
||||
[
|
||||
opened,
|
||||
edited,
|
||||
labeled,
|
||||
unlabeled,
|
||||
assigned,
|
||||
unassigned,
|
||||
milestoned,
|
||||
demilestoned,
|
||||
reopened,
|
||||
closed,
|
||||
]
|
||||
|
||||
jobs:
|
||||
notion_job:
|
||||
runs-on: ubuntu-latest
|
||||
name: Add GitHub Issues to Notion
|
||||
steps:
|
||||
- name: Add GitHub Issues to Notion
|
||||
uses: vulcanize/notion-github-action@v1.2.4-issueid
|
||||
with:
|
||||
notion-token: ${{ secrets.NOTION_TOKEN }}
|
||||
notion-db: ${{ secrets.NOTION_DATABASE }}
|
4
.github/workflows/on-pr.yml
vendored
4
.github/workflows/on-pr.yml
vendored
@ -24,8 +24,8 @@ on:
|
||||
- ".github/workflows/on-pr.yml"
|
||||
- ".github/workflows/tests.yml"
|
||||
- "**"
|
||||
schedule:
|
||||
- cron: '0 13 * * *' # Must be single quotes!!
|
||||
#schedule:
|
||||
# - cron: '0 13 * * *' # Must be single quotes!!
|
||||
|
||||
jobs:
|
||||
trigger-tests:
|
||||
|
4
.github/workflows/system-tests.yml
vendored
4
.github/workflows/system-tests.yml
vendored
@ -15,7 +15,7 @@ on:
|
||||
required: true
|
||||
env:
|
||||
stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '7fb664270a0ba09e2caa3095e8c91f3fdb5b38af' }}
|
||||
ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
|
||||
ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref || '6b38fe9b18f7b19a803c626b742cafdccc1a2365' }}
|
||||
GOPATH: /tmp/go
|
||||
bc_protocol: "http"
|
||||
bc_address: ${{secrets.BC_ADDRESS}}
|
||||
@ -29,7 +29,7 @@ env:
|
||||
|
||||
jobs:
|
||||
system-testing:
|
||||
runs-on: self-hosted
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create GOPATH
|
||||
run: mkdir -p /tmp/go
|
||||
|
1
.gitignore
vendored
1
.gitignore
vendored
@ -11,3 +11,4 @@ ipld-eth-beacon-indexer.log
|
||||
ipld-eth-beacon-indexer
|
||||
config/local.ipld-eth-beacon-indexer-config.json
|
||||
config/docker.ipld-eth-beacon-indexer-config.json
|
||||
.idea/*
|
@ -12,9 +12,10 @@ COPY . .
|
||||
RUN GCO_ENABLED=0 GOOS=linux go build -race -ldflags="-s -w" -o ipld-eth-beacon-indexer .
|
||||
RUN chmod +x ipld-eth-beacon-indexer
|
||||
|
||||
FROM frolvlad/alpine-bash:latest
|
||||
RUN apk --no-cache add ca-certificates libstdc++
|
||||
FROM alpine:latest
|
||||
RUN apk --no-cache add ca-certificates libstdc++ busybox-extras gettext libintl bash gawk sed grep bc coreutils
|
||||
WORKDIR /root/
|
||||
COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-beacon-indexer/ipld-eth-beacon-indexer /root/ipld-eth-beacon-indexer
|
||||
ADD entrypoint.sh .
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
ADD ipld-eth-beacon-config-docker.json .
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
|
24
Makefile
24
Makefile
@ -65,7 +65,17 @@ integration-test-local-no-race:
|
||||
unit-test-local:
|
||||
go vet ./...
|
||||
go fmt ./...
|
||||
$(GINKGO) -r --label-filter unit \
|
||||
$(GINKGO) -r --label-filter 'unit && !flaky' \
|
||||
--randomize-all --randomize-suites \
|
||||
--flake-attempts=3 \
|
||||
--fail-on-pending --keep-going \
|
||||
--trace
|
||||
|
||||
.PHONY: unit-test-local-bellatrix
|
||||
unit-test-local-bellatrix:
|
||||
go vet ./...
|
||||
go fmt ./...
|
||||
$(GINKGO) -r --label-filter 'unit && !flaky && bellatrix' \
|
||||
--randomize-all --randomize-suites \
|
||||
--flake-attempts=3 \
|
||||
--fail-on-pending --keep-going \
|
||||
@ -101,6 +111,16 @@ system-test-local:
|
||||
--fail-on-pending --keep-going \
|
||||
--trace
|
||||
|
||||
.PHONY: e2e-test-local
|
||||
e2e-test-local:
|
||||
go vet ./...
|
||||
go fmt ./...
|
||||
$(GINKGO) -r --label-filter e2e \
|
||||
--randomize-all --randomize-suites \
|
||||
--fail-on-pending --keep-going \
|
||||
--trace
|
||||
|
||||
|
||||
.PHONY: build
|
||||
build:
|
||||
go fmt ./...
|
||||
@ -109,4 +129,4 @@ build:
|
||||
## Build docker image
|
||||
.PHONY: docker-build
|
||||
docker-build:
|
||||
docker build -t vulcanize/ipld-eth-beacon-indexer .
|
||||
docker build -t vulcanize/ipld-eth-beacon-indexer .
|
||||
|
@ -47,7 +47,8 @@ func bootApp() {
|
||||
|
||||
Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
|
||||
viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
|
||||
viper.GetInt("kg.increment"), "boot", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
|
||||
viper.GetInt("kg.increment"), "boot", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"),
|
||||
viper.GetBool("bc.performBeaconBlockProcessing"), viper.GetBool("bc.performBeaconStateProcessing"))
|
||||
if err != nil {
|
||||
StopApplicationPreBoot(err, Db)
|
||||
}
|
||||
|
10
cmd/full.go
10
cmd/full.go
@ -19,6 +19,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
|
||||
"strconv"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
@ -34,7 +35,7 @@ import (
|
||||
var fullCmd = &cobra.Command{
|
||||
Use: "full",
|
||||
Short: "Capture all components of the application (head and historical)",
|
||||
Long: `Capture all components of the application (head and historical`,
|
||||
Long: `Capture all components of the application (head and historical)`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
startFullProcessing()
|
||||
},
|
||||
@ -62,7 +63,8 @@ func startFullProcessing() {
|
||||
|
||||
Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
|
||||
viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
|
||||
viper.GetInt("kg.increment"), "head", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
|
||||
viper.GetInt("kg.increment"), "head", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"),
|
||||
viper.GetBool("bc.performBeaconBlockProcessing"), viper.GetBool("bc.performBeaconStateProcessing"))
|
||||
if err != nil {
|
||||
StopApplicationPreBoot(err, Db)
|
||||
}
|
||||
@ -80,7 +82,7 @@ func startFullProcessing() {
|
||||
|
||||
errG, _ := errgroup.WithContext(context.Background())
|
||||
errG.Go(func() error {
|
||||
errs := Bc.CaptureHistoric(hpContext, viper.GetInt("bc.maxHistoricProcessWorker"))
|
||||
errs := Bc.CaptureHistoric(hpContext, viper.GetInt("bc.maxHistoricProcessWorker"), beaconclient.Slot(viper.GetUint64("bc.minimumSlot")))
|
||||
if len(errs) != 0 {
|
||||
if len(errs) != 0 {
|
||||
log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing historic events")
|
||||
@ -94,7 +96,7 @@ func startFullProcessing() {
|
||||
go func() {
|
||||
errG := new(errgroup.Group)
|
||||
errG.Go(func() error {
|
||||
errs := Bc.ProcessKnownGaps(kgCtx, viper.GetInt("kg.maxKnownGapsWorker"))
|
||||
errs := Bc.ProcessKnownGaps(kgCtx, viper.GetInt("kg.maxKnownGapsWorker"), beaconclient.Slot(viper.GetUint64("kg.minimumSlot")))
|
||||
if len(errs) != 0 {
|
||||
log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
|
||||
return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
|
||||
|
@ -19,6 +19,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
@ -50,7 +51,8 @@ func startHeadTracking() {
|
||||
|
||||
Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
|
||||
viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
|
||||
viper.GetInt("kg.increment"), "head", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
|
||||
viper.GetInt("kg.increment"), "head", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"),
|
||||
viper.GetBool("bc.performBeaconBlockProcessing"), viper.GetBool("bc.performBeaconStateProcessing"))
|
||||
if err != nil {
|
||||
StopApplicationPreBoot(err, Db)
|
||||
}
|
||||
@ -68,7 +70,7 @@ func startHeadTracking() {
|
||||
go func() {
|
||||
errG := new(errgroup.Group)
|
||||
errG.Go(func() error {
|
||||
errs := Bc.ProcessKnownGaps(kgCtx, viper.GetInt("kg.maxKnownGapsWorker"))
|
||||
errs := Bc.ProcessKnownGaps(kgCtx, viper.GetInt("kg.maxKnownGapsWorker"), beaconclient.Slot(viper.GetUint64("kg.minimumSlot")))
|
||||
if len(errs) != 0 {
|
||||
log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
|
||||
return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
|
||||
|
@ -19,6 +19,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
@ -50,7 +51,8 @@ func startHistoricProcessing() {
|
||||
|
||||
Bc, Db, err := boot.BootApplicationWithRetry(ctx, viper.GetString("db.address"), viper.GetInt("db.port"), viper.GetString("db.name"), viper.GetString("db.username"), viper.GetString("db.password"), viper.GetString("db.driver"),
|
||||
viper.GetString("bc.address"), viper.GetInt("bc.port"), viper.GetString("bc.connectionProtocol"), viper.GetString("bc.type"), viper.GetInt("bc.bootRetryInterval"), viper.GetInt("bc.bootMaxRetry"),
|
||||
viper.GetInt("kg.increment"), "historic", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"))
|
||||
viper.GetInt("kg.increment"), "historic", viper.GetBool("t.skipSync"), viper.GetInt("bc.uniqueNodeIdentifier"), viper.GetBool("bc.checkDb"),
|
||||
viper.GetBool("bc.performBeaconBlockProcessing"), viper.GetBool("bc.performBeaconStateProcessing"))
|
||||
if err != nil {
|
||||
StopApplicationPreBoot(err, Db)
|
||||
}
|
||||
@ -64,7 +66,7 @@ func startHistoricProcessing() {
|
||||
|
||||
errG, _ := errgroup.WithContext(context.Background())
|
||||
errG.Go(func() error {
|
||||
errs := Bc.CaptureHistoric(hpContext, viper.GetInt("bc.maxHistoricProcessWorker"))
|
||||
errs := Bc.CaptureHistoric(hpContext, viper.GetInt("bc.maxHistoricProcessWorker"), beaconclient.Slot(viper.GetUint64("bc.minimumSlot")))
|
||||
if len(errs) != 0 {
|
||||
if len(errs) != 0 {
|
||||
log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing historic events")
|
||||
@ -79,7 +81,7 @@ func startHistoricProcessing() {
|
||||
go func() {
|
||||
errG := new(errgroup.Group)
|
||||
errG.Go(func() error {
|
||||
errs := Bc.ProcessKnownGaps(kgContext, viper.GetInt("kg.maxKnownGapsWorker"))
|
||||
errs := Bc.ProcessKnownGaps(kgContext, viper.GetInt("kg.maxKnownGapsWorker"), beaconclient.Slot(viper.GetUint64("kg.minimumSlot")))
|
||||
if len(errs) != 0 {
|
||||
log.WithFields(log.Fields{"errs": errs}).Error("All errors when processing knownGaps")
|
||||
return fmt.Errorf("Application ended because there were too many error when attempting to process knownGaps")
|
||||
|
@ -1,40 +0,0 @@
|
||||
{
|
||||
"db": {
|
||||
"address": "localhost",
|
||||
"password": "password",
|
||||
"port": 8076,
|
||||
"username": "vdbm",
|
||||
"name": "vulcanize_testing",
|
||||
"driver": "PGX"
|
||||
},
|
||||
"bc": {
|
||||
"address": "localhost",
|
||||
"port": 5052,
|
||||
"type": "lighthouse",
|
||||
"bootRetryInterval": 30,
|
||||
"bootMaxRetry": 5,
|
||||
"maxHistoricProcessWorker": 2,
|
||||
"connectionProtocol": "http",
|
||||
"uniqueNodeIdentifier": 100,
|
||||
"checkDb": true
|
||||
},
|
||||
"t": {
|
||||
"skipSync": true
|
||||
},
|
||||
"log": {
|
||||
"level": "debug",
|
||||
"output": true,
|
||||
"file": "./ipld-eth-beacon-indexer.log",
|
||||
"format": "json"
|
||||
},
|
||||
"kg": {
|
||||
"increment": 10000,
|
||||
"processKnownGaps": true,
|
||||
"maxKnownGapsWorker": 2
|
||||
},
|
||||
"pm": {
|
||||
"address": "localhost",
|
||||
"port": 9000,
|
||||
"metrics": true
|
||||
}
|
||||
}
|
1
config/example.ipld-eth-beacon-indexer-config.json
Symbolic link
1
config/example.ipld-eth-beacon-indexer-config.json
Symbolic link
@ -0,0 +1 @@
|
||||
../ipld-eth-beacon-config.json
|
40
config/tel.ipld-eth-beacon-indexer-config.json
Normal file
40
config/tel.ipld-eth-beacon-indexer-config.json
Normal file
@ -0,0 +1,40 @@
|
||||
{
|
||||
"db": {
|
||||
"address": "localhost",
|
||||
"password": "secret12",
|
||||
"port": 45432,
|
||||
"username": "postgres",
|
||||
"name": "postgres",
|
||||
"driver": "PGX"
|
||||
},
|
||||
"bc": {
|
||||
"address": "localhost",
|
||||
"port": 8001,
|
||||
"type": "lighthouse",
|
||||
"bootRetryInterval": 30,
|
||||
"bootMaxRetry": 5,
|
||||
"maxHistoricProcessWorker": 2,
|
||||
"connectionProtocol": "http",
|
||||
"uniqueNodeIdentifier": 100,
|
||||
"checkDb": true
|
||||
},
|
||||
"t": {
|
||||
"skipSync": true
|
||||
},
|
||||
"log": {
|
||||
"level": "debug",
|
||||
"output": true,
|
||||
"file": "./ipld-eth-beacon-indexer.log",
|
||||
"format": "json"
|
||||
},
|
||||
"kg": {
|
||||
"increment": 10000,
|
||||
"processKnownGaps": true,
|
||||
"maxKnownGapsWorker": 2
|
||||
},
|
||||
"pm": {
|
||||
"address": "localhost",
|
||||
"port": 9000,
|
||||
"metrics": true
|
||||
}
|
||||
}
|
@ -3,7 +3,10 @@
|
||||
sleep 10
|
||||
echo "Starting ipld-eth-beacon-indexer"
|
||||
|
||||
cat /root/ipld-eth-beacon-config-docker.json | envsubst > /root/ipld-eth-beacon-config.json
|
||||
|
||||
echo /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
|
||||
env
|
||||
|
||||
if [ ${CAPTURE_MODE} == "boot" ]; then
|
||||
/root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
|
||||
@ -14,11 +17,13 @@ if [ ${CAPTURE_MODE} == "boot" ]; then
|
||||
else
|
||||
echo "ipld-eth-beacon-indexer boot succeeded"
|
||||
fi
|
||||
echo $rv > /root/HEALTH
|
||||
echo $rv
|
||||
cat /root/ipld-eth-beacon-indexer.output
|
||||
|
||||
echo $rv > /root/HEALTH
|
||||
echo $rv
|
||||
cat /root/ipld-eth-beacon-indexer.output
|
||||
|
||||
tail -f /dev/null
|
||||
else
|
||||
exec /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /root/ipld-eth-beacon-indexer.output
|
||||
fi
|
||||
exec /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json > /dev/null &
|
||||
tail -F ipld-eth-beacon-indexer.log
|
||||
fi
|
||||
|
115
go.mod
115
go.mod
@ -5,106 +5,109 @@ go 1.18
|
||||
require (
|
||||
github.com/ipfs/go-ipfs-blockstore v1.2.0
|
||||
github.com/ipfs/go-ipfs-ds-help v1.1.0
|
||||
github.com/jackc/pgconn v1.12.0
|
||||
github.com/multiformats/go-multihash v0.1.0
|
||||
github.com/jackc/pgconn v1.13.0
|
||||
github.com/multiformats/go-multihash v0.2.1
|
||||
github.com/onsi/ginkgo/v2 v2.1.4
|
||||
github.com/onsi/gomega v1.19.0
|
||||
github.com/prometheus/client_golang v1.12.1
|
||||
github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.13.0
|
||||
github.com/protolambda/zrnt v0.28.0
|
||||
github.com/protolambda/ztyp v0.2.2
|
||||
github.com/r3labs/sse/v2 v2.8.1
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
|
||||
github.com/benbjohnson/clock v1.3.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
|
||||
github.com/ethereum/go-ethereum v1.10.17 // indirect
|
||||
github.com/deckarep/golang-set v1.8.0 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
|
||||
github.com/ethereum/go-ethereum v1.10.25 // indirect
|
||||
github.com/go-ole/go-ole v1.2.1 // indirect
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 // indirect
|
||||
github.com/gorilla/websocket v1.4.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
|
||||
github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e // indirect
|
||||
github.com/holiman/uint256 v1.2.0 // indirect
|
||||
github.com/ipfs/bbloom v0.0.4 // indirect
|
||||
github.com/ipfs/go-block-format v0.0.3 // indirect
|
||||
github.com/ipfs/go-cid v0.1.0 // indirect
|
||||
github.com/ipfs/go-datastore v0.5.0 // indirect
|
||||
github.com/ipfs/go-cid v0.3.2 // indirect
|
||||
github.com/ipfs/go-datastore v0.6.0 // indirect
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
|
||||
github.com/ipfs/go-ipld-format v0.3.0 // indirect
|
||||
github.com/ipfs/go-ipld-format v0.4.0 // indirect
|
||||
github.com/ipfs/go-log v1.0.5 // indirect
|
||||
github.com/ipfs/go-log/v2 v2.5.0 // indirect
|
||||
github.com/ipfs/go-log/v2 v2.5.1 // indirect
|
||||
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgio v1.0.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgproto3/v2 v2.3.0 // indirect
|
||||
github.com/jackc/pgproto3/v2 v2.3.1 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
|
||||
github.com/jackc/pgtype v1.11.0 // indirect
|
||||
github.com/jackc/puddle v1.2.1 // indirect
|
||||
github.com/jackc/pgtype v1.12.0 // indirect
|
||||
github.com/jackc/puddle v1.3.0 // indirect
|
||||
github.com/jbenet/goprocess v0.1.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
|
||||
github.com/kilic/bls12-381 v0.1.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
|
||||
github.com/lib/pq v1.10.5 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
|
||||
github.com/minio/highwayhash v1.0.1 // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/multiformats/go-base32 v0.0.4 // indirect
|
||||
github.com/multiformats/go-base32 v0.1.0 // indirect
|
||||
github.com/multiformats/go-base36 v0.1.0 // indirect
|
||||
github.com/multiformats/go-multibase v0.0.3 // indirect
|
||||
github.com/multiformats/go-multibase v0.1.1 // indirect
|
||||
github.com/multiformats/go-varint v0.0.6 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.32.1 // indirect
|
||||
github.com/prometheus/procfs v0.7.3 // indirect
|
||||
github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect
|
||||
github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220303211031-f753e083138c // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.8.0 // indirect
|
||||
github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e // indirect
|
||||
github.com/rogpeppe/go-internal v1.8.0 // indirect
|
||||
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/supranational/blst v0.3.5 // indirect
|
||||
github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect
|
||||
github.com/urfave/cli/v2 v2.3.0 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.5 // indirect
|
||||
github.com/tklauser/numcpus v0.2.2 // indirect
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.21.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 // indirect
|
||||
google.golang.org/grpc v1.46.0 // indirect
|
||||
google.golang.org/protobuf v1.28.0 // indirect
|
||||
go.uber.org/zap v1.23.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
|
||||
golang.org/x/net v0.0.0-20220907135653-1e95f45603a7 // indirect
|
||||
golang.org/x/tools v0.1.10 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
|
||||
lukechampine.com/blake3 v1.1.7 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ferranbt/fastssz v0.0.0-20220303160658-88bb965b6747 // indirect
|
||||
github.com/fsnotify/fsnotify v1.5.4 // indirect
|
||||
github.com/georgysavva/scany v0.3.0
|
||||
github.com/georgysavva/scany v1.2.0
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/jackc/pgx/v4 v4.16.0
|
||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||
github.com/jackc/pgx/v4 v4.17.2
|
||||
github.com/jarcoal/httpmock v1.2.0
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.0 // indirect
|
||||
github.com/prysmaticlabs/prysm v1.4.2-0.20220504145118-df695346a53c
|
||||
github.com/spf13/afero v1.8.2 // indirect
|
||||
github.com/spf13/cast v1.4.1 // indirect
|
||||
github.com/spf13/cobra v1.4.0
|
||||
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
|
||||
github.com/spf13/afero v1.9.2 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/cobra v1.5.0
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/spf13/viper v1.11.0
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
|
||||
github.com/spf13/viper v1.13.0
|
||||
github.com/subosito/gotenv v1.4.1 // indirect
|
||||
golang.org/x/sync v0.0.0-20220907140024-f12130a52804
|
||||
golang.org/x/sys v0.0.0-20220907062415-87db552b00fd // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
gopkg.in/ini.v1 v1.66.4 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
@ -42,11 +42,11 @@ var (
|
||||
//
|
||||
// 3. Make sure the node is synced, unless disregardSync is true.
|
||||
func BootApplication(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string,
|
||||
bcAddress string, bcPort int, bcConnectionProtocol string, bcKgTableIncrement int, disregardSync bool, uniqueNodeIdentifier int, checkDb bool) (*beaconclient.BeaconClient, sql.Database, error) {
|
||||
bcAddress string, bcPort int, bcConnectionProtocol string, bcKgTableIncrement int, disregardSync bool, uniqueNodeIdentifier int, checkDb bool, performBeaconBlockProcessing bool, performBeaconStateProcessing bool) (*beaconclient.BeaconClient, sql.Database, error) {
|
||||
log.Info("Booting the Application")
|
||||
|
||||
log.Debug("Creating the Beacon Client")
|
||||
Bc, err := beaconclient.CreateBeaconClient(ctx, bcConnectionProtocol, bcAddress, bcPort, bcKgTableIncrement, uniqueNodeIdentifier, checkDb)
|
||||
Bc, err := beaconclient.CreateBeaconClient(ctx, bcConnectionProtocol, bcAddress, bcPort, bcKgTableIncrement, uniqueNodeIdentifier, checkDb, performBeaconBlockProcessing, performBeaconStateProcessing)
|
||||
if err != nil {
|
||||
return Bc, nil, err
|
||||
}
|
||||
@ -86,14 +86,15 @@ func BootApplication(ctx context.Context, dbHostname string, dbPort int, dbName
|
||||
// Add retry logic to ensure that we are give the Beacon Client and the DB time to start.
|
||||
func BootApplicationWithRetry(ctx context.Context, dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string,
|
||||
bcAddress string, bcPort int, bcConnectionProtocol string, bcType string, bcRetryInterval int, bcMaxRetry int, bcKgTableIncrement int,
|
||||
startUpMode string, disregardSync bool, uniqueNodeIdentifier int, checkDb bool) (*beaconclient.BeaconClient, sql.Database, error) {
|
||||
startUpMode string, disregardSync bool, uniqueNodeIdentifier int, checkDb bool, performBeaconBlockProcessing bool, performBeaconStateProcessing bool) (*beaconclient.BeaconClient, sql.Database, error) {
|
||||
var err error
|
||||
|
||||
if bcMaxRetry < 0 {
|
||||
i := 0
|
||||
for {
|
||||
BC, DB, err = BootApplication(ctx, dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName,
|
||||
bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, disregardSync, uniqueNodeIdentifier, checkDb)
|
||||
bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, disregardSync, uniqueNodeIdentifier, checkDb,
|
||||
performBeaconBlockProcessing, performBeaconStateProcessing)
|
||||
if err != nil {
|
||||
log.WithFields(log.Fields{
|
||||
"retryNumber": i,
|
||||
@ -108,7 +109,8 @@ func BootApplicationWithRetry(ctx context.Context, dbHostname string, dbPort int
|
||||
} else {
|
||||
for i := 0; i < bcMaxRetry; i++ {
|
||||
BC, DB, err = BootApplication(ctx, dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName,
|
||||
bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, disregardSync, uniqueNodeIdentifier, checkDb)
|
||||
bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, disregardSync, uniqueNodeIdentifier, checkDb,
|
||||
performBeaconBlockProcessing, performBeaconStateProcessing)
|
||||
if err != nil {
|
||||
log.WithFields(log.Fields{
|
||||
"retryNumber": i,
|
||||
|
@ -25,66 +25,68 @@ import (
|
||||
|
||||
var _ = Describe("Boot", func() {
|
||||
var (
|
||||
dbAddress string = "localhost"
|
||||
dbPort int = 8076
|
||||
dbName string = "vulcanize_testing"
|
||||
dbUsername string = "vdbm"
|
||||
dbPassword string = "password"
|
||||
dbDriver string = "PGX"
|
||||
bcAddress string = "localhost"
|
||||
bcPort int = 5052
|
||||
bcConnectionProtocol string = "http"
|
||||
bcType string = "lighthouse"
|
||||
bcBootRetryInterval int = 1
|
||||
bcBootMaxRetry int = 5
|
||||
bcKgTableIncrement int = 10
|
||||
bcUniqueIdentifier int = 100
|
||||
bcCheckDb bool = false
|
||||
dbAddress string = "localhost"
|
||||
dbPort int = 8076
|
||||
dbName string = "vulcanize_testing"
|
||||
dbUsername string = "vdbm"
|
||||
dbPassword string = "password"
|
||||
dbDriver string = "PGX"
|
||||
bcAddress string = "localhost"
|
||||
bcPort int = 5052
|
||||
bcConnectionProtocol string = "http"
|
||||
bcType string = "lighthouse"
|
||||
bcBootRetryInterval int = 1
|
||||
bcBootMaxRetry int = 5
|
||||
bcKgTableIncrement int = 10
|
||||
bcUniqueIdentifier int = 100
|
||||
bcCheckDb bool = false
|
||||
bcProcessBeaconBlocks bool = true
|
||||
bcProcessBeaconState bool = true
|
||||
)
|
||||
Describe("Booting the application", Label("integration"), func() {
|
||||
Context("When the DB and BC are both up and running, we skip checking for a synced head, and we are processing head", func() {
|
||||
It("Should connect successfully", func() {
|
||||
_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", true, bcUniqueIdentifier, bcCheckDb)
|
||||
_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", true, bcUniqueIdentifier, bcCheckDb, bcProcessBeaconBlocks, bcProcessBeaconState)
|
||||
defer db.Close()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
})
|
||||
Context("When the DB and BC are both up and running, we skip checking for a synced head, and we are processing historic ", func() {
|
||||
It("Should connect successfully", func() {
|
||||
_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "historic", true, bcUniqueIdentifier, bcCheckDb)
|
||||
_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "historic", true, bcUniqueIdentifier, bcCheckDb, bcProcessBeaconBlocks, bcProcessBeaconState)
|
||||
defer db.Close()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
})
|
||||
Context("When the DB and BC are both up and running, and we check for a synced head", func() {
|
||||
It("Should not connect successfully", func() {
|
||||
_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", false, bcUniqueIdentifier, bcCheckDb)
|
||||
_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", false, bcUniqueIdentifier, bcCheckDb, bcProcessBeaconBlocks, bcProcessBeaconState)
|
||||
defer db.Close()
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
Context("When the DB and BC are both up and running, we skip checking for a synced head, but the unique identifier is 0", func() {
|
||||
It("Should not connect successfully", func() {
|
||||
_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", false, 0, bcCheckDb)
|
||||
_, db, err := boot.BootApplicationWithRetry(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", false, 0, bcCheckDb, bcProcessBeaconBlocks, bcProcessBeaconState)
|
||||
defer db.Close()
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
Context("When the DB is running but not the BC", func() {
|
||||
It("Should not connect successfully", func() {
|
||||
_, _, err := boot.BootApplication(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
|
||||
_, _, err := boot.BootApplication(context.Background(), dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb, bcProcessBeaconBlocks, bcProcessBeaconState)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
Context("When the BC is running but not the DB", func() {
|
||||
It("Should not connect successfully", func() {
|
||||
_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
|
||||
_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb, bcProcessBeaconBlocks, bcProcessBeaconState)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
Context("When neither the BC or DB are running", func() {
|
||||
It("Should not connect successfully", func() {
|
||||
_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb)
|
||||
_, _, err := boot.BootApplication(context.Background(), "hi", 10, dbName, dbUsername, dbPassword, dbDriver, "hi", 100, bcConnectionProtocol, bcKgTableIncrement, true, bcUniqueIdentifier, bcCheckDb, bcProcessBeaconBlocks, bcProcessBeaconState)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
@ -26,7 +26,7 @@ import (
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/r3labs/sse"
|
||||
"github.com/r3labs/sse/v2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/internal/boot"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/internal/shutdown"
|
||||
@ -51,6 +51,8 @@ var (
|
||||
bcKgTableIncrement int = 10
|
||||
bcUniqueIdentifier int = 100
|
||||
bcCheckDb bool = false
|
||||
bcProcessBeaconBlocks bool = true
|
||||
bcProcessBeaconState bool = true
|
||||
maxWaitSecondsShutdown time.Duration = time.Duration(1) * time.Second
|
||||
DB sql.Database
|
||||
BC *beaconclient.BeaconClient
|
||||
@ -63,7 +65,7 @@ var _ = Describe("Shutdown", func() {
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
BC, DB, err = boot.BootApplicationWithRetry(ctx, dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress,
|
||||
bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", true, bcUniqueIdentifier, bcCheckDb)
|
||||
bcPort, bcConnectionProtocol, bcType, bcBootRetryInterval, bcBootMaxRetry, bcKgTableIncrement, "head", true, bcUniqueIdentifier, bcCheckDb, bcProcessBeaconBlocks, bcProcessBeaconState)
|
||||
notifierCh = make(chan os.Signal, 1)
|
||||
Expect(err).To(BeNil())
|
||||
})
|
||||
|
44
ipld-eth-beacon-config-docker.json
Normal file
44
ipld-eth-beacon-config-docker.json
Normal file
@ -0,0 +1,44 @@
|
||||
{
|
||||
"db": {
|
||||
"address": "${POSTGRES_HOST}",
|
||||
"password": "${POSTGRES_PASSWORD}",
|
||||
"port": ${POSTGRES_PORT},
|
||||
"username": "${POSTGRES_USER}",
|
||||
"name": "${POSTGRES_DB}",
|
||||
"driver": "PGX"
|
||||
},
|
||||
"bc": {
|
||||
"address": "${LIGHTHOUSE_HOST}",
|
||||
"port": ${LIGHTHOUSE_PORT},
|
||||
"type": "lighthouse",
|
||||
"bootRetryInterval": 30,
|
||||
"bootMaxRetry": 5,
|
||||
"maxHistoricProcessWorker": ${BC_MAX_HISTORIC_PROCESS_WORKER},
|
||||
"connectionProtocol": "${LIGHTHOUSE_PROTOCOL}",
|
||||
"uniqueNodeIdentifier": ${BC_UNIQUE_NODE_IDENTIFIER},
|
||||
"checkDb": ${BC_CHECK_DB},
|
||||
"performBeaconStateProcessing": ${BC_BEACON_STATE_PROCESSING_ENABLED},
|
||||
"performBeaconBlockProcessing": ${BC_BEACON_BLOCK_PROCESSING_ENABLED},
|
||||
"minimumSlot": ${BC_MINIMUM_SLOT}
|
||||
},
|
||||
"t": {
|
||||
"skipSync": true
|
||||
},
|
||||
"log": {
|
||||
"level": "${LOG_LEVEL}",
|
||||
"output": true,
|
||||
"file": "./ipld-eth-beacon-indexer.log",
|
||||
"format": "json"
|
||||
},
|
||||
"kg": {
|
||||
"increment": ${KG_INCREMENT},
|
||||
"processKnownGaps": ${KG_PROCESS_KNOWN_GAPS_ENABLED},
|
||||
"maxKnownGapsWorker": ${KG_MAX_KNOWN_GAPS_WORKER},
|
||||
"minimumSlot": ${KG_MINIMUM_SLOT}
|
||||
},
|
||||
"pm": {
|
||||
"address": "${PROM_HOST}",
|
||||
"port": ${PROM_PORT},
|
||||
"metrics": ${PROM_METRICS_ENABLED}
|
||||
}
|
||||
}
|
@ -18,11 +18,11 @@ package beaconclient
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
|
||||
"github.com/r3labs/sse"
|
||||
"github.com/r3labs/sse/v2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
|
||||
"math/rand"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TODO: Use prysms config values instead of hardcoding them here.
|
||||
@ -37,27 +37,29 @@ var (
|
||||
BcBlockRootEndpoint = func(slot string) string {
|
||||
return "/eth/v1/beacon/blocks/" + slot + "/root"
|
||||
}
|
||||
bcSlotsPerEpoch = 32 // Number of slots in a single Epoch
|
||||
bcSlotsPerEpoch uint64 = 32 // Number of slots in a single Epoch
|
||||
//bcSlotPerHistoricalVector = 8192 // The number of slots in a historic vector.
|
||||
//bcFinalizedTopicEndpoint = "/eth/v1/events?topics=finalized_checkpoint" // Endpoint used to subscribe to the head of the chain
|
||||
)
|
||||
|
||||
// A struct that capture the Beacon Server that the Beacon Client will be interacting with and querying.
|
||||
type BeaconClient struct {
|
||||
Context context.Context // A context generic context with multiple uses.
|
||||
ServerEndpoint string // What is the endpoint of the beacon server.
|
||||
Db sql.Database // Database object used for reads and writes.
|
||||
Metrics *BeaconClientMetrics // An object used to keep track of certain BeaconClient Metrics.
|
||||
KnownGapTableIncrement int // The max number of slots within a single known_gaps table entry.
|
||||
UniqueNodeIdentifier int // The unique identifier within the cluster of this individual node.
|
||||
KnownGapsProcess KnownGapsProcessing // object keeping track of knowngaps processing
|
||||
CheckDb bool // Should we check the DB to see if the slot exists before processing it?
|
||||
Context context.Context // A context generic context with multiple uses.
|
||||
ServerEndpoint string // What is the endpoint of the beacon server.
|
||||
Db sql.Database // Database object used for reads and writes.
|
||||
Metrics *BeaconClientMetrics // An object used to keep track of certain BeaconClient Metrics.
|
||||
KnownGapTableIncrement int // The max number of slots within a single known_gaps table entry.
|
||||
UniqueNodeIdentifier int // The unique identifier within the cluster of this individual node.
|
||||
KnownGapsProcess KnownGapsProcessing // object keeping track of knowngaps processing
|
||||
CheckDb bool // Should we check the DB to see if the slot exists before processing it?
|
||||
PerformBeaconStateProcessing bool // Should we process BeaconStates?
|
||||
PerformBeaconBlockProcessing bool // Should we process BeaconBlocks?
|
||||
|
||||
// Used for Head Tracking
|
||||
|
||||
PerformHeadTracking bool // Should we track head?
|
||||
StartingSlot int // If we're performing head tracking. What is the first slot we processed.
|
||||
PreviousSlot int // Whats the previous slot we processed
|
||||
StartingSlot Slot // If we're performing head tracking. What is the first slot we processed.
|
||||
PreviousSlot Slot // Whats the previous slot we processed
|
||||
PreviousBlockRoot string // Whats the previous block root, used to check the next blocks parent.
|
||||
HeadTracking *SseEvents[Head] // Track the head block
|
||||
ReOrgTracking *SseEvents[ChainReorg] // Track all Reorgs
|
||||
@ -78,7 +80,7 @@ type SseEvents[P ProcessedEvents] struct {
|
||||
MessagesCh chan *sse.Event // Contains all the messages from the SSE Channel
|
||||
ErrorCh chan *SseError // Contains any errors while SSE streaming occurred
|
||||
ProcessCh chan *P // Used to capture processed data in its proper struct.
|
||||
SseClient *sse.Client // sse.Client object that is used to interact with the SSE stream
|
||||
sseClient *sse.Client // sse.Client object that is used to interact with the SSE stream
|
||||
}
|
||||
|
||||
// An object to capture any errors when turning an SSE message to JSON.
|
||||
@ -88,7 +90,8 @@ type SseError struct {
|
||||
}
|
||||
|
||||
// A Function to create the BeaconClient.
|
||||
func CreateBeaconClient(ctx context.Context, connectionProtocol string, bcAddress string, bcPort int, bcKgTableIncrement int, uniqueNodeIdentifier int, checkDb bool) (*BeaconClient, error) {
|
||||
func CreateBeaconClient(ctx context.Context, connectionProtocol string, bcAddress string, bcPort int,
|
||||
bcKgTableIncrement int, uniqueNodeIdentifier int, checkDb bool, performBeaconBlockProcessing bool, performBeaconStateProcessing bool) (*BeaconClient, error) {
|
||||
if uniqueNodeIdentifier == 0 {
|
||||
uniqueNodeIdentifier := rand.Int()
|
||||
log.WithField("randomUniqueNodeIdentifier", uniqueNodeIdentifier).Warn("No uniqueNodeIdentifier provided, we are going to use a randomly generated one.")
|
||||
@ -102,14 +105,16 @@ func CreateBeaconClient(ctx context.Context, connectionProtocol string, bcAddres
|
||||
endpoint := fmt.Sprintf("%s://%s:%d", connectionProtocol, bcAddress, bcPort)
|
||||
log.Info("Creating the BeaconClient")
|
||||
return &BeaconClient{
|
||||
Context: ctx,
|
||||
ServerEndpoint: endpoint,
|
||||
KnownGapTableIncrement: bcKgTableIncrement,
|
||||
HeadTracking: createSseEvent[Head](endpoint, BcHeadTopicEndpoint),
|
||||
ReOrgTracking: createSseEvent[ChainReorg](endpoint, bcReorgTopicEndpoint),
|
||||
Metrics: metrics,
|
||||
UniqueNodeIdentifier: uniqueNodeIdentifier,
|
||||
CheckDb: checkDb,
|
||||
Context: ctx,
|
||||
ServerEndpoint: endpoint,
|
||||
KnownGapTableIncrement: bcKgTableIncrement,
|
||||
HeadTracking: createSseEvent[Head](endpoint, BcHeadTopicEndpoint),
|
||||
ReOrgTracking: createSseEvent[ChainReorg](endpoint, bcReorgTopicEndpoint),
|
||||
Metrics: metrics,
|
||||
UniqueNodeIdentifier: uniqueNodeIdentifier,
|
||||
CheckDb: checkDb,
|
||||
PerformBeaconBlockProcessing: performBeaconBlockProcessing,
|
||||
PerformBeaconStateProcessing: performBeaconStateProcessing,
|
||||
//FinalizationTracking: createSseEvent[FinalizedCheckpoint](endpoint, bcFinalizedTopicEndpoint),
|
||||
}, nil
|
||||
}
|
||||
@ -122,10 +127,40 @@ func createSseEvent[P ProcessedEvents](baseEndpoint string, path string) *SseEve
|
||||
MessagesCh: make(chan *sse.Event, 1),
|
||||
ErrorCh: make(chan *SseError),
|
||||
ProcessCh: make(chan *P),
|
||||
SseClient: func(endpoint string) *sse.Client {
|
||||
log.WithFields(log.Fields{"endpoint": endpoint}).Info("Creating SSE client")
|
||||
return sse.NewClient(endpoint)
|
||||
}(endpoint),
|
||||
}
|
||||
return sseEvents
|
||||
}
|
||||
|
||||
func (se *SseEvents[P]) Connect() error {
|
||||
if nil == se.sseClient {
|
||||
se.initClient()
|
||||
}
|
||||
return se.sseClient.SubscribeChanRaw(se.MessagesCh)
|
||||
}
|
||||
|
||||
func (se *SseEvents[P]) Disconnect() {
|
||||
if nil == se.sseClient {
|
||||
return
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{"endpoint": se.Endpoint}).Info("Disconnecting and destroying SSE client")
|
||||
se.sseClient.Unsubscribe(se.MessagesCh)
|
||||
se.sseClient.Connection.CloseIdleConnections()
|
||||
se.sseClient = nil
|
||||
}
|
||||
|
||||
func (se *SseEvents[P]) initClient() {
|
||||
if nil != se.sseClient {
|
||||
se.Disconnect()
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{"endpoint": se.Endpoint}).Info("Creating SSE client")
|
||||
client := sse.NewClient(se.Endpoint)
|
||||
client.ReconnectNotify = func(err error, duration time.Duration) {
|
||||
log.WithFields(log.Fields{"endpoint": se.Endpoint}).Debug("Reconnecting SSE client")
|
||||
}
|
||||
client.OnDisconnect(func(c *sse.Client) {
|
||||
log.WithFields(log.Fields{"endpoint": se.Endpoint}).Debug("SSE client disconnected")
|
||||
})
|
||||
se.sseClient = client
|
||||
}
|
||||
|
@ -50,7 +50,7 @@ func (bc *BeaconClient) StopHeadTracking() error {
|
||||
// This function closes the SSE subscription, but waits until the MessagesCh is empty
|
||||
func (se *SseEvents[ProcessedEvents]) finishProcessingChannel(finish chan<- bool) {
|
||||
loghelper.LogEndpoint(se.Endpoint).Info("Received a close event.")
|
||||
se.SseClient.Unsubscribe(se.MessagesCh)
|
||||
se.Disconnect()
|
||||
for len(se.MessagesCh) != 0 || len(se.ProcessCh) != 0 {
|
||||
time.Sleep(time.Duration(shutdownWaitInterval) * time.Millisecond)
|
||||
}
|
||||
|
@ -20,6 +20,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/protolambda/zrnt/eth2/beacon/common"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@ -29,12 +30,7 @@ import (
|
||||
|
||||
"github.com/jarcoal/httpmock"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
"github.com/prysmaticlabs/prysm/beacon-chain/state"
|
||||
si "github.com/prysmaticlabs/prysm/consensus-types/interfaces"
|
||||
types "github.com/prysmaticlabs/prysm/consensus-types/primitives"
|
||||
dt "github.com/prysmaticlabs/prysm/encoding/ssz/detect"
|
||||
st "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
|
||||
"github.com/r3labs/sse"
|
||||
"github.com/r3labs/sse/v2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
@ -75,7 +71,7 @@ var (
|
||||
CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA2GINRRGFSDKYRZGNTGIYLCGY4TAMJTME3WMMDBGJTDSNRRMNQWGYJQMM4DKM3GHA3WGZTFHE2TSNLGMU2TAMBTHAYTMMZQG44TGNRQ",
|
||||
CorrectBeaconStateMhKey: "/blocks/QLVAEQRQPA3WKNZWHA4DAZLCGY3WEYTEMM4DMMRVGBQWCNJXHA4TKODFHFSDANRXGVSTMNDFG4YTIMZTG44DKNJSGA2GMYRVMFRGCYLGHAZGGMTC",
|
||||
CorrectParentRoot: "0x0000000000000000000000000000000000000000000000000000000000000000",
|
||||
CorrectEth1BlockHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
|
||||
CorrectEth1DataBlockHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
|
||||
},
|
||||
"100-dummy": {
|
||||
HeadMessage: beaconclient.Head{
|
||||
@ -141,7 +137,7 @@ var (
|
||||
CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA2TQMRRHA3WKOJXMY3TKMRQMJRDMOLFMVQTAMJUMMZTQMZUMM4TMNDDGQ2TENJZGM3TEYJQMVQWCZLBGNTDAMZSGAYTGNZZG44TSNTC",
|
||||
CorrectBeaconStateMhKey: "/blocks/QLVAEQRQPBTDEOBWMEYDGNZZMMYDGOBWMEZWGN3CMUZDQZBQGVSDQMRZMY4GKYRXMIZDQMDDMM4WKZDFGE2TINBZMFTDEMDFMJRWIMBWME3WCNJW",
|
||||
CorrectParentRoot: "0x629ae1587895043076500f4f5dcb202a47c2fc95d5b5c548cb83bc97bd2dbfe1",
|
||||
CorrectEth1BlockHash: "0x8d3f027beef5cbd4f8b29fc831aba67a5d74768edca529f5596f07fd207865e1",
|
||||
CorrectEth1DataBlockHash: "0x8d3f027beef5cbd4f8b29fc831aba67a5d74768edca529f5596f07fd207865e1",
|
||||
},
|
||||
"101": {
|
||||
HeadMessage: beaconclient.Head{
|
||||
@ -156,7 +152,7 @@ var (
|
||||
TestNotes: "An easy to process Phase 0 block",
|
||||
SignedBeaconBlock: filepath.Join("ssz-data", "101", "signed-beacon-block.ssz"),
|
||||
BeaconState: filepath.Join("ssz-data", "101", "beacon-state.ssz"),
|
||||
CorrectEth1BlockHash: "0x8d3f027beef5cbd4f8b29fc831aba67a5d74768edca529f5596f07fd207865e1",
|
||||
CorrectEth1DataBlockHash: "0x8d3f027beef5cbd4f8b29fc831aba67a5d74768edca529f5596f07fd207865e1",
|
||||
CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPBQWEZJRME4TOMTFGUYTEMJYGJSDANDGGBSDIYJVMM4WGMRVMY4WKZJVG5RTEZJZMQYGMZRTMY2GGNDDHAZGMZBUGJSDCM3EGMYTAOBT",
|
||||
CorrectBeaconStateMhKey: "/blocks/QLVAEQRQPBRWEMBUMFQTEZLEMJTDCM3DG5RGEN3FG5RGIOLCGYZDCY3FMQ3DQMZSMUYDANZVMU4DSMJUG4ZTKMTFMFRTGMBRHFQTQMRUMNSTQNBX",
|
||||
},
|
||||
@ -196,44 +192,99 @@ var (
|
||||
},
|
||||
"2375703": {
|
||||
HeadMessage: beaconclient.Head{
|
||||
Slot: "2375703",
|
||||
Block: "0x4392372c5f6e39499e31bf924388b5815639103149f0f54f8a453773b1802301",
|
||||
State: "0xb6215b560273af63ec7e011572b60ec1ca0b0232f8ff44fcd4ed55c7526e964e",
|
||||
CurrentDutyDependentRoot: "", PreviousDutyDependentRoot: "", EpochTransition: false, ExecutionOptimistic: false},
|
||||
Slot: "2375703",
|
||||
Block: "0x4392372c5f6e39499e31bf924388b5815639103149f0f54f8a453773b1802301",
|
||||
State: "0xb6215b560273af63ec7e011572b60ec1ca0b0232f8ff44fcd4ed55c7526e964e",
|
||||
CurrentDutyDependentRoot: "",
|
||||
PreviousDutyDependentRoot: "",
|
||||
EpochTransition: false,
|
||||
ExecutionOptimistic: false,
|
||||
},
|
||||
TestNotes: "An easy to process Altair Block",
|
||||
SignedBeaconBlock: filepath.Join("ssz-data", "2375703", "signed-beacon-block.ssz"),
|
||||
BeaconState: filepath.Join("ssz-data", "2375703", "beacon-state.ssz"),
|
||||
CorrectEth1BlockHash: "0xd74b1c60423651624de6bb301ac25808951c167ba6ecdd9b2e79b4315aee8202",
|
||||
CorrectEth1DataBlockHash: "0xd74b1c60423651624de6bb301ac25808951c167ba6ecdd9b2e79b4315aee8202",
|
||||
CorrectParentRoot: "0x08736ddc20b77f65d1aa6301f7e6e856a820ff3ce6430ed2c3694ae35580e740",
|
||||
CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA2DGOJSGM3TEYZVMY3GKMZZGQ4TSZJTGFRGMOJSGQZTQODCGU4DCNJWGM4TCMBTGE2DSZRQMY2TIZRYME2DKMZXG4ZWEMJYGAZDGMBR",
|
||||
CorrectBeaconStateMhKey: "/blocks/QLVAEQRQPBRDMMRRGVRDKNRQGI3TGYLGGYZWKYZXMUYDCMJVG4ZGENRQMVRTCY3BGBRDAMRTGJTDQZTGGQ2GMY3EGRSWINJVMM3TKMRWMU4TMNDF",
|
||||
},
|
||||
"3797056": {
|
||||
HeadMessage: beaconclient.Head{
|
||||
Slot: "3797056",
|
||||
Block: "",
|
||||
State: "",
|
||||
CurrentDutyDependentRoot: "", PreviousDutyDependentRoot: "", EpochTransition: false, ExecutionOptimistic: false},
|
||||
Slot: "3797056",
|
||||
Block: "",
|
||||
State: "",
|
||||
CurrentDutyDependentRoot: "",
|
||||
PreviousDutyDependentRoot: "",
|
||||
EpochTransition: false,
|
||||
ExecutionOptimistic: false,
|
||||
},
|
||||
TestNotes: "An easy to process Altair Block",
|
||||
// The file below should not exist, this will trigger an error message and 404 response from the mock.
|
||||
SignedBeaconBlock: filepath.Join("ssz-data", "3797056", "should-not-exist.txt"),
|
||||
BeaconState: filepath.Join("ssz-data", "3797056", "beacon-state.ssz"),
|
||||
},
|
||||
"4636672": {
|
||||
HeadMessage: beaconclient.Head{
|
||||
Slot: "4636672",
|
||||
Block: "0x9429ce339da8944dd2e1565be8cac5bf634cae2120b6937c081e39148a7f4b1a",
|
||||
State: "0x0067a5d28b38e6e2f59a73046fabbf16a782b978c2c89621a679e7f682b05bd4",
|
||||
CurrentDutyDependentRoot: "",
|
||||
PreviousDutyDependentRoot: "",
|
||||
EpochTransition: true,
|
||||
ExecutionOptimistic: false,
|
||||
},
|
||||
TestNotes: "The first Bellatrix block (empty ExecutionPayload)",
|
||||
SignedBeaconBlock: filepath.Join("ssz-data", "4636672", "signed-beacon-block.ssz"),
|
||||
BeaconState: filepath.Join("ssz-data", "4636672", "beacon-state.ssz"),
|
||||
CorrectEth1DataBlockHash: "0x3b7d392e46db19704d677cadb3310c3776d8c0b8cb2af1c324bb4a394b7f8164",
|
||||
CorrectParentRoot: "0xe7d4f3b7924c30ae047fceabb853b8afdae32b85e0a87ab6c4c37421b353a1da",
|
||||
CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA4TIMRZMNSTGMZZMRQTQOJUGRSGIMTFGE2TMNLCMU4GGYLDGVRGMNRTGRRWCZJSGEZDAYRWHEZTOYZQHAYWKMZZGE2DQYJXMY2GEMLB",
|
||||
CorrectBeaconStateMhKey: "",
|
||||
},
|
||||
"4700013": {
|
||||
HeadMessage: beaconclient.Head{
|
||||
Slot: "4700013",
|
||||
Block: "0x810a00400a80cdffc11ffdcf17ac404ac4dba215b95221955a9dfddf163d0b0d",
|
||||
State: "0x171ef131e0638eddfe1ef73e7b483e344b1cf128b092f2c39e946eb7775b3a2f",
|
||||
CurrentDutyDependentRoot: "",
|
||||
PreviousDutyDependentRoot: "",
|
||||
EpochTransition: true,
|
||||
ExecutionOptimistic: false,
|
||||
},
|
||||
TestNotes: "The first Bellatrix block post-Merge (with ExecutionPayload)",
|
||||
SignedBeaconBlock: filepath.Join("ssz-data", "4700013", "signed-beacon-block.ssz"),
|
||||
BeaconState: filepath.Join("ssz-data", "4700013", "beacon-state.ssz"),
|
||||
CorrectEth1DataBlockHash: "0xb8736ada384707e156f2e0e69d8311ceda11f96806921644a378fd55899894ca",
|
||||
CorrectParentRoot: "0x60e751f7d2cf0ae24b195bda37e9add56a7d8c4b75469c018c0f912518c3bae8",
|
||||
CorrectSignedBeaconBlockMhKey: "/blocks/QLVAEQRQPA4DCMDBGAYDIMBQME4DAY3EMZTGGMJRMZTGIY3GGE3WCYZUGA2GCYZUMRRGCMRRGVRDSNJSGIYTSNJVME4WIZTEMRTDCNRTMQYGEMDE",
|
||||
CorrectBeaconStateMhKey: "",
|
||||
CorrectExecutionPayloadHeader: &beaconclient.DbExecutionPayloadHeader{
|
||||
BlockNumber: 15537394,
|
||||
Timestamp: 1663224179,
|
||||
BlockHash: "0x56a9bb0302da44b8c0b3df540781424684c3af04d0b7a38d72842b762076a664",
|
||||
ParentHash: "0x55b11b918355b1ef9c5db810302ebad0bf2544255b530cdce90674d5887bb286",
|
||||
StateRoot: "0x40c07091e16263270f3579385090fea02dd5f061ba6750228fcc082ff762fda7",
|
||||
ReceiptsRoot: "0x928073fb98ce316265ea35d95ab7e2e1206cecd85242eb841dbbcc4f568fca4b",
|
||||
TransactionsRoot: "0xf9ef008aaf996dccd1c871c7e937f25d66e057e52773fbe2497090c114231acf",
|
||||
},
|
||||
},
|
||||
}
|
||||
TestConfig = Config{
|
||||
protocol: protocol,
|
||||
address: address,
|
||||
port: port,
|
||||
dummyParentRoot: dummyParentRoot,
|
||||
dbHost: dbHost,
|
||||
dbPort: dbPort,
|
||||
dbName: dbName,
|
||||
dbUser: dbUser,
|
||||
dbPassword: dbPassword,
|
||||
dbDriver: dbDriver,
|
||||
knownGapsTableIncrement: knownGapsTableIncrement,
|
||||
bcUniqueIdentifier: bcUniqueIdentifier,
|
||||
checkDb: true,
|
||||
protocol: protocol,
|
||||
address: address,
|
||||
port: port,
|
||||
dummyParentRoot: dummyParentRoot,
|
||||
dbHost: dbHost,
|
||||
dbPort: dbPort,
|
||||
dbName: dbName,
|
||||
dbUser: dbUser,
|
||||
dbPassword: dbPassword,
|
||||
dbDriver: dbDriver,
|
||||
knownGapsTableIncrement: knownGapsTableIncrement,
|
||||
bcUniqueIdentifier: bcUniqueIdentifier,
|
||||
checkDb: true,
|
||||
performBeaconStateProcessing: true,
|
||||
performBeaconBlockProcessing: true,
|
||||
}
|
||||
|
||||
BeaconNodeTester = TestBeaconNode{
|
||||
@ -243,15 +294,16 @@ var (
|
||||
)
|
||||
|
||||
type Message struct {
|
||||
HeadMessage beaconclient.Head // The head messsage that will be streamed to the BeaconClient
|
||||
TestNotes string // A small explanation of the purpose this structure plays in the testing landscape.
|
||||
MimicConfig *MimicConfig // A configuration of parameters that you are trying to
|
||||
SignedBeaconBlock string // The file path output of an SSZ encoded SignedBeaconBlock.
|
||||
BeaconState string // The file path output of an SSZ encoded BeaconState.
|
||||
CorrectSignedBeaconBlockMhKey string // The correct MhKey for the signedBeaconBlock
|
||||
CorrectBeaconStateMhKey string // The correct MhKey beaconState
|
||||
CorrectParentRoot string // The correct parent root
|
||||
CorrectEth1BlockHash string // The correct eth1blockHash
|
||||
HeadMessage beaconclient.Head // The head messsage that will be streamed to the BeaconClient
|
||||
TestNotes string // A small explanation of the purpose this structure plays in the testing landscape.
|
||||
MimicConfig *MimicConfig // A configuration of parameters that you are trying to
|
||||
SignedBeaconBlock string // The file path output of an SSZ encoded SignedBeaconBlock.
|
||||
BeaconState string // The file path output of an SSZ encoded BeaconState.
|
||||
CorrectSignedBeaconBlockMhKey string // The correct MhKey for the signedBeaconBlock
|
||||
CorrectBeaconStateMhKey string // The correct MhKey beaconState
|
||||
CorrectParentRoot string // The correct parent root
|
||||
CorrectEth1DataBlockHash string // The correct eth1blockHash
|
||||
CorrectExecutionPayloadHeader *beaconclient.DbExecutionPayloadHeader // The correct ExecutionPayload details.
|
||||
}
|
||||
|
||||
// A structure that can be utilized to mimic and existing SSZ object but change it ever so slightly.
|
||||
@ -270,8 +322,12 @@ var _ = Describe("Capturehead", Label("head"), func() {
|
||||
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
|
||||
defer httpmock.DeactivateAndReset()
|
||||
BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, 3, maxRetry, 1, 0, 0)
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectParentRoot, BeaconNodeTester.TestEvents["100"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["100"].CorrectSignedBeaconBlockMhKey)
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectBeaconStateMhKey)
|
||||
if bc.PerformBeaconBlockProcessing {
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectParentRoot, BeaconNodeTester.TestEvents["100"].CorrectEth1DataBlockHash, BeaconNodeTester.TestEvents["100"].CorrectSignedBeaconBlockMhKey, BeaconNodeTester.TestEvents["100"].CorrectExecutionPayloadHeader)
|
||||
}
|
||||
if bc.PerformBeaconStateProcessing {
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectBeaconStateMhKey)
|
||||
}
|
||||
|
||||
})
|
||||
})
|
||||
@ -281,8 +337,12 @@ var _ = Describe("Capturehead", Label("head"), func() {
|
||||
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
|
||||
defer httpmock.DeactivateAndReset()
|
||||
BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, maxRetry, 1, 0, 0)
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectParentRoot, BeaconNodeTester.TestEvents["2375703"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["2375703"].CorrectSignedBeaconBlockMhKey)
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectBeaconStateMhKey)
|
||||
if bc.PerformBeaconBlockProcessing {
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectParentRoot, BeaconNodeTester.TestEvents["2375703"].CorrectEth1DataBlockHash, BeaconNodeTester.TestEvents["2375703"].CorrectSignedBeaconBlockMhKey, BeaconNodeTester.TestEvents["2375703"].CorrectExecutionPayloadHeader)
|
||||
}
|
||||
if bc.PerformBeaconStateProcessing {
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectBeaconStateMhKey)
|
||||
}
|
||||
})
|
||||
})
|
||||
Context("Correctly formatted Altair Test Blocks", func() {
|
||||
@ -299,6 +359,26 @@ var _ = Describe("Capturehead", Label("head"), func() {
|
||||
|
||||
})
|
||||
})
|
||||
Context("Correctly formatted Bellatrix Test Blocks", Label("unit", "bellatrix"), func() {
|
||||
It("Should turn it into a struct successfully (pre-Merge).", func() {
|
||||
bc := setUpTest(BeaconNodeTester.TestConfig, "4636672")
|
||||
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
|
||||
defer httpmock.DeactivateAndReset()
|
||||
BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["4636672"].HeadMessage, 144896, maxRetry, 1, 0, 0)
|
||||
if bc.PerformBeaconBlockProcessing {
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["4636672"].HeadMessage, BeaconNodeTester.TestEvents["4636672"].CorrectParentRoot, BeaconNodeTester.TestEvents["4636672"].CorrectEth1DataBlockHash, BeaconNodeTester.TestEvents["4636672"].CorrectSignedBeaconBlockMhKey, BeaconNodeTester.TestEvents["4636672"].CorrectExecutionPayloadHeader)
|
||||
}
|
||||
})
|
||||
It("Should turn it into a struct successfully (post-Merge).", func() {
|
||||
bc := setUpTest(BeaconNodeTester.TestConfig, "4700013")
|
||||
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
|
||||
defer httpmock.DeactivateAndReset()
|
||||
BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["4700013"].HeadMessage, 146875, maxRetry, 1, 0, 0)
|
||||
if bc.PerformBeaconBlockProcessing {
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["4700013"].HeadMessage, BeaconNodeTester.TestEvents["4700013"].CorrectParentRoot, BeaconNodeTester.TestEvents["4700013"].CorrectEth1DataBlockHash, BeaconNodeTester.TestEvents["4700013"].CorrectSignedBeaconBlockMhKey, BeaconNodeTester.TestEvents["4700013"].CorrectExecutionPayloadHeader)
|
||||
}
|
||||
})
|
||||
})
|
||||
Context("Correctly formatted Phase0 Test Blocks", func() {
|
||||
It("Should turn it into a struct successfully.", func() {
|
||||
bc := setUpTest(BeaconNodeTester.TestConfig, "99")
|
||||
@ -311,7 +391,6 @@ var _ = Describe("Capturehead", Label("head"), func() {
|
||||
defer httpmock.DeactivateAndReset()
|
||||
BeaconNodeTester.testProcessBlock(bc, BeaconNodeTester.TestEvents["100-dummy-2"].HeadMessage, 3, maxRetry, 1, 0, 0)
|
||||
})
|
||||
|
||||
})
|
||||
Context("Two consecutive correct blocks", func() {
|
||||
It("Should handle both blocks correctly, without any reorgs or known_gaps", func() {
|
||||
@ -332,12 +411,12 @@ var _ = Describe("Capturehead", Label("head"), func() {
|
||||
})
|
||||
})
|
||||
Context("Phase 0: We have a correctly formated SSZ SignedBeaconBlock and BeaconState", func() {
|
||||
It("Should be able to get each objects root hash.", func() {
|
||||
It("Should be able to get each objects root hash (100).", func() {
|
||||
testSszRoot(BeaconNodeTester.TestEvents["100"])
|
||||
})
|
||||
})
|
||||
Context("Altair: We have a correctly formated SSZ SignedBeaconBlock and BeaconState", func() {
|
||||
It("Should be able to get each objects root hash.", func() {
|
||||
It("Should be able to get each objects root hash (2375703).", func() {
|
||||
testSszRoot(BeaconNodeTester.TestEvents["2375703"])
|
||||
})
|
||||
})
|
||||
@ -446,19 +525,21 @@ var _ = Describe("Capturehead", Label("head"), func() {
|
||||
})
|
||||
|
||||
type Config struct {
|
||||
protocol string
|
||||
address string
|
||||
port int
|
||||
dummyParentRoot string
|
||||
dbHost string
|
||||
dbPort int
|
||||
dbName string
|
||||
dbUser string
|
||||
dbPassword string
|
||||
dbDriver string
|
||||
knownGapsTableIncrement int
|
||||
bcUniqueIdentifier int
|
||||
checkDb bool
|
||||
protocol string
|
||||
address string
|
||||
port int
|
||||
dummyParentRoot string
|
||||
dbHost string
|
||||
dbPort int
|
||||
dbName string
|
||||
dbUser string
|
||||
dbPassword string
|
||||
dbDriver string
|
||||
knownGapsTableIncrement int
|
||||
bcUniqueIdentifier int
|
||||
checkDb bool
|
||||
performBeaconBlockProcessing bool
|
||||
performBeaconStateProcessing bool
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
@ -468,7 +549,7 @@ type Config struct {
|
||||
// Must run before each test. We can't use the beforeEach because of the way
|
||||
// Gingko treats race conditions.
|
||||
func setUpTest(config Config, maxSlot string) *beaconclient.BeaconClient {
|
||||
bc, err := beaconclient.CreateBeaconClient(context.Background(), config.protocol, config.address, config.port, config.knownGapsTableIncrement, config.bcUniqueIdentifier, config.checkDb)
|
||||
bc, err := beaconclient.CreateBeaconClient(context.Background(), config.protocol, config.address, config.port, config.knownGapsTableIncrement, config.bcUniqueIdentifier, config.checkDb, config.performBeaconBlockProcessing, config.performBeaconStateProcessing)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
db, err := postgres.SetupPostgresDb(config.dbHost, config.dbPort, config.dbName, config.dbUser, config.dbPassword, config.dbDriver)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@ -484,10 +565,10 @@ func setUpTest(config Config, maxSlot string) *beaconclient.BeaconClient {
|
||||
}
|
||||
|
||||
// A helper function to validate the expected output from the eth_beacon.slots table.
|
||||
func validateSlot(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctEpoch int, correctStatus string) {
|
||||
func validateSlot(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctEpoch beaconclient.Epoch, correctStatus string) {
|
||||
epoch, dbSlot, blockRoot, stateRoot, status := queryDbSlotAndBlock(bc.Db, headMessage.Slot, headMessage.Block)
|
||||
log.Info("validateSlot: ", headMessage)
|
||||
baseSlot, err := strconv.Atoi(headMessage.Slot)
|
||||
baseSlot, err := beaconclient.ParseSlot(headMessage.Slot)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(dbSlot).To(Equal(baseSlot))
|
||||
Expect(epoch).To(Equal(correctEpoch))
|
||||
@ -497,29 +578,30 @@ func validateSlot(bc *beaconclient.BeaconClient, headMessage beaconclient.Head,
|
||||
}
|
||||
|
||||
// A helper function to validate the expected output from the eth_beacon.signed_block table.
|
||||
func validateSignedBeaconBlock(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctParentRoot string, correctEth1BlockHash string, correctMhKey string) {
|
||||
dbSlot, blockRoot, parentRoot, eth1BlockHash, mhKey := queryDbSignedBeaconBlock(bc.Db, headMessage.Slot, headMessage.Block)
|
||||
func validateSignedBeaconBlock(bc *beaconclient.BeaconClient, headMessage beaconclient.Head,
|
||||
correctParentRoot string, correctEth1DataBlockHash string, correctMhKey string,
|
||||
correctExecutionPayloadHeader *beaconclient.DbExecutionPayloadHeader) {
|
||||
dbSignedBlock := queryDbSignedBeaconBlock(bc.Db, headMessage.Slot, headMessage.Block)
|
||||
log.Info("validateSignedBeaconBlock: ", headMessage)
|
||||
baseSlot, err := strconv.Atoi(headMessage.Slot)
|
||||
baseSlot, err := beaconclient.ParseSlot(headMessage.Slot)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(dbSlot).To(Equal(baseSlot))
|
||||
Expect(blockRoot).To(Equal(headMessage.Block))
|
||||
Expect(parentRoot).To(Equal(correctParentRoot))
|
||||
Expect(eth1BlockHash).To(Equal(correctEth1BlockHash))
|
||||
Expect(mhKey).To(Equal(correctMhKey))
|
||||
|
||||
Expect(dbSignedBlock.Slot).To(Equal(baseSlot.Number()))
|
||||
Expect(dbSignedBlock.BlockRoot).To(Equal(headMessage.Block))
|
||||
Expect(dbSignedBlock.ParentBlock).To(Equal(correctParentRoot))
|
||||
Expect(dbSignedBlock.Eth1DataBlockHash).To(Equal(correctEth1DataBlockHash))
|
||||
Expect(dbSignedBlock.MhKey).To(Equal(correctMhKey))
|
||||
Expect(dbSignedBlock.ExecutionPayloadHeader).To(Equal(correctExecutionPayloadHeader))
|
||||
}
|
||||
|
||||
// A helper function to validate the expected output from the eth_beacon.state table.
|
||||
func validateBeaconState(bc *beaconclient.BeaconClient, headMessage beaconclient.Head, correctMhKey string) {
|
||||
dbSlot, stateRoot, mhKey := queryDbBeaconState(bc.Db, headMessage.Slot, headMessage.State)
|
||||
log.Info("validateBeaconState: ", headMessage)
|
||||
baseSlot, err := strconv.Atoi(headMessage.Slot)
|
||||
baseSlot, err := beaconclient.ParseSlot(headMessage.Slot)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(dbSlot).To(Equal(baseSlot))
|
||||
Expect(stateRoot).To(Equal(headMessage.State))
|
||||
Expect(mhKey).To(Equal(correctMhKey))
|
||||
|
||||
}
|
||||
|
||||
// Wrapper function to send a head message to the beaconclient
|
||||
@ -551,9 +633,10 @@ func sendHeadMessage(bc *beaconclient.BeaconClient, head beaconclient.Head, maxR
|
||||
}
|
||||
|
||||
// A helper function to query the eth_beacon.slots table based on the slot and block_root
|
||||
func queryDbSlotAndBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, int, string, string, string) {
|
||||
func queryDbSlotAndBlock(db sql.Database, querySlot string, queryBlockRoot string) (beaconclient.Epoch, beaconclient.Slot, string, string, string) {
|
||||
sqlStatement := `SELECT epoch, slot, block_root, state_root, status FROM eth_beacon.slots WHERE slot=$1 AND block_root=$2;`
|
||||
var epoch, slot int
|
||||
var epoch beaconclient.Epoch
|
||||
var slot beaconclient.Slot
|
||||
var blockRoot, stateRoot, status string
|
||||
log.Debug("Starting to query the eth_beacon.slots table, ", querySlot, " ", queryBlockRoot)
|
||||
err := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot).Scan(&epoch, &slot, &blockRoot, &stateRoot, &status)
|
||||
@ -563,25 +646,56 @@ func queryDbSlotAndBlock(db sql.Database, querySlot string, queryBlockRoot strin
|
||||
}
|
||||
|
||||
// A helper function to query the eth_beacon.signed_block table based on the slot and block_root.
|
||||
func queryDbSignedBeaconBlock(db sql.Database, querySlot string, queryBlockRoot string) (int, string, string, string, string) {
|
||||
sqlStatement := `SELECT slot, block_root, parent_block_root, eth1_block_hash, mh_key FROM eth_beacon.signed_block WHERE slot=$1 AND block_root=$2;`
|
||||
var slot int
|
||||
var blockRoot, parent_block_root, eth1_block_hash, mh_key string
|
||||
func queryDbSignedBeaconBlock(db sql.Database, querySlot string, queryBlockRoot string) beaconclient.DbSignedBeaconBlock {
|
||||
sqlStatement := `SELECT slot, block_root, parent_block_root, eth1_data_block_hash, mh_key,
|
||||
payload_block_number, payload_timestamp, payload_block_hash,
|
||||
payload_parent_hash, payload_state_root, payload_receipts_root,
|
||||
payload_transactions_root FROM eth_beacon.signed_block WHERE slot=$1 AND block_root=$2;`
|
||||
|
||||
var slot beaconclient.Slot
|
||||
var payloadBlockNumber, payloadTimestamp *uint64
|
||||
var blockRoot, parentBlockRoot, eth1DataBlockHash, mhKey string
|
||||
var payloadBlockHash, payloadParentHash, payloadStateRoot, payloadReceiptsRoot, payloadTransactionsRoot *string
|
||||
|
||||
row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryBlockRoot)
|
||||
err := row.Scan(&slot, &blockRoot, &parent_block_root, ð1_block_hash, &mh_key)
|
||||
err := row.Scan(&slot, &blockRoot, &parentBlockRoot, ð1DataBlockHash, &mhKey,
|
||||
&payloadBlockNumber, &payloadTimestamp, &payloadBlockHash,
|
||||
&payloadParentHash, &payloadStateRoot, &payloadReceiptsRoot, &payloadTransactionsRoot)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return slot, blockRoot, parent_block_root, eth1_block_hash, mh_key
|
||||
|
||||
signedBlock := beaconclient.DbSignedBeaconBlock{
|
||||
Slot: slot.Number(),
|
||||
BlockRoot: blockRoot,
|
||||
ParentBlock: parentBlockRoot,
|
||||
Eth1DataBlockHash: eth1DataBlockHash,
|
||||
MhKey: mhKey,
|
||||
ExecutionPayloadHeader: nil,
|
||||
}
|
||||
|
||||
if nil != payloadBlockNumber {
|
||||
signedBlock.ExecutionPayloadHeader = &beaconclient.DbExecutionPayloadHeader{
|
||||
BlockNumber: *payloadBlockNumber,
|
||||
Timestamp: *payloadTimestamp,
|
||||
BlockHash: *payloadBlockHash,
|
||||
ParentHash: *payloadParentHash,
|
||||
StateRoot: *payloadStateRoot,
|
||||
ReceiptsRoot: *payloadReceiptsRoot,
|
||||
TransactionsRoot: *payloadTransactionsRoot,
|
||||
}
|
||||
}
|
||||
|
||||
return signedBlock
|
||||
}
|
||||
|
||||
// A helper function to query the eth_beacon.signed_block table based on the slot and block_root.
|
||||
func queryDbBeaconState(db sql.Database, querySlot string, queryStateRoot string) (int, string, string) {
|
||||
func queryDbBeaconState(db sql.Database, querySlot string, queryStateRoot string) (beaconclient.Slot, string, string) {
|
||||
sqlStatement := `SELECT slot, state_root, mh_key FROM eth_beacon.state WHERE slot=$1 AND state_root=$2;`
|
||||
var slot int
|
||||
var stateRoot, mh_key string
|
||||
var slot beaconclient.Slot
|
||||
var stateRoot, mhKey string
|
||||
row := db.QueryRow(context.Background(), sqlStatement, querySlot, queryStateRoot)
|
||||
err := row.Scan(&slot, &stateRoot, &mh_key)
|
||||
err := row.Scan(&slot, &stateRoot, &mhKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return slot, stateRoot, mh_key
|
||||
return slot, stateRoot, mhKey
|
||||
}
|
||||
|
||||
// Count the entries in the knownGaps table.
|
||||
@ -620,55 +734,27 @@ func writeSlot(db sql.Database, slot string) {
|
||||
|
||||
// Read a file with the SignedBeaconBlock in SSZ and return the SSZ object. This is used for testing only.
|
||||
// We can't use the readSignedBeaconBlockInterface to update struct fields so this is the workaround.
|
||||
func readSignedBeaconBlock(slotFile string) (*st.SignedBeaconBlock, error) {
|
||||
func readSignedBeaconBlock(slotFile string) (*beaconclient.SignedBeaconBlock, error) {
|
||||
dat, err := os.ReadFile(slotFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
|
||||
}
|
||||
block := &st.SignedBeaconBlock{}
|
||||
var block beaconclient.SignedBeaconBlock
|
||||
err = block.UnmarshalSSZ(dat)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return block, nil
|
||||
}
|
||||
|
||||
// Read a file with the SignedBeaconBlock in SSZ and return the SSZ object. This is used for testing only.
|
||||
// We can't use the readSignedBeaconBlockInterface to update struct fields so this is the workaround.
|
||||
func readSignedBeaconBlockAltair(slotFile string) (*st.SignedBeaconBlockAltair, error) {
|
||||
dat, err := os.ReadFile(slotFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
|
||||
}
|
||||
block := &st.SignedBeaconBlockAltair{}
|
||||
err = block.UnmarshalSSZ(dat)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return block, nil
|
||||
}
|
||||
|
||||
// Read a file with the SignedBeaconBlock in SSZ and return the SSZ objects interface. This is production like.
|
||||
// It will provide the correct struct for the given fork.
|
||||
func readSignedBeaconBlockInterface(slotFile string, vm *dt.VersionedUnmarshaler) (si.SignedBeaconBlock, error) {
|
||||
dat, err := os.ReadFile(slotFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
|
||||
}
|
||||
|
||||
block, err := vm.UnmarshalBeaconBlock(dat)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return block, nil
|
||||
|
||||
return &block, nil
|
||||
}
|
||||
|
||||
// Read a file with the BeaconState in SSZ and return the SSZ object
|
||||
func readBeaconState(slotFile string) (state.BeaconState, *dt.VersionedUnmarshaler, error) {
|
||||
func readBeaconState(slotFile string) (*beaconclient.BeaconState, error) {
|
||||
dat, err := os.ReadFile(slotFile)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
|
||||
return nil, fmt.Errorf("Can't find the slot file, %s", slotFile)
|
||||
}
|
||||
versionedUnmarshaler, err := dt.FromState(dat)
|
||||
var beaconState beaconclient.BeaconState
|
||||
err = beaconState.UnmarshalSSZ(dat)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
state, err := versionedUnmarshaler.UnmarshalBeaconState(dat)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return state, versionedUnmarshaler, nil
|
||||
return &beaconState, nil
|
||||
}
|
||||
|
||||
// An object that is used to aggregate test functions. Test functions are needed because we need to
|
||||
@ -768,52 +854,62 @@ func (tbc TestBeaconNode) provideSsz(slotIdentifier string, sszIdentifier string
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
Expect(block.IsPhase0()).To(BeTrue())
|
||||
var phase0 = block.GetPhase0()
|
||||
|
||||
slot, err := strconv.ParseUint(Message.HeadMessage.Slot, 10, 64)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
block.Block.Slot = types.Slot(slot)
|
||||
phase0.Message.Slot = common.Slot(slot)
|
||||
|
||||
block.Block.StateRoot, err = hex.DecodeString(Message.HeadMessage.State)
|
||||
phase0.Message.StateRoot, err = decodeRoot(Message.HeadMessage.State)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
if Message.MimicConfig.ParentRoot == "" {
|
||||
block.Block.ParentRoot, err = hex.DecodeString(dummyParentRoot)
|
||||
phase0.Message.ParentRoot, err = decodeRoot(dummyParentRoot)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
} else {
|
||||
block.Block.ParentRoot, err = hex.DecodeString(Message.MimicConfig.ParentRoot)
|
||||
phase0.Message.ParentRoot, err = decodeRoot(Message.MimicConfig.ParentRoot)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
return block.MarshalSSZ()
|
||||
case "altair":
|
||||
block, err := readSignedBeaconBlockAltair(slotFile)
|
||||
block, err := readSignedBeaconBlock(slotFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
Expect(block.IsAltair()).To(BeTrue())
|
||||
var altair = block.GetAltair()
|
||||
slot, err := strconv.ParseUint(Message.HeadMessage.Slot, 10, 64)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
block.Block.Slot = types.Slot(slot)
|
||||
altair.Message.Slot = common.Slot(slot)
|
||||
|
||||
block.Block.StateRoot, err = hex.DecodeString(Message.HeadMessage.State)
|
||||
altair.Message.StateRoot, err = decodeRoot(Message.HeadMessage.State)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
if Message.MimicConfig.ParentRoot == "" {
|
||||
block.Block.ParentRoot, err = hex.DecodeString(dummyParentRoot)
|
||||
altair.Message.ParentRoot, err = decodeRoot(dummyParentRoot)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
} else {
|
||||
block.Block.ParentRoot, err = hex.DecodeString(Message.MimicConfig.ParentRoot)
|
||||
altair.Message.ParentRoot, err = decodeRoot(Message.MimicConfig.ParentRoot)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
return block.MarshalSSZ()
|
||||
}
|
||||
}
|
||||
if sszIdentifier == "state" {
|
||||
state, _, err := readBeaconState(slotFile)
|
||||
state, err := readBeaconState(slotFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slot, err := strconv.ParseUint(Message.HeadMessage.Slot, 10, 64)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = state.SetSlot(types.Slot(slot))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
if state.IsBellatrix() {
|
||||
state.GetBellatrix().Slot = common.Slot(slot)
|
||||
} else if state.IsAltair() {
|
||||
state.GetAltair().Slot = common.Slot(slot)
|
||||
} else {
|
||||
state.GetPhase0().Slot = common.Slot(slot)
|
||||
}
|
||||
return state.MarshalSSZ()
|
||||
}
|
||||
}
|
||||
@ -831,7 +927,7 @@ func (tbc TestBeaconNode) provideSsz(slotIdentifier string, sszIdentifier string
|
||||
|
||||
// Helper function to test three reorg messages. There are going to be many functions like this,
|
||||
// Because we need to test the same logic for multiple phases.
|
||||
func (tbc TestBeaconNode) testMultipleReorgs(bc *beaconclient.BeaconClient, firstHead beaconclient.Head, secondHead beaconclient.Head, thirdHead beaconclient.Head, epoch int, maxRetry int) {
|
||||
func (tbc TestBeaconNode) testMultipleReorgs(bc *beaconclient.BeaconClient, firstHead beaconclient.Head, secondHead beaconclient.Head, thirdHead beaconclient.Head, epoch beaconclient.Epoch, maxRetry int) {
|
||||
go bc.CaptureHead()
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
@ -863,7 +959,7 @@ func (tbc TestBeaconNode) testMultipleReorgs(bc *beaconclient.BeaconClient, firs
|
||||
NewHeadBlock: secondHead.Block,
|
||||
OldHeadState: thirdHead.State,
|
||||
NewHeadState: secondHead.State,
|
||||
Epoch: strconv.Itoa(epoch),
|
||||
Epoch: epoch.Format(),
|
||||
ExecutionOptimistic: false,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@ -893,7 +989,7 @@ func (tbc TestBeaconNode) testMultipleReorgs(bc *beaconclient.BeaconClient, firs
|
||||
}
|
||||
|
||||
// A test to validate a single block was processed correctly
|
||||
func (tbc TestBeaconNode) testProcessBlock(bc *beaconclient.BeaconClient, head beaconclient.Head, epoch int, maxRetry int, expectedSuccessInsert uint64, expectedKnownGaps uint64, expectedReorgs uint64) {
|
||||
func (tbc TestBeaconNode) testProcessBlock(bc *beaconclient.BeaconClient, head beaconclient.Head, epoch beaconclient.Epoch, maxRetry int, expectedSuccessInsert uint64, expectedKnownGaps uint64, expectedReorgs uint64) {
|
||||
go bc.CaptureHead()
|
||||
time.Sleep(1 * time.Second)
|
||||
sendHeadMessage(bc, head, maxRetry, expectedSuccessInsert)
|
||||
@ -923,7 +1019,7 @@ func (tbc TestBeaconNode) testProcessBlock(bc *beaconclient.BeaconClient, head b
|
||||
|
||||
// A test that ensures that if two HeadMessages occur for a single slot they are marked
|
||||
// as proposed and forked correctly.
|
||||
func (tbc TestBeaconNode) testMultipleHead(bc *beaconclient.BeaconClient, firstHead beaconclient.Head, secondHead beaconclient.Head, epoch int, maxRetry int) {
|
||||
func (tbc TestBeaconNode) testMultipleHead(bc *beaconclient.BeaconClient, firstHead beaconclient.Head, secondHead beaconclient.Head, epoch beaconclient.Epoch, maxRetry int) {
|
||||
go bc.CaptureHead()
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
@ -979,15 +1075,25 @@ func (tbc TestBeaconNode) testKnownGapsMessages(bc *beaconclient.BeaconClient, t
|
||||
|
||||
// This function will make sure we are properly able to get the SszRoot of the SignedBeaconBlock and the BeaconState.
|
||||
func testSszRoot(msg Message) {
|
||||
state, vm, err := readBeaconState(msg.BeaconState)
|
||||
state, err := readBeaconState(msg.BeaconState)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
stateRoot, err := state.HashTreeRoot(context.Background())
|
||||
stateRoot := state.HashTreeRoot()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(msg.HeadMessage.State).To(Equal("0x" + hex.EncodeToString(stateRoot[:])))
|
||||
|
||||
block, err := readSignedBeaconBlockInterface(msg.SignedBeaconBlock, vm)
|
||||
block, err := readSignedBeaconBlock(msg.SignedBeaconBlock)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
blockRoot, err := block.Block().HashTreeRoot()
|
||||
blockRoot := block.Block().HashTreeRoot()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(msg.HeadMessage.Block).To(Equal("0x" + hex.EncodeToString(blockRoot[:])))
|
||||
}
|
||||
|
||||
func decodeRoot(raw string) (common.Root, error) {
|
||||
value, err := hex.DecodeString(raw)
|
||||
if err != nil {
|
||||
return common.Root{}, err
|
||||
}
|
||||
var root common.Root
|
||||
copy(root[:], value[:32])
|
||||
return root, nil
|
||||
}
|
||||
|
@ -22,16 +22,15 @@ import (
|
||||
"fmt"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// This function will perform all the heavy lifting for tracking the head of the chain.
|
||||
func (bc *BeaconClient) CaptureHistoric(ctx context.Context, maxWorkers int) []error {
|
||||
func (bc *BeaconClient) CaptureHistoric(ctx context.Context, maxWorkers int, minimumSlot Slot) []error {
|
||||
log.Info("We are starting the historical processing service.")
|
||||
bc.HistoricalProcess = HistoricProcessing{db: bc.Db, metrics: bc.Metrics, uniqueNodeIdentifier: bc.UniqueNodeIdentifier}
|
||||
errs := handleBatchProcess(ctx, maxWorkers, bc.HistoricalProcess, bc.HistoricalProcess.db, bc.ServerEndpoint, bc.Metrics, bc.CheckDb, bc.Metrics.IncrementHistoricSlotProcessed)
|
||||
errs := handleBatchProcess(ctx, maxWorkers, bc.HistoricalProcess, bc.SlotProcessingDetails(), bc.Metrics.IncrementHistoricSlotProcessed, minimumSlot)
|
||||
log.Debug("Exiting Historical")
|
||||
return errs
|
||||
}
|
||||
@ -53,7 +52,7 @@ func (bc *BeaconClient) StopHistoric(cancel context.CancelFunc) error {
|
||||
//
|
||||
// 2. Known Gaps Processing
|
||||
type BatchProcessing interface {
|
||||
getSlotRange(context.Context, chan<- slotsToProcess) []error // Write the slots to process in a channel, return an error if you cant get the next slots to write.
|
||||
getSlotRange(context.Context, chan<- slotsToProcess, Slot) []error // Write the slots to process in a channel, return an error if you cant get the next slots to write.
|
||||
handleProcessingErrors(context.Context, <-chan batchHistoricError) // Custom logic to handle errors.
|
||||
removeTableEntry(context.Context, <-chan slotsToProcess) error // With the provided start and end slot, remove the entry from the database.
|
||||
releaseDbLocks() error // Update the checked_out column to false for whatever table is being updated.
|
||||
@ -67,14 +66,14 @@ type BatchProcessing interface {
|
||||
|
||||
// A struct to pass around indicating a table entry for slots to process.
|
||||
type slotsToProcess struct {
|
||||
startSlot int // The start slot
|
||||
endSlot int // The end slot
|
||||
startSlot Slot // The start slot
|
||||
endSlot Slot // The end slot
|
||||
}
|
||||
|
||||
type batchHistoricError struct {
|
||||
err error // The error that occurred when attempting to a slot
|
||||
errProcess string // The process that caused the error.
|
||||
slot int // The slot which the error is for.
|
||||
slot Slot // The slot which the error is for.
|
||||
}
|
||||
|
||||
// Wrapper function for the BatchProcessing interface.
|
||||
@ -91,9 +90,9 @@ type batchHistoricError struct {
|
||||
// 4. Remove the slot entry from the DB.
|
||||
//
|
||||
// 5. Handle any errors.
|
||||
func handleBatchProcess(ctx context.Context, maxWorkers int, bp BatchProcessing, db sql.Database, serverEndpoint string, metrics *BeaconClientMetrics, checkDb bool, incrementTracker func(uint64)) []error {
|
||||
func handleBatchProcess(ctx context.Context, maxWorkers int, bp BatchProcessing, spd SlotProcessingDetails, incrementTracker func(uint64), minimumSlot Slot) []error {
|
||||
slotsCh := make(chan slotsToProcess)
|
||||
workCh := make(chan int)
|
||||
workCh := make(chan Slot)
|
||||
processedCh := make(chan slotsToProcess)
|
||||
errCh := make(chan batchHistoricError)
|
||||
finalErrCh := make(chan []error, 1)
|
||||
@ -108,7 +107,7 @@ func handleBatchProcess(ctx context.Context, maxWorkers int, bp BatchProcessing,
|
||||
for w := 1; w <= maxWorkers; w++ {
|
||||
log.WithFields(log.Fields{"maxWorkers": maxWorkers}).Debug("Starting batch processing workers")
|
||||
|
||||
go processSlotRangeWorker(ctx, workCh, errCh, db, serverEndpoint, metrics, checkDb, incrementTracker)
|
||||
go processSlotRangeWorker(ctx, workCh, errCh, spd, incrementTracker)
|
||||
}
|
||||
|
||||
// Process all ranges and send each individual slot to the worker.
|
||||
@ -161,7 +160,7 @@ func handleBatchProcess(ctx context.Context, maxWorkers int, bp BatchProcessing,
|
||||
|
||||
// Get slots from the DB.
|
||||
go func() {
|
||||
errs := bp.getSlotRange(ctx, slotsCh) // Periodically adds new entries....
|
||||
errs := bp.getSlotRange(ctx, slotsCh, minimumSlot) // Periodically adds new entries....
|
||||
if errs != nil {
|
||||
finalErrCh <- errs
|
||||
}
|
||||
|
@ -54,8 +54,12 @@ var _ = Describe("Capturehistoric", func() {
|
||||
BeaconNodeTester.writeEventToHistoricProcess(bc, 0, 0, 10)
|
||||
BeaconNodeTester.runHistoricalProcess(bc, 2, 1, 0, 0, 0)
|
||||
validateSlot(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, 0, "proposed")
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, BeaconNodeTester.TestEvents["0"].CorrectParentRoot, BeaconNodeTester.TestEvents["0"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["0"].CorrectSignedBeaconBlockMhKey)
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, BeaconNodeTester.TestEvents["0"].CorrectBeaconStateMhKey)
|
||||
if bc.PerformBeaconBlockProcessing {
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, BeaconNodeTester.TestEvents["0"].CorrectParentRoot, BeaconNodeTester.TestEvents["0"].CorrectEth1DataBlockHash, BeaconNodeTester.TestEvents["0"].CorrectSignedBeaconBlockMhKey, BeaconNodeTester.TestEvents["0"].CorrectExecutionPayloadHeader)
|
||||
}
|
||||
if bc.PerformBeaconStateProcessing {
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["0"].HeadMessage, BeaconNodeTester.TestEvents["0"].CorrectBeaconStateMhKey)
|
||||
}
|
||||
})
|
||||
})
|
||||
Context("When there is a skipped slot", func() {
|
||||
@ -94,7 +98,7 @@ var _ = Describe("Capturehistoric", func() {
|
||||
BeaconNodeTester.runKnownGapsProcess(bc, 2, 2, 0, 2, 0)
|
||||
})
|
||||
})
|
||||
Context("When theres a reprocessing error", Label("reprocessingError"), func() {
|
||||
Context("When theres a reprocessing error", Label("reprocessingError", "flaky"), func() {
|
||||
It("Should update the reprocessing error.", func() {
|
||||
bc := setUpTest(BeaconNodeTester.TestConfig, "99")
|
||||
BeaconNodeTester.SetupBeaconNodeMock(BeaconNodeTester.TestEvents, BeaconNodeTester.TestConfig.protocol, BeaconNodeTester.TestConfig.address, BeaconNodeTester.TestConfig.port, BeaconNodeTester.TestConfig.dummyParentRoot)
|
||||
@ -201,7 +205,7 @@ func (tbc TestBeaconNode) writeEventToHistoricProcess(bc *beaconclient.BeaconCli
|
||||
// Start the CaptureHistoric function, and check for the correct inserted slots.
|
||||
func (tbc TestBeaconNode) runHistoricalProcess(bc *beaconclient.BeaconClient, maxWorkers int, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go bc.CaptureHistoric(ctx, maxWorkers)
|
||||
go bc.CaptureHistoric(ctx, maxWorkers, 0)
|
||||
validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
|
||||
log.Debug("Calling the stop function for historical processing..")
|
||||
err := bc.StopHistoric(cancel)
|
||||
@ -213,7 +217,7 @@ func (tbc TestBeaconNode) runHistoricalProcess(bc *beaconclient.BeaconClient, ma
|
||||
// Wrapper function that processes knownGaps
|
||||
func (tbc TestBeaconNode) runKnownGapsProcess(bc *beaconclient.BeaconClient, maxWorkers int, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go bc.ProcessKnownGaps(ctx, maxWorkers)
|
||||
go bc.ProcessKnownGaps(ctx, maxWorkers, 0)
|
||||
validateMetrics(bc, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError)
|
||||
err := bc.StopKnownGapsProcessing(cancel)
|
||||
time.Sleep(5 * time.Second)
|
||||
@ -268,16 +272,28 @@ func validateMetrics(bc *beaconclient.BeaconClient, expectedInserts, expectedReo
|
||||
// A wrapper function to validate a few popular blocks
|
||||
func validatePopularBatchBlocks(bc *beaconclient.BeaconClient) {
|
||||
validateSlot(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, 3, "proposed")
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectParentRoot, BeaconNodeTester.TestEvents["100"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["100"].CorrectSignedBeaconBlockMhKey)
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectBeaconStateMhKey)
|
||||
if bc.PerformBeaconBlockProcessing {
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectParentRoot, BeaconNodeTester.TestEvents["100"].CorrectEth1DataBlockHash, BeaconNodeTester.TestEvents["100"].CorrectSignedBeaconBlockMhKey, BeaconNodeTester.TestEvents["100"].CorrectExecutionPayloadHeader)
|
||||
}
|
||||
if bc.PerformBeaconStateProcessing {
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["100"].HeadMessage, BeaconNodeTester.TestEvents["100"].CorrectBeaconStateMhKey)
|
||||
}
|
||||
|
||||
validateSlot(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, 3, "proposed")
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, BeaconNodeTester.TestEvents["100"].HeadMessage.Block, BeaconNodeTester.TestEvents["101"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["101"].CorrectSignedBeaconBlockMhKey)
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, BeaconNodeTester.TestEvents["101"].CorrectBeaconStateMhKey)
|
||||
if bc.PerformBeaconBlockProcessing {
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, BeaconNodeTester.TestEvents["100"].HeadMessage.Block, BeaconNodeTester.TestEvents["101"].CorrectEth1DataBlockHash, BeaconNodeTester.TestEvents["101"].CorrectSignedBeaconBlockMhKey, BeaconNodeTester.TestEvents["101"].CorrectExecutionPayloadHeader)
|
||||
}
|
||||
if bc.PerformBeaconStateProcessing {
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["101"].HeadMessage, BeaconNodeTester.TestEvents["101"].CorrectBeaconStateMhKey)
|
||||
}
|
||||
|
||||
validateSlot(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, 74240, "proposed")
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectParentRoot, BeaconNodeTester.TestEvents["2375703"].CorrectEth1BlockHash, BeaconNodeTester.TestEvents["2375703"].CorrectSignedBeaconBlockMhKey)
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectBeaconStateMhKey)
|
||||
if bc.PerformBeaconBlockProcessing {
|
||||
validateSignedBeaconBlock(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectParentRoot, BeaconNodeTester.TestEvents["2375703"].CorrectEth1DataBlockHash, BeaconNodeTester.TestEvents["2375703"].CorrectSignedBeaconBlockMhKey, BeaconNodeTester.TestEvents["2375703"].CorrectExecutionPayloadHeader)
|
||||
}
|
||||
if bc.PerformBeaconStateProcessing {
|
||||
validateBeaconState(bc, BeaconNodeTester.TestEvents["2375703"].HeadMessage, BeaconNodeTester.TestEvents["2375703"].CorrectBeaconStateMhKey)
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure all rows have checked_out as false.
|
||||
|
@ -16,9 +16,11 @@
|
||||
package beaconclient
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@ -46,7 +48,7 @@ type SyncData struct {
|
||||
}
|
||||
|
||||
// This function will check to see if we are synced up with the head of chain.
|
||||
//{"data":{"is_syncing":true,"head_slot":"62528","sync_distance":"3734299"}}
|
||||
// {"data":{"is_syncing":true,"head_slot":"62528","sync_distance":"3734299"}}
|
||||
func (bc BeaconClient) CheckHeadSync() (bool, error) {
|
||||
syncStatus, err := bc.QueryHeadSync()
|
||||
if err != nil {
|
||||
@ -71,14 +73,17 @@ func (bc BeaconClient) QueryHeadSync() (Sync, error) {
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
var body bytes.Buffer
|
||||
buf := bufio.NewWriter(&body)
|
||||
_, err = io.Copy(buf, resp.Body)
|
||||
|
||||
if err != nil {
|
||||
return syncStatus, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &syncStatus); err != nil {
|
||||
if err := json.Unmarshal(body.Bytes(), &syncStatus); err != nil {
|
||||
loghelper.LogEndpoint(bcSync).WithFields(log.Fields{
|
||||
"rawMessage": string(body),
|
||||
"rawMessage": body.String(),
|
||||
"err": err,
|
||||
}).Error("Unable to unmarshal sync status")
|
||||
return syncStatus, err
|
||||
@ -149,14 +154,16 @@ func (bc BeaconClient) queryLighthouseDbInfo() (LighthouseDatabaseInfo, error) {
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
var body bytes.Buffer
|
||||
buf := bufio.NewWriter(&body)
|
||||
_, err = io.Copy(buf, resp.Body)
|
||||
if err != nil {
|
||||
return dbInfo, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &dbInfo); err != nil {
|
||||
if err := json.Unmarshal(body.Bytes(), &dbInfo); err != nil {
|
||||
loghelper.LogEndpoint(lhDbInfo).WithFields(log.Fields{
|
||||
"rawMessage": string(body),
|
||||
"rawMessage": body.String(),
|
||||
"err": err,
|
||||
}).Error("Unable to unmarshal the lighthouse database information")
|
||||
return dbInfo, err
|
||||
|
441
pkg/beaconclient/consensus.go
Normal file
441
pkg/beaconclient/consensus.go
Normal file
@ -0,0 +1,441 @@
|
||||
package beaconclient
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"github.com/protolambda/zrnt/eth2/beacon/altair"
|
||||
"github.com/protolambda/zrnt/eth2/beacon/bellatrix"
|
||||
"github.com/protolambda/zrnt/eth2/beacon/common"
|
||||
"github.com/protolambda/zrnt/eth2/beacon/phase0"
|
||||
"github.com/protolambda/zrnt/eth2/configs"
|
||||
"github.com/protolambda/ztyp/codec"
|
||||
"github.com/protolambda/ztyp/tree"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Eth1Data common.Eth1Data
|
||||
type Root common.Root
|
||||
type Signature common.BLSSignature
|
||||
type Slot uint64
|
||||
type Epoch uint64
|
||||
type ExecutionPayloadHeader common.ExecutionPayloadHeader
|
||||
|
||||
func ParseSlot(v string) (Slot, error) {
|
||||
slotNum, err := strconv.ParseUint(v, 10, 64)
|
||||
return Slot(slotNum), err
|
||||
}
|
||||
|
||||
func (s *Slot) Format() string {
|
||||
return strconv.FormatUint(uint64(*s), 10)
|
||||
}
|
||||
|
||||
func (s *Slot) Number() uint64 {
|
||||
return uint64(*s)
|
||||
}
|
||||
|
||||
func (s *Slot) Plus(v uint64) Slot {
|
||||
return Slot(v + s.Number())
|
||||
}
|
||||
|
||||
func (s *Slot) PlusInt(v int) Slot {
|
||||
return s.Plus(uint64(v))
|
||||
}
|
||||
|
||||
func (e *Epoch) Format() string {
|
||||
return strconv.FormatUint(uint64(*e), 10)
|
||||
}
|
||||
|
||||
type BeaconBlock struct {
|
||||
spec *common.Spec
|
||||
bellatrix *bellatrix.BeaconBlock
|
||||
altair *altair.BeaconBlock
|
||||
phase0 *phase0.BeaconBlock
|
||||
}
|
||||
|
||||
type BeaconBlockBody struct {
|
||||
spec *common.Spec
|
||||
bellatrix *bellatrix.BeaconBlockBody
|
||||
altair *altair.BeaconBlockBody
|
||||
phase0 *phase0.BeaconBlockBody
|
||||
}
|
||||
|
||||
type BeaconState struct {
|
||||
spec *common.Spec
|
||||
bellatrix *bellatrix.BeaconState
|
||||
altair *altair.BeaconState
|
||||
phase0 *phase0.BeaconState
|
||||
}
|
||||
|
||||
type SignedBeaconBlock struct {
|
||||
spec *common.Spec
|
||||
bellatrix *bellatrix.SignedBeaconBlock
|
||||
altair *altair.SignedBeaconBlock
|
||||
phase0 *phase0.SignedBeaconBlock
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) UnmarshalSSZ(ssz []byte) error {
|
||||
spec := chooseSpec(s.spec)
|
||||
|
||||
var bellatrix bellatrix.SignedBeaconBlock
|
||||
err := bellatrix.Deserialize(spec, makeDecodingReader(ssz))
|
||||
if nil == err {
|
||||
s.bellatrix = &bellatrix
|
||||
s.altair = nil
|
||||
s.phase0 = nil
|
||||
log.Info("Unmarshalled Bellatrix SignedBeaconBlock")
|
||||
return nil
|
||||
}
|
||||
|
||||
var altair altair.SignedBeaconBlock
|
||||
err = altair.Deserialize(spec, makeDecodingReader(ssz))
|
||||
if nil == err {
|
||||
s.bellatrix = nil
|
||||
s.altair = &altair
|
||||
s.phase0 = nil
|
||||
log.Info("Unmarshalled Altair SignedBeaconBlock")
|
||||
return nil
|
||||
}
|
||||
|
||||
var phase0 phase0.SignedBeaconBlock
|
||||
err = phase0.Deserialize(spec, makeDecodingReader(ssz))
|
||||
if nil == err {
|
||||
s.bellatrix = nil
|
||||
s.altair = nil
|
||||
s.phase0 = &phase0
|
||||
log.Info("Unmarshalled Phase0 SignedBeaconBlock")
|
||||
return nil
|
||||
}
|
||||
|
||||
s.bellatrix = nil
|
||||
s.altair = nil
|
||||
s.phase0 = nil
|
||||
|
||||
log.Warning("Unable to unmarshal SignedBeaconBlock")
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) MarshalSSZ() ([]byte, error) {
|
||||
spec := chooseSpec(s.spec)
|
||||
var err error
|
||||
var buf bytes.Buffer
|
||||
encodingWriter := codec.NewEncodingWriter(&buf)
|
||||
|
||||
if s.IsBellatrix() {
|
||||
err = s.bellatrix.Serialize(spec, encodingWriter)
|
||||
}
|
||||
if s.IsAltair() {
|
||||
err = s.altair.Serialize(spec, encodingWriter)
|
||||
}
|
||||
if s.IsPhase0() {
|
||||
err = s.phase0.Serialize(spec, encodingWriter)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), err
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) IsBellatrix() bool {
|
||||
return s.bellatrix != nil
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) IsAltair() bool {
|
||||
return s.altair != nil
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) IsPhase0() bool {
|
||||
return s.phase0 != nil
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) GetBellatrix() *bellatrix.SignedBeaconBlock {
|
||||
return s.bellatrix
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) GetAltair() *altair.SignedBeaconBlock {
|
||||
return s.altair
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) GetPhase0() *phase0.SignedBeaconBlock {
|
||||
return s.phase0
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) Signature() Signature {
|
||||
if s.IsBellatrix() {
|
||||
return Signature(s.bellatrix.Signature)
|
||||
}
|
||||
|
||||
if s.IsAltair() {
|
||||
return Signature(s.altair.Signature)
|
||||
}
|
||||
|
||||
if s.IsPhase0() {
|
||||
return Signature(s.phase0.Signature)
|
||||
}
|
||||
|
||||
return Signature{}
|
||||
}
|
||||
|
||||
func (s *SignedBeaconBlock) Block() *BeaconBlock {
|
||||
if s.IsBellatrix() {
|
||||
return &BeaconBlock{bellatrix: &s.bellatrix.Message, spec: s.spec}
|
||||
}
|
||||
|
||||
if s.IsAltair() {
|
||||
return &BeaconBlock{altair: &s.altair.Message, spec: s.spec}
|
||||
}
|
||||
|
||||
if s.IsPhase0() {
|
||||
return &BeaconBlock{phase0: &s.phase0.Message, spec: s.spec}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BeaconBlock) IsBellatrix() bool {
|
||||
return b.bellatrix != nil
|
||||
}
|
||||
|
||||
func (b *BeaconBlock) IsAltair() bool {
|
||||
return b.altair != nil
|
||||
}
|
||||
|
||||
func (b *BeaconBlock) IsPhase0() bool {
|
||||
return b.phase0 != nil
|
||||
}
|
||||
|
||||
func (s *BeaconBlock) GetBellatrix() *bellatrix.BeaconBlock {
|
||||
return s.bellatrix
|
||||
}
|
||||
|
||||
func (s *BeaconBlock) GetAltair() *altair.BeaconBlock {
|
||||
return s.altair
|
||||
}
|
||||
|
||||
func (s *BeaconBlock) GetPhase0() *phase0.BeaconBlock {
|
||||
return s.phase0
|
||||
}
|
||||
|
||||
func (b *BeaconBlock) ParentRoot() Root {
|
||||
if b.IsBellatrix() {
|
||||
return Root(b.bellatrix.ParentRoot)
|
||||
}
|
||||
|
||||
if b.IsAltair() {
|
||||
return Root(b.altair.ParentRoot)
|
||||
}
|
||||
|
||||
if b.IsPhase0() {
|
||||
return Root(b.phase0.ParentRoot)
|
||||
}
|
||||
|
||||
return Root{}
|
||||
}
|
||||
|
||||
func (b *BeaconBlock) StateRoot() Root {
|
||||
if b.IsBellatrix() {
|
||||
return Root(b.bellatrix.StateRoot)
|
||||
}
|
||||
|
||||
if b.IsAltair() {
|
||||
return Root(b.altair.StateRoot)
|
||||
}
|
||||
|
||||
if b.IsPhase0() {
|
||||
return Root(b.phase0.StateRoot)
|
||||
}
|
||||
|
||||
return Root{}
|
||||
}
|
||||
|
||||
func (b *BeaconBlock) Body() *BeaconBlockBody {
|
||||
if b.IsBellatrix() {
|
||||
return &BeaconBlockBody{bellatrix: &b.bellatrix.Body, spec: b.spec}
|
||||
}
|
||||
|
||||
if b.IsAltair() {
|
||||
return &BeaconBlockBody{altair: &b.altair.Body, spec: b.spec}
|
||||
}
|
||||
|
||||
if b.IsPhase0() {
|
||||
return &BeaconBlockBody{phase0: &b.phase0.Body, spec: b.spec}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BeaconBlockBody) IsBellatrix() bool {
|
||||
return b.bellatrix != nil
|
||||
}
|
||||
|
||||
func (b *BeaconBlockBody) IsAltair() bool {
|
||||
return b.altair != nil
|
||||
}
|
||||
|
||||
func (b *BeaconBlockBody) IsPhase0() bool {
|
||||
return b.phase0 != nil
|
||||
}
|
||||
|
||||
func (b *BeaconBlockBody) Eth1Data() Eth1Data {
|
||||
if b.IsBellatrix() {
|
||||
return Eth1Data(b.bellatrix.Eth1Data)
|
||||
}
|
||||
|
||||
if b.IsAltair() {
|
||||
return Eth1Data(b.altair.Eth1Data)
|
||||
}
|
||||
|
||||
if b.IsPhase0() {
|
||||
return Eth1Data(b.phase0.Eth1Data)
|
||||
}
|
||||
|
||||
return Eth1Data{}
|
||||
}
|
||||
|
||||
func (b *BeaconBlockBody) ExecutionPayloadHeader() *ExecutionPayloadHeader {
|
||||
if b.IsBellatrix() {
|
||||
payloadHeader := b.bellatrix.ExecutionPayload.Header(chooseSpec(b.spec))
|
||||
return (*ExecutionPayloadHeader)(payloadHeader)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BeaconBlock) HashTreeRoot() Root {
|
||||
spec := chooseSpec(b.spec)
|
||||
hashFn := tree.GetHashFn()
|
||||
|
||||
if b.IsBellatrix() {
|
||||
return Root(b.bellatrix.HashTreeRoot(spec, hashFn))
|
||||
}
|
||||
|
||||
if b.IsAltair() {
|
||||
return Root(b.altair.HashTreeRoot(spec, hashFn))
|
||||
}
|
||||
|
||||
if b.IsPhase0() {
|
||||
return Root(b.phase0.HashTreeRoot(spec, hashFn))
|
||||
}
|
||||
|
||||
return Root{}
|
||||
}
|
||||
|
||||
func (s *BeaconState) UnmarshalSSZ(ssz []byte) error {
|
||||
spec := chooseSpec(s.spec)
|
||||
|
||||
var bellatrix bellatrix.BeaconState
|
||||
err := bellatrix.Deserialize(spec, makeDecodingReader(ssz))
|
||||
if nil == err {
|
||||
s.bellatrix = &bellatrix
|
||||
s.altair = nil
|
||||
s.phase0 = nil
|
||||
log.Info("Unmarshalled Bellatrix BeaconState")
|
||||
return nil
|
||||
}
|
||||
|
||||
var altair altair.BeaconState
|
||||
err = altair.Deserialize(spec, makeDecodingReader(ssz))
|
||||
if nil == err {
|
||||
s.bellatrix = nil
|
||||
s.altair = &altair
|
||||
s.phase0 = nil
|
||||
log.Info("Unmarshalled Altair BeaconState")
|
||||
return nil
|
||||
}
|
||||
|
||||
var phase0 phase0.BeaconState
|
||||
err = phase0.Deserialize(spec, makeDecodingReader(ssz))
|
||||
if nil == err {
|
||||
s.bellatrix = nil
|
||||
s.altair = nil
|
||||
s.phase0 = &phase0
|
||||
log.Info("Unmarshalled Phase0 BeaconState")
|
||||
return nil
|
||||
}
|
||||
|
||||
s.bellatrix = nil
|
||||
s.altair = nil
|
||||
s.phase0 = nil
|
||||
|
||||
log.Warning("Unable to unmarshal BeaconState")
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *BeaconState) MarshalSSZ() ([]byte, error) {
|
||||
spec := chooseSpec(s.spec)
|
||||
var err error
|
||||
var buf bytes.Buffer
|
||||
encodingWriter := codec.NewEncodingWriter(&buf)
|
||||
|
||||
if s.IsBellatrix() {
|
||||
err = s.bellatrix.Serialize(spec, encodingWriter)
|
||||
} else if s.IsAltair() {
|
||||
err = s.altair.Serialize(spec, encodingWriter)
|
||||
} else if s.IsPhase0() {
|
||||
err = s.phase0.Serialize(spec, encodingWriter)
|
||||
} else {
|
||||
err = errors.New("BeaconState not set")
|
||||
}
|
||||
|
||||
if nil != err {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (s *BeaconState) IsBellatrix() bool {
|
||||
return s.bellatrix != nil
|
||||
}
|
||||
|
||||
func (s *BeaconState) IsAltair() bool {
|
||||
return s.altair != nil
|
||||
}
|
||||
|
||||
func (s *BeaconState) IsPhase0() bool {
|
||||
return s.phase0 != nil
|
||||
}
|
||||
|
||||
func (s *BeaconState) HashTreeRoot() Root {
|
||||
spec := chooseSpec(s.spec)
|
||||
hashFn := tree.GetHashFn()
|
||||
|
||||
if s.IsBellatrix() {
|
||||
return Root(s.bellatrix.HashTreeRoot(spec, hashFn))
|
||||
}
|
||||
|
||||
if s.IsAltair() {
|
||||
return Root(s.altair.HashTreeRoot(spec, hashFn))
|
||||
}
|
||||
|
||||
if s.IsPhase0() {
|
||||
return Root(s.phase0.HashTreeRoot(spec, hashFn))
|
||||
}
|
||||
|
||||
return Root{}
|
||||
}
|
||||
|
||||
func (s *BeaconState) GetBellatrix() *bellatrix.BeaconState {
|
||||
return s.bellatrix
|
||||
}
|
||||
|
||||
func (s *BeaconState) GetAltair() *altair.BeaconState {
|
||||
return s.altair
|
||||
}
|
||||
|
||||
func (s *BeaconState) GetPhase0() *phase0.BeaconState {
|
||||
return s.phase0
|
||||
}
|
||||
|
||||
func chooseSpec(spec *common.Spec) *common.Spec {
|
||||
if nil == spec {
|
||||
return configs.Mainnet
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
func makeDecodingReader(ssz []byte) *codec.DecodingReader {
|
||||
return codec.NewDecodingReader(bytes.NewReader(ssz), uint64(len(ssz)))
|
||||
}
|
@ -18,8 +18,6 @@ package beaconclient
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/jackc/pgx/v4"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
|
||||
@ -34,8 +32,14 @@ INSERT INTO eth_beacon.slots (epoch, slot, block_root, state_root, status)
|
||||
VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
|
||||
// Statement to upsert to the eth_beacon.signed_blocks table.
|
||||
UpsertSignedBeaconBlockStmt string = `
|
||||
INSERT INTO eth_beacon.signed_block (slot, block_root, parent_block_root, eth1_block_hash, mh_key)
|
||||
INSERT INTO eth_beacon.signed_block (slot, block_root, parent_block_root, eth1_data_block_hash, mh_key)
|
||||
VALUES ($1, $2, $3, $4, $5) ON CONFLICT (slot, block_root) DO NOTHING`
|
||||
UpsertSignedBeaconBlockWithPayloadStmt string = `
|
||||
INSERT INTO eth_beacon.signed_block (slot, block_root, parent_block_root, eth1_data_block_hash, mh_key,
|
||||
payload_block_number, payload_timestamp, payload_block_hash,
|
||||
payload_parent_hash, payload_state_root, payload_receipts_root,
|
||||
payload_transactions_root)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (slot, block_root) DO NOTHING`
|
||||
// Statement to upsert to the eth_beacon.state table.
|
||||
UpsertBeaconState string = `
|
||||
INSERT INTO eth_beacon.state (slot, state_root, mh_key)
|
||||
@ -94,8 +98,8 @@ type DatabaseWriter struct {
|
||||
rawSignedBeaconBlock *[]byte
|
||||
}
|
||||
|
||||
func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot string, parentBlockRoot string,
|
||||
eth1BlockHash string, status string, rawSignedBeaconBlock *[]byte, rawBeaconState *[]byte, metrics *BeaconClientMetrics) (*DatabaseWriter, error) {
|
||||
func CreateDatabaseWrite(db sql.Database, slot Slot, stateRoot string, blockRoot string, parentBlockRoot string,
|
||||
eth1DataBlockHash string, payloadHeader *ExecutionPayloadHeader, status string, rawSignedBeaconBlock *[]byte, rawBeaconState *[]byte, metrics *BeaconClientMetrics) (*DatabaseWriter, error) {
|
||||
ctx := context.Background()
|
||||
tx, err := db.Begin(ctx)
|
||||
if err != nil {
|
||||
@ -110,7 +114,7 @@ func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot
|
||||
Metrics: metrics,
|
||||
}
|
||||
dw.prepareSlotsModel(slot, stateRoot, blockRoot, status)
|
||||
err = dw.prepareSignedBeaconBlockModel(slot, blockRoot, parentBlockRoot, eth1BlockHash)
|
||||
err = dw.prepareSignedBeaconBlockModel(slot, blockRoot, parentBlockRoot, eth1DataBlockHash, payloadHeader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -124,10 +128,10 @@ func CreateDatabaseWrite(db sql.Database, slot int, stateRoot string, blockRoot
|
||||
// Write functions to write each all together...
|
||||
// Should I do one atomic write?
|
||||
// Create the model for the eth_beacon.slots table
|
||||
func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoot string, status string) {
|
||||
func (dw *DatabaseWriter) prepareSlotsModel(slot Slot, stateRoot string, blockRoot string, status string) {
|
||||
dw.DbSlots = &DbSlots{
|
||||
Epoch: calculateEpoch(slot, bcSlotsPerEpoch),
|
||||
Slot: strconv.Itoa(slot),
|
||||
Slot: slot.Number(),
|
||||
StateRoot: stateRoot,
|
||||
BlockRoot: blockRoot,
|
||||
Status: status,
|
||||
@ -137,30 +141,45 @@ func (dw *DatabaseWriter) prepareSlotsModel(slot int, stateRoot string, blockRoo
|
||||
}
|
||||
|
||||
// Create the model for the eth_beacon.signed_block table.
|
||||
func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot int, blockRoot string, parentBlockRoot string, eth1BlockHash string) error {
|
||||
func (dw *DatabaseWriter) prepareSignedBeaconBlockModel(slot Slot, blockRoot string, parentBlockRoot string, eth1DataBlockHash string,
|
||||
payloadHeader *ExecutionPayloadHeader) error {
|
||||
mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.BlockRoot))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dw.DbSignedBeaconBlock = &DbSignedBeaconBlock{
|
||||
Slot: strconv.Itoa(slot),
|
||||
BlockRoot: blockRoot,
|
||||
ParentBlock: parentBlockRoot,
|
||||
Eth1BlockHash: eth1BlockHash,
|
||||
MhKey: mhKey,
|
||||
Slot: slot.Number(),
|
||||
BlockRoot: blockRoot,
|
||||
ParentBlock: parentBlockRoot,
|
||||
Eth1DataBlockHash: eth1DataBlockHash,
|
||||
MhKey: mhKey,
|
||||
ExecutionPayloadHeader: nil,
|
||||
}
|
||||
|
||||
if nil != payloadHeader {
|
||||
dw.DbSignedBeaconBlock.ExecutionPayloadHeader = &DbExecutionPayloadHeader{
|
||||
BlockNumber: uint64(payloadHeader.BlockNumber),
|
||||
Timestamp: uint64(payloadHeader.Timestamp),
|
||||
BlockHash: toHex(payloadHeader.BlockHash),
|
||||
ParentHash: toHex(payloadHeader.ParentHash),
|
||||
StateRoot: toHex(payloadHeader.StateRoot),
|
||||
ReceiptsRoot: toHex(payloadHeader.ReceiptsRoot),
|
||||
TransactionsRoot: toHex(payloadHeader.TransactionsRoot),
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug("dw.DbSignedBeaconBlock: ", dw.DbSignedBeaconBlock)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create the model for the eth_beacon.state table.
|
||||
func (dw *DatabaseWriter) prepareBeaconStateModel(slot int, stateRoot string) error {
|
||||
func (dw *DatabaseWriter) prepareBeaconStateModel(slot Slot, stateRoot string) error {
|
||||
mhKey, err := MultihashKeyFromSSZRoot([]byte(dw.DbSlots.StateRoot))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dw.DbBeaconState = &DbBeaconState{
|
||||
Slot: strconv.Itoa(slot),
|
||||
Slot: slot.Number(),
|
||||
StateRoot: stateRoot,
|
||||
MhKey: mhKey,
|
||||
}
|
||||
@ -229,6 +248,11 @@ func (dw *DatabaseWriter) upsertSlots() error {
|
||||
|
||||
// Add the information for the signed_block to a transaction.
|
||||
func (dw *DatabaseWriter) transactSignedBeaconBlocks() error {
|
||||
if nil == dw.rawSignedBeaconBlock || len(*dw.rawSignedBeaconBlock) == 0 {
|
||||
log.Warn("Skipping writing of empty BeaconBlock.")
|
||||
return nil
|
||||
}
|
||||
|
||||
err := dw.upsertPublicBlocks(dw.DbSignedBeaconBlock.MhKey, dw.rawSignedBeaconBlock)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -252,9 +276,36 @@ func (dw *DatabaseWriter) upsertPublicBlocks(key string, data *[]byte) error {
|
||||
|
||||
// Upsert to the eth_beacon.signed_block table.
|
||||
func (dw *DatabaseWriter) upsertSignedBeaconBlock() error {
|
||||
_, err := dw.Tx.Exec(dw.Ctx, UpsertSignedBeaconBlockStmt, dw.DbSignedBeaconBlock.Slot, dw.DbSignedBeaconBlock.BlockRoot, dw.DbSignedBeaconBlock.ParentBlock, dw.DbSignedBeaconBlock.Eth1BlockHash, dw.DbSignedBeaconBlock.MhKey)
|
||||
block := dw.DbSignedBeaconBlock
|
||||
var err error
|
||||
if nil != block.ExecutionPayloadHeader {
|
||||
_, err = dw.Tx.Exec(dw.Ctx,
|
||||
UpsertSignedBeaconBlockWithPayloadStmt,
|
||||
block.Slot,
|
||||
block.BlockRoot,
|
||||
block.ParentBlock,
|
||||
block.Eth1DataBlockHash,
|
||||
block.MhKey,
|
||||
block.ExecutionPayloadHeader.BlockNumber,
|
||||
block.ExecutionPayloadHeader.Timestamp,
|
||||
block.ExecutionPayloadHeader.BlockHash,
|
||||
block.ExecutionPayloadHeader.ParentHash,
|
||||
block.ExecutionPayloadHeader.StateRoot,
|
||||
block.ExecutionPayloadHeader.ReceiptsRoot,
|
||||
block.ExecutionPayloadHeader.TransactionsRoot,
|
||||
)
|
||||
} else {
|
||||
_, err = dw.Tx.Exec(dw.Ctx,
|
||||
UpsertSignedBeaconBlockStmt,
|
||||
block.Slot,
|
||||
block.BlockRoot,
|
||||
block.ParentBlock,
|
||||
block.Eth1DataBlockHash,
|
||||
block.MhKey,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(dw.DbSlots.Slot, err).WithFields(log.Fields{"block_root": dw.DbSignedBeaconBlock.BlockRoot}).Error("Unable to write to the slot to the eth_beacon.signed_block table")
|
||||
loghelper.LogSlotError(dw.DbSlots.Slot, err).WithFields(log.Fields{"block_root": block.BlockRoot}).Error("Unable to write to the slot to the eth_beacon.signed_block table")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@ -262,6 +313,11 @@ func (dw *DatabaseWriter) upsertSignedBeaconBlock() error {
|
||||
|
||||
// Add the information for the state to a transaction.
|
||||
func (dw *DatabaseWriter) transactBeaconState() error {
|
||||
if nil == dw.rawBeaconState || len(*dw.rawBeaconState) == 0 {
|
||||
log.Warn("Skipping writing of empty BeaconState.")
|
||||
return nil
|
||||
}
|
||||
|
||||
err := dw.upsertPublicBlocks(dw.DbBeaconState.MhKey, dw.rawBeaconState)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -285,56 +341,51 @@ func (dw *DatabaseWriter) upsertBeaconState() error {
|
||||
|
||||
// Update a given slot to be marked as forked within a transaction. Provide the slot and the latest latestBlockRoot.
|
||||
// We will mark all entries for the given slot that don't match the provided latestBlockRoot as forked.
|
||||
func transactReorgs(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) {
|
||||
slotNum, strErr := strconv.Atoi(slot)
|
||||
if strErr != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, strErr).Error("We can't convert the slot to an int...")
|
||||
}
|
||||
|
||||
func transactReorgs(tx sql.Tx, ctx context.Context, slot Slot, latestBlockRoot string, metrics *BeaconClientMetrics) {
|
||||
forkCount, err := updateForked(tx, ctx, slot, latestBlockRoot)
|
||||
if err != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while updating all forks.")
|
||||
transactKnownGaps(tx, ctx, 1, slotNum, slotNum, err, "reorg", metrics)
|
||||
loghelper.LogReorgError(slot.Number(), latestBlockRoot, err).Error("We ran into some trouble while updating all forks.")
|
||||
transactKnownGaps(tx, ctx, 1, slot, slot, err, "reorg", metrics)
|
||||
}
|
||||
proposedCount, err := updateProposed(tx, ctx, slot, latestBlockRoot)
|
||||
if err != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We ran into some trouble while trying to update the proposed slot.")
|
||||
transactKnownGaps(tx, ctx, 1, slotNum, slotNum, err, "reorg", metrics)
|
||||
loghelper.LogReorgError(slot.Number(), latestBlockRoot, err).Error("We ran into some trouble while trying to update the proposed slot.")
|
||||
transactKnownGaps(tx, ctx, 1, slot, slot, err, "reorg", metrics)
|
||||
}
|
||||
|
||||
if forkCount > 0 {
|
||||
loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
|
||||
loghelper.LogReorg(slot.Number(), latestBlockRoot).WithFields(log.Fields{
|
||||
"forkCount": forkCount,
|
||||
}).Info("Updated rows that were forked.")
|
||||
} else {
|
||||
loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
|
||||
loghelper.LogReorg(slot.Number(), latestBlockRoot).WithFields(log.Fields{
|
||||
"forkCount": forkCount,
|
||||
}).Warn("There were no forked rows to update.")
|
||||
}
|
||||
|
||||
if proposedCount == 1 {
|
||||
loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
|
||||
loghelper.LogReorg(slot.Number(), latestBlockRoot).WithFields(log.Fields{
|
||||
"proposedCount": proposedCount,
|
||||
}).Info("Updated the row that should have been marked as proposed.")
|
||||
} else if proposedCount > 1 {
|
||||
loghelper.LogReorg(slot, latestBlockRoot).WithFields(log.Fields{
|
||||
loghelper.LogReorg(slot.Number(), latestBlockRoot).WithFields(log.Fields{
|
||||
"proposedCount": proposedCount,
|
||||
}).Error("Too many rows were marked as proposed!")
|
||||
transactKnownGaps(tx, ctx, 1, slotNum, slotNum, fmt.Errorf("Too many rows were marked as unproposed."), "reorg", metrics)
|
||||
transactKnownGaps(tx, ctx, 1, slot, slot, fmt.Errorf("Too many rows were marked as unproposed."), "reorg", metrics)
|
||||
} else if proposedCount == 0 {
|
||||
transactKnownGaps(tx, ctx, 1, slotNum, slotNum, fmt.Errorf("Unable to find properly proposed row in DB"), "reorg", metrics)
|
||||
loghelper.LogReorg(slot, latestBlockRoot).Info("Updated the row that should have been marked as proposed.")
|
||||
transactKnownGaps(tx, ctx, 1, slot, slot, fmt.Errorf("Unable to find properly proposed row in DB"), "reorg", metrics)
|
||||
loghelper.LogReorg(slot.Number(), latestBlockRoot).Info("Updated the row that should have been marked as proposed.")
|
||||
}
|
||||
|
||||
metrics.IncrementReorgsInsert(1)
|
||||
}
|
||||
|
||||
// Wrapper function that will create a transaction and execute the function.
|
||||
func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *BeaconClientMetrics) {
|
||||
func writeReorgs(db sql.Database, slot Slot, latestBlockRoot string, metrics *BeaconClientMetrics) {
|
||||
ctx := context.Background()
|
||||
tx, err := db.Begin(ctx)
|
||||
if err != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, err).Fatal("Unable to create a new transaction for reorgs")
|
||||
loghelper.LogReorgError(slot.Number(), latestBlockRoot, err).Fatal("Unable to create a new transaction for reorgs")
|
||||
}
|
||||
defer func() {
|
||||
err := tx.Rollback(ctx)
|
||||
@ -344,35 +395,35 @@ func writeReorgs(db sql.Database, slot string, latestBlockRoot string, metrics *
|
||||
}()
|
||||
transactReorgs(tx, ctx, slot, latestBlockRoot, metrics)
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, err).Fatal("Unable to execute the transaction for reorgs")
|
||||
loghelper.LogReorgError(slot.Number(), latestBlockRoot, err).Fatal("Unable to execute the transaction for reorgs")
|
||||
}
|
||||
}
|
||||
|
||||
// Update the slots table by marking the old slot's as forked.
|
||||
func updateForked(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) {
|
||||
func updateForked(tx sql.Tx, ctx context.Context, slot Slot, latestBlockRoot string) (int64, error) {
|
||||
res, err := tx.Exec(ctx, UpdateForkedStmt, slot, latestBlockRoot)
|
||||
if err != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the forked slots")
|
||||
loghelper.LogReorgError(slot.Number(), latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the forked slots")
|
||||
return 0, err
|
||||
}
|
||||
count, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, err).Error("Unable to figure out how many entries were marked as forked.")
|
||||
loghelper.LogReorgError(slot.Number(), latestBlockRoot, err).Error("Unable to figure out how many entries were marked as forked.")
|
||||
return 0, err
|
||||
}
|
||||
return count, err
|
||||
}
|
||||
|
||||
// Mark a slot as proposed.
|
||||
func updateProposed(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot string) (int64, error) {
|
||||
func updateProposed(tx sql.Tx, ctx context.Context, slot Slot, latestBlockRoot string) (int64, error) {
|
||||
res, err := tx.Exec(ctx, UpdateProposedStmt, slot, latestBlockRoot)
|
||||
if err != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the proposed slot.")
|
||||
loghelper.LogReorgError(slot.Number(), latestBlockRoot, err).Error("We are unable to update the eth_beacon.slots table with the proposed slot.")
|
||||
return 0, err
|
||||
}
|
||||
count, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
loghelper.LogReorgError(slot, latestBlockRoot, err).Error("Unable to figure out how many entries were marked as proposed")
|
||||
loghelper.LogReorgError(slot.Number(), latestBlockRoot, err).Error("Unable to figure out how many entries were marked as proposed")
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@ -382,17 +433,17 @@ func updateProposed(tx sql.Tx, ctx context.Context, slot string, latestBlockRoot
|
||||
// A wrapper function to call upsertKnownGaps. This function will break down the range of known_gaps into
|
||||
// smaller chunks. For example, instead of having an entry of 1-101, if we increment the entries by 10 slots, we would
|
||||
// have 10 entries as follows: 1-10, 11-20, etc...
|
||||
func transactKnownGaps(tx sql.Tx, ctx context.Context, tableIncrement int, startSlot int, endSlot int, entryError error, entryProcess string, metric *BeaconClientMetrics) {
|
||||
func transactKnownGaps(tx sql.Tx, ctx context.Context, tableIncrement int, startSlot Slot, endSlot Slot, entryError error, entryProcess string, metric *BeaconClientMetrics) {
|
||||
var entryErrorMsg string
|
||||
if entryError == nil {
|
||||
entryErrorMsg = ""
|
||||
} else {
|
||||
entryErrorMsg = entryError.Error()
|
||||
}
|
||||
if endSlot-startSlot <= tableIncrement {
|
||||
if endSlot.Number()-startSlot.Number() <= uint64(tableIncrement) {
|
||||
kgModel := DbKnownGaps{
|
||||
StartSlot: strconv.Itoa(startSlot),
|
||||
EndSlot: strconv.Itoa(endSlot),
|
||||
StartSlot: startSlot.Number(),
|
||||
EndSlot: endSlot.Number(),
|
||||
CheckedOut: false,
|
||||
ReprocessingError: "",
|
||||
EntryError: entryErrorMsg,
|
||||
@ -400,24 +451,24 @@ func transactKnownGaps(tx sql.Tx, ctx context.Context, tableIncrement int, start
|
||||
}
|
||||
upsertKnownGaps(tx, ctx, kgModel, metric)
|
||||
} else {
|
||||
totalSlots := endSlot - startSlot
|
||||
totalSlots := endSlot.Number() - startSlot.Number()
|
||||
var chunks int
|
||||
chunks = totalSlots / tableIncrement
|
||||
if totalSlots%tableIncrement != 0 {
|
||||
chunks = int(totalSlots / uint64(tableIncrement))
|
||||
if totalSlots%uint64(tableIncrement) != 0 {
|
||||
chunks = chunks + 1
|
||||
}
|
||||
|
||||
for i := 0; i < chunks; i++ {
|
||||
var tempStart, tempEnd int
|
||||
tempStart = startSlot + (i * tableIncrement)
|
||||
var tempStart, tempEnd Slot
|
||||
tempStart = startSlot.PlusInt(i * tableIncrement)
|
||||
if i+1 == chunks {
|
||||
tempEnd = endSlot
|
||||
} else {
|
||||
tempEnd = startSlot + ((i + 1) * tableIncrement)
|
||||
tempEnd = startSlot.PlusInt((i + 1) * tableIncrement)
|
||||
}
|
||||
kgModel := DbKnownGaps{
|
||||
StartSlot: strconv.Itoa(tempStart),
|
||||
EndSlot: strconv.Itoa(tempEnd),
|
||||
StartSlot: tempStart.Number(),
|
||||
EndSlot: tempEnd.Number(),
|
||||
CheckedOut: false,
|
||||
ReprocessingError: "",
|
||||
EntryError: entryErrorMsg,
|
||||
@ -430,11 +481,11 @@ func transactKnownGaps(tx sql.Tx, ctx context.Context, tableIncrement int, start
|
||||
|
||||
// Wrapper function, instead of adding the knownGaps entries to a transaction, it will
|
||||
// create the transaction and write it.
|
||||
func writeKnownGaps(db sql.Database, tableIncrement int, startSlot int, endSlot int, entryError error, entryProcess string, metric *BeaconClientMetrics) {
|
||||
func writeKnownGaps(db sql.Database, tableIncrement int, startSlot Slot, endSlot Slot, entryError error, entryProcess string, metric *BeaconClientMetrics) {
|
||||
ctx := context.Background()
|
||||
tx, err := db.Begin(ctx)
|
||||
if err != nil {
|
||||
loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Fatal("Unable to create a new transaction for knownGaps")
|
||||
loghelper.LogSlotRangeError(startSlot.Number(), endSlot.Number(), err).Fatal("Unable to create a new transaction for knownGaps")
|
||||
}
|
||||
defer func() {
|
||||
err := tx.Rollback(ctx)
|
||||
@ -444,7 +495,7 @@ func writeKnownGaps(db sql.Database, tableIncrement int, startSlot int, endSlot
|
||||
}()
|
||||
transactKnownGaps(tx, ctx, tableIncrement, startSlot, endSlot, entryError, entryProcess, metric)
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Fatal("Unable to execute the transaction for knownGaps")
|
||||
loghelper.LogSlotRangeError(startSlot.Number(), endSlot.Number(), err).Fatal("Unable to execute the transaction for knownGaps")
|
||||
}
|
||||
}
|
||||
|
||||
@ -467,8 +518,8 @@ func upsertKnownGaps(tx sql.Tx, ctx context.Context, knModel DbKnownGaps, metric
|
||||
}
|
||||
|
||||
// A function to write the gap between the highest slot in the DB and the first processed slot.
|
||||
func writeStartUpGaps(db sql.Database, tableIncrement int, firstSlot int, metric *BeaconClientMetrics) {
|
||||
var maxSlot int
|
||||
func writeStartUpGaps(db sql.Database, tableIncrement int, firstSlot Slot, metric *BeaconClientMetrics) {
|
||||
var maxSlot Slot
|
||||
err := db.QueryRow(context.Background(), QueryHighestSlotStmt).Scan(&maxSlot)
|
||||
if err != nil {
|
||||
loghelper.LogError(err).Fatal("Unable to get the max block from the DB. We must close the application or we might have undetected gaps.")
|
||||
@ -496,19 +547,19 @@ func writeStartUpGaps(db sql.Database, tableIncrement int, firstSlot int, metric
|
||||
}
|
||||
|
||||
// A function to update a knownGap range with a reprocessing error.
|
||||
func updateKnownGapErrors(db sql.Database, startSlot int, endSlot int, reprocessingErr error, metric *BeaconClientMetrics) error {
|
||||
func updateKnownGapErrors(db sql.Database, startSlot Slot, endSlot Slot, reprocessingErr error, metric *BeaconClientMetrics) error {
|
||||
res, err := db.Exec(context.Background(), UpsertKnownGapsErrorStmt, startSlot, endSlot, reprocessingErr.Error())
|
||||
if err != nil {
|
||||
loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Error("Unable to update reprocessing_error")
|
||||
loghelper.LogSlotRangeError(startSlot.Number(), endSlot.Number(), err).Error("Unable to update reprocessing_error")
|
||||
return err
|
||||
}
|
||||
row, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).Error("Unable to count rows affected when trying to update reprocessing_error.")
|
||||
loghelper.LogSlotRangeError(startSlot.Number(), endSlot.Number(), err).Error("Unable to count rows affected when trying to update reprocessing_error.")
|
||||
return err
|
||||
}
|
||||
if row != 1 {
|
||||
loghelper.LogSlotRangeError(strconv.Itoa(startSlot), strconv.Itoa(endSlot), err).WithFields(log.Fields{
|
||||
loghelper.LogSlotRangeError(startSlot.Number(), endSlot.Number(), err).WithFields(log.Fields{
|
||||
"rowCount": row,
|
||||
}).Error("The rows affected by the upsert for reprocessing_error is not 1.")
|
||||
metric.IncrementKnownGapsReprocessError(1)
|
||||
@ -519,13 +570,12 @@ func updateKnownGapErrors(db sql.Database, startSlot int, endSlot int, reprocess
|
||||
}
|
||||
|
||||
// A quick helper function to calculate the epoch.
|
||||
func calculateEpoch(slot int, slotPerEpoch int) string {
|
||||
epoch := slot / slotPerEpoch
|
||||
return strconv.Itoa(epoch)
|
||||
func calculateEpoch(slot Slot, slotPerEpoch uint64) uint64 {
|
||||
return slot.Number() / slotPerEpoch
|
||||
}
|
||||
|
||||
// A helper function to check to see if the slot is processed.
|
||||
func isSlotProcessed(db sql.Database, checkProcessStmt string, slot string) (bool, error) {
|
||||
func isSlotProcessed(db sql.Database, checkProcessStmt string, slot Slot) (bool, error) {
|
||||
processRow, err := db.Exec(context.Background(), checkProcessStmt, slot)
|
||||
if err != nil {
|
||||
return false, err
|
||||
@ -542,7 +592,7 @@ func isSlotProcessed(db sql.Database, checkProcessStmt string, slot string) (boo
|
||||
|
||||
// Check to see if this slot is in the DB. Check eth_beacon.slots, eth_beacon.signed_block
|
||||
// and eth_beacon.state. If the slot exists, return true
|
||||
func IsSlotInDb(ctx context.Context, db sql.Database, slot string, blockRoot string, stateRoot string) (bool, error) {
|
||||
func IsSlotInDb(ctx context.Context, db sql.Database, slot Slot, blockRoot string, stateRoot string) (bool, error) {
|
||||
var (
|
||||
isInBeaconState bool
|
||||
isInSignedBeaconBlock bool
|
||||
@ -585,7 +635,7 @@ func IsSlotInDb(ctx context.Context, db sql.Database, slot string, blockRoot str
|
||||
|
||||
// Provide a statement, slot, and root, and this function will check to see
|
||||
// if the slot and root exist in the table.
|
||||
func checkSlotAndRoot(db sql.Database, statement, slot, root string) (bool, error) {
|
||||
func checkSlotAndRoot(db sql.Database, statement string, slot Slot, root string) (bool, error) {
|
||||
processRow, err := db.Exec(context.Background(), statement, slot, root)
|
||||
if err != nil {
|
||||
return false, err
|
||||
|
225
pkg/beaconclient/e2emerge_test.go
Normal file
225
pkg/beaconclient/e2emerge_test.go
Normal file
@ -0,0 +1,225 @@
|
||||
package beaconclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/ethclient"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pkg/errors"
|
||||
zcommon "github.com/protolambda/zrnt/eth2/beacon/common"
|
||||
"github.com/protolambda/zrnt/eth2/configs"
|
||||
"github.com/protolambda/ztyp/tree"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
|
||||
"math/big"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ = Describe("e2emerge", Label("e2e"), func() {
|
||||
e2eConfig := TestConfig
|
||||
e2eConfig.port = getEnvInt("TEST_E2E_LIGHTHOUSE_PORT", 5052)
|
||||
e2eConfig.performBeaconStateProcessing = false
|
||||
e2eConfig.performBeaconBlockProcessing = true
|
||||
|
||||
level, _ := log.ParseLevel("debug")
|
||||
log.SetLevel(level)
|
||||
|
||||
Describe("Run the application against a Merge testnet", func() {
|
||||
Context("When we send a TX to geth", func() {
|
||||
It("We should see the TX included in the ExecutionPayload of a BeaconBlock", func() {
|
||||
bc := setUpTest(e2eConfig, "0")
|
||||
go bc.CaptureHead()
|
||||
|
||||
tx, _ := sendTestTx()
|
||||
beaconBlock := waitForTxToBeIndexed(bc.Db, tx)
|
||||
Expect(beaconBlock).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
type SentTx struct {
|
||||
hash string
|
||||
raw []byte
|
||||
blockNo uint64
|
||||
blockHash string
|
||||
txIndex uint
|
||||
}
|
||||
|
||||
func (tx *SentTx) RawHex() string {
|
||||
return "0x" + hex.EncodeToString(tx.raw)
|
||||
}
|
||||
|
||||
func waitForTxToBeIndexed(db sql.Database, tx *SentTx) *beaconclient.DbSignedBeaconBlock {
|
||||
var beaconBlock *beaconclient.DbSignedBeaconBlock = nil
|
||||
for i := 0; i < 30; i++ {
|
||||
time.Sleep(time.Second)
|
||||
record := lookForTxInDb(db, tx)
|
||||
if nil != record {
|
||||
beaconBlock = record
|
||||
log.Debugf("Found ETH1 TX %s in SignedBeaconBlock %d/%s", tx.hash, beaconBlock.Slot, beaconBlock.BlockRoot)
|
||||
break
|
||||
}
|
||||
}
|
||||
return beaconBlock
|
||||
}
|
||||
|
||||
func lookForTxInDb(db sql.Database, tx *SentTx) *beaconclient.DbSignedBeaconBlock {
|
||||
sqlStatement := `SELECT * FROM eth_beacon.signed_block WHERE
|
||||
payload_block_number = $1 AND
|
||||
payload_block_hash = $2 AND
|
||||
payload_transactions_root = $3`
|
||||
|
||||
// We can pre-calculate the root and query on it because we are only sending a single TX at a time.
|
||||
// Otherwise we would need to lookup the root by block num+hash, then do a proof that its txs
|
||||
// root includes our TX.
|
||||
var ptxs = zcommon.PayloadTransactions{tx.raw}
|
||||
txRoot := ptxs.HashTreeRoot(configs.Mainnet, tree.GetHashFn())
|
||||
|
||||
var slot uint64
|
||||
var blockRoot, parentBlock, eth1DataBlockHash, mhKey string
|
||||
|
||||
var blockNumber, timestamp uint64
|
||||
var blockHash, parentHash, stateRoot, receiptsRoot, transactionsRoot string
|
||||
|
||||
err := db.
|
||||
QueryRow(context.Background(), sqlStatement, tx.blockNo, tx.blockHash, "0x"+hex.EncodeToString(txRoot[:])).
|
||||
Scan(&slot, &blockRoot, &parentBlock, ð1DataBlockHash, &mhKey,
|
||||
&blockNumber, ×tamp, &blockHash, &parentHash, &stateRoot,
|
||||
&receiptsRoot, &transactionsRoot)
|
||||
if nil != err {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &beaconclient.DbSignedBeaconBlock{
|
||||
Slot: slot,
|
||||
BlockRoot: blockRoot,
|
||||
ParentBlock: parentBlock,
|
||||
Eth1DataBlockHash: eth1DataBlockHash,
|
||||
MhKey: mhKey,
|
||||
ExecutionPayloadHeader: &beaconclient.DbExecutionPayloadHeader{
|
||||
BlockNumber: blockNumber,
|
||||
Timestamp: timestamp,
|
||||
BlockHash: blockHash,
|
||||
ParentHash: parentHash,
|
||||
StateRoot: stateRoot,
|
||||
ReceiptsRoot: receiptsRoot,
|
||||
TransactionsRoot: transactionsRoot,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func sendTestTx() (*SentTx, error) {
|
||||
ctx := context.Background()
|
||||
eth, err := createClient()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
//TODO: Pull from test config / env.
|
||||
tx, err := sendTransaction(
|
||||
ctx,
|
||||
eth,
|
||||
getEnvStr("TEST_E2E_FROM_ADDR", "0xe6ce22afe802caf5ff7d3845cec8c736ecc8d61f"),
|
||||
getEnvStr("TEST_E2E_TO_ADDR", "0xe22AD83A0dE117bA0d03d5E94Eb4E0d80a69C62a"),
|
||||
int64(getEnvInt("TEST_E2E_AMOUNT", 10)),
|
||||
getEnvStr("TEST_E2E_SIGNING_KEY", "0x888814df89c4358d7ddb3fa4b0213e7331239a80e1f013eaa7b2deca2a41a218"),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
txBin, err := tx.MarshalBinary()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for i := 0; i <= 30; i++ {
|
||||
time.Sleep(time.Second)
|
||||
receipt, _ := eth.TransactionReceipt(ctx, tx.Hash())
|
||||
if nil != receipt {
|
||||
sentTx := &SentTx{
|
||||
hash: tx.Hash().String(),
|
||||
raw: txBin,
|
||||
blockNo: receipt.BlockNumber.Uint64(),
|
||||
blockHash: receipt.BlockHash.String(),
|
||||
txIndex: receipt.TransactionIndex,
|
||||
}
|
||||
log.Debugf("Sent ETH1 TX %s (Block No: %d, Block Hash: %s)", sentTx.hash, sentTx.blockNo, sentTx.blockHash)
|
||||
return sentTx, nil
|
||||
}
|
||||
}
|
||||
|
||||
err = errors.New("Timed out waiting for TX.")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func createClient() (*ethclient.Client, error) {
|
||||
return ethclient.Dial(getEnvStr("TEST_E2E_GETH_URL", "http://localhost:8545"))
|
||||
}
|
||||
|
||||
// sendTransaction sends a transaction with 1 ETH to a specified address.
|
||||
func sendTransaction(ctx context.Context, eth *ethclient.Client, fromAddr string, toAddr string, amount int64, signingKey string) (*types.Transaction, error) {
|
||||
var (
|
||||
from = common.HexToAddress(fromAddr)
|
||||
to = common.HexToAddress(toAddr)
|
||||
sk = crypto.ToECDSAUnsafe(common.FromHex(signingKey))
|
||||
value = big.NewInt(amount)
|
||||
gasLimit = uint64(getEnvInt("TEST_E2E_GAS_LIMIT", 21000))
|
||||
)
|
||||
// Retrieve the chainid (needed for signer)
|
||||
chainid, err := eth.ChainID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Retrieve the pending nonce
|
||||
nonce, err := eth.PendingNonceAt(ctx, from)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Get suggested gas price
|
||||
tipCap, _ := eth.SuggestGasTipCap(ctx)
|
||||
feeCap, _ := eth.SuggestGasPrice(ctx)
|
||||
log.Info("Tip cap: ", tipCap)
|
||||
log.Info("Fee cap: ", feeCap)
|
||||
// Create a new transaction
|
||||
tx := types.NewTx(
|
||||
&types.DynamicFeeTx{
|
||||
ChainID: chainid,
|
||||
Nonce: nonce,
|
||||
GasTipCap: tipCap,
|
||||
GasFeeCap: feeCap,
|
||||
Gas: gasLimit,
|
||||
To: &to,
|
||||
Value: value,
|
||||
Data: nil,
|
||||
})
|
||||
// Sign the transaction using our keys
|
||||
signedTx, _ := types.SignTx(tx, types.NewLondonSigner(chainid), sk)
|
||||
// Send the transaction to our node
|
||||
return signedTx, eth.SendTransaction(ctx, signedTx)
|
||||
}
|
||||
|
||||
func getEnvStr(envVar string, def string) string {
|
||||
value, set := os.LookupEnv(envVar)
|
||||
if set {
|
||||
return value
|
||||
} else {
|
||||
return def
|
||||
}
|
||||
}
|
||||
|
||||
func getEnvInt(envVar string, def int) int {
|
||||
value, set := os.LookupEnv(envVar)
|
||||
if set {
|
||||
number, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
return number
|
||||
} else {
|
||||
return def
|
||||
}
|
||||
}
|
1
pkg/beaconclient/eth2.0-spec-tests
Symbolic link
1
pkg/beaconclient/eth2.0-spec-tests
Symbolic link
@ -0,0 +1 @@
|
||||
../../../external/eth2.0-spec-tests
|
@ -31,11 +31,10 @@ var _ = Describe("Healthcheck", func() {
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
Bc, err = beaconclient.CreateBeaconClient(context.Background(), "http", "localhost", 5052, 10, bcUniqueIdentifier, false)
|
||||
Bc, err = beaconclient.CreateBeaconClient(context.Background(), "http", "localhost", 5052, 10, bcUniqueIdentifier, false, true, true)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
errBc, err = beaconclient.CreateBeaconClient(context.Background(), "http", "blah-blah", 1010, 10, bcUniqueIdentifier, false)
|
||||
errBc, err = beaconclient.CreateBeaconClient(context.Background(), "http", "blah-blah", 1010, 10, bcUniqueIdentifier, false, true, true)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
})
|
||||
Describe("Connecting to the lighthouse client", Label("integration"), func() {
|
||||
Context("When the client is running", func() {
|
||||
|
@ -19,6 +19,7 @@ package beaconclient
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/pkg/errors"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
@ -33,11 +34,11 @@ var (
|
||||
// This function will capture all the SSE events for a given SseEvents object.
|
||||
// When new messages come in, it will ensure that they are decoded into JSON.
|
||||
// If any errors occur, it log the error information.
|
||||
func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P], errMetricInc func(uint64)) {
|
||||
func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P], errMetricInc func(uint64), idleTimeout time.Duration) {
|
||||
go func() {
|
||||
errG := new(errgroup.Group)
|
||||
errG.Go(func() error {
|
||||
err := eventHandler.SseClient.SubscribeChanRaw(eventHandler.MessagesCh)
|
||||
err := eventHandler.Connect()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -54,9 +55,21 @@ func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P], errMe
|
||||
}
|
||||
|
||||
}()
|
||||
|
||||
// TODO(telackey): Doesn't there need to be a check here that the handler hasn't been shutdown?
|
||||
for {
|
||||
var idleTimer *time.Timer = nil
|
||||
var idleTimerC <-chan time.Time = nil
|
||||
if idleTimeout > 0 {
|
||||
idleTimer = time.NewTimer(idleTimeout)
|
||||
idleTimerC = idleTimer.C
|
||||
}
|
||||
|
||||
select {
|
||||
case message := <-eventHandler.MessagesCh:
|
||||
if nil != idleTimer {
|
||||
idleTimer.Stop()
|
||||
}
|
||||
// Message can be nil if its a keep-alive message
|
||||
if len(message.Data) != 0 {
|
||||
log.WithFields(log.Fields{"msg": string(message.Data)}).Debug("We are going to send the following message to be processed.")
|
||||
@ -64,6 +77,9 @@ func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P], errMe
|
||||
}
|
||||
|
||||
case headErr := <-eventHandler.ErrorCh:
|
||||
if nil != idleTimer {
|
||||
idleTimer.Stop()
|
||||
}
|
||||
log.WithFields(log.Fields{
|
||||
"endpoint": eventHandler.Endpoint,
|
||||
"err": headErr.err,
|
||||
@ -71,6 +87,21 @@ func handleIncomingSseEvent[P ProcessedEvents](eventHandler *SseEvents[P], errMe
|
||||
},
|
||||
).Error("Unable to handle event.")
|
||||
errMetricInc(1)
|
||||
|
||||
case <-idleTimerC:
|
||||
err := errors.New("SSE idle timeout")
|
||||
log.WithFields(log.Fields{
|
||||
"endpoint": eventHandler.Endpoint,
|
||||
"err": err,
|
||||
"msg": err.Error(),
|
||||
},
|
||||
).Error("TIMEOUT - Attempting to resubscribe")
|
||||
errMetricInc(1)
|
||||
eventHandler.Disconnect()
|
||||
err = eventHandler.Connect()
|
||||
if err != nil {
|
||||
log.Error("Unable to re-subscribe.", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -93,6 +124,6 @@ func processMsg[P ProcessedEvents](msg []byte, processCh chan<- *P, errorCh chan
|
||||
// Capture all of the event topics.
|
||||
func (bc *BeaconClient) captureEventTopic() {
|
||||
log.Info("We are capturing all SSE events")
|
||||
go handleIncomingSseEvent(bc.HeadTracking, bc.Metrics.IncrementHeadError)
|
||||
go handleIncomingSseEvent(bc.ReOrgTracking, bc.Metrics.IncrementReorgError)
|
||||
go handleIncomingSseEvent(bc.HeadTracking, bc.Metrics.IncrementHeadError, time.Second*30)
|
||||
go handleIncomingSseEvent(bc.ReOrgTracking, bc.Metrics.IncrementReorgError, 0)
|
||||
}
|
||||
|
@ -23,7 +23,7 @@ import (
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
|
||||
)
|
||||
|
||||
//Create a metric struct and register each channel with prometheus
|
||||
// Create a metric struct and register each channel with prometheus
|
||||
func CreateBeaconClientMetrics() (*BeaconClientMetrics, error) {
|
||||
metrics := &BeaconClientMetrics{
|
||||
SlotInserts: 0,
|
||||
@ -138,7 +138,7 @@ func (m *BeaconClientMetrics) IncrementReorgError(inc uint64) {
|
||||
}
|
||||
|
||||
// Wrapper function to increment the number of knownGaps that were updated with reprocessing errors.
|
||||
//If we want to use mutexes later we can easily update all occurrences here.
|
||||
// If we want to use mutexes later we can easily update all occurrences here.
|
||||
func (m *BeaconClientMetrics) IncrementKnownGapsReprocessError(inc uint64) {
|
||||
atomic.AddUint64(&m.KnownGapsReprocessError, inc)
|
||||
}
|
||||
|
@ -53,34 +53,45 @@ type ChainReorg struct {
|
||||
|
||||
// A struct to capture whats being written to the eth-beacon.slots table.
|
||||
type DbSlots struct {
|
||||
Epoch string // The epoch.
|
||||
Slot string // The slot.
|
||||
Epoch uint64 // The epoch.
|
||||
Slot uint64 // The slot.
|
||||
BlockRoot string // The block root
|
||||
StateRoot string // The state root
|
||||
Status string // The status, it can be proposed | forked | skipped.
|
||||
}
|
||||
|
||||
// A struct to handle the details of an embedded Eth1-block (ie, the ExecutionPayload)
|
||||
type DbExecutionPayloadHeader struct {
|
||||
BlockNumber uint64
|
||||
Timestamp uint64
|
||||
BlockHash string
|
||||
ParentHash string
|
||||
StateRoot string
|
||||
ReceiptsRoot string
|
||||
TransactionsRoot string
|
||||
}
|
||||
|
||||
// A struct to capture whats being written to eth-beacon.signed_block table.
|
||||
type DbSignedBeaconBlock struct {
|
||||
Slot string // The slot.
|
||||
BlockRoot string // The block root
|
||||
ParentBlock string // The parent block root.
|
||||
Eth1BlockHash string // The eth1 block_hash
|
||||
MhKey string // The ipld multihash key.
|
||||
|
||||
Slot uint64 // The slot.
|
||||
BlockRoot string // The block root
|
||||
ParentBlock string // The parent block root.
|
||||
Eth1DataBlockHash string // The eth1 block_hash
|
||||
MhKey string // The ipld multihash key.
|
||||
ExecutionPayloadHeader *DbExecutionPayloadHeader // The ExecutionPayloadHeader (after Bellatrix only).
|
||||
}
|
||||
|
||||
// A struct to capture whats being written to eth-beacon.state table.
|
||||
type DbBeaconState struct {
|
||||
Slot string // The slot.
|
||||
Slot uint64 // The slot.
|
||||
StateRoot string // The state root
|
||||
MhKey string // The ipld multihash key.
|
||||
}
|
||||
|
||||
// A structure to capture whats being written to the eth-beacon.known_gaps table.
|
||||
type DbKnownGaps struct {
|
||||
StartSlot string // The start slot for known_gaps, inclusive.
|
||||
EndSlot string // The end slot for known_gaps, inclusive.
|
||||
StartSlot uint64 // The start slot for known_gaps, inclusive.
|
||||
EndSlot uint64 // The end slot for known_gaps, inclusive.
|
||||
CheckedOut bool // Indicates if any process is currently processing this entry.
|
||||
ReprocessingError string // The error that occurred when attempting to reprocess these entries.
|
||||
EntryError string // The error that caused this entry to be added to the table. Could be null.
|
||||
|
@ -20,9 +20,8 @@ package beaconclient
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
|
||||
)
|
||||
|
||||
// This function will perform the necessary steps to handle a reorg.
|
||||
@ -31,7 +30,11 @@ func (bc *BeaconClient) handleReorg() {
|
||||
for {
|
||||
reorg := <-bc.ReOrgTracking.ProcessCh
|
||||
log.WithFields(log.Fields{"reorg": reorg}).Debug("Received a new reorg message.")
|
||||
writeReorgs(bc.Db, reorg.Slot, reorg.NewHeadBlock, bc.Metrics)
|
||||
slot, err := ParseSlot(reorg.Slot)
|
||||
if nil != err {
|
||||
loghelper.LogSlotError(slot.Number(), err)
|
||||
}
|
||||
writeReorgs(bc.Db, slot, reorg.NewHeadBlock, bc.Metrics)
|
||||
}
|
||||
}
|
||||
|
||||
@ -42,7 +45,7 @@ func (bc *BeaconClient) handleHead() {
|
||||
for {
|
||||
head := <-bc.HeadTracking.ProcessCh
|
||||
// Process all the work here.
|
||||
slot, err := strconv.Atoi(head.Slot)
|
||||
slot, err := ParseSlot(head.Slot)
|
||||
if err != nil {
|
||||
bc.HeadTracking.ErrorCh <- &SseError{
|
||||
err: fmt.Errorf("Unable to turn the slot from string to int: %s", head.Slot),
|
||||
@ -66,7 +69,7 @@ func (bc *BeaconClient) handleHead() {
|
||||
bc.StartingSlot = slot
|
||||
}
|
||||
|
||||
go processHeadSlot(bc.Db, bc.ServerEndpoint, slot, head.Block, head.State, bc.PreviousSlot, bc.PreviousBlockRoot, bc.Metrics, bc.KnownGapTableIncrement, bc.CheckDb)
|
||||
go processHeadSlot(slot, head.Block, head.State, bc.SlotProcessingDetails())
|
||||
|
||||
log.WithFields(log.Fields{"head": head.Slot}).Debug("We finished calling processHeadSlot.")
|
||||
|
||||
|
@ -33,11 +33,11 @@ import (
|
||||
var (
|
||||
// Get a single highest priority and non-checked out row row from eth_beacon.historical_process
|
||||
getHpEntryStmt string = `SELECT start_slot, end_slot FROM eth_beacon.historic_process
|
||||
WHERE checked_out=false
|
||||
WHERE checked_out=false AND end_slot >= $1
|
||||
ORDER BY priority ASC
|
||||
LIMIT 1;`
|
||||
// Used to periodically check to see if there is a new entry in the eth_beacon.historic_process table.
|
||||
checkHpEntryStmt string = `SELECT * FROM eth_beacon.historic_process WHERE checked_out=false;`
|
||||
checkHpEntryStmt string = `SELECT * FROM eth_beacon.historic_process WHERE checked_out=false AND end_slot >= $1;`
|
||||
// Used to checkout a row from the eth_beacon.historic_process table
|
||||
lockHpEntryStmt string = `UPDATE eth_beacon.historic_process
|
||||
SET checked_out=true, checked_out_by=$3
|
||||
@ -58,8 +58,8 @@ type HistoricProcessing struct {
|
||||
}
|
||||
|
||||
// Get a single row of historical slots from the table.
|
||||
func (hp HistoricProcessing) getSlotRange(ctx context.Context, slotCh chan<- slotsToProcess) []error {
|
||||
return getBatchProcessRow(ctx, hp.db, getHpEntryStmt, checkHpEntryStmt, lockHpEntryStmt, slotCh, strconv.Itoa(hp.uniqueNodeIdentifier))
|
||||
func (hp HistoricProcessing) getSlotRange(ctx context.Context, slotCh chan<- slotsToProcess, minimumSlot Slot) []error {
|
||||
return getBatchProcessRow(ctx, hp.db, getHpEntryStmt, checkHpEntryStmt, lockHpEntryStmt, slotCh, strconv.Itoa(hp.uniqueNodeIdentifier), minimumSlot)
|
||||
}
|
||||
|
||||
// Remove the table entry.
|
||||
@ -74,7 +74,7 @@ func (hp HistoricProcessing) handleProcessingErrors(ctx context.Context, errMess
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case errMs := <-errMessages:
|
||||
loghelper.LogSlotError(strconv.Itoa(errMs.slot), errMs.err)
|
||||
loghelper.LogSlotError(errMs.slot.Number(), errMs.err)
|
||||
writeKnownGaps(hp.db, 1, errMs.slot, errMs.slot, errMs.err, errMs.errProcess, hp.metrics)
|
||||
}
|
||||
}
|
||||
@ -97,14 +97,14 @@ func (hp HistoricProcessing) releaseDbLocks() error {
|
||||
}
|
||||
|
||||
// Process the slot range.
|
||||
func processSlotRangeWorker(ctx context.Context, workCh <-chan int, errCh chan<- batchHistoricError, db sql.Database, serverAddress string, metrics *BeaconClientMetrics, checkDb bool, incrementTracker func(uint64)) {
|
||||
func processSlotRangeWorker(ctx context.Context, workCh <-chan Slot, errCh chan<- batchHistoricError, spd SlotProcessingDetails, incrementTracker func(uint64)) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case slot := <-workCh:
|
||||
log.Debug("Handling slot: ", slot)
|
||||
err, errProcess := handleHistoricSlot(ctx, db, serverAddress, slot, metrics, checkDb)
|
||||
err, errProcess := handleHistoricSlot(ctx, slot, spd)
|
||||
if err != nil {
|
||||
errMs := batchHistoricError{
|
||||
err: err,
|
||||
@ -123,7 +123,7 @@ func processSlotRangeWorker(ctx context.Context, workCh <-chan int, errCh chan<-
|
||||
// It also locks the row by updating the checked_out column.
|
||||
// The statement for getting the start_slot and end_slot must be provided.
|
||||
// The statement for "locking" the row must also be provided.
|
||||
func getBatchProcessRow(ctx context.Context, db sql.Database, getStartEndSlotStmt string, checkNewRowsStmt string, checkOutRowStmt string, slotCh chan<- slotsToProcess, uniqueNodeIdentifier string) []error {
|
||||
func getBatchProcessRow(ctx context.Context, db sql.Database, getStartEndSlotStmt string, checkNewRowsStmt string, checkOutRowStmt string, slotCh chan<- slotsToProcess, uniqueNodeIdentifier string, minimumSlot Slot) []error {
|
||||
errCount := make([]error, 0)
|
||||
|
||||
// 5 is an arbitrary number. It allows us to retry a few times before
|
||||
@ -139,7 +139,7 @@ func getBatchProcessRow(ctx context.Context, db sql.Database, getStartEndSlotStm
|
||||
"errCount": errCount,
|
||||
}).Error("New error entry added")
|
||||
}
|
||||
processRow, err := db.Exec(context.Background(), checkNewRowsStmt)
|
||||
processRow, err := db.Exec(context.Background(), checkNewRowsStmt, minimumSlot)
|
||||
if err != nil {
|
||||
errCount = append(errCount, err)
|
||||
}
|
||||
@ -172,13 +172,13 @@ func getBatchProcessRow(ctx context.Context, db sql.Database, getStartEndSlotStm
|
||||
|
||||
// Query the DB for slots.
|
||||
sp := slotsToProcess{}
|
||||
err = tx.QueryRow(dbCtx, getStartEndSlotStmt).Scan(&sp.startSlot, &sp.endSlot)
|
||||
err = tx.QueryRow(dbCtx, getStartEndSlotStmt, minimumSlot).Scan(&sp.startSlot, &sp.endSlot)
|
||||
if err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
time.Sleep(1 * time.Second)
|
||||
break
|
||||
}
|
||||
loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), getStartEndSlotStmt, err).Error("Unable to get a row")
|
||||
loghelper.LogSlotRangeStatementError(sp.startSlot.Number(), sp.endSlot.Number(), getStartEndSlotStmt, err).Error("Unable to get a row")
|
||||
errCount = append(errCount, err)
|
||||
break
|
||||
}
|
||||
@ -186,25 +186,25 @@ func getBatchProcessRow(ctx context.Context, db sql.Database, getStartEndSlotStm
|
||||
// Checkout the Row
|
||||
res, err := tx.Exec(dbCtx, checkOutRowStmt, sp.startSlot, sp.endSlot, uniqueNodeIdentifier)
|
||||
if err != nil {
|
||||
loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).Error("Unable to checkout the row")
|
||||
loghelper.LogSlotRangeStatementError(sp.startSlot.Number(), sp.endSlot.Number(), checkOutRowStmt, err).Error("Unable to checkout the row")
|
||||
errCount = append(errCount, err)
|
||||
break
|
||||
}
|
||||
rows, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, fmt.Errorf("Unable to determine the rows affected when trying to checkout a row."))
|
||||
loghelper.LogSlotRangeStatementError(sp.startSlot.Number(), sp.endSlot.Number(), checkOutRowStmt, fmt.Errorf("Unable to determine the rows affected when trying to checkout a row."))
|
||||
errCount = append(errCount, err)
|
||||
break
|
||||
}
|
||||
if rows > 1 {
|
||||
loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).WithFields(log.Fields{
|
||||
loghelper.LogSlotRangeStatementError(sp.startSlot.Number(), sp.endSlot.Number(), checkOutRowStmt, err).WithFields(log.Fields{
|
||||
"rowsReturn": rows,
|
||||
}).Error("We locked too many rows.....")
|
||||
errCount = append(errCount, err)
|
||||
break
|
||||
}
|
||||
if rows == 0 {
|
||||
loghelper.LogSlotRangeStatementError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), checkOutRowStmt, err).WithFields(log.Fields{
|
||||
loghelper.LogSlotRangeStatementError(sp.startSlot.Number(), sp.endSlot.Number(), checkOutRowStmt, err).WithFields(log.Fields{
|
||||
"rowsReturn": rows,
|
||||
}).Error("We did not lock a single row.")
|
||||
errCount = append(errCount, err)
|
||||
@ -212,7 +212,7 @@ func getBatchProcessRow(ctx context.Context, db sql.Database, getStartEndSlotStm
|
||||
}
|
||||
err = tx.Commit(dbCtx)
|
||||
if err != nil {
|
||||
loghelper.LogSlotRangeError(strconv.Itoa(sp.startSlot), strconv.Itoa(sp.endSlot), err).Error("Unable commit transactions.")
|
||||
loghelper.LogSlotRangeError(sp.startSlot.Number(), sp.endSlot.Number(), err).Error("Unable commit transactions.")
|
||||
errCount = append(errCount, err)
|
||||
break
|
||||
}
|
||||
@ -241,11 +241,11 @@ func removeRowPostProcess(ctx context.Context, db sql.Database, processCh <-chan
|
||||
"endSlot": slots.endSlot,
|
||||
}).Debug("Starting to check to see if the following slots have been processed")
|
||||
for {
|
||||
isStartProcess, err := isSlotProcessed(db, checkProcessedStmt, strconv.Itoa(slots.startSlot))
|
||||
isStartProcess, err := isSlotProcessed(db, checkProcessedStmt, slots.startSlot)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
}
|
||||
isEndProcess, err := isSlotProcessed(db, checkProcessedStmt, strconv.Itoa(slots.endSlot))
|
||||
isEndProcess, err := isSlotProcessed(db, checkProcessedStmt, slots.endSlot)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
}
|
||||
@ -255,7 +255,7 @@ func removeRowPostProcess(ctx context.Context, db sql.Database, processCh <-chan
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
|
||||
_, err := db.Exec(context.Background(), removeStmt, strconv.Itoa(slots.startSlot), strconv.Itoa(slots.endSlot))
|
||||
_, err := db.Exec(context.Background(), removeStmt, slots.startSlot.Number(), slots.endSlot.Number())
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
}
|
||||
|
@ -30,11 +30,11 @@ import (
|
||||
var (
|
||||
// Get a single non-checked out row row from eth_beacon.known_gaps.
|
||||
getKgEntryStmt string = `SELECT start_slot, end_slot FROM eth_beacon.known_gaps
|
||||
WHERE checked_out=false
|
||||
WHERE checked_out=false AND end_slot >= $1
|
||||
ORDER BY priority ASC
|
||||
LIMIT 1;`
|
||||
// Used to periodically check to see if there is a new entry in the eth_beacon.known_gaps table.
|
||||
checkKgEntryStmt string = `SELECT * FROM eth_beacon.known_gaps WHERE checked_out=false;`
|
||||
checkKgEntryStmt string = `SELECT * FROM eth_beacon.known_gaps WHERE checked_out=false AND end_slot >= $1;`
|
||||
// Used to checkout a row from the eth_beacon.known_gaps table
|
||||
lockKgEntryStmt string = `UPDATE eth_beacon.known_gaps
|
||||
SET checked_out=true, checked_out_by=$3
|
||||
@ -58,10 +58,10 @@ type KnownGapsProcessing struct {
|
||||
}
|
||||
|
||||
// This function will perform all the heavy lifting for tracking the head of the chain.
|
||||
func (bc *BeaconClient) ProcessKnownGaps(ctx context.Context, maxWorkers int) []error {
|
||||
func (bc *BeaconClient) ProcessKnownGaps(ctx context.Context, maxWorkers int, minimumSlot Slot) []error {
|
||||
log.Info("We are starting the known gaps processing service.")
|
||||
bc.KnownGapsProcess = KnownGapsProcessing{db: bc.Db, uniqueNodeIdentifier: bc.UniqueNodeIdentifier, metrics: bc.Metrics}
|
||||
errs := handleBatchProcess(ctx, maxWorkers, bc.KnownGapsProcess, bc.KnownGapsProcess.db, bc.ServerEndpoint, bc.Metrics, bc.CheckDb, bc.Metrics.IncrementKnownGapsProcessed)
|
||||
errs := handleBatchProcess(ctx, maxWorkers, bc.KnownGapsProcess, bc.SlotProcessingDetails(), bc.Metrics.IncrementKnownGapsProcessed, minimumSlot)
|
||||
log.Debug("Exiting known gaps processing service")
|
||||
return errs
|
||||
}
|
||||
@ -78,8 +78,8 @@ func (bc *BeaconClient) StopKnownGapsProcessing(cancel context.CancelFunc) error
|
||||
}
|
||||
|
||||
// Get a single row of historical slots from the table.
|
||||
func (kgp KnownGapsProcessing) getSlotRange(ctx context.Context, slotCh chan<- slotsToProcess) []error {
|
||||
return getBatchProcessRow(ctx, kgp.db, getKgEntryStmt, checkKgEntryStmt, lockKgEntryStmt, slotCh, strconv.Itoa(kgp.uniqueNodeIdentifier))
|
||||
func (kgp KnownGapsProcessing) getSlotRange(ctx context.Context, slotCh chan<- slotsToProcess, minimumSlot Slot) []error {
|
||||
return getBatchProcessRow(ctx, kgp.db, getKgEntryStmt, checkKgEntryStmt, lockKgEntryStmt, slotCh, strconv.Itoa(kgp.uniqueNodeIdentifier), minimumSlot)
|
||||
}
|
||||
|
||||
// Remove the table entry.
|
||||
@ -97,21 +97,21 @@ func (kgp KnownGapsProcessing) handleProcessingErrors(ctx context.Context, errMe
|
||||
// Check to see if this if this entry already exists.
|
||||
res, err := kgp.db.Exec(context.Background(), checkKgSingleSlotStmt, errMs.slot, errMs.slot)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).Error("Unable to see if this slot is in the eth_beacon.known_gaps table")
|
||||
loghelper.LogSlotError(errMs.slot.Number(), err).Error("Unable to see if this slot is in the eth_beacon.known_gaps table")
|
||||
}
|
||||
|
||||
rows, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).WithFields(log.Fields{
|
||||
loghelper.LogSlotError(errMs.slot.Number(), err).WithFields(log.Fields{
|
||||
"queryStatement": checkKgSingleSlotStmt,
|
||||
}).Error("Unable to get the number of rows affected by this statement.")
|
||||
}
|
||||
|
||||
if rows > 0 {
|
||||
loghelper.LogSlotError(strconv.Itoa(errMs.slot), errMs.err).Error("We received an error when processing a knownGap")
|
||||
loghelper.LogSlotError(errMs.slot.Number(), errMs.err).Error("We received an error when processing a knownGap")
|
||||
err = updateKnownGapErrors(kgp.db, errMs.slot, errMs.slot, errMs.err, kgp.metrics)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(strconv.Itoa(errMs.slot), err).Error("Error processing known gap")
|
||||
loghelper.LogSlotError(errMs.slot.Number(), err).Error("Error processing known gap")
|
||||
}
|
||||
} else {
|
||||
writeKnownGaps(kgp.db, 1, errMs.slot, errMs.slot, errMs.err, errMs.errProcess, kgp.metrics)
|
||||
|
@ -23,36 +23,55 @@ import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v4"
|
||||
si "github.com/prysmaticlabs/prysm/consensus-types/interfaces"
|
||||
"github.com/prysmaticlabs/prysm/consensus-types/wrapper"
|
||||
dt "github.com/prysmaticlabs/prysm/encoding/ssz/detect"
|
||||
|
||||
// The below is temporary, once https://github.com/prysmaticlabs/prysm/issues/10006 has been resolved we wont need it.
|
||||
// pb "github.com/prysmaticlabs/prysm/proto/prysm/v2"
|
||||
|
||||
state "github.com/prysmaticlabs/prysm/beacon-chain/state"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/database/sql"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var (
|
||||
ParentRootUnmarshalError = "Unable to properly unmarshal the ParentRoot field in the SignedBeaconBlock."
|
||||
MissingEth1Data = "Can't get the Eth1 block_hash"
|
||||
VersionedUnmarshalerError = "Unable to create a versioned unmarshaler"
|
||||
)
|
||||
type SlotProcessingDetails struct {
|
||||
Context context.Context // A context generic context with multiple uses.
|
||||
ServerEndpoint string // What is the endpoint of the beacon server.
|
||||
Db sql.Database // Database object used for reads and writes.
|
||||
Metrics *BeaconClientMetrics // An object used to keep track of certain BeaconClient Metrics.
|
||||
KnownGapTableIncrement int // The max number of slots within a single known_gaps table entry.
|
||||
CheckDb bool // Should we check the DB to see if the slot exists before processing it?
|
||||
PerformBeaconStateProcessing bool // Should we process BeaconStates?
|
||||
PerformBeaconBlockProcessing bool // Should we process BeaconBlocks?
|
||||
|
||||
StartingSlot Slot // If we're performing head tracking. What is the first slot we processed.
|
||||
PreviousSlot Slot // Whats the previous slot we processed
|
||||
PreviousBlockRoot string // Whats the previous block root, used to check the next blocks parent.
|
||||
}
|
||||
|
||||
func (bc *BeaconClient) SlotProcessingDetails() SlotProcessingDetails {
|
||||
return SlotProcessingDetails{
|
||||
Context: bc.Context,
|
||||
ServerEndpoint: bc.ServerEndpoint,
|
||||
Db: bc.Db,
|
||||
Metrics: bc.Metrics,
|
||||
|
||||
CheckDb: bc.CheckDb,
|
||||
PerformBeaconBlockProcessing: bc.PerformBeaconBlockProcessing,
|
||||
PerformBeaconStateProcessing: bc.PerformBeaconStateProcessing,
|
||||
|
||||
KnownGapTableIncrement: bc.KnownGapTableIncrement,
|
||||
StartingSlot: bc.StartingSlot,
|
||||
PreviousSlot: bc.PreviousSlot,
|
||||
PreviousBlockRoot: bc.PreviousBlockRoot,
|
||||
}
|
||||
}
|
||||
|
||||
type ProcessSlot struct {
|
||||
// Generic
|
||||
|
||||
Slot int // The slot number.
|
||||
Epoch int // The epoch number.
|
||||
Slot Slot // The slot number.
|
||||
Epoch Epoch // The epoch number.
|
||||
BlockRoot string // The hex encoded string of the BlockRoot.
|
||||
StateRoot string // The hex encoded string of the StateRoot.
|
||||
ParentBlockRoot string // The hex encoded string of the parent block.
|
||||
@ -63,12 +82,12 @@ type ProcessSlot struct {
|
||||
PerformanceMetrics PerformanceMetrics // An object to keep track of performance metrics.
|
||||
// BeaconBlock
|
||||
|
||||
SszSignedBeaconBlock []byte // The entire SSZ encoded SignedBeaconBlock
|
||||
FullSignedBeaconBlock si.SignedBeaconBlock // The unmarshaled BeaconState object, the unmarshalling could have errors.
|
||||
SszSignedBeaconBlock []byte // The entire SSZ encoded SignedBeaconBlock
|
||||
FullSignedBeaconBlock *SignedBeaconBlock // The unmarshaled BeaconState object, the unmarshalling could have errors.
|
||||
|
||||
// BeaconState
|
||||
FullBeaconState state.BeaconState // The unmarshaled BeaconState object, the unmarshalling could have errors.
|
||||
SszBeaconState []byte // The entire SSZ encoded BeaconState
|
||||
FullBeaconState *BeaconState // The unmarshaled BeaconState object, the unmarshalling could have errors.
|
||||
SszBeaconState []byte // The entire SSZ encoded BeaconState
|
||||
|
||||
// DB Write objects
|
||||
DbSlotsModel *DbSlots // The model being written to the slots table.
|
||||
@ -92,7 +111,16 @@ type PerformanceMetrics struct {
|
||||
// This function will do all the work to process the slot and write it to the DB.
|
||||
// It will return the error and error process. The error process is used for providing reach detail to the
|
||||
// known_gaps table.
|
||||
func processFullSlot(ctx context.Context, db sql.Database, serverAddress string, slot int, blockRoot string, stateRoot string, previousSlot int, previousBlockRoot string, headOrHistoric string, metrics *BeaconClientMetrics, knownGapsTableIncrement int, checkDb bool) (error, string) {
|
||||
func processFullSlot(
|
||||
ctx context.Context,
|
||||
slot Slot,
|
||||
blockRoot string,
|
||||
stateRoot string,
|
||||
previousSlot Slot,
|
||||
previousBlockRoot string,
|
||||
knownGapsTableIncrement int,
|
||||
headOrHistoric string,
|
||||
spd *SlotProcessingDetails) (error, string) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ""
|
||||
@ -103,8 +131,8 @@ func processFullSlot(ctx context.Context, db sql.Database, serverAddress string,
|
||||
BlockRoot: blockRoot,
|
||||
StateRoot: stateRoot,
|
||||
HeadOrHistoric: headOrHistoric,
|
||||
Db: db,
|
||||
Metrics: metrics,
|
||||
Db: spd.Db,
|
||||
Metrics: spd.Metrics,
|
||||
PerformanceMetrics: PerformanceMetrics{
|
||||
BeaconNodeBlockRetrievalTime: 0,
|
||||
BeaconNodeStateRetrievalTime: 0,
|
||||
@ -120,58 +148,75 @@ func processFullSlot(ctx context.Context, db sql.Database, serverAddress string,
|
||||
}
|
||||
|
||||
g, _ := errgroup.WithContext(context.Background())
|
||||
vUnmarshalerCh := make(chan *dt.VersionedUnmarshaler, 1)
|
||||
|
||||
// Get the BeaconState.
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
default:
|
||||
start := time.Now()
|
||||
err := ps.getBeaconState(serverAddress, vUnmarshalerCh)
|
||||
if err != nil {
|
||||
return err
|
||||
if spd.PerformBeaconStateProcessing {
|
||||
// Get the BeaconState.
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
default:
|
||||
start := time.Now()
|
||||
err := ps.getBeaconState(spd.ServerEndpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ps.PerformanceMetrics.BeaconNodeStateRetrievalTime = time.Since(start)
|
||||
return nil
|
||||
}
|
||||
ps.PerformanceMetrics.BeaconNodeStateRetrievalTime = time.Since(start)
|
||||
return nil
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Get the SignedBeaconBlock.
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
default:
|
||||
start := time.Now()
|
||||
err := ps.getSignedBeaconBlock(serverAddress, vUnmarshalerCh)
|
||||
if err != nil {
|
||||
return err
|
||||
if spd.PerformBeaconBlockProcessing {
|
||||
// Get the SignedBeaconBlock.
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
default:
|
||||
start := time.Now()
|
||||
err := ps.getSignedBeaconBlock(spd.ServerEndpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ps.PerformanceMetrics.BeaconNodeBlockRetrievalTime = time.Since(start)
|
||||
return nil
|
||||
}
|
||||
ps.PerformanceMetrics.BeaconNodeBlockRetrievalTime = time.Since(start)
|
||||
return nil
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return err, "processSlot"
|
||||
}
|
||||
|
||||
parseBeaconTime := time.Now()
|
||||
finalBlockRoot, finalStateRoot, finalEth1BlockHash, err := ps.provideFinalHash()
|
||||
finalBlockRoot, finalStateRoot, _, err := ps.provideFinalHash()
|
||||
if err != nil {
|
||||
return err, "CalculateBlockRoot"
|
||||
}
|
||||
ps.PerformanceMetrics.ParseBeaconObjectForHash = time.Since(parseBeaconTime)
|
||||
|
||||
if checkDb {
|
||||
if spd.CheckDb {
|
||||
checkDbTime := time.Now()
|
||||
inDb, err := IsSlotInDb(ctx, ps.Db, strconv.Itoa(ps.Slot), finalBlockRoot, finalStateRoot)
|
||||
if err != nil {
|
||||
return err, "checkDb"
|
||||
var blockRequired bool
|
||||
if spd.PerformBeaconBlockProcessing {
|
||||
blockExists, err := checkSlotAndRoot(ps.Db, CheckSignedBeaconBlockStmt, ps.Slot, finalBlockRoot)
|
||||
if err != nil {
|
||||
return err, "checkDb"
|
||||
}
|
||||
blockRequired = !blockExists
|
||||
}
|
||||
if inDb {
|
||||
|
||||
var stateRequired bool
|
||||
if spd.PerformBeaconStateProcessing {
|
||||
stateExists, err := checkSlotAndRoot(ps.Db, CheckBeaconStateStmt, ps.Slot, finalStateRoot)
|
||||
if err != nil {
|
||||
return err, "checkDb"
|
||||
}
|
||||
stateRequired = !stateExists
|
||||
}
|
||||
|
||||
if !blockRequired && !stateRequired {
|
||||
log.WithField("slot", slot).Info("Slot already in the DB.")
|
||||
return nil, ""
|
||||
}
|
||||
@ -180,7 +225,7 @@ func processFullSlot(ctx context.Context, db sql.Database, serverAddress string,
|
||||
|
||||
// Get this object ready to write
|
||||
createDbWriteTime := time.Now()
|
||||
dw, err := ps.createWriteObjects(finalBlockRoot, finalStateRoot, finalEth1BlockHash)
|
||||
dw, err := ps.createWriteObjects()
|
||||
if err != nil {
|
||||
return err, "blockRoot"
|
||||
}
|
||||
@ -206,7 +251,7 @@ func processFullSlot(ctx context.Context, db sql.Database, serverAddress string,
|
||||
reorgTime := time.Now()
|
||||
headOrHistoric = strings.ToLower(headOrHistoric)
|
||||
if headOrHistoric != "head" && headOrHistoric != "historic" {
|
||||
return fmt.Errorf("headOrHistoric must be either historic or head!"), ""
|
||||
return fmt.Errorf("headOrHistoric must be either historic or head"), ""
|
||||
}
|
||||
if ps.HeadOrHistoric == "head" && previousSlot != 0 && previousBlockRoot != "" && ps.Status != "skipped" {
|
||||
ps.checkPreviousSlot(dw.Tx, dw.Ctx, previousSlot, previousBlockRoot, knownGapsTableIncrement)
|
||||
@ -234,97 +279,111 @@ func processFullSlot(ctx context.Context, db sql.Database, serverAddress string,
|
||||
}
|
||||
|
||||
// Handle a slot that is at head. A wrapper function for calling `handleFullSlot`.
|
||||
func processHeadSlot(db sql.Database, serverAddress string, slot int, blockRoot string, stateRoot string, previousSlot int, previousBlockRoot string, metrics *BeaconClientMetrics, knownGapsTableIncrement int, checkDb bool) {
|
||||
// Get the knownGaps at startUp.
|
||||
if previousSlot == 0 && previousBlockRoot == "" {
|
||||
writeStartUpGaps(db, knownGapsTableIncrement, slot, metrics)
|
||||
func processHeadSlot(slot Slot, blockRoot string, stateRoot string, spd SlotProcessingDetails) {
|
||||
// Get the knownGaps at startUp
|
||||
if spd.PreviousSlot == 0 && spd.PreviousBlockRoot == "" {
|
||||
writeStartUpGaps(spd.Db, spd.KnownGapTableIncrement, slot, spd.Metrics)
|
||||
}
|
||||
err, errReason := processFullSlot(context.Background(), db, serverAddress, slot, blockRoot, stateRoot, previousSlot, previousBlockRoot, "head", metrics, knownGapsTableIncrement, checkDb)
|
||||
// TODO(telackey): Why context.Background()?
|
||||
err, errReason := processFullSlot(context.Background(), slot, blockRoot, stateRoot,
|
||||
spd.PreviousSlot, spd.PreviousBlockRoot, spd.KnownGapTableIncrement, "head", &spd)
|
||||
if err != nil {
|
||||
writeKnownGaps(db, knownGapsTableIncrement, slot, slot, err, errReason, metrics)
|
||||
writeKnownGaps(spd.Db, spd.KnownGapTableIncrement, slot, slot, err, errReason, spd.Metrics)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle a historic slot. A wrapper function for calling `handleFullSlot`.
|
||||
func handleHistoricSlot(ctx context.Context, db sql.Database, serverAddress string, slot int, metrics *BeaconClientMetrics, checkDb bool) (error, string) {
|
||||
return processFullSlot(ctx, db, serverAddress, slot, "", "", 0, "", "historic", metrics, 1, checkDb)
|
||||
func handleHistoricSlot(ctx context.Context, slot Slot, spd SlotProcessingDetails) (error, string) {
|
||||
return processFullSlot(ctx, slot, "", "", 0, "",
|
||||
1, "historic", &spd)
|
||||
}
|
||||
|
||||
// Update the SszSignedBeaconBlock and FullSignedBeaconBlock object with their respective values.
|
||||
func (ps *ProcessSlot) getSignedBeaconBlock(serverAddress string, vmCh <-chan *dt.VersionedUnmarshaler) error {
|
||||
func (ps *ProcessSlot) getSignedBeaconBlock(serverAddress string) error {
|
||||
var blockIdentifier string // Used to query the block
|
||||
if ps.BlockRoot != "" {
|
||||
blockIdentifier = ps.BlockRoot
|
||||
} else {
|
||||
blockIdentifier = strconv.Itoa(ps.Slot)
|
||||
}
|
||||
blockEndpoint := serverAddress + BcBlockQueryEndpoint + blockIdentifier
|
||||
var err error
|
||||
var rc int
|
||||
ps.SszSignedBeaconBlock, rc, err = querySsz(blockEndpoint, strconv.Itoa(ps.Slot))
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error("Unable to properly query the slot.")
|
||||
return err
|
||||
blockIdentifier = ps.Slot.Format()
|
||||
}
|
||||
|
||||
vm := <-vmCh
|
||||
if rc != 200 {
|
||||
ps.FullSignedBeaconBlock = &wrapper.Phase0SignedBeaconBlock{}
|
||||
blockEndpoint := serverAddress + BcBlockQueryEndpoint + blockIdentifier
|
||||
sszSignedBeaconBlock, rc, err := querySsz(blockEndpoint, ps.Slot)
|
||||
|
||||
if err != nil || rc != 200 {
|
||||
loghelper.LogSlotError(ps.Slot.Number(), err).Error("Unable to properly query the slot.")
|
||||
ps.FullSignedBeaconBlock = nil
|
||||
ps.SszSignedBeaconBlock = []byte{}
|
||||
ps.ParentBlockRoot = ""
|
||||
ps.Status = "skipped"
|
||||
return nil
|
||||
|
||||
// A 404 is normal in the case of a "skipped" slot.
|
||||
if rc == 404 {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if vm == nil {
|
||||
return fmt.Errorf(VersionedUnmarshalerError)
|
||||
}
|
||||
|
||||
ps.FullSignedBeaconBlock, err = vm.UnmarshalBeaconBlock(ps.SszSignedBeaconBlock)
|
||||
var signedBeaconBlock SignedBeaconBlock
|
||||
err = signedBeaconBlock.UnmarshalSSZ(sszSignedBeaconBlock)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Warn("Unable to process the slots SignedBeaconBlock")
|
||||
return nil
|
||||
loghelper.LogSlotError(ps.Slot.Number(), err).Error("Unable to unmarshal SignedBeaconBlock for slot.")
|
||||
ps.FullSignedBeaconBlock = nil
|
||||
ps.SszSignedBeaconBlock = []byte{}
|
||||
ps.ParentBlockRoot = ""
|
||||
ps.Status = "skipped"
|
||||
return err
|
||||
}
|
||||
ps.ParentBlockRoot = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().ParentRoot())
|
||||
|
||||
ps.FullSignedBeaconBlock = &signedBeaconBlock
|
||||
ps.SszSignedBeaconBlock = sszSignedBeaconBlock
|
||||
|
||||
ps.ParentBlockRoot = toHex(ps.FullSignedBeaconBlock.Block().ParentRoot())
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update the SszBeaconState and FullBeaconState object with their respective values.
|
||||
func (ps *ProcessSlot) getBeaconState(serverEndpoint string, vmCh chan<- *dt.VersionedUnmarshaler) error {
|
||||
func (ps *ProcessSlot) getBeaconState(serverEndpoint string) error {
|
||||
var stateIdentifier string // Used to query the state
|
||||
if ps.StateRoot != "" {
|
||||
stateIdentifier = ps.StateRoot
|
||||
} else {
|
||||
stateIdentifier = strconv.Itoa(ps.Slot)
|
||||
stateIdentifier = ps.Slot.Format()
|
||||
}
|
||||
stateEndpoint := serverEndpoint + BcStateQueryEndpoint + stateIdentifier
|
||||
ps.SszBeaconState, _, _ = querySsz(stateEndpoint, strconv.Itoa(ps.Slot))
|
||||
|
||||
versionedUnmarshaler, err := dt.FromState(ps.SszBeaconState)
|
||||
stateEndpoint := serverEndpoint + BcStateQueryEndpoint + stateIdentifier
|
||||
sszBeaconState, _, err := querySsz(stateEndpoint, ps.Slot)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error(VersionedUnmarshalerError)
|
||||
vmCh <- nil
|
||||
return fmt.Errorf(VersionedUnmarshalerError)
|
||||
}
|
||||
vmCh <- versionedUnmarshaler
|
||||
ps.FullBeaconState, err = versionedUnmarshaler.UnmarshalBeaconState(ps.SszBeaconState)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(strconv.Itoa(ps.Slot), err).Error("Unable to process the slots BeaconState")
|
||||
loghelper.LogSlotError(ps.Slot.Number(), err).Error("Unable to properly query the BeaconState.")
|
||||
return err
|
||||
}
|
||||
|
||||
var beaconState BeaconState
|
||||
err = beaconState.UnmarshalSSZ(sszBeaconState)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(ps.Slot.Number(), err).Error("Unable to unmarshal the BeaconState.")
|
||||
return err
|
||||
}
|
||||
|
||||
ps.FullBeaconState = &beaconState
|
||||
ps.SszBeaconState = sszBeaconState
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check to make sure that the previous block we processed is the parent of the current block.
|
||||
func (ps *ProcessSlot) checkPreviousSlot(tx sql.Tx, ctx context.Context, previousSlot int, previousBlockRoot string, knownGapsTableIncrement int) {
|
||||
parentRoot := "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().ParentRoot())
|
||||
slot := int(ps.FullBeaconState.Slot())
|
||||
func (ps *ProcessSlot) checkPreviousSlot(tx sql.Tx, ctx context.Context, previousSlot Slot, previousBlockRoot string, knownGapsTableIncrement int) {
|
||||
if nil == ps.FullSignedBeaconBlock {
|
||||
log.Debug("Can't check block root, no current block.")
|
||||
return
|
||||
}
|
||||
parentRoot := toHex(ps.FullSignedBeaconBlock.Block().ParentRoot())
|
||||
slot := ps.Slot
|
||||
if previousSlot == slot {
|
||||
log.WithFields(log.Fields{
|
||||
"slot": slot,
|
||||
"fork": true,
|
||||
}).Warn("A fork occurred! The previous slot and current slot match.")
|
||||
transactReorgs(tx, ctx, strconv.Itoa(ps.Slot), ps.BlockRoot, ps.Metrics)
|
||||
transactReorgs(tx, ctx, ps.Slot, ps.BlockRoot, ps.Metrics)
|
||||
} else if previousSlot > slot {
|
||||
log.WithFields(log.Fields{
|
||||
"previousSlot": previousSlot,
|
||||
@ -335,20 +394,20 @@ func (ps *ProcessSlot) checkPreviousSlot(tx sql.Tx, ctx context.Context, previou
|
||||
"previousSlot": previousSlot,
|
||||
"currentSlot": slot,
|
||||
}).Error("We skipped a few slots.")
|
||||
transactKnownGaps(tx, ctx, knownGapsTableIncrement, previousSlot+1, slot-1, fmt.Errorf("Gaps during head processing"), "headGaps", ps.Metrics)
|
||||
transactKnownGaps(tx, ctx, knownGapsTableIncrement, previousSlot+1, slot-1, fmt.Errorf("gaps during head processing"), "headGaps", ps.Metrics)
|
||||
} else if previousBlockRoot != parentRoot {
|
||||
log.WithFields(log.Fields{
|
||||
"previousBlockRoot": previousBlockRoot,
|
||||
"currentBlockParent": parentRoot,
|
||||
}).Error("The previousBlockRoot does not match the current blocks parent, an unprocessed fork might have occurred.")
|
||||
transactReorgs(tx, ctx, strconv.Itoa(previousSlot), parentRoot, ps.Metrics)
|
||||
transactReorgs(tx, ctx, previousSlot, parentRoot, ps.Metrics)
|
||||
} else {
|
||||
log.Debug("Previous Slot and Current Slot are one distance from each other.")
|
||||
}
|
||||
}
|
||||
|
||||
// Transforms all the raw data into DB models that can be written to the DB.
|
||||
func (ps *ProcessSlot) createWriteObjects(blockRoot, stateRoot, eth1BlockHash string) (*DatabaseWriter, error) {
|
||||
func (ps *ProcessSlot) createWriteObjects() (*DatabaseWriter, error) {
|
||||
var status string
|
||||
if ps.Status != "" {
|
||||
status = ps.Status
|
||||
@ -356,7 +415,18 @@ func (ps *ProcessSlot) createWriteObjects(blockRoot, stateRoot, eth1BlockHash st
|
||||
status = "proposed"
|
||||
}
|
||||
|
||||
dw, err := CreateDatabaseWrite(ps.Db, ps.Slot, stateRoot, blockRoot, ps.ParentBlockRoot, eth1BlockHash, status, &ps.SszSignedBeaconBlock, &ps.SszBeaconState, ps.Metrics)
|
||||
parseBeaconTime := time.Now()
|
||||
// These will normally be pre-calculated by this point.
|
||||
blockRoot, stateRoot, eth1DataBlockHash, err := ps.provideFinalHash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ps.PerformanceMetrics.ParseBeaconObjectForHash = time.Since(parseBeaconTime)
|
||||
|
||||
payloadHeader := ps.provideExecutionPayloadDetails()
|
||||
|
||||
dw, err := CreateDatabaseWrite(ps.Db, ps.Slot, stateRoot, blockRoot, ps.ParentBlockRoot, eth1DataBlockHash,
|
||||
payloadHeader, status, &ps.SszSignedBeaconBlock, &ps.SszBeaconState, ps.Metrics)
|
||||
if err != nil {
|
||||
return dw, err
|
||||
}
|
||||
@ -364,39 +434,65 @@ func (ps *ProcessSlot) createWriteObjects(blockRoot, stateRoot, eth1BlockHash st
|
||||
return dw, nil
|
||||
}
|
||||
|
||||
// This function will return the final blockRoot, stateRoot, and eth1BlockHash that will be
|
||||
// This function will return the final blockRoot, stateRoot, and eth1DataBlockHash that will be
|
||||
// used to write to a DB
|
||||
func (ps *ProcessSlot) provideFinalHash() (string, string, string, error) {
|
||||
var (
|
||||
stateRoot string
|
||||
blockRoot string
|
||||
eth1BlockHash string
|
||||
stateRoot string
|
||||
blockRoot string
|
||||
eth1DataBlockHash string
|
||||
)
|
||||
if ps.Status == "skipped" {
|
||||
stateRoot = ""
|
||||
blockRoot = ""
|
||||
eth1BlockHash = ""
|
||||
eth1DataBlockHash = ""
|
||||
} else {
|
||||
if ps.StateRoot != "" {
|
||||
stateRoot = ps.StateRoot
|
||||
} else {
|
||||
stateRoot = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().StateRoot())
|
||||
log.Debug("StateRoot: ", stateRoot)
|
||||
if nil != ps.FullSignedBeaconBlock {
|
||||
stateRoot = toHex(ps.FullSignedBeaconBlock.Block().StateRoot())
|
||||
log.Debug("BeaconBlock StateRoot: ", stateRoot)
|
||||
} else {
|
||||
log.Debug("BeaconBlock StateRoot: <nil beacon block>")
|
||||
}
|
||||
}
|
||||
|
||||
if ps.BlockRoot != "" {
|
||||
blockRoot = ps.BlockRoot
|
||||
} else {
|
||||
var err error
|
||||
rawBlockRoot, err := ps.FullSignedBeaconBlock.Block().HashTreeRoot()
|
||||
//blockRoot, err = queryBlockRoot(blockRootEndpoint, strconv.Itoa(ps.Slot))
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
if nil != ps.FullSignedBeaconBlock {
|
||||
rawBlockRoot := ps.FullSignedBeaconBlock.Block().HashTreeRoot()
|
||||
blockRoot = toHex(rawBlockRoot)
|
||||
log.WithFields(log.Fields{"blockRoot": blockRoot}).Debug("Block Root from ssz")
|
||||
} else {
|
||||
log.Debug("BeaconBlock HashTreeRoot: <nil beacon block>")
|
||||
}
|
||||
blockRoot = "0x" + hex.EncodeToString(rawBlockRoot[:])
|
||||
log.WithFields(log.Fields{"blockRoot": blockRoot}).Debug("Block Root from ssz")
|
||||
}
|
||||
eth1BlockHash = "0x" + hex.EncodeToString(ps.FullSignedBeaconBlock.Block().Body().Eth1Data().BlockHash)
|
||||
if nil != ps.FullSignedBeaconBlock {
|
||||
eth1DataBlockHash = toHex(ps.FullSignedBeaconBlock.Block().Body().Eth1Data().BlockHash)
|
||||
}
|
||||
}
|
||||
return blockRoot, stateRoot, eth1BlockHash, nil
|
||||
return blockRoot, stateRoot, eth1DataBlockHash, nil
|
||||
}
|
||||
|
||||
func (ps *ProcessSlot) provideExecutionPayloadDetails() *ExecutionPayloadHeader {
|
||||
if nil == ps.FullSignedBeaconBlock || !ps.FullSignedBeaconBlock.IsBellatrix() {
|
||||
return nil
|
||||
}
|
||||
|
||||
payload := ps.FullSignedBeaconBlock.Block().Body().ExecutionPayloadHeader()
|
||||
blockNumber := uint64(payload.BlockNumber)
|
||||
|
||||
// The earliest blocks on the Bellatrix fork, pre-Merge, have zeroed ExecutionPayloads.
|
||||
// There is nothing useful to to store in that case, even though the structure exists.
|
||||
if blockNumber == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return payload
|
||||
}
|
||||
|
||||
func toHex(r [32]byte) string {
|
||||
return "0x" + hex.EncodeToString(r[:])
|
||||
}
|
||||
|
@ -18,40 +18,16 @@
|
||||
package beaconclient
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/loghelper"
|
||||
)
|
||||
|
||||
// A helper function to query endpoints that utilize slots.
|
||||
func querySsz(endpoint string, slot string) ([]byte, int, error) {
|
||||
log.WithFields(log.Fields{"endpoint": endpoint}).Debug("Querying endpoint")
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("GET", endpoint, nil)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(slot, err).Error("Unable to create a request!")
|
||||
return nil, 0, fmt.Errorf("Unable to create a request!: %s", err.Error())
|
||||
}
|
||||
req.Header.Set("Accept", "application/octet-stream")
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(slot, err).Error("Unable to query Beacon Node!")
|
||||
return nil, 0, fmt.Errorf("Unable to query Beacon Node: %s", err.Error())
|
||||
}
|
||||
defer response.Body.Close()
|
||||
rc := response.StatusCode
|
||||
body, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(slot, err).Error("Unable to turn response into a []bytes array!")
|
||||
return nil, rc, fmt.Errorf("Unable to turn response into a []bytes array!: %s", err.Error())
|
||||
}
|
||||
return body, rc, nil
|
||||
}
|
||||
|
||||
// Object to unmarshal the BlockRootResponse
|
||||
type BlockRootResponse struct {
|
||||
Data BlockRootMessage `json:"data"`
|
||||
@ -62,35 +38,36 @@ type BlockRootMessage struct {
|
||||
Root string `json:"root"`
|
||||
}
|
||||
|
||||
// A function to query the blockroot for a given slot.
|
||||
func queryBlockRoot(endpoint string, slot string) (string, error) {
|
||||
// A helper function to query endpoints that utilize slots.
|
||||
func querySsz(endpoint string, slot Slot) ([]byte, int, error) {
|
||||
log.WithFields(log.Fields{"endpoint": endpoint}).Debug("Querying endpoint")
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("GET", endpoint, nil)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(slot, err).Error("Unable to create a request!")
|
||||
return "", fmt.Errorf("Unable to create a request!: %s", err.Error())
|
||||
loghelper.LogSlotError(slot.Number(), err).Error("Unable to create a request!")
|
||||
return nil, 0, fmt.Errorf("Unable to create a request!: %s", err.Error())
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Accept", "application/octet-stream")
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(slot, err).Error("Unable to query Beacon Node!")
|
||||
return "", fmt.Errorf("Unable to query Beacon Node: %s", err.Error())
|
||||
loghelper.LogSlotError(slot.Number(), err).Error("Unable to query Beacon Node!")
|
||||
return nil, 0, fmt.Errorf("Unable to query Beacon Node: %s", err.Error())
|
||||
}
|
||||
defer response.Body.Close()
|
||||
body, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(slot, err).Error("Unable to turn response into a []bytes array!")
|
||||
return "", fmt.Errorf("Unable to turn response into a []bytes array!: %s", err.Error())
|
||||
|
||||
rc := response.StatusCode
|
||||
// Any 2xx code is OK.
|
||||
if rc < 200 || rc >= 300 {
|
||||
return nil, rc, fmt.Errorf("HTTP Error: %d", rc)
|
||||
}
|
||||
|
||||
resp := BlockRootResponse{}
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
loghelper.LogEndpoint(endpoint).WithFields(log.Fields{
|
||||
"rawMessage": string(body),
|
||||
"err": err,
|
||||
}).Error("Unable to unmarshal the block root")
|
||||
return "", err
|
||||
var body bytes.Buffer
|
||||
buf := bufio.NewWriter(&body)
|
||||
_, err = io.Copy(buf, response.Body)
|
||||
if err != nil {
|
||||
loghelper.LogSlotError(slot.Number(), err).Error("Unable to turn response into a []bytes array!")
|
||||
return nil, rc, fmt.Errorf("Unable to turn response into a []bytes array!: %s", err.Error())
|
||||
}
|
||||
return resp.Data.Root, nil
|
||||
|
||||
return body.Bytes(), rc, nil
|
||||
}
|
||||
|
@ -1,32 +1,41 @@
|
||||
package beaconclient_test
|
||||
|
||||
import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
//. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/ipld-eth-beacon-indexer/pkg/beaconclient"
|
||||
)
|
||||
|
||||
var (
|
||||
prodConfig = Config{
|
||||
protocol: os.Getenv("bc_protocol"),
|
||||
address: os.Getenv("bc_address"),
|
||||
port: getEnvInt(os.Getenv("bc_port")),
|
||||
dbHost: os.Getenv("db_host"),
|
||||
dbPort: getEnvInt(os.Getenv("db_port")),
|
||||
dbName: os.Getenv("db_name"),
|
||||
dbUser: os.Getenv("db_user"),
|
||||
dbPassword: os.Getenv("db_password"),
|
||||
dbDriver: os.Getenv("db_driver"),
|
||||
knownGapsTableIncrement: 100000000,
|
||||
bcUniqueIdentifier: 100,
|
||||
checkDb: false,
|
||||
protocol: os.Getenv("bc_protocol"),
|
||||
address: os.Getenv("bc_address"),
|
||||
port: getEnvInt(os.Getenv("bc_port"), 5052),
|
||||
dbHost: os.Getenv("db_host"),
|
||||
dbPort: getEnvInt(os.Getenv("db_port"), 8076),
|
||||
dbName: os.Getenv("db_name"),
|
||||
dbUser: os.Getenv("db_user"),
|
||||
dbPassword: os.Getenv("db_password"),
|
||||
dbDriver: os.Getenv("db_driver"),
|
||||
knownGapsTableIncrement: 100000000,
|
||||
bcUniqueIdentifier: 100,
|
||||
checkDb: false,
|
||||
performBeaconBlockProcessing: true,
|
||||
// As of 2022-09, generating and downloading the full BeaconState is so slow it will cause the tests to fail.
|
||||
performBeaconStateProcessing: false,
|
||||
}
|
||||
)
|
||||
|
||||
// Note: These tests expect to communicate with a fully-synced Beacon node.
|
||||
|
||||
var _ = Describe("Systemvalidation", Label("system"), func() {
|
||||
level, _ := log.ParseLevel("debug")
|
||||
log.SetLevel(level)
|
||||
|
||||
Describe("Run the application against a running lighthouse node", func() {
|
||||
Context("When we receive head messages", func() {
|
||||
It("We should process the messages successfully", func() {
|
||||
@ -52,15 +61,6 @@ var _ = Describe("Systemvalidation", Label("system"), func() {
|
||||
})
|
||||
})
|
||||
|
||||
// Wrapper function to get int env variables.
|
||||
func getEnvInt(envVar string) int {
|
||||
val, err := strconv.Atoi(envVar)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
// Start head tracking and wait for the expected results.
|
||||
func processProdHeadBlocks(bc *beaconclient.BeaconClient, expectedInserts, expectedReorgs, expectedKnownGaps, expectedKnownGapsReprocessError uint64) {
|
||||
go bc.CaptureHead()
|
||||
|
@ -28,21 +28,21 @@ func LogError(err error) *log.Entry {
|
||||
}
|
||||
|
||||
// A simple herlper function to log slot and error.
|
||||
func LogSlotError(slot string, err error) *log.Entry {
|
||||
func LogSlotError(slot uint64, err error) *log.Entry {
|
||||
return log.WithFields(log.Fields{
|
||||
"err": err,
|
||||
"slot": slot,
|
||||
})
|
||||
}
|
||||
|
||||
func LogSlotRangeError(startSlot string, endSlot string, err error) *log.Entry {
|
||||
func LogSlotRangeError(startSlot uint64, endSlot uint64, err error) *log.Entry {
|
||||
return log.WithFields(log.Fields{
|
||||
"err": err,
|
||||
"startSlot": startSlot,
|
||||
"endSlot": endSlot,
|
||||
})
|
||||
}
|
||||
func LogSlotRangeStatementError(startSlot string, endSlot string, statement string, err error) *log.Entry {
|
||||
func LogSlotRangeStatementError(startSlot uint64, endSlot uint64, statement string, err error) *log.Entry {
|
||||
return log.WithFields(log.Fields{
|
||||
"err": err,
|
||||
"startSlot": startSlot,
|
||||
|
@ -20,7 +20,7 @@ import (
|
||||
)
|
||||
|
||||
// A simple helper function that will help wrap the reorg error messages.
|
||||
func LogReorgError(slot string, latestBlockRoot string, err error) *log.Entry {
|
||||
func LogReorgError(slot uint64, latestBlockRoot string, err error) *log.Entry {
|
||||
return log.WithFields(log.Fields{
|
||||
"err": err,
|
||||
"slot": slot,
|
||||
@ -29,7 +29,7 @@ func LogReorgError(slot string, latestBlockRoot string, err error) *log.Entry {
|
||||
}
|
||||
|
||||
// A simple helper function that will help wrap regular reorg messages.
|
||||
func LogReorg(slot string, latestBlockRoot string) *log.Entry {
|
||||
func LogReorg(slot uint64, latestBlockRoot string) *log.Entry {
|
||||
return log.WithFields(log.Fields{
|
||||
"slot": slot,
|
||||
"latestBlockRoot": latestBlockRoot,
|
||||
|
Loading…
Reference in New Issue
Block a user