ci, build, update deps

rm makefile
basic test
add chaindata fixture
simplify publish workflow
This commit is contained in:
Roy Crihfield 2023-08-25 20:10:48 +08:00
parent 7f281419eb
commit 7a03e5ef04
35 changed files with 229 additions and 205 deletions

1
.dockerignore Normal file
View File

@ -0,0 +1 @@
.git

View File

@ -0,0 +1,28 @@
name: Publish Docker image
on:
release:
types: [published]
jobs:
docker-build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- id: vars
name: Output SHA and version tag
run: |
echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: Build and tag image
run: |
docker build . \
-t cerc-io/eth-statediff-service \
-t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} \
-t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
- name: Push image tags
run: |
echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}

View File

@ -0,0 +1,65 @@
name: Tests
on:
pull_request:
branches: '*'
push:
branches:
- main
- ci-test
workflow_call:
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
integration-tests:
name: Run integration tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version-file: go.mod
check-latest: true
- name: Run dockerd
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: Run DB container
run: docker compose -f test/compose.yml up --wait
- name: Configure Gitea access
env:
TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
run: |
git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"
- name: Build package
run: go build .
# Run a sanity test against the fixture data
# Complete integration tests are TODO
- name: Run basic integration test
env:
DATABASE_TYPE: postgres
LEVELDB_PATH: ./fixture/chaindata
LEVELDB_ANCIENT: ./fixture/chaindata/ancient
LOG_FILE_PATH: ./server-log
timeout-minutes: 30
run: |
./eth-statediff-service --config ./test/ci-config.toml serve &
sleep 10
./scripts/request-range.sh 0 32 || (E=$?; cat ./server-log; exit $E)
until grep "Finished processing block 32" ./server-log
do sleep 1; done
count_results() {
query="select count(*) from $1;"
docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
psql -tA cerc_testing -U vdbm -c "$query"
}
set -x
[[ "$(count_results eth.header_cids)" = 33 ]]
[[ "$(count_results eth.state_cids)" = 21 ]]
[[ "$(count_results eth.storage_cids)" = 18 ]]

View File

@ -1,35 +0,0 @@
name: MANUAL Override Publish from release SHA to TAG
on:
workflow_dispatch:
inputs:
giteaPublishTag:
description: 'Release TAG to publish TO on gitea; e.g. v4.1.5-alpha'
required: true
cercContainerTag:
description: 'Container (truncated!!! SHA) to release-tag FROM'
required: true
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
build:
name: Pull SHA and add release-tag
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get the version
id: vars
run: |
echo ::set-output name=sha::$(echo ${cercContainerTag:0:7})
- name: Pull docker image by SHA
run: docker pull git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}}
- name: Tag docker image TAG
run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
- name: Tag docker image TAG
run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest
- name: Docker Login
run: echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
- name: Docker Push Release Tag
run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
- name: Docker Push LATEST Tag
run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest

View File

@ -1,73 +0,0 @@
name: Publish Docker image
on:
release:
types: [published]
pull_request:
jobs:
pre_job:
# continue-on-error: true # Uncomment once integration is finished
runs-on: ubuntu-latest
# Map a step output to a job output
outputs:
should_skip: ${{ steps.skip_check.outputs.should_skip }}
steps:
- id: skip_check
uses: fkirc/skip-duplicate-actions@v4
with:
# All of these options are optional, so you can remove them if you are happy with the defaults
concurrent_skipping: "never"
skip_after_successful_duplicate: "true"
do_not_skip: '["workflow_dispatch", "schedule"]'
run-tests:
if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
needs: pre_job
uses: ./.github/workflows/tests.yml
build:
name: Run docker build
runs-on: ubuntu-latest
if: |
always() &&
(needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') &&
github.event_name == 'release'
needs: run-tests
steps:
- uses: actions/checkout@v2
- name: Get the version
id: vars
run: |
echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
- name: Run docker build
run: make docker-build
- name: Tag docker image
run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
- name: Tag docker image TAG
run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
- name: Docker Login
run: echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
- name: Docker Push
run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
# push_to_registries:
# name: Push Docker image to Docker Hub
# runs-on: ubuntu-latest
# if: |
# always() &&
# (needs.build.result == 'success') &&
# github.event_name == 'release'
# needs: build
# steps:
# - name: Get the version
# id: vars
# run: |
# echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
# echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
# - name: Docker Login to Github Registry
# run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
# - name: Docker Pull
# run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
# - name: Docker Login to Docker Registry
# run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
# - name: Tag docker image
# run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
# - name: Docker Push to Docker Hub
# run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}

View File

@ -1,37 +0,0 @@
name: Tests for Geth that are used in multiple jobs.
on:
workflow_call:
env:
GOPATH: /tmp/go
jobs:
build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Run docker build
run: make docker-build
statediff-unit-test:
name: Run statediff unit tests
runs-on: ubuntu-latest
env:
GO111MODULE: on
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- uses: actions/setup-go@v3
with:
go-version: ">=1.18.0"
check-latest: true
- name: Checkout code
uses: actions/checkout@v2
- name: Run unit tests
run: |
make test

View File

@ -1,20 +1,24 @@
FROM golang:1.19-alpine as builder
RUN apk --update --no-cache add make git g++ linux-headers
RUN apk add --no-cache git gcc musl-dev binutils-gold
# DEBUG
RUN apk add busybox-extras
# Get and build ipfs-blockchain-watcher
ADD . /go/src/github.com/cerc-io/eth-statediff-service
#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
WORKDIR /eth-statediff-service
WORKDIR /go/src/github.com/cerc-io/eth-statediff-service
RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
ARG GIT_VDBTO_TOKEN
COPY go.mod go.sum ./
RUN if [ -n "$GIT_VDBTO_TOKEN" ]; then git config --global url."https://$GIT_VDBTO_TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"; fi && \
go mod download && \
rm -f ~/.gitconfig
COPY . .
RUN go build -ldflags '-extldflags "-static"' -o eth-statediff-service .
# app container
FROM alpine
ARG CONFIG_FILE="./environments/config.toml"
ARG USER="vdbm"
ARG EXPOSE_PORT=8545
RUN apk --no-cache add su-exec bash
@ -23,12 +27,11 @@ WORKDIR /app
# chown first so dir is writable
# note: using $USER is merged, but not in the stable release yet
COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml
COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh .
COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments
COPY --from=builder /eth-statediff-service/startup_script.sh .
COPY --from=builder /eth-statediff-service/environments environments
# keep binaries immutable
COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service
COPY --from=builder /eth-statediff-service/eth-statediff-service eth-statediff-service
EXPOSE $EXPOSE_PORT

View File

@ -1,12 +0,0 @@
## Build docker image
.PHONY: docker-build
docker-build:
docker build -t cerc-io/eth-statediff-service .
.PHONY: test
test:
go test -p 1 ./pkg/... -v
build:
go fmt ./...
go build

View File

@ -2,16 +2,12 @@ package cmd
import (
"context"
"encoding/json"
"fmt"
"os"
statediff "github.com/cerc-io/plugeth-statediff"
"github.com/cerc-io/plugeth-statediff/indexer"
"github.com/cerc-io/plugeth-statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
"github.com/cerc-io/plugeth-statediff/utils"
"github.com/cerc-io/plugeth-statediff/utils/log"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"

View File

@ -1,23 +0,0 @@
version: '3.2'
services:
eth-statediff-service:
build:
context: ./
cache_from:
- alpine:latest
- golang:1.16
dockerfile: ./Dockerfile
args:
USER: "vdbm"
CONFIG_FILE: ./environments/example.toml
EXPOSE_PORT: 8545
environment:
- VDB_COMMAND=serve
volumes:
- eth-statediff-service-data:/root/.ethereum/
ports:
- "127.0.0.1:8545:8545"
volumes:
eth-statediff-service-data:

6
fixture/.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
*/*.log
*/CURRENT*
*/LOCK
*/LOG
*/MANIFEST-*
*/ancient/FLOCK

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><01>

View File

@ -0,0 +1 @@
<EFBFBD><01>

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><01>

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><01>

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><01>

6
go.mod
View File

@ -138,9 +138,9 @@ require (
)
replace (
github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230803115933-6bb6d4e27fd2
// github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.0
github.com/cerc-io/plugeth-statediff => ../plugeth-statediff
github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.1.2
github.com/cerc-io/eth-testing => git.vdb.to/cerc-io/eth-testing v0.3.1
github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.3
github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1
github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46
)

7
go.sum
View File

@ -46,10 +46,13 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230803115933-6bb6d4e27fd2 h1:UImCl/qXSk9WD5hG1PLS4wfjkHIdgr+5BvzsNqeWEv4=
git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230803115933-6bb6d4e27fd2/go.mod h1:VtTq+MbVe5aKTtb/CoSySjz5vXPFOs9HPBTALmEUfj4=
git.vdb.to/cerc-io/eth-iterator-utils v0.1.2 h1:PdMR5B9wrQSYuYpFhN+9Kc8AEZ0pTt5eKCmu8oCtFcY=
git.vdb.to/cerc-io/eth-iterator-utils v0.1.2/go.mod h1:OvXbdWbZ5viBXC/Ui1EkhsSmGB+AUX+TjGa3UDAfjfg=
git.vdb.to/cerc-io/eth-testing v0.3.1 h1:sPnlMev6oEgTjsW7GtUkSsjKNG/+X6P9q0izSejLGpM=
git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1 h1:KLjxHwp9Zp7xhECccmJS00RiL+VwTuUGLU7qeIctg8g=
git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1/go.mod h1:cYXZu70+6xmDgIgrTD81GPasv16piiAFJnKyAbwVPMU=
git.vdb.to/cerc-io/plugeth-statediff v0.1.3 h1:aYzE3JUIXmLQI+/6axxPJ64GT+1JamChGp4J6+CBrLY=
git.vdb.to/cerc-io/plugeth-statediff v0.1.3/go.mod h1:PX1bwTwZKzegQu6zYdpuJf1tHNmU0suapKOUUO2xIzs=
git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46 h1:KYcbbne/RXd7AuxbUd/3hgk1jPN+33k2CKiNsUsMCC0=
git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46/go.mod h1:VpDN61dxy64zGff05F0adujR5enD/JEdXBkTQ+PaIsQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

View File

@ -64,7 +64,7 @@ type StateDiffService interface {
// Service is the underlying struct for the state diffing service
type Service struct {
// Used to build the state diff objects
Builder statediff.Builder
builder statediff.Builder
// Used to read data from LevelDB
lvlDBReader Reader
// Used to signal shutdown of the service
@ -88,7 +88,7 @@ func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer
}
return &Service{
lvlDBReader: lvlDBReader,
Builder: builder,
builder: builder,
indexer: indexer,
workers: conf.ServiceWorkers,
queue: make(chan RangeRequest, conf.WorkerQueueSize),
@ -290,7 +290,7 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params)
// processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) {
stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{
stateDiff, err := sds.builder.BuildStateDiffObject(statediff.Args{
BlockHash: currentBlock.Hash(),
BlockNumber: currentBlock.Number(),
OldStateRoot: parentRoot,
@ -436,7 +436,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
}
prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
t = time.Now()
err = sds.Builder.WriteStateDiff(statediff.Args{
err = sds.builder.WriteStateDiff(statediff.Args{
NewStateRoot: block.Root(),
OldStateRoot: parentRoot,
BlockNumber: block.Number(),

22
scripts/request-range.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -eu
FROM=$1
TO=$2
URL=127.0.0.1:8545
DATA='{
"jsonrpc": "2.0",
"method": "statediff_writeStateDiffsInRange",
"params": ['"$FROM"', '"$TO"', {
"includeBlock": true,
"includeReceipts": true,
"includeTD": true,
"includeCode": true
}
],
"id": 1
}'
exec curl -s $URL -X POST -H 'Content-Type: application/json' --data "$DATA"

16
test/ci-chain.json Normal file
View File

@ -0,0 +1,16 @@
{
"chainId": 41337,
"homesteadBlock": 0,
"eip150Block": 0,
"eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"eip155Block": 0,
"eip158Block": 0,
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0,
"istanbulBlock": 0,
"clique": {
"period": 5,
"epoch": 30000
}
}

36
test/ci-config.toml Normal file
View File

@ -0,0 +1,36 @@
[leveldb]
mode = "local"
url = "http://127.0.0.1:8082/"
[server]
ipcPath = ".ipc"
httpPath = "0.0.0.0:8545"
[statediff]
serviceWorkers = 1
workerQueueSize = 1024
trieWorkers = 4
[log]
level = "debug"
[database]
name = "cerc_testing"
hostname = "localhost"
port = 8077
user = "vdbm"
password = "password"
type = "postgres"
driver = "sqlx"
[cache]
database = 1024
trie = 1024
[ethereum]
chainConfig = "test/ci-chain.json"
nodeID = ""
clientName = "eth-statediff-service"
genesisBlock = "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
networkID = 1
chainID = 41337

23
test/compose.yml Normal file
View File

@ -0,0 +1,23 @@
services:
migrations:
restart: on-failure
depends_on:
- ipld-eth-db
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
environment:
DATABASE_USER: "vdbm"
DATABASE_NAME: "cerc_testing"
DATABASE_PASSWORD: "password"
DATABASE_HOSTNAME: "ipld-eth-db"
DATABASE_PORT: 5432
ipld-eth-db:
image: timescale/timescaledb:latest-pg14
restart: always
command: ["postgres", "-c", "log_statement=all"]
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "cerc_testing"
POSTGRES_PASSWORD: "password"
ports:
- 127.0.0.1:8077:5432