CI, build & docker updates

rm makefile
basic test
add chaindata fixture
simplify publish workflow
This commit is contained in:
Roy Crihfield 2023-09-02 01:20:46 +08:00
parent 7605fb0358
commit b91e854974
32 changed files with 218 additions and 194 deletions

1
.dockerignore Normal file
View File

@ -0,0 +1 @@
.git

View File

@ -0,0 +1,28 @@
name: Publish Docker image
on:
release:
types: [published]
jobs:
docker-build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- id: vars
name: Output SHA and version tag
run: |
echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: Build and tag image
run: |
docker build . \
-t cerc-io/eth-statediff-service \
-t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} \
-t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
- name: Push image tags
run: |
echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}

View File

@ -0,0 +1,65 @@
name: Tests
on:
pull_request:
branches: '*'
push:
branches:
- main
- ci-test
workflow_call:
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
integration-tests:
name: Run integration tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version-file: go.mod
check-latest: true
- name: Run dockerd
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: Run DB container
run: docker compose -f test/compose.yml up --wait
- name: Configure Gitea access
env:
TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
run: |
git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"
- name: Build package
run: go build .
- name: Run server
env:
DATABASE_TYPE: postgres
LEVELDB_PATH: ./fixture/chaindata
LEVELDB_ANCIENT: ./fixture/chaindata/ancient
LOG_FILE_PATH: ./server-log
run: |
./eth-statediff-service --config ./test/ci-config.toml serve &
# Run a sanity test against the fixture data
# Complete integration tests are TODO
- name: Run test
run: |
./scripts/request-range.sh 0 32 || (E=$?; cat ./server-log; exit $E)
until grep "Finished processing block 32" ./server-log
do sleep 1; done
count_results() {
query="select count(*) from $1;"
docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
psql -tA cerc_testing -U vdbm -c "$query"
}
set -x
[[ "$(count_results eth.header_cids)" = 33 ]]
[[ "$(count_results eth.state_cids)" = 21 ]]
[[ "$(count_results eth.storage_cids)" = 18 ]]

View File

@ -1,35 +0,0 @@
name: MANUAL Override Publish from release SHA to TAG
on:
workflow_dispatch:
inputs:
giteaPublishTag:
description: 'Release TAG to publish TO on gitea; e.g. v4.1.5-alpha'
required: true
cercContainerTag:
description: 'Container (truncated!!! SHA) to release-tag FROM'
required: true
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
build:
name: Pull SHA and add release-tag
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get the version
id: vars
run: |
echo ::set-output name=sha::$(echo ${cercContainerTag:0:7})
- name: Pull docker image by SHA
run: docker pull git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}}
- name: Tag docker image TAG
run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
- name: Tag docker image TAG
run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest
- name: Docker Login
run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
- name: Docker Push Release Tag
run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
- name: Docker Push LATEST Tag
run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest

View File

@ -1,73 +0,0 @@
name: Publish Docker image
on:
release:
types: [published]
pull_request:
jobs:
pre_job:
# continue-on-error: true # Uncomment once integration is finished
runs-on: ubuntu-latest
# Map a step output to a job output
outputs:
should_skip: ${{ steps.skip_check.outputs.should_skip }}
steps:
- id: skip_check
uses: fkirc/skip-duplicate-actions@v4
with:
# All of these options are optional, so you can remove them if you are happy with the defaults
concurrent_skipping: "never"
skip_after_successful_duplicate: "true"
do_not_skip: '["workflow_dispatch", "schedule"]'
run-tests:
if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
needs: pre_job
uses: ./.github/workflows/tests.yml
build:
name: Run docker build
runs-on: ubuntu-latest
if: |
always() &&
(needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') &&
github.event_name == 'release'
needs: run-tests
steps:
- uses: actions/checkout@v2
- name: Get the version
id: vars
run: |
echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
- name: Run docker build
run: make docker-build
- name: Tag docker image
run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
- name: Tag docker image TAG
run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
- name: Docker Login
run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
- name: Docker Push
run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
# push_to_registries:
# name: Push Docker image to Docker Hub
# runs-on: ubuntu-latest
# if: |
# always() &&
# (needs.build.result == 'success') &&
# github.event_name == 'release'
# needs: build
# steps:
# - name: Get the version
# id: vars
# run: |
# echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
# echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
# - name: Docker Login to Github Registry
# run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
# - name: Docker Pull
# run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
# - name: Docker Login to Docker Registry
# run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
# - name: Tag docker image
# run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
# - name: Docker Push to Docker Hub
# run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}

View File

@ -1,37 +0,0 @@
name: Tests for Geth that are used in multiple jobs.
on:
workflow_call:
env:
GOPATH: /tmp/go
jobs:
build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Run docker build
run: make docker-build
statediff-unit-test:
name: Run statediff unit tests
runs-on: ubuntu-latest
env:
GO111MODULE: on
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- uses: actions/setup-go@v3
with:
go-version: ">=1.18.0"
check-latest: true
- name: Checkout code
uses: actions/checkout@v2
- name: Run unit tests
run: |
make test

View File

@ -1,21 +1,24 @@
FROM golang:1.19-alpine as builder FROM golang:1.19-alpine as builder
RUN apk --update --no-cache add make git g++ linux-headers RUN apk add --no-cache git gcc musl-dev binutils-gold
# DEBUG # DEBUG
RUN apk add busybox-extras RUN apk add busybox-extras
# Get and build ipfs-blockchain-watcher WORKDIR /eth-statediff-service
ADD . /go/src/github.com/cerc-io/eth-statediff-service
#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
WORKDIR /go/src/github.com/cerc-io/eth-statediff-service ARG GIT_VDBTO_TOKEN
RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
COPY go.mod go.sum ./
RUN if [ -n "$GIT_VDBTO_TOKEN" ]; then git config --global url."https://$GIT_VDBTO_TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"; fi && \
go mod download && \
rm -f ~/.gitconfig
COPY . .
RUN go build -ldflags '-extldflags "-static"' -o eth-statediff-service .
# app container
FROM alpine FROM alpine
ARG USER="vdm" ARG USER="vdbm"
ARG CONFIG_FILE="./environments/config.toml"
ARG EXPOSE_PORT=8545 ARG EXPOSE_PORT=8545
RUN adduser -Du 5000 $USER adm RUN adduser -Du 5000 $USER adm
@ -27,12 +30,11 @@ USER $USER
# chown first so dir is writable # chown first so dir is writable
# note: using $USER is merged, but not in the stable release yet # note: using $USER is merged, but not in the stable release yet
COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml COPY --chown=5000:5000 --from=builder /eth-statediff-service/startup_script.sh .
COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh . COPY --chown=5000:5000 --from=builder /eth-statediff-service/environments environments
COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments
# keep binaries immutable # keep binaries immutable
COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service COPY --from=builder /eth-statediff-service/eth-statediff-service eth-statediff-service
EXPOSE $EXPOSE_PORT EXPOSE $EXPOSE_PORT

View File

@ -1,12 +0,0 @@
## Build docker image
.PHONY: docker-build
docker-build:
docker build -t cerc-io/eth-statediff-service .
.PHONY: test
test:
go test -p 1 ./pkg/... -v
build:
go fmt ./...
go build

View File

@ -1,23 +0,0 @@
version: '3.2'
services:
eth-statediff-service:
build:
context: ./
cache_from:
- alpine:latest
- golang:1.16
dockerfile: ./Dockerfile
args:
USER: "vdbm"
CONFIG_FILE: ./environments/example.toml
EXPOSE_PORT: 8545
environment:
- VDB_COMMAND=serve
volumes:
- eth-statediff-service-data:/root/.ethereum/
ports:
- "127.0.0.1:8545:8545"
volumes:
eth-statediff-service-data:

6
fixture/.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
*/*.log
*/CURRENT*
*/LOCK
*/LOG
*/MANIFEST-*
*/ancient/FLOCK

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><01>

View File

@ -0,0 +1 @@
<EFBFBD><01>

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><01>

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><01>

Binary file not shown.

View File

@ -0,0 +1 @@
<EFBFBD><01>

22
scripts/request-range.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -eu
FROM=$1
TO=$2
URL=127.0.0.1:8545
DATA='{
"jsonrpc": "2.0",
"method": "statediff_writeStateDiffsInRange",
"params": ['"$FROM"', '"$TO"', {
"includeBlock": true,
"includeReceipts": true,
"includeTD": true,
"includeCode": true
}
],
"id": 1
}'
exec curl -s $URL -X POST -H 'Content-Type: application/json' --data "$DATA"

View File

@ -17,4 +17,4 @@ mkdir -p /app/geth-rw && \
sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \ sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \
echo "Running the statediff service" && \ echo "Running the statediff service" && \
sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml exec sudo ./eth-statediff-service "$VDB_COMMAND" --config=/config/config.toml

16
test/ci-chain.json Normal file
View File

@ -0,0 +1,16 @@
{
"chainId": 41337,
"homesteadBlock": 0,
"eip150Block": 0,
"eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"eip155Block": 0,
"eip158Block": 0,
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0,
"istanbulBlock": 0,
"clique": {
"period": 5,
"epoch": 30000
}
}

36
test/ci-config.toml Normal file
View File

@ -0,0 +1,36 @@
[leveldb]
mode = "local"
url = "http://127.0.0.1:8082/"
[server]
ipcPath = ".ipc"
httpPath = "0.0.0.0:8545"
[statediff]
serviceWorkers = 1
workerQueueSize = 1024
trieWorkers = 4
[log]
level = "debug"
[database]
name = "cerc_testing"
hostname = "localhost"
port = 8077
user = "vdbm"
password = "password"
type = "postgres"
driver = "sqlx"
[cache]
database = 1024
trie = 1024
[ethereum]
chainConfig = "test/ci-chain.json"
nodeID = ""
clientName = "eth-statediff-service"
genesisBlock = "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
networkID = 1
chainID = 41337

23
test/compose.yml Normal file
View File

@ -0,0 +1,23 @@
services:
migrations:
restart: on-failure
depends_on:
- ipld-eth-db
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
environment:
DATABASE_USER: "vdbm"
DATABASE_NAME: "cerc_testing"
DATABASE_PASSWORD: "password"
DATABASE_HOSTNAME: "ipld-eth-db"
DATABASE_PORT: 5432
ipld-eth-db:
image: timescale/timescaledb:latest-pg14
restart: always
command: ["postgres", "-c", "log_statement=all"]
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "cerc_testing"
POSTGRES_PASSWORD: "password"
ports:
- 127.0.0.1:8077:5432