CI, build & docker updates

rm makefile
basic test
add chaindata fixture
simplify publish workflow

.dockerignore

@@ -0,0 +1 @@
+.git

.gitea/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  docker-build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: vars
+        name: Output SHA and version tag
+        run: |
+          echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
+          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - name: Build and tag image
+        run: |
+          docker build . \
+          -t cerc-io/eth-statediff-service \
+          -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} \
+          -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
+      - name: Push image tags
+        run: |
+          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+          docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+          docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}

.gitea/workflows/tests.yml

@@ -0,0 +1,65 @@
+name: Tests
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - ci-test
+  workflow_call:
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+jobs:
+  integration-tests:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          check-latest: true
+      - name: Run dockerd
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: Run DB container
+        run: docker compose -f test/compose.yml up --wait
+      - name: Configure Gitea access
+        env:
+          TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
+        run: |
+          git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"
+      - name: Build package
+        run: go build .
+      - name: Run server
+        env:
+          DATABASE_TYPE: postgres
+          LEVELDB_PATH: ./fixture/chaindata
+          LEVELDB_ANCIENT: ./fixture/chaindata/ancient
+          LOG_FILE_PATH: ./server-log
+        run: |
+          ./eth-statediff-service --config ./test/ci-config.toml serve &
+
+        # Run a sanity test against the fixture data
+        # Complete integration tests are TODO
+      - name: Run test
+        run: |
+          ./scripts/request-range.sh 0 32 || (E=$?; cat ./server-log; exit $E)
+
+          until grep "Finished processing block 32" ./server-log
+          do sleep 1; done
+
+          count_results() {
+              query="select count(*) from $1;"
+              docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
+                  psql -tA cerc_testing -U vdbm -c "$query"
+          }
+          set -x
+          [[ "$(count_results eth.header_cids)" = 33 ]]
+          [[ "$(count_results eth.state_cids)" = 21 ]]
+          [[ "$(count_results eth.storage_cids)" = 18 ]]

.github/workflows/manual_publish.yml

@@ -1,35 +0,0 @@
-name: MANUAL Override Publish from release SHA to TAG 
-on:
-  workflow_dispatch:
-    inputs:
-      giteaPublishTag:
-        description: 'Release TAG to publish TO on gitea; e.g. v4.1.5-alpha'
-        required: true
-      cercContainerTag:
-        description: 'Container (truncated!!! SHA) to release-tag FROM'
-        required: true
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-  # This workflow contains a single job called "build"
-  build:
-    name: Pull SHA and add release-tag
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: |
-          echo ::set-output name=sha::$(echo ${cercContainerTag:0:7})
-      - name: Pull docker image by SHA
-        run: docker pull git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}}
-      - name: Tag docker image TAG
-        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
-      - name: Tag docker image TAG
-        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest
-      - name: Docker Login
-        run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
-      - name: Docker Push Release Tag
-        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
-      - name: Docker Push LATEST Tag
-        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest

.github/workflows/on-publish-pr.yml

@@ -1,73 +0,0 @@
-name: Publish Docker image
-on:
-  release:
-    types: [published]
-  pull_request:
-jobs:
-  pre_job:
-    # continue-on-error: true # Uncomment once integration is finished
-    runs-on: ubuntu-latest
-    # Map a step output to a job output
-    outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
-    steps:
-      - id: skip_check
-        uses: fkirc/skip-duplicate-actions@v4
-        with:
-          # All of these options are optional, so you can remove them if you are happy with the defaults
-          concurrent_skipping: "never"
-          skip_after_successful_duplicate: "true"
-          do_not_skip: '["workflow_dispatch", "schedule"]'
-  run-tests:
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
-    needs: pre_job
-    uses: ./.github/workflows/tests.yml
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    if: |
-      always() &&
-      (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') &&
-      github.event_name == 'release'
-    needs: run-tests
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: |
-          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-      - name: Tag docker image TAG
-        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
-      - name: Docker Push
-        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-#  push_to_registries:
-#    name: Push Docker image to Docker Hub
-#    runs-on: ubuntu-latest
-#    if: |
-#      always() &&
-#      (needs.build.result == 'success') &&
-#      github.event_name == 'release'
-#    needs: build
-#    steps:
-#      - name: Get the version
-#        id: vars
-#        run: |
-#          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-#          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-#      - name: Docker Login to Github Registry
-#        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-#      - name: Docker Pull
-#        run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-#      - name: Docker Login to Docker Registry
-#        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
-#      - name: Tag docker image
-#        run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
-#      - name: Docker Push to Docker Hub
-#        run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}

.github/workflows/tests.yml

@@ -1,37 +0,0 @@
-name: Tests for Geth that are used in multiple jobs.
-
-on:
-  workflow_call:
-
-env:
-  GOPATH: /tmp/go
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Run docker build
-        run: make docker-build
-
-  statediff-unit-test:
-    name: Run statediff unit tests
-    runs-on: ubuntu-latest
-    env:
-      GO111MODULE: on
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.18.0"
-          check-latest: true
-
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Run unit tests
-        run: |
-          make test

Dockerfile

@@ -1,21 +1,24 @@
 FROM golang:1.19-alpine as builder
 
-RUN apk --update --no-cache add make git g++ linux-headers
+RUN apk add --no-cache git gcc musl-dev binutils-gold
 # DEBUG
 RUN apk add busybox-extras
 
-# Get and build ipfs-blockchain-watcher
+WORKDIR /eth-statediff-service
-ADD . /go/src/github.com/cerc-io/eth-statediff-service
-#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
 
-WORKDIR /go/src/github.com/cerc-io/eth-statediff-service
+ARG GIT_VDBTO_TOKEN
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
+
+COPY go.mod go.sum ./
+RUN if [ -n "$GIT_VDBTO_TOKEN" ]; then git config --global url."https://$GIT_VDBTO_TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"; fi && \
+    go mod download && \
+    rm -f ~/.gitconfig
+COPY . .
+
+RUN go build -ldflags '-extldflags "-static"' -o eth-statediff-service .
 
-# app container
 FROM alpine
 
-ARG USER="vdm"
+ARG USER="vdbm"
-ARG CONFIG_FILE="./environments/config.toml"
 ARG EXPOSE_PORT=8545
 
 RUN adduser -Du 5000 $USER adm
@@ -27,12 +30,11 @@ USER $USER
 
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml
+COPY --chown=5000:5000 --from=builder /eth-statediff-service/startup_script.sh .
-COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh .
+COPY --chown=5000:5000 --from=builder /eth-statediff-service/environments environments
-COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments
 
 # keep binaries immutable
-COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service
+COPY --from=builder /eth-statediff-service/eth-statediff-service eth-statediff-service
 
 EXPOSE $EXPOSE_PORT
 

Makefile

@@ -1,12 +0,0 @@
-## Build docker image
-.PHONY: docker-build
-docker-build:
-	docker build -t cerc-io/eth-statediff-service .
-
-.PHONY: test
-test:
-	go test -p 1 ./pkg/... -v
-
-build:
-	go fmt ./...
-	go build

docker-compose.yml

@@ -1,23 +0,0 @@
-version: '3.2'
-
-services:
-  eth-statediff-service:
-    build:
-      context: ./
-      cache_from:
-      - alpine:latest
-      - golang:1.16
-      dockerfile: ./Dockerfile
-      args:
-        USER: "vdbm"
-        CONFIG_FILE: ./environments/example.toml
-        EXPOSE_PORT: 8545
-    environment:
-    - VDB_COMMAND=serve
-    volumes:
-    - eth-statediff-service-data:/root/.ethereum/
-    ports:
-    - "127.0.0.1:8545:8545"
-
-volumes:
-  eth-statediff-service-data:

fixture/.gitignore

@@ -0,0 +1,6 @@
+*/*.log
+*/CURRENT*
+*/LOCK
+*/LOG
+*/MANIFEST-*
+*/ancient/FLOCK

fixture/chaindata/ancient/bodies.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/diffs.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/hashes.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/headers.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/receipts.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

scripts/request-range.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eu
+
+FROM=$1
+TO=$2
+URL=127.0.0.1:8545
+
+DATA='{
+    "jsonrpc": "2.0",
+      "method": "statediff_writeStateDiffsInRange",
+      "params": ['"$FROM"', '"$TO"', {
+          "includeBlock": true,
+          "includeReceipts": true,
+          "includeTD": true,
+          "includeCode": true
+        }
+      ],
+      "id": 1
+}'
+
+exec curl -s $URL -X POST -H 'Content-Type: application/json' --data "$DATA"

startup_script.sh

@@ -17,4 +17,4 @@ mkdir -p /app/geth-rw && \
 sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \
 
 echo "Running the statediff service" && \
-sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml
+exec sudo ./eth-statediff-service "$VDB_COMMAND" --config=/config/config.toml

test/ci-chain.json

@@ -0,0 +1,16 @@
+{
+  "chainId": 41337,
+  "homesteadBlock": 0,
+  "eip150Block": 0,
+  "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "eip155Block": 0,
+  "eip158Block": 0,
+  "byzantiumBlock": 0,
+  "constantinopleBlock": 0,
+  "petersburgBlock": 0,
+  "istanbulBlock": 0,
+  "clique": {
+    "period": 5,
+    "epoch": 30000
+  }
+}

test/ci-config.toml

@@ -0,0 +1,36 @@
+[leveldb]
+    mode = "local"
+    url = "http://127.0.0.1:8082/"
+
+[server]
+    ipcPath = ".ipc"
+    httpPath = "0.0.0.0:8545"
+
+[statediff]
+    serviceWorkers = 1
+    workerQueueSize = 1024
+    trieWorkers = 4
+
+[log]
+    level = "debug"
+
+[database]
+    name            = "cerc_testing"
+    hostname        = "localhost"
+    port            = 8077
+    user            = "vdbm"
+    password        = "password"
+    type            = "postgres"
+    driver          = "sqlx"
+
+[cache]
+    database = 1024
+    trie = 1024
+
+[ethereum]
+    chainConfig = "test/ci-chain.json"
+    nodeID = ""
+    clientName = "eth-statediff-service"
+    genesisBlock = "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
+    networkID = 1
+    chainID = 41337

test/compose.yml

@@ -0,0 +1,23 @@
+services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "cerc_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
+  ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
+    restart: always
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "cerc_testing"
+      POSTGRES_PASSWORD: "password"
+    ports:
+      - 127.0.0.1:8077:5432