ci, build, update deps

rm makefile
basic test
add chaindata fixture
simplify publish workflow

.dockerignore

@@ -0,0 +1 @@
+.git

.gitea/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  docker-build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: vars
+        name: Output SHA and version tag
+        run: |
+          echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
+          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - name: Build and tag image
+        run: |
+          docker build . \
+          -t cerc-io/eth-statediff-service \
+          -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} \
+          -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
+      - name: Push image tags
+        run: |
+          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+          docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+          docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}

.gitea/workflows/tests.yml

@@ -0,0 +1,65 @@
+name: Tests
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - ci-test
+  workflow_call:
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+jobs:
+  integration-tests:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          check-latest: true
+      - name: Run dockerd
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: Run DB container
+        run: docker compose -f test/compose.yml up --wait
+      - name: Configure Gitea access
+        env:
+          TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
+        run: |
+          git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"
+      - name: Build package
+        run: go build .
+        # Run a sanity test against the fixture data
+        # Complete integration tests are TODO
+      - name: Run basic integration test
+        env:
+          DATABASE_TYPE: postgres
+          LEVELDB_PATH: ./fixture/chaindata
+          LEVELDB_ANCIENT: ./fixture/chaindata/ancient
+          LOG_FILE_PATH: ./server-log
+        timeout-minutes: 30
+        run: |
+          ./eth-statediff-service --config ./test/ci-config.toml serve &
+          sleep 10
+
+          ./scripts/request-range.sh 0 32 || (E=$?; cat ./server-log; exit $E)
+
+          until grep "Finished processing block 32" ./server-log
+          do sleep 1; done
+
+          count_results() {
+              query="select count(*) from $1;"
+              docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
+                  psql -tA cerc_testing -U vdbm -c "$query"
+          }
+          set -x
+          [[ "$(count_results eth.header_cids)" = 33 ]]
+          [[ "$(count_results eth.state_cids)" = 21 ]]
+          [[ "$(count_results eth.storage_cids)" = 18 ]]

.github/workflows/manual_publish.yml

@@ -1,35 +0,0 @@
-name: MANUAL Override Publish from release SHA to TAG 
-on:
-  workflow_dispatch:
-    inputs:
-      giteaPublishTag:
-        description: 'Release TAG to publish TO on gitea; e.g. v4.1.5-alpha'
-        required: true
-      cercContainerTag:
-        description: 'Container (truncated!!! SHA) to release-tag FROM'
-        required: true
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-  # This workflow contains a single job called "build"
-  build:
-    name: Pull SHA and add release-tag
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: |
-          echo ::set-output name=sha::$(echo ${cercContainerTag:0:7})
-      - name: Pull docker image by SHA
-        run: docker pull git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}}
-      - name: Tag docker image TAG
-        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
-      - name: Tag docker image TAG
-        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest
-      - name: Docker Login
-        run: echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
-      - name: Docker Push Release Tag
-        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
-      - name: Docker Push LATEST Tag
-        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest

.github/workflows/on-publish-pr.yml

@@ -1,73 +0,0 @@
-name: Publish Docker image
-on:
-  release:
-    types: [published]
-  pull_request:
-jobs:
-  pre_job:
-    # continue-on-error: true # Uncomment once integration is finished
-    runs-on: ubuntu-latest
-    # Map a step output to a job output
-    outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
-    steps:
-      - id: skip_check
-        uses: fkirc/skip-duplicate-actions@v4
-        with:
-          # All of these options are optional, so you can remove them if you are happy with the defaults
-          concurrent_skipping: "never"
-          skip_after_successful_duplicate: "true"
-          do_not_skip: '["workflow_dispatch", "schedule"]'
-  run-tests:
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
-    needs: pre_job
-    uses: ./.github/workflows/tests.yml
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    if: |
-      always() &&
-      (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') &&
-      github.event_name == 'release'
-    needs: run-tests
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: |
-          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-      - name: Tag docker image TAG
-        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
-      - name: Docker Push
-        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-#  push_to_registries:
-#    name: Push Docker image to Docker Hub
-#    runs-on: ubuntu-latest
-#    if: |
-#      always() &&
-#      (needs.build.result == 'success') &&
-#      github.event_name == 'release'
-#    needs: build
-#    steps:
-#      - name: Get the version
-#        id: vars
-#        run: |
-#          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-#          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-#      - name: Docker Login to Github Registry
-#        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-#      - name: Docker Pull
-#        run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-#      - name: Docker Login to Docker Registry
-#        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
-#      - name: Tag docker image
-#        run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
-#      - name: Docker Push to Docker Hub
-#        run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}

.github/workflows/tests.yml

@@ -1,37 +0,0 @@
-name: Tests for Geth that are used in multiple jobs.
-
-on:
-  workflow_call:
-
-env:
-  GOPATH: /tmp/go
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Run docker build
-        run: make docker-build
-
-  statediff-unit-test:
-    name: Run statediff unit tests
-    runs-on: ubuntu-latest
-    env:
-      GO111MODULE: on
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.18.0"
-          check-latest: true
-
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Run unit tests
-        run: |
-          make test

Dockerfile

@@ -1,20 +1,24 @@
 FROM golang:1.19-alpine as builder
 
-RUN apk --update --no-cache add make git g++ linux-headers
+RUN apk add --no-cache git gcc musl-dev binutils-gold
 # DEBUG
 RUN apk add busybox-extras
 
-# Get and build ipfs-blockchain-watcher
+WORKDIR /eth-statediff-service
-ADD . /go/src/github.com/cerc-io/eth-statediff-service
-#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
 
-WORKDIR /go/src/github.com/cerc-io/eth-statediff-service
+ARG GIT_VDBTO_TOKEN
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
+
+COPY go.mod go.sum ./
+RUN if [ -n "$GIT_VDBTO_TOKEN" ]; then git config --global url."https://$GIT_VDBTO_TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"; fi && \
+    go mod download && \
+    rm -f ~/.gitconfig
+COPY . .
+
+RUN go build -ldflags '-extldflags "-static"' -o eth-statediff-service .
 
-# app container
 FROM alpine
 
-ARG CONFIG_FILE="./environments/config.toml"
+ARG USER="vdbm"
 ARG EXPOSE_PORT=8545
 
 RUN apk --no-cache add su-exec bash
@@ -23,12 +27,11 @@ WORKDIR /app
 
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml
+COPY --from=builder /eth-statediff-service/startup_script.sh .
-COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh .
+COPY --from=builder /eth-statediff-service/environments environments
-COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments
 
 # keep binaries immutable
-COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service
+COPY --from=builder /eth-statediff-service/eth-statediff-service eth-statediff-service
 
 EXPOSE $EXPOSE_PORT
 

Makefile

@@ -1,12 +0,0 @@
-## Build docker image
-.PHONY: docker-build
-docker-build:
-	docker build -t cerc-io/eth-statediff-service .
-
-.PHONY: test
-test:
-	go test -p 1 ./pkg/... -v
-
-build:
-	go fmt ./...
-	go build

cmd/util.go

@@ -2,16 +2,12 @@ package cmd
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
-	"os"
 
 	statediff "github.com/cerc-io/plugeth-statediff"
 	"github.com/cerc-io/plugeth-statediff/indexer"
 	"github.com/cerc-io/plugeth-statediff/indexer/node"
 	"github.com/cerc-io/plugeth-statediff/indexer/shared"
 	"github.com/cerc-io/plugeth-statediff/utils"
-	"github.com/cerc-io/plugeth-statediff/utils/log"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"

docker-compose.yml

@@ -1,23 +0,0 @@
-version: '3.2'
-
-services:
-  eth-statediff-service:
-    build:
-      context: ./
-      cache_from:
-      - alpine:latest
-      - golang:1.16
-      dockerfile: ./Dockerfile
-      args:
-        USER: "vdbm"
-        CONFIG_FILE: ./environments/example.toml
-        EXPOSE_PORT: 8545
-    environment:
-    - VDB_COMMAND=serve
-    volumes:
-    - eth-statediff-service-data:/root/.ethereum/
-    ports:
-    - "127.0.0.1:8545:8545"
-
-volumes:
-  eth-statediff-service-data:

fixture/.gitignore

@@ -0,0 +1,6 @@
+*/*.log
+*/CURRENT*
+*/LOCK
+*/LOG
+*/MANIFEST-*
+*/ancient/FLOCK

fixture/chaindata/ancient/bodies.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/diffs.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/hashes.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/headers.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/receipts.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

go.mod

@@ -138,9 +138,9 @@ require (
 )
 
 replace (
-	github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230803115933-6bb6d4e27fd2
+	github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.1.2
-	// github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.0
+	github.com/cerc-io/eth-testing => git.vdb.to/cerc-io/eth-testing v0.3.1
-	github.com/cerc-io/plugeth-statediff => ../plugeth-statediff
+	github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.3
 	github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1
 	github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46
 )

go.sum

@@ -46,10 +46,13 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230803115933-6bb6d4e27fd2 h1:UImCl/qXSk9WD5hG1PLS4wfjkHIdgr+5BvzsNqeWEv4=
+git.vdb.to/cerc-io/eth-iterator-utils v0.1.2 h1:PdMR5B9wrQSYuYpFhN+9Kc8AEZ0pTt5eKCmu8oCtFcY=
-git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230803115933-6bb6d4e27fd2/go.mod h1:VtTq+MbVe5aKTtb/CoSySjz5vXPFOs9HPBTALmEUfj4=
+git.vdb.to/cerc-io/eth-iterator-utils v0.1.2/go.mod h1:OvXbdWbZ5viBXC/Ui1EkhsSmGB+AUX+TjGa3UDAfjfg=
+git.vdb.to/cerc-io/eth-testing v0.3.1 h1:sPnlMev6oEgTjsW7GtUkSsjKNG/+X6P9q0izSejLGpM=
 git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1 h1:KLjxHwp9Zp7xhECccmJS00RiL+VwTuUGLU7qeIctg8g=
 git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1/go.mod h1:cYXZu70+6xmDgIgrTD81GPasv16piiAFJnKyAbwVPMU=
+git.vdb.to/cerc-io/plugeth-statediff v0.1.3 h1:aYzE3JUIXmLQI+/6axxPJ64GT+1JamChGp4J6+CBrLY=
+git.vdb.to/cerc-io/plugeth-statediff v0.1.3/go.mod h1:PX1bwTwZKzegQu6zYdpuJf1tHNmU0suapKOUUO2xIzs=
 git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46 h1:KYcbbne/RXd7AuxbUd/3hgk1jPN+33k2CKiNsUsMCC0=
 git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46/go.mod h1:VpDN61dxy64zGff05F0adujR5enD/JEdXBkTQ+PaIsQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

pkg/service.go

@@ -64,7 +64,7 @@ type StateDiffService interface {
 // Service is the underlying struct for the state diffing service
 type Service struct {
 	// Used to build the state diff objects
-	Builder statediff.Builder
+	builder statediff.Builder
 	// Used to read data from LevelDB
 	lvlDBReader Reader
 	// Used to signal shutdown of the service
@@ -88,7 +88,7 @@ func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer
 	}
 	return &Service{
 		lvlDBReader: lvlDBReader,
-		Builder:     builder,
+		builder:     builder,
 		indexer:     indexer,
 		workers:     conf.ServiceWorkers,
 		queue:       make(chan RangeRequest, conf.WorkerQueueSize),
@@ -290,7 +290,7 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params)
 
 // processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
 func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) {
-	stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{
+	stateDiff, err := sds.builder.BuildStateDiffObject(statediff.Args{
 		BlockHash:    currentBlock.Hash(),
 		BlockNumber:  currentBlock.Number(),
 		OldStateRoot: parentRoot,
@@ -436,7 +436,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 	}
 	prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = sds.Builder.WriteStateDiff(statediff.Args{
+	err = sds.builder.WriteStateDiff(statediff.Args{
 		NewStateRoot: block.Root(),
 		OldStateRoot: parentRoot,
 		BlockNumber:  block.Number(),

scripts/request-range.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eu
+
+FROM=$1
+TO=$2
+URL=127.0.0.1:8545
+
+DATA='{
+    "jsonrpc": "2.0",
+      "method": "statediff_writeStateDiffsInRange",
+      "params": ['"$FROM"', '"$TO"', {
+          "includeBlock": true,
+          "includeReceipts": true,
+          "includeTD": true,
+          "includeCode": true
+        }
+      ],
+      "id": 1
+}'
+
+exec curl -s $URL -X POST -H 'Content-Type: application/json' --data "$DATA"

test/ci-chain.json

@@ -0,0 +1,16 @@
+{
+  "chainId": 41337,
+  "homesteadBlock": 0,
+  "eip150Block": 0,
+  "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "eip155Block": 0,
+  "eip158Block": 0,
+  "byzantiumBlock": 0,
+  "constantinopleBlock": 0,
+  "petersburgBlock": 0,
+  "istanbulBlock": 0,
+  "clique": {
+    "period": 5,
+    "epoch": 30000
+  }
+}

test/ci-config.toml

@@ -0,0 +1,36 @@
+[leveldb]
+    mode = "local"
+    url = "http://127.0.0.1:8082/"
+
+[server]
+    ipcPath = ".ipc"
+    httpPath = "0.0.0.0:8545"
+
+[statediff]
+    serviceWorkers = 1
+    workerQueueSize = 1024
+    trieWorkers = 4
+
+[log]
+    level = "debug"
+
+[database]
+    name            = "cerc_testing"
+    hostname        = "localhost"
+    port            = 8077
+    user            = "vdbm"
+    password        = "password"
+    type            = "postgres"
+    driver          = "sqlx"
+
+[cache]
+    database = 1024
+    trie = 1024
+
+[ethereum]
+    chainConfig = "test/ci-chain.json"
+    nodeID = ""
+    clientName = "eth-statediff-service"
+    genesisBlock = "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
+    networkID = 1
+    chainID = 41337

test/compose.yml

@@ -0,0 +1,23 @@
+services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "cerc_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
+  ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
+    restart: always
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "cerc_testing"
+      POSTGRES_PASSWORD: "password"
+    ports:
+      - 127.0.0.1:8077:5432