diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..191381e --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.git \ No newline at end of file diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml new file mode 100644 index 0000000..03a3513 --- /dev/null +++ b/.gitea/workflows/publish.yml @@ -0,0 +1,28 @@ +name: Publish Docker image + +on: + release: + types: [published] + +jobs: + docker-build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - id: vars + name: Output SHA and version tag + run: | + echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT + echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + - name: Build and tag image + run: | + docker build . \ + -t cerc-io/eth-statediff-service \ + -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} \ + -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}} + - name: Push image tags + run: | + echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin + docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} + docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}} diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml new file mode 100644 index 0000000..2515480 --- /dev/null +++ b/.gitea/workflows/tests.yml @@ -0,0 +1,65 @@ +name: Tests + +on: + pull_request: + branches: '*' + push: + branches: + - main + - ci-test + workflow_call: + +# Needed until we can incorporate docker startup into the executor container +env: + DOCKER_HOST: unix:///var/run/dind.sock + +jobs: + integration-tests: + name: Run integration tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version-file: go.mod + check-latest: true + - name: Run dockerd + run: | + dockerd -H $DOCKER_HOST --userland-proxy=false & + sleep 5 + - name: Run DB container + run: docker compose -f test/compose.yml up --wait + - name: Configure Gitea access + env: + TOKEN: ${{ secrets.CICD_REPO_TOKEN }} + run: | + git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/" + - name: Build package + run: go build . + - name: Run server + env: + DATABASE_TYPE: postgres + LEVELDB_PATH: ./fixture/chaindata + LEVELDB_ANCIENT: ./fixture/chaindata/ancient + LOG_FILE_PATH: ./server-log + run: | + ./eth-statediff-service --config ./test/ci-config.toml serve & + + # Run a sanity test against the fixture data + # Complete integration tests are TODO + - name: Run test + run: | + ./scripts/request-range.sh 0 32 || (E=$?; cat ./server-log; exit $E) + + until grep "Finished processing block 32" ./server-log + do sleep 1; done + + count_results() { + query="select count(*) from $1;" + docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \ + psql -tA cerc_testing -U vdbm -c "$query" + } + set -x + [[ "$(count_results eth.header_cids)" = 33 ]] + [[ "$(count_results eth.state_cids)" = 21 ]] + [[ "$(count_results eth.storage_cids)" = 18 ]] diff --git a/.github/workflows/manual_publish.yml b/.github/workflows/manual_publish.yml deleted file mode 100644 index f884a29..0000000 --- a/.github/workflows/manual_publish.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: MANUAL Override Publish from release SHA to TAG -on: - workflow_dispatch: - inputs: - giteaPublishTag: - description: 'Release TAG to publish TO on gitea; e.g. v4.1.5-alpha' - required: true - cercContainerTag: - description: 'Container (truncated!!! SHA) to release-tag FROM' - required: true - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - name: Pull SHA and add release-tag - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get the version - id: vars - run: | - echo ::set-output name=sha::$(echo ${cercContainerTag:0:7}) - - name: Pull docker image by SHA - run: docker pull git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} - - name: Tag docker image TAG - run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}} - - name: Tag docker image TAG - run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest - - name: Docker Login - run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin - - name: Docker Push Release Tag - run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}} - - name: Docker Push LATEST Tag - run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest diff --git a/.github/workflows/on-publish-pr.yml b/.github/workflows/on-publish-pr.yml deleted file mode 100644 index 952be1d..0000000 --- a/.github/workflows/on-publish-pr.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Publish Docker image -on: - release: - types: [published] - pull_request: -jobs: - pre_job: - # continue-on-error: true # Uncomment once integration is finished - runs-on: ubuntu-latest - # Map a step output to a job output - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@v4 - with: - # All of these options are optional, so you can remove them if you are happy with the defaults - concurrent_skipping: "never" - skip_after_successful_duplicate: "true" - do_not_skip: '["workflow_dispatch", "schedule"]' - run-tests: - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} - needs: pre_job - uses: ./.github/workflows/tests.yml - build: - name: Run docker build - runs-on: ubuntu-latest - if: | - always() && - (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') && - github.event_name == 'release' - needs: run-tests - steps: - - uses: actions/checkout@v2 - - name: Get the version - id: vars - run: | - echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) - echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/}) - - name: Run docker build - run: make docker-build - - name: Tag docker image - run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} - - name: Tag docker image TAG - run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}} - - name: Docker Login - run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin - - name: Docker Push - run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} -# push_to_registries: -# name: Push Docker image to Docker Hub -# runs-on: ubuntu-latest -# if: | -# always() && -# (needs.build.result == 'success') && -# github.event_name == 'release' -# needs: build -# steps: -# - name: Get the version -# id: vars -# run: | -# echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) -# echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/}) -# - name: Docker Login to Github Registry -# run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin -# - name: Docker Pull -# run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} -# - name: Docker Login to Docker Registry -# run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin -# - name: Tag docker image -# run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}} -# - name: Docker Push to Docker Hub -# run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 17ed729..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Tests for Geth that are used in multiple jobs. - -on: - workflow_call: - -env: - GOPATH: /tmp/go - -jobs: - build: - name: Run docker build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Run docker build - run: make docker-build - - statediff-unit-test: - name: Run statediff unit tests - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - uses: actions/setup-go@v3 - with: - go-version: ">=1.18.0" - check-latest: true - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Run unit tests - run: | - make test diff --git a/Dockerfile b/Dockerfile index 65a9172..1f98491 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,24 @@ FROM golang:1.19-alpine as builder -RUN apk --update --no-cache add make git g++ linux-headers +RUN apk add --no-cache git gcc musl-dev binutils-gold # DEBUG RUN apk add busybox-extras -# Get and build ipfs-blockchain-watcher -ADD . /go/src/github.com/cerc-io/eth-statediff-service -#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service +WORKDIR /eth-statediff-service -WORKDIR /go/src/github.com/cerc-io/eth-statediff-service -RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service . +ARG GIT_VDBTO_TOKEN + +COPY go.mod go.sum ./ +RUN if [ -n "$GIT_VDBTO_TOKEN" ]; then git config --global url."https://$GIT_VDBTO_TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"; fi && \ + go mod download && \ + rm -f ~/.gitconfig +COPY . . + +RUN go build -ldflags '-extldflags "-static"' -o eth-statediff-service . -# app container FROM alpine -ARG USER="vdm" -ARG CONFIG_FILE="./environments/config.toml" +ARG USER="vdbm" ARG EXPOSE_PORT=8545 RUN adduser -Du 5000 $USER adm @@ -27,12 +30,11 @@ USER $USER # chown first so dir is writable # note: using $USER is merged, but not in the stable release yet -COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml -COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh . -COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments +COPY --chown=5000:5000 --from=builder /eth-statediff-service/startup_script.sh . +COPY --chown=5000:5000 --from=builder /eth-statediff-service/environments environments # keep binaries immutable -COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service +COPY --from=builder /eth-statediff-service/eth-statediff-service eth-statediff-service EXPOSE $EXPOSE_PORT diff --git a/Makefile b/Makefile deleted file mode 100644 index 48bd006..0000000 --- a/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -## Build docker image -.PHONY: docker-build -docker-build: - docker build -t cerc-io/eth-statediff-service . - -.PHONY: test -test: - go test -p 1 ./pkg/... -v - -build: - go fmt ./... - go build diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 74f6334..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: '3.2' - -services: - eth-statediff-service: - build: - context: ./ - cache_from: - - alpine:latest - - golang:1.16 - dockerfile: ./Dockerfile - args: - USER: "vdbm" - CONFIG_FILE: ./environments/example.toml - EXPOSE_PORT: 8545 - environment: - - VDB_COMMAND=serve - volumes: - - eth-statediff-service-data:/root/.ethereum/ - ports: - - "127.0.0.1:8545:8545" - -volumes: - eth-statediff-service-data: diff --git a/fixture/.gitignore b/fixture/.gitignore new file mode 100644 index 0000000..c95f9f9 --- /dev/null +++ b/fixture/.gitignore @@ -0,0 +1,6 @@ +*/*.log +*/CURRENT* +*/LOCK +*/LOG +*/MANIFEST-* +*/ancient/FLOCK diff --git a/fixture/chaindata/000002.ldb b/fixture/chaindata/000002.ldb new file mode 100644 index 0000000..c7bb1f9 Binary files /dev/null and b/fixture/chaindata/000002.ldb differ diff --git a/fixture/chaindata/000004.ldb b/fixture/chaindata/000004.ldb new file mode 100644 index 0000000..bdea3f6 Binary files /dev/null and b/fixture/chaindata/000004.ldb differ diff --git a/fixture/chaindata/ancient/bodies.0000.cdat b/fixture/chaindata/ancient/bodies.0000.cdat new file mode 100644 index 0000000..e69de29 diff --git a/fixture/chaindata/ancient/bodies.cidx b/fixture/chaindata/ancient/bodies.cidx new file mode 100644 index 0000000..ab2c684 Binary files /dev/null and b/fixture/chaindata/ancient/bodies.cidx differ diff --git a/fixture/chaindata/ancient/bodies.meta b/fixture/chaindata/ancient/bodies.meta new file mode 100644 index 0000000..59ee616 --- /dev/null +++ b/fixture/chaindata/ancient/bodies.meta @@ -0,0 +1 @@ +€ \ No newline at end of file diff --git a/fixture/chaindata/ancient/diffs.0000.rdat b/fixture/chaindata/ancient/diffs.0000.rdat new file mode 100644 index 0000000..e69de29 diff --git a/fixture/chaindata/ancient/diffs.meta b/fixture/chaindata/ancient/diffs.meta new file mode 100644 index 0000000..59ee616 --- /dev/null +++ b/fixture/chaindata/ancient/diffs.meta @@ -0,0 +1 @@ +€ \ No newline at end of file diff --git a/fixture/chaindata/ancient/diffs.ridx b/fixture/chaindata/ancient/diffs.ridx new file mode 100644 index 0000000..ab2c684 Binary files /dev/null and b/fixture/chaindata/ancient/diffs.ridx differ diff --git a/fixture/chaindata/ancient/hashes.0000.rdat b/fixture/chaindata/ancient/hashes.0000.rdat new file mode 100644 index 0000000..e69de29 diff --git a/fixture/chaindata/ancient/hashes.meta b/fixture/chaindata/ancient/hashes.meta new file mode 100644 index 0000000..59ee616 --- /dev/null +++ b/fixture/chaindata/ancient/hashes.meta @@ -0,0 +1 @@ +€ \ No newline at end of file diff --git a/fixture/chaindata/ancient/hashes.ridx b/fixture/chaindata/ancient/hashes.ridx new file mode 100644 index 0000000..ab2c684 Binary files /dev/null and b/fixture/chaindata/ancient/hashes.ridx differ diff --git a/fixture/chaindata/ancient/headers.0000.cdat b/fixture/chaindata/ancient/headers.0000.cdat new file mode 100644 index 0000000..e69de29 diff --git a/fixture/chaindata/ancient/headers.cidx b/fixture/chaindata/ancient/headers.cidx new file mode 100644 index 0000000..ab2c684 Binary files /dev/null and b/fixture/chaindata/ancient/headers.cidx differ diff --git a/fixture/chaindata/ancient/headers.meta b/fixture/chaindata/ancient/headers.meta new file mode 100644 index 0000000..59ee616 --- /dev/null +++ b/fixture/chaindata/ancient/headers.meta @@ -0,0 +1 @@ +€ \ No newline at end of file diff --git a/fixture/chaindata/ancient/receipts.0000.cdat b/fixture/chaindata/ancient/receipts.0000.cdat new file mode 100644 index 0000000..e69de29 diff --git a/fixture/chaindata/ancient/receipts.cidx b/fixture/chaindata/ancient/receipts.cidx new file mode 100644 index 0000000..ab2c684 Binary files /dev/null and b/fixture/chaindata/ancient/receipts.cidx differ diff --git a/fixture/chaindata/ancient/receipts.meta b/fixture/chaindata/ancient/receipts.meta new file mode 100644 index 0000000..59ee616 --- /dev/null +++ b/fixture/chaindata/ancient/receipts.meta @@ -0,0 +1 @@ +€ \ No newline at end of file diff --git a/scripts/request-range.sh b/scripts/request-range.sh new file mode 100755 index 0000000..6924fe5 --- /dev/null +++ b/scripts/request-range.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -eu + +FROM=$1 +TO=$2 +URL=127.0.0.1:8545 + +DATA='{ + "jsonrpc": "2.0", + "method": "statediff_writeStateDiffsInRange", + "params": ['"$FROM"', '"$TO"', { + "includeBlock": true, + "includeReceipts": true, + "includeTD": true, + "includeCode": true + } + ], + "id": 1 +}' + +exec curl -s $URL -X POST -H 'Content-Type: application/json' --data "$DATA" diff --git a/startup_script.sh b/startup_script.sh index cfd63cc..3c5c472 100755 --- a/startup_script.sh +++ b/startup_script.sh @@ -17,4 +17,4 @@ mkdir -p /app/geth-rw && \ sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \ echo "Running the statediff service" && \ -sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml +exec sudo ./eth-statediff-service "$VDB_COMMAND" --config=/config/config.toml diff --git a/test/ci-chain.json b/test/ci-chain.json new file mode 100644 index 0000000..d2ee2f5 --- /dev/null +++ b/test/ci-chain.json @@ -0,0 +1,16 @@ +{ + "chainId": 41337, + "homesteadBlock": 0, + "eip150Block": 0, + "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "clique": { + "period": 5, + "epoch": 30000 + } +} diff --git a/test/ci-config.toml b/test/ci-config.toml new file mode 100644 index 0000000..1fb40f0 --- /dev/null +++ b/test/ci-config.toml @@ -0,0 +1,36 @@ +[leveldb] + mode = "local" + url = "http://127.0.0.1:8082/" + +[server] + ipcPath = ".ipc" + httpPath = "0.0.0.0:8545" + +[statediff] + serviceWorkers = 1 + workerQueueSize = 1024 + trieWorkers = 4 + +[log] + level = "debug" + +[database] + name = "cerc_testing" + hostname = "localhost" + port = 8077 + user = "vdbm" + password = "password" + type = "postgres" + driver = "sqlx" + +[cache] + database = 1024 + trie = 1024 + +[ethereum] + chainConfig = "test/ci-chain.json" + nodeID = "" + clientName = "eth-statediff-service" + genesisBlock = "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05" + networkID = 1 + chainID = 41337 diff --git a/test/compose.yml b/test/compose.yml new file mode 100644 index 0000000..1715384 --- /dev/null +++ b/test/compose.yml @@ -0,0 +1,23 @@ +services: + migrations: + restart: on-failure + depends_on: + - ipld-eth-db + image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha + environment: + DATABASE_USER: "vdbm" + DATABASE_NAME: "cerc_testing" + DATABASE_PASSWORD: "password" + DATABASE_HOSTNAME: "ipld-eth-db" + DATABASE_PORT: 5432 + + ipld-eth-db: + image: timescale/timescaledb:latest-pg14 + restart: always + command: ["postgres", "-c", "log_statement=all"] + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "cerc_testing" + POSTGRES_PASSWORD: "password" + ports: + - 127.0.0.1:8077:5432