[wip] test updates

.gitea/workflows/test.yml

@@ -8,10 +8,8 @@ on:
       - main
       - ci-test
 
-# Needed until we can incorporate docker startup into the executor container
 env:
-  DOCKER_HOST: unix:///var/run/dind.sock
+  SO_VERSION: roy/fnet-eth-deneb-upgrade # TODO update after PR merged
-  SO_VERSION: v1.1.0-c30c779-202309082138
 
 jobs:
   unit-tests:
@@ -23,18 +21,9 @@ jobs:
         with:
           go-version-file: 'go.mod'
           check-latest: true
-      - name: Run dockerd
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: Run DB container
         run: |
           docker compose -f test/compose.yml up --wait
-      - name: Set up Gitea access token
-        env:
-          TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
-        run: |
-          git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf https://git.vdb.to/
       - name: Run tests
         run: go test -p 1 -v ./...
 
@@ -48,41 +37,41 @@ jobs:
       - uses: actions/checkout@v3
         with:
           repository: cerc-io/plugeth
-          ref: statediff
+          ref: aa7607d4ad25988cd055381b9bd590dd397b3329 # TODO update after PR
           path: ./plugeth
-      - name: Run dockerd
-        run: dockerd -H $DOCKER_HOST --userland-proxy=false &
-        # These images need access tokens configured
       - name: Build docker image
-        env:
-          TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
         run: |
-          [[ -n "$TOKEN" ]]
+          docker build ./plugeth-statediff -t cerc/plugeth-statediff:local
-          docker build ./plugeth-statediff -t cerc/plugeth-statediff:local \
+          docker build ./plugeth -t cerc/plugeth:local
-            --build-arg GIT_VDBTO_TOKEN="$TOKEN"
-          docker build ./plugeth -t cerc/plugeth:local \
-            --build-arg GIT_VDBTO_TOKEN="$TOKEN"
 
-      - name: Install stack-orchestrator
+      - name: "Checkout stack-orchestrator"
+        uses: actions/checkout@v3
+        with:
+          repository: cerc-io/stack-orchestrator
+          ref: ${{ env.SO_VERSION }}
+          path: ./stack-orchestrator
+      - name: "Install stack-orchestrator"
         run: |
-          curl -L -O https://github.com/cerc-io/stack-orchestrator/releases/download/$SO_VERSION/laconic-so
+          apt-get update && apt-get install -y python3-pip
-          chmod +x laconic-so
+          pip3 install -e ./stack-orchestrator
+      # - name: Install stack-orchestrator
+      #   run: |
+      #     curl -L -O https://github.com/cerc-io/stack-orchestrator/releases/download/$SO_VERSION/laconic-so
+      #     chmod +x laconic-so
       - name: Clone system-tests
         uses: actions/checkout@v3
         with:
           repository: cerc-io/system-tests
-          ref: plugeth-compat
+          ref: roy/bump-web3    # TODO update
           path: ./system-tests
           token: ${{ secrets.CICD_REPO_TOKEN }}
       - name: Run testnet stack
         working-directory: ./plugeth-statediff
         env:
-          LACONIC_SO: ../laconic-so
+          LACONIC_SO: laconic-so
         run: ./scripts/integration-setup.sh
       - name: Install Python
         uses: actions/setup-python@v4
-        with:
-          python-version: 3.10
       - name: Run tests
         working-directory: ./system-tests
         run: |
@@ -100,7 +89,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           repository: cerc-io/eth-statediff-compliance
-          ref: v0.1.0
+          ref: update-geth      # TODO update
           path: ./eth-statediff-compliance
           token: ${{ secrets.CICD_REPO_TOKEN }}
       - uses: actions/setup-go@v4
@@ -109,24 +98,17 @@ jobs:
           check-latest: true
       - name: Install jq
         run: apt-get update && apt-get install -yq jq
-      - name: Set up Gitea access token
-        env:
-          TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
-        run: |
-          git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf https://git.vdb.to/
 
-      - name: Update go.mod for dumpdiff-geth
+      - name: Update go.mod in nested modules
         working-directory: ./eth-statediff-compliance/
-        run: ./scripts/update-mod.sh ../plugeth-statediff dumpdiff-geth/
+        run: |
-      - name: Update go.mod for dumpdiff-plugeth
+          set -x
-        working-directory: ./eth-statediff-compliance/
+          ./scripts/update-mod.sh ../plugeth-statediff dumpdiff-plugeth/
-        run: ./scripts/update-mod.sh ../plugeth-statediff dumpdiff-plugeth/
+          ./scripts/update-mod.sh ../plugeth-statediff dumpdiff-plugeth-parallel/
-      - name: Update go.mod for dumpdiff-plugeth-parallel
-        working-directory: ./eth-statediff-compliance/
-        run: ./scripts/update-mod.sh ../plugeth-statediff dumpdiff-plugeth-parallel/
       - name: Build tools
         working-directory: ./eth-statediff-compliance/
         run: make all
+
       - name: Compare output of geth and plugeth
         working-directory: ./eth-statediff-compliance/
         run: ./scripts/compare-diffs.sh geth plugeth

go.mod

@@ -3,7 +3,7 @@ module github.com/cerc-io/plugeth-statediff
 go 1.21
 
 require (
-	github.com/cerc-io/eth-iterator-utils v0.1.2
+	github.com/cerc-io/eth-iterator-utils v0.2.0
 	github.com/cerc-io/eth-testing v0.4.0
 	github.com/ethereum/go-ethereum v1.13.14
 	github.com/georgysavva/scany v0.2.9
@@ -140,8 +140,8 @@ require (
 )
 
 replace (
-	github.com/cerc-io/eth-iterator-utils => github.com/cerc-io/eth-iterator-utils v0.1.3-0.20240319064619-152c5f646824
+	// TODO
-	github.com/cerc-io/eth-testing => github.com/cerc-io/eth-testing v0.3.2-0.20240315081427-e027e285e4f5
+	// github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v1.13.14.0.0-cerc-0
-	github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20240328095627-4556585a56ee
+	github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20240402020250-c50b6cc02620
-	github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20240328092714-2bd23e1f3c1b
+	github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0
 )

go.sum

@@ -1,8 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-git.vdb.to/cerc-io/plugeth v0.0.0-20240328095627-4556585a56ee h1:fDkuDAweXHt+8Dn3C5bVd//nlPExsla84ZMjARxishA=
+git.vdb.to/cerc-io/plugeth v0.0.0-20240402020250-c50b6cc02620 h1:WRYHy9TG2QoUqbk4522BWLP7/nl87WzwdLGM+DAeiyU=
-git.vdb.to/cerc-io/plugeth v0.0.0-20240328095627-4556585a56ee/go.mod h1:Py4WlRjIiBzAKfvpflglzh/XshTAV0oXSOV8mkft10I=
+git.vdb.to/cerc-io/plugeth v0.0.0-20240402020250-c50b6cc02620/go.mod h1:1AMBRYYMPWAWPCK3ui469ymYlEsciWBrRJWjPX5nxy8=
-git.vdb.to/cerc-io/plugeth-utils v0.0.0-20240328092714-2bd23e1f3c1b h1:b8B9IyC2aLPfCAznK+19uwI6i9ZX3VvhojuzAnUkWdc=
+git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0 h1:4GwCBbdLB8mCZINDzoUqpPq7aP4Ha5PPYCyG2h6ee6s=
-git.vdb.to/cerc-io/plugeth-utils v0.0.0-20240328092714-2bd23e1f3c1b/go.mod h1:COwKAuTZIsCouCOrIDBhvHZqpbOO1Ojgdy5KTvL8mJg=
+git.vdb.to/cerc-io/plugeth-utils v1.5.0-cerc-0/go.mod h1:COwKAuTZIsCouCOrIDBhvHZqpbOO1Ojgdy5KTvL8mJg=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
@@ -35,10 +35,10 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cerc-io/eth-iterator-utils v0.1.3-0.20240319064619-152c5f646824 h1:YGcOMd3g6XRwokIN44VLlb51Y0lNAoh/HLTGmBESgEQ=
+github.com/cerc-io/eth-iterator-utils v0.2.0 h1:wikAfWZ0fAqLqUy/Ud/a1n9p/arVeKG8P8tRjZE7oBg=
-github.com/cerc-io/eth-iterator-utils v0.1.3-0.20240319064619-152c5f646824/go.mod h1:O4QE+mgtYjCtzqcTXo/dC7HURb++PZTdrq6MtPdCtoE=
+github.com/cerc-io/eth-iterator-utils v0.2.0/go.mod h1:wDUJvwKDSOdqTIyeG+yXJ2ckzc9f2Fem614fV61DBcg=
-github.com/cerc-io/eth-testing v0.3.2-0.20240315081427-e027e285e4f5 h1:hS6ASI3zxdoSTSzdLaPVbn0zOTDcHuA69BwYnjzCAKI=
+github.com/cerc-io/eth-testing v0.4.0 h1:ivGbXnEqlXMt/3m3jbsPJaVT7ZDTenFQWCryt1Rd/Jk=
-github.com/cerc-io/eth-testing v0.3.2-0.20240315081427-e027e285e4f5/go.mod h1:CVsmHjFldX9gwaQSQwGmKbmh0g6Dq+bsqB2CxBf9zbk=
+github.com/cerc-io/eth-testing v0.4.0/go.mod h1:CVsmHjFldX9gwaQSQwGmKbmh0g6Dq+bsqB2CxBf9zbk=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=

scripts/integration-setup.sh

@@ -4,8 +4,7 @@
 
 set -e
 
-cluster="${1:-test}"
+laconic_so="${LACONIC_SO:-laconic-so} --stack $(readlink -f test) --verbose --debug"
-laconic_so="${LACONIC_SO:-laconic-so} --stack fixturenet-plugeth-tx --verbose"
 
 CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
 
@@ -14,19 +13,19 @@ export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel
 
 # v5 migrations only go up to version 18
 echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 >> $CONFIG_DIR/stack.env
+# don't run plugeth in the debugger
+echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
 
 set -x
 
 if [[ -z $SKIP_BUILD ]]; then
-    $laconic_so setup-repositories \
+  $laconic_so setup-repositories \
-        --exclude github.com/dboreham/foundry,github.com/cerc-io/tx-spammer,github.com/cerc-io/ipld-eth-server,git.vdb.to/cerc-io/plugeth,git.vdb.to/cerc-io/plugeth-statediff \
+    --exclude git.vdb.to/cerc-io/plugeth,git.vdb.to/cerc-io/plugeth-statediff
-        --branches-file ./test/stack-refs.txt
 
-    $laconic_so build-containers \
+  $laconic_so build-containers \
-        --exclude cerc/ipld-eth-server,cerc/keycloak,cerc/tx-spammer,cerc/foundry,cerc/plugeth,cerc/plugeth-statediff
+    --exclude cerc/plugeth,cerc/plugeth-statediff
 fi
 
 $laconic_so deploy \
-    --exclude foundry,keycloak,tx-spammer,ipld-eth-server \
     --env-file $CONFIG_DIR/stack.env \
-    --cluster "$cluster" up
+    --cluster test up

test/stack-refs.txt

@@ -1 +1 @@
-github.com/cerc-io/ipld-eth-db v5.0.5-alpha
+github.com/cerc-io/ipld-eth-db v5.2.1-alpha

test/stack.yml

@@ -0,0 +1,20 @@
+version: "1.2"
+name: fixturenet-plugeth-tx
+description: "plugeth Ethereum Fixturenet"
+repos:
+  - git.vdb.to/cerc-io/plugeth@statediff
+  - git.vdb.to/cerc-io/plugeth-statediff
+  - git.vdb.to/cerc-io/lighthouse
+  - git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha
+containers:
+  - cerc/plugeth-statediff
+  - cerc/plugeth
+  - cerc/fixturenet-eth-genesis
+  - cerc/fixturenet-plugeth-plugeth
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-lighthouse
+  - cerc/ipld-eth-db
+pods:
+  - fixturenet-plugeth
+  - ipld-eth-db