update int tests

.github/workflows/test.yml

@@ -2,6 +2,11 @@ name: Unit and integration tests
 on:
   workflow_call:
 
+env:
+  # Needed until we can incorporate docker startup into the executor container
+  DOCKER_HOST: unix:///var/run/dind.sock
+  SO_VERSION: v1.1.0-c30c779-202309082138
+
 jobs:
   unit-tests:
     name: "Run unit tests"
@@ -12,9 +17,18 @@ jobs:
         with:
           go-version-file: 'go.mod'
           check-latest: true
+      - name: "Run dockerd"
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
       - name: "Run DB container"
         run: |
           docker compose -f test/compose-db.yml up --wait --quiet-pull
+      - name: "Set Gitea credentials"
+        env:
+          TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
+        run: |
+          git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"
       - name: "Run tests"
         run: |
           go test -v ./pkg/...
@@ -28,21 +42,30 @@ jobs:
         with:
           go-version-file: 'go.mod'
           check-latest: true
+      - name: "Run dockerd"
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
       - name: "Install stack-orchestrator"
-        uses: actions/checkout@v3
+        run: |
-        with:
+          curl -L -O https://github.com/cerc-io/stack-orchestrator/releases/download/$SO_VERSION/laconic-so
-          repository: cerc-io/stack-orchestrator
+          chmod +x laconic-so
-          ref: v1.1.0-ff616db-202305310811 # first tag with merged changes for this stack
-          path: ./stack-orchestrator
       - run: pip install ./stack-orchestrator
+      - name: "Set Gitea credentials"
+        env:
+          TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
+        run: |
+          git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"
+
       - name: "Run testnet stack"
+        env:
+          CERC_GO_AUTH_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
         run: ./scripts/integration-setup.sh
       - name: "Run contract deployer"
         run: |
           docker compose -f test/compose-deployer.yml up --wait --quiet-pull
-      - name: "Build package and wait for testnet"
+      - name: "Wait for testnet"
         run: |
-          go build ./... &
           # Start validator at current head, but not before Merge (block 1 on test chain)
           echo "Waiting for chain head to progress..."
           while
@@ -51,7 +74,6 @@ jobs:
           do sleep 5; done
           echo "Chain has reached block $height"
           echo VALIDATE_FROM_BLOCK=$height >> "$GITHUB_ENV"
-          wait $!
       - name: "Run tests"
         run: |
-          go test -v ./integration/...
+          go test ./integration/... -v -timeout=20m

go.mod

@@ -287,7 +287,7 @@ replace (
 	github.com/cerc-io/eth-ipfs-state-validator/v5 => git.vdb.to/cerc-io/eth-ipfs-state-validator/v5 v5.0.0-20230910103736-6fcae158de32
 	github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230910102631-c8330d8c7fb3
 	github.com/cerc-io/eth-testing => git.vdb.to/cerc-io/eth-testing v0.1.2-0.20230909172706-0d908a1e61f6
-	github.com/cerc-io/ipld-eth-server/v5 => git.vdb.to/cerc-io/ipld-eth-server 49fbc43b1518c552debe58f4739a6ba22fb16dd7
+	github.com/cerc-io/ipld-eth-server/v5 => git.vdb.to/cerc-io/ipld-eth-server/v5 v5.0.0-20230911133532-49fbc43b1518
 	github.com/cerc-io/ipld-eth-statedb => git.vdb.to/cerc-io/ipld-eth-statedb v0.0.5-alpha.0.20230910103157-6cd44545ba8a
 	github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.1-0.20230910102119-e43cbb30f919
 	github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1

go.sum

@@ -50,6 +50,8 @@ git.vdb.to/cerc-io/eth-ipfs-state-validator/v5 v5.0.0-20230910103736-6fcae158de3
 git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230910102631-c8330d8c7fb3 h1:EW5Zw4vKRUJb/N095DuGSzNPGsv889muFW4RZj03HOw=
 git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230910102631-c8330d8c7fb3/go.mod h1:LLNVVr1r5kirXXY6bCjWAT1mrqJ3TV51J8pdyR07Auk=
 git.vdb.to/cerc-io/eth-testing v0.1.2-0.20230909172706-0d908a1e61f6 h1:kAvdKNfJmzIHk8/cEGuXnKCU4Xfy42nr2a2zg3IVthE=
+git.vdb.to/cerc-io/ipld-eth-server/v5 v5.0.0-20230911133532-49fbc43b1518 h1:E1DxCwQYihs7MxZHYYQtkN9RxZ/p6f8hVaO1KGYRDik=
+git.vdb.to/cerc-io/ipld-eth-server/v5 v5.0.0-20230911133532-49fbc43b1518/go.mod h1:NShKIHmnO3MgJ3/oQAv3vUpdbpIRYQsvXhXaqYRmGYc=
 git.vdb.to/cerc-io/ipld-eth-statedb v0.0.5-alpha.0.20230910103157-6cd44545ba8a h1:u/VBGhKoqDui2NFJ/8uATr5iDqC7JLSNO3SYUiT3jTo=
 git.vdb.to/cerc-io/ipld-eth-statedb v0.0.5-alpha.0.20230910103157-6cd44545ba8a/go.mod h1:uwWvDtI9nUbXtyrUPqbseizF/9ZoLzhbnbecOHMyHow=
 git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1 h1:KLjxHwp9Zp7xhECccmJS00RiL+VwTuUGLU7qeIctg8g=

integration/integration_test.go

@@ -16,7 +16,7 @@ import (
 )
 
 const (
-	timeout            = 10 * time.Minute
+	timeout            = 20 * time.Minute
 	pollInterval       = time.Second
 	progressBufferSize = 200
 )
@@ -56,7 +56,7 @@ func setup(t *testing.T, progressChan chan uint64) {
 		cfg.FromBlock = lastValidated
 	}
 	// default trail is unnecessarily long
-	cfg.Trail = 16
+	cfg.Trail = 8
 
 	service, err := validator.NewService(cfg, progressChan)
 	if err != nil {
@@ -103,7 +103,7 @@ func TestValidateContracts(t *testing.T) {
 		g := gomega.NewWithT(t)
 
 		var blocks []uint64
-		for i := 0; i < 5; i++ {
+		for i := 0; i < 3; i++ {
 			res, err := integration.PutTestValue(contract.Address, i)
 			if err != nil {
 				t.Fatal(err)

scripts/integration-setup.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
+# Build and deploy a cluster with only what we need from the stack
 
-set -ex
+set -e
 
 CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
 
@@ -9,20 +10,21 @@ export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-..}"
 # v5 migrations only go up to version 18
 echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 >> $CONFIG_DIR/stack.env
 
-laconic_so="${LACONIC_SO:-laconic-so} --stack fixturenet-eth-loaded --quiet"
+laconic_so="${LACONIC_SO:-laconic-so} --stack fixturenet-plugeth-tx --quiet"
 
 set -x
 
-# Build and deploy a cluster with only what we need from the stack
+if [[ -z $SKIP_BUILD ]]; then
-$laconic_so setup-repositories \
+    $laconic_so setup-repositories \
-    --exclude github.com/cerc-io/ipld-eth-server,github.com/cerc-io/tx-spammer \
+        --exclude github.com/cerc-io/ipld-eth-server,github.com/cerc-io/tx-spammer \
-    --branches-file ./test/stack-refs.txt
+        --branches-file ./test/stack-refs.txt
 
-$laconic_so build-containers \
+    $laconic_so build-containers \
-    --exclude cerc/ipld-eth-server,cerc/keycloak,cerc/tx-spammer
+        --exclude cerc/ipld-eth-server,cerc/keycloak,cerc/tx-spammer
+fi
 
 $laconic_so deploy \
-    --include fixturenet-eth,ipld-eth-db \
+    --include fixturenet-plugeth,ipld-eth-db \
     --env-file $CONFIG_DIR/stack.env \
     --cluster test up
 
@@ -47,6 +49,6 @@ export PGPASSWORD=password
 query_blocks_exist='SELECT exists(SELECT block_number FROM ipld.blocks LIMIT 1);'
 
 echo "Waiting until we have some data written..."
-until [[ "$(psql -qtA cerc_testing -h localhost -U vdbm -p 8077 -c "$query_blocks_exist")" = 't' ]]; do
+until [[ "$(psql -qtA cerc_testing -h localhost -U vdbm -p 8077 -c "$query_blocks_exist")" -eq 't' ]]; do
     sleep 2
 done

test/compose-deployer.yml

@@ -3,7 +3,7 @@
 services:
   contract-deployer:
     restart: on-failure
-    image: cerc/ipld-eth-db-validator/contract-deployer
+    image: cerc/ipld-eth-db-validator/contract-deployer:local
     build: ./contract
     networks:
       - test_default