Merge pull request #19 from vulcanize/geth_1_10_21_wip

geth 1.10.21 update
go-quic not 1.19 ready for both checks
2022-08-03 14:51:29 -04:00 · 2022-08-03 14:42:06 -04:00 · 2022-08-03 14:36:47 -04:00 · 2022-08-03 11:37:55 -04:00 · 2022-08-03 11:11:22 -04:00 · 2022-07-20 17:18:07 -04:00
18 changed files with 1353 additions and 617 deletions
@@ -1,29 +0,0 @@
-name: Notion Sync
-
-on:
-  workflow_dispatch:
-  issues:
-    types:
-      [
-        opened,
-        edited,
-        labeled,
-        unlabeled,
-        assigned,
-        unassigned,
-        milestoned,
-        demilestoned,
-        reopened,
-        closed,
-      ]
-
-jobs:
-  notion_job:
-    runs-on: ubuntu-latest
-    name: Add GitHub Issues to Notion
-    steps:
-      - name: Add GitHub Issues to Notion
-        uses: vulcanize/notion-github-action@v1.2.4-issueid
-        with:
-          notion-token: ${{ secrets.NOTION_TOKEN }}
-          notion-db: ${{ secrets.NOTION_DATABASE }}
@@ -3,8 +3,9 @@ on:
  workflow_call:

 env:
-  STACK_ORCHESTRATOR_REF: "main"
-  GO_ETHEREUM_REF: "v1.10.18-statediff-v3"
+  STACK_ORCHESTRATOR_REF: "f2fd766f5400fcb9eb47b50675d2e3b1f2753702"
+  GO_ETHEREUM_REF: "v1.10.19-statediff-4.1.0-alpha" # Use the tag, we are going to download the bin not build it.
+  IPLD_ETH_DB_REF: "b59505eab252670c622b42ce60621e9747fb64f9"

 jobs:
  integrationtest:
@@ -16,43 +17,50 @@ jobs:
    steps:
      - name: Create GOPATH
        run: mkdir -p /tmp/go
+
      - uses: actions/setup-go@v3
        with:
-          go-version: ">=1.18.0"
+          go-version: "1.18.0"
          check-latest: true
+
      - uses: actions/checkout@v2
        with:
          path: "./ipld-eth-db-validator"
+
      - uses: actions/checkout@v2
        with:
          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
          path: "./stack-orchestrator/"
          repository: vulcanize/stack-orchestrator
+
      - uses: actions/checkout@v2
        with:
-          ref: ${{ env.GO_ETHEREUM_REF }}
-          repository: vulcanize/go-ethereum
-          path: "./go-ethereum/"
+          ref: ${{ env.IPLD_ETH_DB_REF }}
+          repository: vulcanize/ipld-eth-db
+          path: "./ipld-eth-db/"
+
      - name: Create config file
        run: |
-          echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ > ./config.sh
          echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh
+          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
+          echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> ./config.sh
          echo db_write=$DB_WRITE >> ./config.sh
          cat ./config.sh
-      - name: Build geth
+
+      - name: Download Geth
        run: |
          cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
-          ./compile-geth.sh \
-          -p "$GITHUB_WORKSPACE/config.sh" \
-          -e docker
+          wget https://github.com/vulcanize/go-ethereum/releases/download/${{env.GO_ETHEREUM_REF}}/geth-linux-amd64
+
      - name: Run docker compose
        run: |
          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
          --env-file "$GITHUB_WORKSPACE/config.sh" \
          up -d --build
+
      - name: Run integration test.
        run: |
          cd $GITHUB_WORKSPACE/ipld-eth-db-validator
@@ -60,31 +68,24 @@ jobs:

  unittest:
    name: Run unit tests
-    env:
-      GOPATH: /tmp/go
-      DB_WRITE: true
    runs-on: ubuntu-latest
    steps:
      - name: Create GOPATH
        run: mkdir -p /tmp/go
+
      - uses: actions/setup-go@v3
        with:
-          go-version: ">=1.18.0"
+          go-version: "1.18.0"
          check-latest: true
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-eth-db-validator"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
-          path: "./stack-orchestrator/"
-          repository: vulcanize/stack-orchestrator
-      - name: Run docker compose
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Spin up database
        run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-db.yml" \
-          up -d --build
-      - name: Run unit test.
+          docker-compose up -d
+
+      - name: Run unit tests
        run: |
-          cd $GITHUB_WORKSPACE/ipld-eth-db-validator
+          sleep 30
          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test
@@ -0,0 +1,3 @@
+ipld-eth-db-validator
+.vscode
+.idea
@@ -1,18 +1,30 @@
 version: '3.2'

 services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: vulcanize/ipld-eth-db:v4.2.0-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
  ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
    restart: always
-    image: vulcanize/ipld-eth-db:v3.2.0
+    command: ["postgres", "-c", "log_statement=all"]
    environment:
      POSTGRES_USER: "vdbm"
      POSTGRES_DB: "vulcanize_testing"
      POSTGRES_PASSWORD: "password"
-    volumes:
-      - vdb_db_eth_validator:/var/lib/postgresql/data
    ports:
      - "127.0.0.1:8077:5432"
-    command: ["postgres", "-c", "log_statement=all"]
+    volumes:
+      - vdb_db_eth_validator:/var/lib/postgresql/data

 volumes:
-  vdb_db_eth_validator:
+  vdb_db_eth_validator:
@@ -3,57 +3,67 @@ module github.com/vulcanize/ipld-eth-db-validator
 go 1.18

 require (
-	github.com/ethereum/go-ethereum v1.10.18
+	github.com/ethereum/go-ethereum v1.10.21
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.11.0
-	github.com/vulcanize/ipfs-ethdb/v3 v3.0.3
-	github.com/vulcanize/ipld-eth-server/v3 v3.2.1
+	github.com/vulcanize/ipfs-ethdb/v4 v4.0.5-alpha
+	github.com/vulcanize/ipld-eth-server/v4 v4.1.4-alpha
 )

 require (
 	bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
-	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
 	github.com/Stebalien/go-bitfield v0.0.1 // indirect
 	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
-	github.com/benbjohnson/clock v1.1.0 // indirect
+	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/btcsuite/btcd v0.22.1 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
-	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/containerd/cgroups v1.0.3 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
 	github.com/cskr/pubsub v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
 	github.com/georgysavva/scany v0.2.9 // indirect
-	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-bexpr v0.1.10 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
@@ -63,115 +73,100 @@ require (
 	github.com/huin/goupnp v1.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/go-bitswap v0.4.0 // indirect
+	github.com/ipfs/go-bitfield v1.0.0 // indirect
+	github.com/ipfs/go-bitswap v0.8.0 // indirect
 	github.com/ipfs/go-block-format v0.0.3 // indirect
-	github.com/ipfs/go-blockservice v0.1.7 // indirect
-	github.com/ipfs/go-cid v0.0.7 // indirect
-	github.com/ipfs/go-cidutil v0.0.2 // indirect
-	github.com/ipfs/go-datastore v0.4.6 // indirect
-	github.com/ipfs/go-ds-measure v0.1.0 // indirect
-	github.com/ipfs/go-fetcher v1.5.0 // indirect
-	github.com/ipfs/go-filestore v1.0.0 // indirect
+	github.com/ipfs/go-blockservice v0.4.0 // indirect
+	github.com/ipfs/go-cid v0.2.0 // indirect
+	github.com/ipfs/go-cidutil v0.1.0 // indirect
+	github.com/ipfs/go-datastore v0.5.1 // indirect
+	github.com/ipfs/go-delegated-routing v0.3.0 // indirect
+	github.com/ipfs/go-ds-measure v0.2.0 // indirect
+	github.com/ipfs/go-fetcher v1.6.1 // indirect
+	github.com/ipfs/go-filestore v1.2.0 // indirect
 	github.com/ipfs/go-fs-lock v0.0.7 // indirect
-	github.com/ipfs/go-graphsync v0.8.0 // indirect
-	github.com/ipfs/go-ipfs v0.10.0 // indirect
-	github.com/ipfs/go-ipfs-blockstore v1.0.1 // indirect
+	github.com/ipfs/go-graphsync v0.13.1 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
 	github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
-	github.com/ipfs/go-ipfs-config v0.16.0 // indirect
 	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
-	github.com/ipfs/go-ipfs-ds-help v1.0.0 // indirect
-	github.com/ipfs/go-ipfs-exchange-interface v0.0.1 // indirect
-	github.com/ipfs/go-ipfs-exchange-offline v0.0.1 // indirect
-	github.com/ipfs/go-ipfs-files v0.0.8 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect
+	github.com/ipfs/go-ipfs-files v0.1.1 // indirect
 	github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
-	github.com/ipfs/go-ipfs-pinner v0.1.2 // indirect
+	github.com/ipfs/go-ipfs-pinner v0.2.1 // indirect
 	github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
 	github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
-	github.com/ipfs/go-ipfs-provider v0.6.1 // indirect
-	github.com/ipfs/go-ipfs-routing v0.1.0 // indirect
+	github.com/ipfs/go-ipfs-provider v0.7.1 // indirect
+	github.com/ipfs/go-ipfs-routing v0.2.1 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
 	github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
-	github.com/ipfs/go-ipld-format v0.2.0 // indirect
-	github.com/ipfs/go-ipld-legacy v0.1.0 // indirect
+	github.com/ipfs/go-ipld-format v0.4.0 // indirect
+	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
 	github.com/ipfs/go-ipns v0.1.2 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
-	github.com/ipfs/go-log/v2 v2.3.0 // indirect
-	github.com/ipfs/go-merkledag v0.4.0 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/ipfs/go-merkledag v0.6.0 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
-	github.com/ipfs/go-mfs v0.1.2 // indirect
-	github.com/ipfs/go-namesys v0.3.1 // indirect
-	github.com/ipfs/go-path v0.1.2 // indirect
-	github.com/ipfs/go-peertaskqueue v0.4.0 // indirect
-	github.com/ipfs/go-unixfs v0.2.5 // indirect
-	github.com/ipfs/go-unixfsnode v1.1.3 // indirect
+	github.com/ipfs/go-mfs v0.2.1 // indirect
+	github.com/ipfs/go-namesys v0.5.0 // indirect
+	github.com/ipfs/go-path v0.3.0 // indirect
+	github.com/ipfs/go-peertaskqueue v0.7.1 // indirect
+	github.com/ipfs/go-unixfs v0.4.0 // indirect
+	github.com/ipfs/go-unixfsnode v1.4.0 // indirect
 	github.com/ipfs/go-verifcid v0.0.1 // indirect
-	github.com/ipfs/interface-go-ipfs-core v0.5.1 // indirect
-	github.com/ipld/go-codec-dagpb v1.3.0 // indirect
-	github.com/ipld/go-ipld-prime v0.12.2 // indirect
+	github.com/ipfs/interface-go-ipfs-core v0.7.0 // indirect
+	github.com/ipfs/kubo v0.14.0 // indirect
+	github.com/ipld/edelweiss v0.1.4 // indirect
+	github.com/ipld/go-codec-dagpb v1.4.0 // indirect
+	github.com/ipld/go-ipld-prime v0.17.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.10.0 // indirect
+	github.com/jackc/pgconn v1.12.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.1.1 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.8.1 // indirect
-	github.com/jackc/pgx/v4 v4.13.0 // indirect
-	github.com/jackc/puddle v1.1.3 // indirect
+	github.com/jackc/pgtype v1.11.0 // indirect
+	github.com/jackc/pgx/v4 v4.16.1 // indirect
+	github.com/jackc/puddle v1.2.1 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
-	github.com/klauspost/compress v1.11.7 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.4 // indirect
+	github.com/klauspost/compress v1.15.1 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
 	github.com/koron/go-ssdp v0.0.2 // indirect
-	github.com/lib/pq v1.10.5 // indirect
-	github.com/libp2p/go-addr-util v0.1.0 // indirect
+	github.com/lib/pq v1.10.6 // indirect
 	github.com/libp2p/go-buffer-pool v0.0.2 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
-	github.com/libp2p/go-conn-security-multistream v0.2.1 // indirect
-	github.com/libp2p/go-doh-resolver v0.3.1 // indirect
+	github.com/libp2p/go-doh-resolver v0.4.0 // indirect
 	github.com/libp2p/go-eventbus v0.2.1 // indirect
 	github.com/libp2p/go-flow-metrics v0.0.3 // indirect
-	github.com/libp2p/go-libp2p v0.15.0 // indirect
-	github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 // indirect
-	github.com/libp2p/go-libp2p-autonat v0.4.2 // indirect
-	github.com/libp2p/go-libp2p-blankhost v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-circuit v0.4.0 // indirect
-	github.com/libp2p/go-libp2p-connmgr v0.2.4 // indirect
-	github.com/libp2p/go-libp2p-core v0.9.0 // indirect
-	github.com/libp2p/go-libp2p-discovery v0.5.1 // indirect
-	github.com/libp2p/go-libp2p-kad-dht v0.13.1 // indirect
+	github.com/libp2p/go-libp2p v0.20.3 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-core v0.16.1 // indirect
+	github.com/libp2p/go-libp2p-discovery v0.7.0 // indirect
+	github.com/libp2p/go-libp2p-kad-dht v0.16.0 // indirect
 	github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect
 	github.com/libp2p/go-libp2p-loggables v0.1.0 // indirect
-	github.com/libp2p/go-libp2p-mplex v0.4.1 // indirect
-	github.com/libp2p/go-libp2p-nat v0.0.6 // indirect
-	github.com/libp2p/go-libp2p-noise v0.2.2 // indirect
-	github.com/libp2p/go-libp2p-peerstore v0.2.8 // indirect
-	github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-pubsub v0.5.4 // indirect
-	github.com/libp2p/go-libp2p-pubsub-router v0.4.0 // indirect
-	github.com/libp2p/go-libp2p-quic-transport v0.12.0 // indirect
+	github.com/libp2p/go-libp2p-peerstore v0.6.0 // indirect
+	github.com/libp2p/go-libp2p-pubsub v0.6.1 // indirect
+	github.com/libp2p/go-libp2p-pubsub-router v0.5.0 // indirect
 	github.com/libp2p/go-libp2p-record v0.1.3 // indirect
+	github.com/libp2p/go-libp2p-resource-manager v0.3.0 // indirect
 	github.com/libp2p/go-libp2p-routing-helpers v0.2.3 // indirect
-	github.com/libp2p/go-libp2p-swarm v0.5.3 // indirect
-	github.com/libp2p/go-libp2p-tls v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-transport-upgrader v0.4.6 // indirect
-	github.com/libp2p/go-libp2p-xor v0.0.0-20210714161855-5c005aca55db // indirect
-	github.com/libp2p/go-libp2p-yamux v0.5.4 // indirect
-	github.com/libp2p/go-maddr-filter v0.1.0 // indirect
-	github.com/libp2p/go-mplex v0.3.0 // indirect
-	github.com/libp2p/go-msgio v0.0.6 // indirect
-	github.com/libp2p/go-nat v0.0.5 // indirect
-	github.com/libp2p/go-netroute v0.1.6 // indirect
+	github.com/libp2p/go-libp2p-swarm v0.11.0 // indirect
+	github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
+	github.com/libp2p/go-mplex v0.7.0 // indirect
+	github.com/libp2p/go-msgio v0.2.0 // indirect
+	github.com/libp2p/go-nat v0.1.0 // indirect
+	github.com/libp2p/go-netroute v0.2.0 // indirect
 	github.com/libp2p/go-openssl v0.0.7 // indirect
-	github.com/libp2p/go-reuseport v0.0.2 // indirect
-	github.com/libp2p/go-reuseport-transport v0.0.5 // indirect
-	github.com/libp2p/go-sockaddr v0.1.1 // indirect
-	github.com/libp2p/go-stream-muxer-multistream v0.3.0 // indirect
-	github.com/libp2p/go-tcp-transport v0.2.8 // indirect
-	github.com/libp2p/go-ws-transport v0.5.0 // indirect
-	github.com/libp2p/go-yamux/v2 v2.2.0 // indirect
-	github.com/libp2p/zeroconf/v2 v2.0.0 // indirect
-	github.com/lucas-clemente/quic-go v0.26.0 // indirect
+	github.com/libp2p/go-reuseport v0.2.0 // indirect
+	github.com/libp2p/go-yamux/v3 v3.1.2 // indirect
+	github.com/libp2p/zeroconf/v2 v2.1.1 // indirect
+	github.com/lucas-clemente/quic-go v0.27.1 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailgun/groupcache/v2 v2.3.0 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
@@ -182,43 +177,47 @@ require (
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/miekg/dns v1.1.43 // indirect
+	github.com/miekg/dns v1.1.48 // indirect
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
-	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/mitchellh/pointerstructure v1.2.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
-	github.com/multiformats/go-base32 v0.0.3 // indirect
+	github.com/multiformats/go-base32 v0.0.4 // indirect
 	github.com/multiformats/go-base36 v0.1.0 // indirect
-	github.com/multiformats/go-multiaddr v0.4.0 // indirect
+	github.com/multiformats/go-multiaddr v0.5.0 // indirect
 	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
-	github.com/multiformats/go-multibase v0.0.3 // indirect
-	github.com/multiformats/go-multicodec v0.3.0 // indirect
-	github.com/multiformats/go-multihash v0.1.0 // indirect
-	github.com/multiformats/go-multistream v0.2.2 // indirect
+	github.com/multiformats/go-multibase v0.1.0 // indirect
+	github.com/multiformats/go-multicodec v0.5.0 // indirect
+	github.com/multiformats/go-multihash v0.2.0 // indirect
+	github.com/multiformats/go-multistream v0.3.3 // indirect
 	github.com/multiformats/go-varint v0.0.6 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
 	github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
-	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.30.0 // indirect
+	github.com/prometheus/common v0.33.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/prometheus/tsdb v0.7.1 // indirect
+	github.com/prometheus/tsdb v0.10.0 // indirect
+	github.com/raulk/clock v1.1.0 // indirect
+	github.com/raulk/go-watchdog v1.2.0 // indirect
 	github.com/rjeczalik/notify v0.9.1 // indirect
 	github.com/rs/cors v1.7.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/segmentio/fasthash v1.0.3 // indirect
-	github.com/shirou/gopsutil v3.21.5+incompatible // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
@@ -227,48 +226,54 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
 	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.7.1 // indirect
+	github.com/stretchr/testify v1.7.2 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
 	github.com/thoas/go-funk v0.9.2 // indirect
+	github.com/tidwall/gjson v1.14.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.6 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
-	github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.2 // indirect
+	github.com/urfave/cli/v2 v2.10.2 // indirect
+	github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.6-alpha // indirect
+	github.com/wI2L/jsondiff v0.2.0 // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
-	github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 // indirect
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
 	github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/otel v0.20.0 // indirect
-	go.opentelemetry.io/otel/metric v0.20.0 // indirect
-	go.opentelemetry.io/otel/trace v0.20.0 // indirect
+	go.opentelemetry.io/otel v1.7.0 // indirect
+	go.opentelemetry.io/otel/trace v1.7.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/dig v1.10.0 // indirect
-	go.uber.org/fx v1.13.1 // indirect
-	go.uber.org/multierr v1.7.0 // indirect
-	go.uber.org/zap v1.19.0 // indirect
+	go.uber.org/dig v1.14.0 // indirect
+	go.uber.org/fx v1.16.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/zap v1.21.0 // indirect
 	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
-	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
-	golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	golang.org/x/tools v0.1.8 // indirect
-	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	lukechampine.com/blake3 v1.1.6 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/driver/postgres v1.3.7 // indirect
+	gorm.io/gorm v1.23.5 // indirect
+	lukechampine.com/blake3 v1.1.7 // indirect
 )

-replace github.com/ethereum/go-ethereum v1.10.18 => github.com/vulcanize/go-ethereum v1.10.18-statediff-3.2.2
+replace github.com/ethereum/go-ethereum v1.10.21 => github.com/vulcanize/go-ethereum v1.10.21-statediff-4.1.2-alpha
@@ -10,7 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/jmoiron/sqlx"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
 )

 var (
@@ -0,0 +1,224 @@
+package validator
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+)
+
+// ValidateReferentialIntegrity validates referential integrity at the given height
+func ValidateReferentialIntegrity(db *sqlx.DB, blockNumber uint64) error {
+
+	err := ValidateHeaderCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateUncleCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateTransactionCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateReceiptCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStateCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStorageCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStateAccountsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateAccessListElementsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateLogCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateHeaderCIDsRef does a reference integrity check on references in eth.header_cids table
+func ValidateHeaderCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	err := ValidateIPFSBlocks(db, blockNumber, "eth.header_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateUncleCIDsRef does a reference integrity check on references in eth.uncle_cids table
+func ValidateUncleCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, UncleCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.uncle_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateTransactionCIDsRef does a reference integrity check on references in eth.header_cids table
+func ValidateTransactionCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, TransactionCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.transaction_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateReceiptCIDsRef does a reference integrity check on references in eth.receipt_cids table
+func ValidateReceiptCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, ReceiptCIDsRefTransactionCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.receipt_cids", "leaf_mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStateCIDsRef does a reference integrity check on references in eth.state_cids table
+func ValidateStateCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StateCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.state_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStorageCIDsRef does a reference integrity check on references in eth.storage_cids table
+func ValidateStorageCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StorageCIDsRefStateCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.storage_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStateAccountsRef does a reference integrity check on references in eth.state_accounts table
+func ValidateStateAccountsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StateAccountsRefStateCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
+	}
+
+	return nil
+}
+
+// ValidateAccessListElementsRef does a reference integrity check on references in eth.access_list_elements table
+func ValidateAccessListElementsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, AccessListElementsRefTransactionCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
+	}
+
+	return nil
+}
+
+// ValidateLogCIDsRef does a reference integrity check on references in eth.log_cids table
+func ValidateLogCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, LogCIDsRefReceiptCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.receipt_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.log_cids", "leaf_mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateIPFSBlocks does a reference integrity check between the given CID table and IPFS blocks table on MHKey and block number
+func ValidateIPFSBlocks(db *sqlx.DB, blockNumber uint64, CIDTable string, mhKeyField string) error {
+	var exists bool
+	err := db.Get(&exists, fmt.Sprintf(CIDsRefIPLDBlocks, CIDTable, mhKeyField), blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "public.blocks")
+	}
+
+	return nil
+}
@@ -0,0 +1,121 @@
+package validator
+
+// Queries to validate referential integrity in the indexed data:
+// At the given block number,
+// In each table, for each (would be) foreign key reference, perform left join with the referenced table on the foreign key fields.
+// Select rows where there are no matching rows in the referenced table.
+// If any such rows exist, there are missing entries in the referenced table.
+
+const (
+	CIDsRefIPLDBlocks = `SELECT EXISTS (
+						SELECT *
+						FROM %[1]s
+						LEFT JOIN public.blocks ON (
+							%[1]s.%[2]s = blocks.key
+							AND %[1]s.block_number = blocks.block_number
+						)
+						WHERE
+							%[1]s.block_number = $1
+							AND blocks.key IS NULL
+					)`
+
+	UncleCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.uncle_cids
+						LEFT JOIN eth.header_cids ON (
+							uncle_cids.header_id = header_cids.block_hash
+							AND uncle_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							uncle_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	TransactionCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.transaction_cids
+						LEFT JOIN eth.header_cids ON (
+							transaction_cids.header_id = header_cids.block_hash
+							AND transaction_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							transaction_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	ReceiptCIDsRefTransactionCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.receipt_cids
+						LEFT JOIN eth.transaction_cids ON (
+							receipt_cids.tx_id = transaction_cids.tx_hash
+							AND receipt_cids.header_id = transaction_cids.header_id
+							AND receipt_cids.block_number = transaction_cids.block_number
+						)
+						WHERE
+							receipt_cids.block_number = $1
+							AND transaction_cids.tx_hash IS NULL
+					)`
+
+	StateCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.state_cids
+						LEFT JOIN eth.header_cids ON (
+							state_cids.header_id = header_cids.block_hash
+							AND state_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							state_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	StorageCIDsRefStateCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.storage_cids
+						LEFT JOIN eth.state_cids ON (
+							storage_cids.state_path = state_cids.state_path
+							AND storage_cids.header_id = state_cids.header_id
+							AND storage_cids.block_number = state_cids.block_number
+						)
+						WHERE
+							storage_cids.block_number = $1
+							AND state_cids.state_path IS NULL
+					)`
+
+	StateAccountsRefStateCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.state_accounts
+						LEFT JOIN eth.state_cids ON (
+							state_accounts.state_path = state_cids.state_path
+							AND state_accounts.header_id = state_cids.header_id
+							AND state_accounts.block_number = state_cids.block_number
+						)
+						WHERE
+							state_accounts.block_number = $1
+							AND state_cids.state_path IS NULL
+					)`
+
+	AccessListElementsRefTransactionCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.access_list_elements
+						LEFT JOIN eth.transaction_cids ON (
+							access_list_elements.tx_id = transaction_cids.tx_hash
+							AND access_list_elements.block_number = transaction_cids.block_number
+						)
+						WHERE
+							access_list_elements.block_number = $1
+							AND transaction_cids.tx_hash IS NULL
+					)`
+
+	LogCIDsRefReceiptCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.log_cids
+						LEFT JOIN eth.receipt_cids ON (
+							log_cids.rct_id = receipt_cids.tx_id
+							AND log_cids.header_id = receipt_cids.header_id
+							AND log_cids.block_number = receipt_cids.block_number
+						)
+						WHERE
+							log_cids.block_number = $1
+							AND receipt_cids.tx_id IS NULL
+					)`
+)
@@ -0,0 +1,334 @@
+package validator_test
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
+)
+
+var _ = Describe("RefIntegrity", func() {
+	var (
+		ctx = context.Background()
+
+		db          *sqlx.DB
+		diffIndexer interfaces.StateDiffIndexer
+	)
+
+	BeforeEach(func() {
+		db = shared.SetupDB()
+		diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
+	})
+
+	AfterEach(func() {
+		shared.TearDownDB(db)
+	})
+
+	Describe("ValidateHeaderCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of header_cids table", func() {
+			err := validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateUncleCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of uncle_cids table", func() {
+			err := validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cid entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding uncle IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateTransactionCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of transaction_cids table", func() {
+			err := validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cid entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding transaction IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateReceiptCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of receipt_cids table", func() {
+			err := validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding transaction_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.transaction_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
+		})
+
+		It("Throws an error if corresponding receipt IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStateCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of state_cids table", func() {
+			err := validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding state IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStorageCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of storage_cids table", func() {
+			err := validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding state_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.state_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
+		})
+
+		It("Throws an error if corresponding storage IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStateAccountsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of state_accounts table", func() {
+			err := validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding state_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.state_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
+		})
+	})
+
+	Describe("ValidateAccessListElementsRef", func() {
+		BeforeEach(func() {
+			indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, mocks.TestConfig, test_helpers.Genesis.Hash())
+
+			tx, err := indexAndPublisher.PushBlock(mocks.MockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of access_list_elements table", func() {
+			err := validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding transaction_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.transaction_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
+		})
+	})
+
+	Describe("ValidateLogCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of log_cids table", func() {
+			err := validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding receipt_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.receipt_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.receipt_cids"))
+		})
+
+		It("Throws an error if corresponding log IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+})
+
+func deleteEntriesFrom(db *sqlx.DB, tableName string) error {
+	pgStr := "DELETE FROM %s"
+	_, err := db.Exec(fmt.Sprintf(pgStr, tableName))
+	return err
+}
@@ -20,14 +20,18 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
-	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres"
-	ipldEth "github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-	ethServerShared "github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres"
+	ipldEth "github.com/vulcanize/ipld-eth-server/v4/pkg/eth"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
 )

 var (
 	big8  = big.NewInt(8)
 	big32 = big.NewInt(32)
+
+	ReferentialIntegrityErr = "referential integrity check failed at block %d, entry for %s not found"
+	EntryNotFoundErr        = "entry for %s not found"
 )

 type service struct {
@@ -134,6 +138,14 @@ func (s *service) Validate(ctx context.Context, api *ipldEth.PublicEthAPI, idxBl
 		}

 		s.logger.Infof("state root verified for block %d", idxBlockNum)
+
+		err = ValidateReferentialIntegrity(s.db, idxBlockNum)
+		if err != nil {
+			s.logger.Errorf("failed to verify referential integrity at block %d", idxBlockNum)
+			return idxBlockNum, err
+		}
+		s.logger.Infof("referential integrity verified for block %d", idxBlockNum)
+
 		if s.progressChan != nil {
 			s.progressChan <- idxBlockNum
 		}
@@ -0,0 +1,13 @@
+package validator_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestValidator(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Validator Suite")
+}
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 set -o xtrace

@@ -1,9 +1,17 @@
+#!/bin/bash
+
+set -e
+
 # Clear up existing docker images and volume.
 docker-compose down --remove-orphans --volumes

-docker-compose -f docker-compose.yml up -d ipld-eth-db
-sleep 10
+# Spin up TimescaleDB
+docker-compose -f docker-compose.yml up -d migrations ipld-eth-db
+sleep 45

+# Run unit tests
+go clean -testcache
 PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test

+# Clean up
 docker-compose down --remove-orphans --volumes
@@ -1,23 +1,30 @@
-# Test Insructions
+# Test Instructions

 ## Setup

 - For running integration tests:

-  - Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator) and [go-ethereum](https://github.com/vulcanize/go-ethereum) repositories.
+  - Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator), [go-ethereum](https://github.com/vulcanize/go-ethereum) and [ipld-eth-db](https://github.com/vulcanize/ipld-eth-db) repositories.

-  - Checkout [v3 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-3.2.2) in go-ethereum repo.
+  - Checkout [v4 release](https://github.com/vulcanize/ipld-eth-db/releases/tag/v4.2.0-alpha) in ipld-eth-db repo.
+
+    ```bash
+    # In ipld-eth-db repo.
+    git checkout v4.2.0-alpha
+    ```
+
+  - Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.19-statediff-4.1.0-alpha) in go-ethereum repo.

    ```bash
    # In go-ethereum repo.
-    git checkout v1.10.18-statediff-3.2.2
+    git checkout v1.10.19-statediff-4.1.0-alpha
    ```

  - Checkout working commit in stack-orchestrator repo.

    ```bash
    # In stack-orchestrator repo.
-    git checkout main
+    git checkout f2fd766f5400fcb9eb47b50675d2e3b1f2753702
    ```

 ## Run
@@ -48,12 +55,16 @@
      ```bash
      #!/bin/bash

+      # Path to ipld-eth-server repo.
+      vulcanize_ipld_eth_db=~/ipld-eth-db/
+
      # Path to go-ethereum repo.
      vulcanize_go_ethereum=~/go-ethereum

      # Path to contract folder.
      vulcanize_test_contract=~/ipld-eth-db-validator/test/contract

+      genesis_file_path='start-up-files/go-ethereum/genesis.json'
      db_write=true
      ```

@@ -65,7 +76,7 @@

      ./wrapper.sh \
      -e docker \
-      -d ../docker/latest/docker-compose-db.yml \
+      -d ../docker/local/docker-compose-db-sharding.yml \
      -d ../docker/local/docker-compose-go-ethereum.yml \
      -d ../docker/local/docker-compose-contract.yml \
      -v remove \
@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"net/http"

-	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v3/test"
+	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
 )

 type PutResult struct {
@@ -11,8 +11,8 @@ import (
 	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
 	integration "github.com/vulcanize/ipld-eth-db-validator/test"

-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
-	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v3/test"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
+	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
 )

 const (
@@ -12,8 +12,8 @@ import (
 	"github.com/jmoiron/sqlx"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"

 	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
 	"github.com/vulcanize/ipld-eth-db-validator/validator_test"
@@ -120,6 +120,9 @@ var _ = Describe("eth state reading tests", func() {
 			for i := uint64(blockHeight); i <= chainLength-trail; i++ {
 				err = validator.ValidateBlock(context.Background(), api, i)
 				Expect(err).ToNot(HaveOccurred())
+
+				err = validator.ValidateReferentialIntegrity(db, i)
+				Expect(err).ToNot(HaveOccurred())
 			}
 		})
 	})
Author	SHA1	Message	Date
Michael	716978e23c	Merge pull request #19 from vulcanize/geth_1_10_21_wip geth 1.10.21 update	2022-08-03 14:51:29 -04:00
Michael Shaw	e1a92a978e	go-quic not 1.19 ready for both checks	2022-08-03 14:42:06 -04:00
Michael Shaw	8e3bc49407	go-quic not 1.19 ready	2022-08-03 14:36:47 -04:00
Michael Shaw	3804f52aaf	ipld-eth-server missed version update	2022-08-03 11:37:55 -04:00
Michael Shaw	47ef12426d	geth 1.10.21 update	2022-08-03 11:11:22 -04:00
Michael	5c7b15f304	Merge pull request #18 from vulcanize/release-v4.1.1-alpha update for go-ethereum 1.10.20	2022-07-20 17:18:07 -04:00
Michael Shaw	55248fdd0d	update for go-ethereum 1.10.20	2022-07-19 14:57:44 -04:00
prathamesh0	5fb86ba963	Upgrade dependencies and update referential integrity checks (#17 ) * Upgrade dependencies and update referential integrity queries * Update GitHub workflow to run tests	2022-07-13 13:12:59 +05:30
Abdul Rabbani	7ae4886b9a	update geth and CICD refs (#15 ) * update geth and CICD refs * Update stack-orch * Download bin instead of building it * Fix integration Tests * Update DB Ref	2022-06-20 09:37:15 -04:00
nabarun	b11ace0711	Revert changes to perform referential integrity tests (#16 )	2022-06-20 14:20:14 +05:30
Abdul Rabbani prathamesh Ashwin Phatak	d111dd85db	Feature/update go geth sharding (#13 ) * Upgrade Geth, Go and add CICD * update ipld-eth-server version * Track validation progress on a channel * Add integration tests * Setup validator config and update instructions to run tests locally * Update readme and tests * Update test to use v4 Infrastructure * Inlcude the env file * Fix config file write * Update DB configuration Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com> Co-authored-by: Ashwin Phatak <ashwinpphatak@gmail.com>	2022-06-03 09:28:02 -04:00
prathamesh0	cc935dc97b	Add checks to validate referential integrity in the data (#7 ) * Add checks to validate referential integrity * Add unit tests for referential integrity validation * Use EXISTS in referential integrity validation queries	2022-06-01 11:29:53 +05:30
Ashwin Phatak	effbdc3f58	Merge pull request #9 from deep-stack/pm-merge-perpetual Merge changes to make validation process perpetual	2022-05-31 13:44:36 +05:30
prathamesh	31cc17b70f	Merge changes to make validation process perpetual	2022-05-31 13:09:45 +05:30
Ashwin Phatak	e89d64aa72	Merge pull request #5 from deep-stack/pm-v4-changes Upgrade dependencies to v4	2022-05-23 20:01:52 +05:30
prathamesh	a56da1782a	Update instructions to run tests	2022-05-23 19:43:21 +05:30
prathamesh	e102a2b5ac	Upgrade dependencies to v4	2022-05-23 19:08:17 +05:30