Feature/update go geth sharding (#13)

* Upgrade Geth, Go and add CICD

* update ipld-eth-server version

* Track validation progress on a channel

* Add integration tests

* Setup validator config and update instructions to run tests locally

* Update readme and tests

* Update test to use v4 Infrastructure

* Inlcude the env file

* Fix config file write

* Update DB configuration

Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
Co-authored-by: Ashwin Phatak <ashwinpphatak@gmail.com>

.github/workflows/issues-notion-sync.yml

@@ -1,29 +0,0 @@
-name: Notion Sync
-
-on:
-  workflow_dispatch:
-  issues:
-    types:
-      [
-        opened,
-        edited,
-        labeled,
-        unlabeled,
-        assigned,
-        unassigned,
-        milestoned,
-        demilestoned,
-        reopened,
-        closed,
-      ]
-
-jobs:
-  notion_job:
-    runs-on: ubuntu-latest
-    name: Add GitHub Issues to Notion
-    steps:
-      - name: Add GitHub Issues to Notion
-        uses: vulcanize/notion-github-action@v1.2.4-issueid
-        with:
-          notion-token: ${{ secrets.NOTION_TOKEN }}
-          notion-db: ${{ secrets.NOTION_DATABASE }}

.github/workflows/tests.yml

@@ -3,8 +3,9 @@ on:
   workflow_call:
 
 env:
-  STACK_ORCHESTRATOR_REF: "main"
-  GO_ETHEREUM_REF: "v1.10.18-statediff-v3"
+  STACK_ORCHESTRATOR_REF: 418957a1f745c921b21286c13bb033f922a91ae9
+  GO_ETHEREUM_REF: "v1.10.18-statediff-4.0.2-alpha"
+  IPLD_ETH_DB_REF: 91d30b9ea1acecd0a7f4307390a98bf3e289b8d7
 
 jobs:
   integrationtest:
@@ -33,11 +34,19 @@ jobs:
           ref: ${{ env.GO_ETHEREUM_REF }}
           repository: vulcanize/go-ethereum
           path: "./go-ethereum/"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.IPLD_ETH_DB_REF }}
+          repository: vulcanize/ipld-eth-db
+          path: "./ipld-eth-db/"
       - name: Create config file
         run: |
-          echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ > ./config.sh
+          echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> ./config.sh
           echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh
+          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
           echo db_write=$DB_WRITE >> ./config.sh
+          echo go_ethereum_db_dependency=access-node >> ./config.sh
+          echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
           cat ./config.sh
       - name: Build geth
         run: |
@@ -48,9 +57,10 @@ jobs:
       - name: Run docker compose
         run: |
           docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
           --env-file "$GITHUB_WORKSPACE/config.sh" \
           up -d --build
       - name: Run integration test.
@@ -79,12 +89,25 @@ jobs:
           ref: ${{ env.STACK_ORCHESTRATOR_REF }}
           path: "./stack-orchestrator/"
           repository: vulcanize/stack-orchestrator
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.IPLD_ETH_DB_REF }}
+          repository: vulcanize/ipld-eth-db
+          path: "./ipld-eth-db/"
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
+          echo db_write=$DB_WRITE >> ./config.sh
+          echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
+          cat ./config.sh
       - name: Run docker compose
         run: |
           docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
+          --env-file "$GITHUB_WORKSPACE/config.sh" \
           up -d --build
       - name: Run unit test.
         run: |
           cd $GITHUB_WORKSPACE/ipld-eth-db-validator
-          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test
+          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8066 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing_v4 make test

docker-compose.yml

@@ -1,18 +1,30 @@
 version: '3.2'
 
 services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: vulcanize/ipld-eth-db:v4.1.1-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
   ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
     restart: always
-    image: vulcanize/ipld-eth-db:v3.2.0
+    command: ["postgres", "-c", "log_statement=all"]
     environment:
       POSTGRES_USER: "vdbm"
       POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
-    volumes:
-      - vdb_db_eth_validator:/var/lib/postgresql/data
     ports:
       - "127.0.0.1:8077:5432"
-    command: ["postgres", "-c", "log_statement=all"]
+    volumes:
+      - vdb_db_eth_validator:/var/lib/postgresql/data
 
 volumes:
-  vdb_db_eth_validator:
+  vdb_db_eth_validator:

go.mod

@@ -10,8 +10,9 @@ require (
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.11.0
-	github.com/vulcanize/ipfs-ethdb/v3 v3.0.3
+	github.com/vulcanize/ipfs-ethdb/v4 v4.0.1-alpha
 	github.com/vulcanize/ipld-eth-server/v3 v3.2.1
+	github.com/vulcanize/ipld-eth-server/v4 v4.0.2-alpha
 )
 
 require (
@@ -234,7 +235,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.6 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
-	github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.2 // indirect
+	github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.1-alpha // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
@@ -271,4 +272,4 @@ require (
 	lukechampine.com/blake3 v1.1.6 // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.10.18 => github.com/vulcanize/go-ethereum v1.10.18-statediff-3.2.2
+replace github.com/ethereum/go-ethereum v1.10.18 => github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha

go.sum

@@ -1674,13 +1674,17 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.2 h1:YZ24WFrrnTC3of2EFfMIQsGUCk1hkDu++l0TIw6RrFw=
-github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.2/go.mod h1:PrVSnKWJhnbCYUAIHESAwnXITN5df3IBckmkmQIBv7o=
-github.com/vulcanize/go-ethereum v1.10.18-statediff-3.2.2 h1:hAZFOtnfdHtVgiwtcNFZDxCEN01c4W5Xsw+1K/IlHBA=
-github.com/vulcanize/go-ethereum v1.10.18-statediff-3.2.2/go.mod h1:HelXH7UT1uWdb+St6UAj4pPf93GOggjIV7pVbrWIZ3o=
+github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.1-alpha h1:3q3siR+ZzbpJ7b0LsMsVyHj7/PJYNf+0XM48LinLUTw=
+github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.1-alpha/go.mod h1:L+0oYQrOUXO7vly1WNT9CUi1ydk5+JEW70q6EqLS4SI=
+github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha h1:gTcfBHoaRpsRbP1/dnurg5LFUaqsQMTNqs4R4S2wm9U=
+github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha/go.mod h1:HelXH7UT1uWdb+St6UAj4pPf93GOggjIV7pVbrWIZ3o=
 github.com/vulcanize/ipfs-ethdb/v3 v3.0.3 h1:KyNMtSKZk2ZuU+mwAv/SdNEeigFkCxvsTDqxHDpwUL8=
-github.com/vulcanize/ipfs-ethdb/v3 v3.0.3/go.mod h1:hi3c5YadpoJc0A6Du9OzEAhGxtk0S1pTSAY/rApvuA0=
+github.com/vulcanize/ipfs-ethdb/v4 v4.0.1-alpha h1:VxZONVJMxAC6vLhmlQsC3zhmY0D2A3DUoGAn8az6pXY=
+github.com/vulcanize/ipfs-ethdb/v4 v4.0.1-alpha/go.mod h1:K2ayGLsvsHcm+4A9oD3mwMLElNjw7yn70h+KXo6JdrU=
 github.com/vulcanize/ipld-eth-server/v3 v3.2.1 h1:mJSUWG0LGP8zey/i6Dm544MzESce9yZ2uHvQ5PF+hy0=
 github.com/vulcanize/ipld-eth-server/v3 v3.2.1/go.mod h1:xvnmxc6h9f70XfvEB6mV8kzTFUEdssS6gidlPWNpQeg=
+github.com/vulcanize/ipld-eth-server/v4 v4.0.2-alpha h1:UDcRi9qvLWgDMKuUesEKEl/Mt94LVO0GLkgg0xVzRxs=
+github.com/vulcanize/ipld-eth-server/v4 v4.0.2-alpha/go.mod h1:k/ZzHvWBAHYmnLNY2uHIRc5WCaVPlqXiBbDMvCYlhMU=
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
 github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA=
 github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=

pkg/validator/config.go

@@ -10,7 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/jmoiron/sqlx"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
 )
 
 var (

pkg/validator/ref_integrity.go

@@ -0,0 +1,224 @@
+package validator
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+)
+
+// ValidateReferentialIntegrity validates referential integrity at the given height
+func ValidateReferentialIntegrity(db *sqlx.DB, blockNumber uint64) error {
+
+	err := ValidateHeaderCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateUncleCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateTransactionCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateReceiptCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStateCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStorageCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStateAccountsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateAccessListElementsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateLogCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateHeaderCIDsRef does a reference integrity check on references in eth.header_cids table
+func ValidateHeaderCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	err := ValidateIPFSBlocks(db, blockNumber, "eth.header_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateUncleCIDsRef does a reference integrity check on references in eth.uncle_cids table
+func ValidateUncleCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, UncleCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.uncle_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateTransactionCIDsRef does a reference integrity check on references in eth.header_cids table
+func ValidateTransactionCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, TransactionCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.transaction_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateReceiptCIDsRef does a reference integrity check on references in eth.receipt_cids table
+func ValidateReceiptCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, ReceiptCIDsRefTransactionCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.receipt_cids", "leaf_mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStateCIDsRef does a reference integrity check on references in eth.state_cids table
+func ValidateStateCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StateCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.state_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStorageCIDsRef does a reference integrity check on references in eth.storage_cids table
+func ValidateStorageCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StorageCIDsRefStateCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.storage_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStateAccountsRef does a reference integrity check on references in eth.state_accounts table
+func ValidateStateAccountsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StateAccountsRefStateCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
+	}
+
+	return nil
+}
+
+// ValidateAccessListElementsRef does a reference integrity check on references in eth.access_list_elements table
+func ValidateAccessListElementsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, AccessListElementsRefTransactionCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
+	}
+
+	return nil
+}
+
+// ValidateLogCIDsRef does a reference integrity check on references in eth.log_cids table
+func ValidateLogCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, LogCIDsRefReceiptCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.receipt_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.log_cids", "leaf_mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateIPFSBlocks does a reference integrity check between the given CID table and IPFS blocks table on MHKey and block number
+func ValidateIPFSBlocks(db *sqlx.DB, blockNumber uint64, CIDTable string, mhKeyField string) error {
+	var exists bool
+	err := db.Get(&exists, fmt.Sprintf(CIDsRefIPLDBlocks, CIDTable, mhKeyField), blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "public.blocks")
+	}
+
+	return nil
+}

pkg/validator/ref_integrity_queries.go

@@ -0,0 +1,119 @@
+package validator
+
+// Queries to validate referential integrity in the indexed data:
+// At the given block number,
+// In each table, for each (would be) foreign key reference, perform left join with the referenced table on the foreign key fields.
+// Select rows where there are no matching rows in the referenced table.
+// If any such rows exist, there are missing entries in the referenced table.
+
+const (
+	CIDsRefIPLDBlocks = `SELECT EXISTS (
+						SELECT *
+						FROM %[1]s
+						LEFT JOIN public.blocks ON (
+							%[1]s.%[2]s = blocks.key
+							AND %[1]s.block_number = blocks.block_number
+						)
+						WHERE
+							%[1]s.block_number = $1
+							AND blocks.key IS NULL
+					)`
+
+	UncleCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.uncle_cids
+						LEFT JOIN eth.header_cids ON (
+							uncle_cids.header_id = header_cids.block_hash
+							AND uncle_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							uncle_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	TransactionCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.transaction_cids
+						LEFT JOIN eth.header_cids ON (
+							transaction_cids.header_id = header_cids.block_hash
+							AND transaction_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							transaction_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	ReceiptCIDsRefTransactionCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.receipt_cids
+						LEFT JOIN eth.transaction_cids ON (
+							receipt_cids.tx_id = transaction_cids.tx_hash
+							AND receipt_cids.block_number = transaction_cids.block_number
+						)
+						WHERE
+							receipt_cids.block_number = $1
+							AND transaction_cids.tx_hash IS NULL
+					)`
+
+	StateCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.state_cids
+						LEFT JOIN eth.header_cids ON (
+							state_cids.header_id = header_cids.block_hash
+							AND state_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							state_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	StorageCIDsRefStateCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.storage_cids
+						LEFT JOIN eth.state_cids ON (
+							storage_cids.state_path = state_cids.state_path
+							AND storage_cids.header_id = state_cids.header_id
+							AND storage_cids.block_number = state_cids.block_number
+						)
+						WHERE
+							storage_cids.block_number = $1
+							AND state_cids.state_path IS NULL
+					)`
+
+	StateAccountsRefStateCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.state_accounts
+						LEFT JOIN eth.state_cids ON (
+							state_accounts.state_path = state_cids.state_path
+							AND state_accounts.header_id = state_cids.header_id
+							AND state_accounts.block_number = state_cids.block_number
+						)
+						WHERE
+							state_accounts.block_number = $1
+							AND state_cids.state_path IS NULL
+					)`
+
+	AccessListElementsRefTransactionCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.access_list_elements
+						LEFT JOIN eth.transaction_cids ON (
+							access_list_elements.tx_id = transaction_cids.tx_hash
+							AND access_list_elements.block_number = transaction_cids.block_number
+						)
+						WHERE
+							access_list_elements.block_number = $1
+							AND transaction_cids.tx_hash IS NULL
+					)`
+
+	LogCIDsRefReceiptCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.log_cids
+						LEFT JOIN eth.receipt_cids ON (
+							log_cids.rct_id = receipt_cids.tx_id
+							AND log_cids.block_number = receipt_cids.block_number
+						)
+						WHERE
+							log_cids.block_number = $1
+							AND receipt_cids.tx_id IS NULL
+					)`
+)

pkg/validator/ref_integrity_test.go

@@ -0,0 +1,334 @@
+package validator_test
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
+)
+
+var _ = Describe("RefIntegrity", func() {
+	var (
+		ctx = context.Background()
+
+		db          *sqlx.DB
+		diffIndexer interfaces.StateDiffIndexer
+	)
+
+	BeforeEach(func() {
+		db = shared.SetupDB()
+		diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
+	})
+
+	AfterEach(func() {
+		shared.TearDownDB(db)
+	})
+
+	Describe("ValidateHeaderCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of header_cids table", func() {
+			err := validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateUncleCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of uncle_cids table", func() {
+			err := validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cid entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding uncle IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateTransactionCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of transaction_cids table", func() {
+			err := validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cid entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding transaction IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateReceiptCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of receipt_cids table", func() {
+			err := validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding transaction_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.transaction_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
+		})
+
+		It("Throws an error if corresponding receipt IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStateCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of state_cids table", func() {
+			err := validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding state IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStorageCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of storage_cids table", func() {
+			err := validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding state_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.state_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
+		})
+
+		It("Throws an error if corresponding storage IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStateAccountsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of state_accounts table", func() {
+			err := validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding state_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.state_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
+		})
+	})
+
+	Describe("ValidateAccessListElementsRef", func() {
+		BeforeEach(func() {
+			indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, mocks.TestConfig, test_helpers.Genesis.Hash())
+
+			tx, err := indexAndPublisher.PushBlock(mocks.MockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of access_list_elements table", func() {
+			err := validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding transaction_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.transaction_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
+		})
+	})
+
+	Describe("ValidateLogCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of log_cids table", func() {
+			err := validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding receipt_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.receipt_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.receipt_cids"))
+		})
+
+		It("Throws an error if corresponding log IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+})
+
+func deleteEntriesFrom(db *sqlx.DB, tableName string) error {
+	pgStr := "DELETE FROM %s"
+	_, err := db.Exec(fmt.Sprintf(pgStr, tableName))
+	return err
+}

pkg/validator/validator.go

@@ -20,14 +20,18 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
-	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres"
-	ipldEth "github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-	ethServerShared "github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres"
+	ipldEth "github.com/vulcanize/ipld-eth-server/v4/pkg/eth"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
 )
 
 var (
 	big8  = big.NewInt(8)
 	big32 = big.NewInt(32)
+
+	ReferentialIntegrityErr = "referential integrity check failed at block %d, entry for %s not found"
+	EntryNotFoundErr        = "entry for %s not found"
 )
 
 type service struct {

pkg/validator/validator_suite_test.go

@@ -0,0 +1,13 @@
+package validator_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestValidator(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Validator Suite")
+}

scripts/run_integration_test.sh

@@ -1,12 +1,14 @@
+#!/bin/bash
+
 set -e
 set -o xtrace
 
 export PGPASSWORD=password
 export DATABASE_USER=vdbm
-export DATABASE_PORT=8077
+export DATABASE_PORT=8066
 export DATABASE_PASSWORD=password
 export DATABASE_HOSTNAME=127.0.0.1
-export DATABASE_NAME=vulcanize_testing
+export DATABASE_NAME=vulcanize_testing_v4
 
 # Wait for containers to be up and execute the integration test.
 while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \

scripts/run_unit_test.sh

@@ -1,9 +1,17 @@
+#!/bin/bash
+
+set -e
+
 # Clear up existing docker images and volume.
 docker-compose down --remove-orphans --volumes
 
-docker-compose -f docker-compose.yml up -d ipld-eth-db
-sleep 10
+# Spin up TimescaleDB
+docker-compose -f docker-compose.yml up -d migrations ipld-eth-db
+sleep 45
 
+# Run unit tests
+go clean -testcache
 PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test
 
+# Clean up
 docker-compose down --remove-orphans --volumes

test/README.md

@@ -1,4 +1,4 @@
-# Test Insructions
+# Test Instructions
 
 ## Setup
 
@@ -6,18 +6,18 @@
 
   - Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator) and [go-ethereum](https://github.com/vulcanize/go-ethereum) repositories.
 
-  - Checkout [v3 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-3.2.2) in go-ethereum repo.
+  - Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-4.0.2-alpha) in go-ethereum repo.
 
     ```bash
     # In go-ethereum repo.
-    git checkout v1.10.18-statediff-3.2.2
+    git checkout v1.10.18-statediff-4.0.2-alpha
     ```
 
   - Checkout working commit in stack-orchestrator repo.
 
     ```bash
     # In stack-orchestrator repo.
-    git checkout main
+    git checkout 418957a1f745c921b21286c13bb033f922a91ae9
     ```
 
 ## Run
@@ -55,6 +55,10 @@
       vulcanize_test_contract=~/ipld-eth-db-validator/test/contract
 
       db_write=true
+      ipld_eth_server_db_dependency=access-node
+      go_ethereum_db_dependency=access-node
+
+      connecting_db_name=vulcanize_testing_v4
       ```
 
     - Run stack-orchestrator:

test/helper.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"net/http"
 
-	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v3/test"
+	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
 )
 
 type PutResult struct {

test/integration_test.go

@@ -11,8 +11,8 @@ import (
 	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
 	integration "github.com/vulcanize/ipld-eth-db-validator/test"
 
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
-	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v3/test"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
+	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
 )
 
 const (

validator_test/validator_test.go

@@ -12,8 +12,8 @@ import (
 	"github.com/jmoiron/sqlx"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
 
 	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
 	"github.com/vulcanize/ipld-eth-db-validator/validator_test"
@@ -120,6 +120,9 @@ var _ = Describe("eth state reading tests", func() {
 			for i := uint64(blockHeight); i <= chainLength-trail; i++ {
 				err = validator.ValidateBlock(context.Background(), api, i)
 				Expect(err).ToNot(HaveOccurred())
+
+				err = validator.ValidateReferentialIntegrity(db, i)
+				Expect(err).ToNot(HaveOccurred())
 			}
 		})
 	})