Feature/update go geth sharding (#13 )

* Upgrade Geth, Go and add CICD * update ipld-eth-server version * Track validation progress on a channel * Add integration tests * Setup validator config and update instructions to run tests locally * Update readme and tests * Update test to use v4 Infrastructure * Inlcude the env file * Fix config file write * Update DB configuration Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com> Co-authored-by: Ashwin Phatak <ashwinpphatak@gmail.com>
Add checks to validate referential integrity in the data (#7 )
2022-06-03 09:28:02 -04:00 · 2022-06-01 11:29:53 +05:30 · 2022-05-31 13:44:36 +05:30 · 2022-05-31 13:09:45 +05:30 · 2022-05-23 20:01:52 +05:30 · 2022-05-23 19:43:21 +05:30
22 changed files with 1164 additions and 368 deletions
@@ -0,0 +1,6 @@
+name: Run all tests
+on: [pull_request]
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/tests.yml
@@ -0,0 +1,113 @@
+name: Test the stack.
+on:
+  workflow_call:
+
+env:
+  STACK_ORCHESTRATOR_REF: 418957a1f745c921b21286c13bb033f922a91ae9
+  GO_ETHEREUM_REF: "v1.10.18-statediff-4.0.2-alpha"
+  IPLD_ETH_DB_REF: 91d30b9ea1acecd0a7f4307390a98bf3e289b8d7
+
+jobs:
+  integrationtest:
+    name: Run integration tests
+    env:
+      GOPATH: /tmp/go
+      DB_WRITE: true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-db-validator"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.GO_ETHEREUM_REF }}
+          repository: vulcanize/go-ethereum
+          path: "./go-ethereum/"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.IPLD_ETH_DB_REF }}
+          repository: vulcanize/ipld-eth-db
+          path: "./ipld-eth-db/"
+      - name: Create config file
+        run: |
+          echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> ./config.sh
+          echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh
+          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
+          echo db_write=$DB_WRITE >> ./config.sh
+          echo go_ethereum_db_dependency=access-node >> ./config.sh
+          echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
+          cat ./config.sh
+      - name: Build geth
+        run: |
+          cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
+          ./compile-geth.sh \
+          -p "$GITHUB_WORKSPACE/config.sh" \
+          -e docker
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
+          --env-file "$GITHUB_WORKSPACE/config.sh" \
+          up -d --build
+      - name: Run integration test.
+        run: |
+          cd $GITHUB_WORKSPACE/ipld-eth-db-validator
+          ./scripts/run_integration_test.sh
+
+  unittest:
+    name: Run unit tests
+    env:
+      GOPATH: /tmp/go
+      DB_WRITE: true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-db-validator"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.IPLD_ETH_DB_REF }}
+          repository: vulcanize/ipld-eth-db
+          path: "./ipld-eth-db/"
+      - name: Create config file
+        run: |
+          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
+          echo db_write=$DB_WRITE >> ./config.sh
+          echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
+          cat ./config.sh
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
+          --env-file "$GITHUB_WORKSPACE/config.sh" \
+          up -d --build
+      - name: Run unit test.
+        run: |
+          cd $GITHUB_WORKSPACE/ipld-eth-db-validator
+          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8066 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing_v4 make test
@@ -3,22 +3,18 @@ BASE = $(GOPATH)/src/$(PACKAGE)
 PKGS = go list ./... | grep -v "^vendor/"

 # Tools
-## Testing library
-GINKGO = $(BIN)/ginkgo
-$(BIN)/ginkgo:
-	go install github.com/onsi/ginkgo/ginkgo

 .PHONY: integrationtest
-integrationtest: | $(GINKGO) $(GOOSE)
+integrationtest: | $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	$(GINKGO) -r test/ -v
+	go run github.com/onsi/ginkgo/ginkgo -r test/ -v

 .PHONY: test
-test: | $(GINKGO) $(GOOSE)
+test: | $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	$(GINKGO) -r validator_test/ -v
+	go run github.com/onsi/ginkgo/ginkgo -r validator_test/ -v

 build:
 	go fmt ./...
@@ -7,7 +7,6 @@ import (
 	"os/signal"
 	"sync"

-	"github.com/ethereum/go-ethereum/statediff"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -34,20 +33,7 @@ func stateValidator() {
 		logWithCommand.Fatal(err)
 	}

-	height := viper.GetUint64("validate.block-height")
-	if height < 1 {
-		logWithCommand.Fatalf("block height cannot be less the 1")
-	}
-	trail := viper.GetUint64("validate.trail")
-	sleepInterval := viper.GetUint("validate.sleepInterval")
-
-	chainConfigPath := viper.GetString("ethereum.chainConfig")
-	chainCfg, err := statediff.LoadConfig(chainConfigPath)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
-	service := validator.NewService(cfg.DB, height, trail, sleepInterval, chainCfg)
+	service := validator.NewService(cfg, nil)

 	wg := new(sync.WaitGroup)
 	wg.Add(1)
@@ -1,18 +1,30 @@
 version: '3.2'

 services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: vulcanize/ipld-eth-db:v4.1.1-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
  ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
    restart: always
-    image: vulcanize/ipld-eth-db:v3.2.0
+    command: ["postgres", "-c", "log_statement=all"]
    environment:
      POSTGRES_USER: "vdbm"
      POSTGRES_DB: "vulcanize_testing"
      POSTGRES_PASSWORD: "password"
-    volumes:
-      - vdb_db_eth_validator:/var/lib/postgresql/data
    ports:
      - "127.0.0.1:8077:5432"
-    command: ["postgres", "-c", "log_statement=all"]
+    volumes:
+      - vdb_db_eth_validator:/var/lib/postgresql/data

 volumes:
-  vdb_db_eth_validator:
+  vdb_db_eth_validator:
@@ -1,17 +1,18 @@
 module github.com/vulcanize/ipld-eth-db-validator

-go 1.17
+go 1.18

 require (
-	github.com/ethereum/go-ethereum v1.10.17
+	github.com/ethereum/go-ethereum v1.10.18
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.11.0
-	github.com/vulcanize/ipfs-ethdb/v3 v3.0.1
-	github.com/vulcanize/ipld-eth-server/v3 v3.1.0
+	github.com/vulcanize/ipfs-ethdb/v4 v4.0.1-alpha
+	github.com/vulcanize/ipld-eth-server/v3 v3.2.1
+	github.com/vulcanize/ipld-eth-server/v4 v4.0.2-alpha
 )

 require (
@@ -60,7 +61,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/holiman/uint256 v1.2.0 // indirect
-	github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204 // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-bitswap v0.4.0 // indirect
@@ -171,11 +172,12 @@ require (
 	github.com/libp2p/go-ws-transport v0.5.0 // indirect
 	github.com/libp2p/go-yamux/v2 v2.2.0 // indirect
 	github.com/libp2p/zeroconf/v2 v2.0.0 // indirect
-	github.com/lucas-clemente/quic-go v0.23.0 // indirect
+	github.com/lucas-clemente/quic-go v0.26.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailgun/groupcache/v2 v2.3.0 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect
+	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
@@ -233,7 +235,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.6 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
-	github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.0 // indirect
+	github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.1-alpha // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
@@ -270,4 +272,4 @@ require (
 	lukechampine.com/blake3 v1.1.6 // indirect
 )

-replace github.com/ethereum/go-ethereum v1.10.17 => github.com/vulcanize/go-ethereum v1.10.17-statediff-3.2.1
+replace github.com/ethereum/go-ethereum v1.10.18 => github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha
@@ -6,10 +6,11 @@ import (
 	"time"

 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/jmoiron/sqlx"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
 )

 var (
@@ -59,11 +60,35 @@ var TestChainConfig = &params.ChainConfig{
 type Config struct {
 	dbConfig postgres.Config
 	DB       *sqlx.DB
+
+	ChainCfg *params.ChainConfig
+
+	BlockNum, Trail uint64
+	SleepInterval   uint
 }

 func NewConfig() (*Config, error) {
 	cfg := new(Config)
-	return cfg, cfg.setupDB()
+	err := cfg.setupDB()
+	if err != nil {
+		return nil, err
+	}
+
+	cfg.BlockNum = viper.GetUint64("validate.block-height")
+	if cfg.BlockNum < 1 {
+		return nil, fmt.Errorf("block height cannot be less the 1")
+	}
+
+	cfg.Trail = viper.GetUint64("validate.trail")
+	cfg.SleepInterval = viper.GetUint("validate.sleepInterval")
+
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+	cfg.ChainCfg, err = statediff.LoadConfig(chainConfigPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
 }

 func (c *Config) setupDB() error {
@@ -1,9 +1,13 @@
 package validator

 import (
+	"errors"
+
 	"github.com/ethereum/go-ethereum/ethdb"
 )

+var errNotSupported = errors.New("this operation is not supported")
+
 type database struct {
 	ethDB ethdb.Database
 }
@@ -105,7 +109,7 @@ func (d *database) NewBatchWithSize(size int) ethdb.Batch {
 	return d.ethDB.NewBatchWithSize(size)
 }

-func (d *database) ReadAncients(fn func(ethdb.AncientReader) error) (err error) {
+func (d *database) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
 	return d.ethDB.ReadAncients(fn)
 }

@@ -117,3 +121,8 @@ func (d *database) Close() error {
 func (d *database) NewSnapshot() (ethdb.Snapshot, error) {
 	return d.NewSnapshot()
 }
+
+// NewSnapshot creates a database snapshot based on the current state.
+func (d *database) AncientDatadir() (string, error) {
+	return "", errNotSupported
+}
@@ -0,0 +1,224 @@
+package validator
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+)
+
+// ValidateReferentialIntegrity validates referential integrity at the given height
+func ValidateReferentialIntegrity(db *sqlx.DB, blockNumber uint64) error {
+
+	err := ValidateHeaderCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateUncleCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateTransactionCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateReceiptCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStateCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStorageCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateStateAccountsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateAccessListElementsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	err = ValidateLogCIDsRef(db, blockNumber)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateHeaderCIDsRef does a reference integrity check on references in eth.header_cids table
+func ValidateHeaderCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	err := ValidateIPFSBlocks(db, blockNumber, "eth.header_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateUncleCIDsRef does a reference integrity check on references in eth.uncle_cids table
+func ValidateUncleCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, UncleCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.uncle_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateTransactionCIDsRef does a reference integrity check on references in eth.header_cids table
+func ValidateTransactionCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, TransactionCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.transaction_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateReceiptCIDsRef does a reference integrity check on references in eth.receipt_cids table
+func ValidateReceiptCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, ReceiptCIDsRefTransactionCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.receipt_cids", "leaf_mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStateCIDsRef does a reference integrity check on references in eth.state_cids table
+func ValidateStateCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StateCIDsRefHeaderCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.state_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStorageCIDsRef does a reference integrity check on references in eth.storage_cids table
+func ValidateStorageCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StorageCIDsRefStateCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.storage_cids", "mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateStateAccountsRef does a reference integrity check on references in eth.state_accounts table
+func ValidateStateAccountsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, StateAccountsRefStateCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
+	}
+
+	return nil
+}
+
+// ValidateAccessListElementsRef does a reference integrity check on references in eth.access_list_elements table
+func ValidateAccessListElementsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, AccessListElementsRefTransactionCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
+	}
+
+	return nil
+}
+
+// ValidateLogCIDsRef does a reference integrity check on references in eth.log_cids table
+func ValidateLogCIDsRef(db *sqlx.DB, blockNumber uint64) error {
+	var exists bool
+	err := db.Get(&exists, LogCIDsRefReceiptCIDs, blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.receipt_cids")
+	}
+
+	err = ValidateIPFSBlocks(db, blockNumber, "eth.log_cids", "leaf_mh_key")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ValidateIPFSBlocks does a reference integrity check between the given CID table and IPFS blocks table on MHKey and block number
+func ValidateIPFSBlocks(db *sqlx.DB, blockNumber uint64, CIDTable string, mhKeyField string) error {
+	var exists bool
+	err := db.Get(&exists, fmt.Sprintf(CIDsRefIPLDBlocks, CIDTable, mhKeyField), blockNumber)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "public.blocks")
+	}
+
+	return nil
+}
@@ -0,0 +1,119 @@
+package validator
+
+// Queries to validate referential integrity in the indexed data:
+// At the given block number,
+// In each table, for each (would be) foreign key reference, perform left join with the referenced table on the foreign key fields.
+// Select rows where there are no matching rows in the referenced table.
+// If any such rows exist, there are missing entries in the referenced table.
+
+const (
+	CIDsRefIPLDBlocks = `SELECT EXISTS (
+						SELECT *
+						FROM %[1]s
+						LEFT JOIN public.blocks ON (
+							%[1]s.%[2]s = blocks.key
+							AND %[1]s.block_number = blocks.block_number
+						)
+						WHERE
+							%[1]s.block_number = $1
+							AND blocks.key IS NULL
+					)`
+
+	UncleCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.uncle_cids
+						LEFT JOIN eth.header_cids ON (
+							uncle_cids.header_id = header_cids.block_hash
+							AND uncle_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							uncle_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	TransactionCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.transaction_cids
+						LEFT JOIN eth.header_cids ON (
+							transaction_cids.header_id = header_cids.block_hash
+							AND transaction_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							transaction_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	ReceiptCIDsRefTransactionCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.receipt_cids
+						LEFT JOIN eth.transaction_cids ON (
+							receipt_cids.tx_id = transaction_cids.tx_hash
+							AND receipt_cids.block_number = transaction_cids.block_number
+						)
+						WHERE
+							receipt_cids.block_number = $1
+							AND transaction_cids.tx_hash IS NULL
+					)`
+
+	StateCIDsRefHeaderCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.state_cids
+						LEFT JOIN eth.header_cids ON (
+							state_cids.header_id = header_cids.block_hash
+							AND state_cids.block_number = header_cids.block_number
+						)
+						WHERE
+							state_cids.block_number = $1
+							AND header_cids.block_hash IS NULL
+					)`
+
+	StorageCIDsRefStateCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.storage_cids
+						LEFT JOIN eth.state_cids ON (
+							storage_cids.state_path = state_cids.state_path
+							AND storage_cids.header_id = state_cids.header_id
+							AND storage_cids.block_number = state_cids.block_number
+						)
+						WHERE
+							storage_cids.block_number = $1
+							AND state_cids.state_path IS NULL
+					)`
+
+	StateAccountsRefStateCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.state_accounts
+						LEFT JOIN eth.state_cids ON (
+							state_accounts.state_path = state_cids.state_path
+							AND state_accounts.header_id = state_cids.header_id
+							AND state_accounts.block_number = state_cids.block_number
+						)
+						WHERE
+							state_accounts.block_number = $1
+							AND state_cids.state_path IS NULL
+					)`
+
+	AccessListElementsRefTransactionCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.access_list_elements
+						LEFT JOIN eth.transaction_cids ON (
+							access_list_elements.tx_id = transaction_cids.tx_hash
+							AND access_list_elements.block_number = transaction_cids.block_number
+						)
+						WHERE
+							access_list_elements.block_number = $1
+							AND transaction_cids.tx_hash IS NULL
+					)`
+
+	LogCIDsRefReceiptCIDs = `SELECT EXISTS (
+						SELECT *
+						FROM eth.log_cids
+						LEFT JOIN eth.receipt_cids ON (
+							log_cids.rct_id = receipt_cids.tx_id
+							AND log_cids.block_number = receipt_cids.block_number
+						)
+						WHERE
+							log_cids.block_number = $1
+							AND receipt_cids.tx_id IS NULL
+					)`
+)
@@ -0,0 +1,334 @@
+package validator_test
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
+)
+
+var _ = Describe("RefIntegrity", func() {
+	var (
+		ctx = context.Background()
+
+		db          *sqlx.DB
+		diffIndexer interfaces.StateDiffIndexer
+	)
+
+	BeforeEach(func() {
+		db = shared.SetupDB()
+		diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
+	})
+
+	AfterEach(func() {
+		shared.TearDownDB(db)
+	})
+
+	Describe("ValidateHeaderCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of header_cids table", func() {
+			err := validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateUncleCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of uncle_cids table", func() {
+			err := validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cid entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding uncle IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateTransactionCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of transaction_cids table", func() {
+			err := validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cid entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding transaction IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateReceiptCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of receipt_cids table", func() {
+			err := validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding transaction_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.transaction_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
+		})
+
+		It("Throws an error if corresponding receipt IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStateCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of state_cids table", func() {
+			err := validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding header_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.header_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
+		})
+
+		It("Throws an error if corresponding state IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStorageCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of storage_cids table", func() {
+			err := validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding state_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.state_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
+		})
+
+		It("Throws an error if corresponding storage IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+	Describe("ValidateStateAccountsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of state_accounts table", func() {
+			err := validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding state_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.state_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
+		})
+	})
+
+	Describe("ValidateAccessListElementsRef", func() {
+		BeforeEach(func() {
+			indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, mocks.TestConfig, test_helpers.Genesis.Hash())
+
+			tx, err := indexAndPublisher.PushBlock(mocks.MockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of access_list_elements table", func() {
+			err := validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding transaction_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.transaction_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
+		})
+	})
+
+	Describe("ValidateLogCIDsRef", func() {
+		BeforeEach(func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Validates referential integrity of log_cids table", func() {
+			err := validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("Throws an error if corresponding receipt_cids entry not found", func() {
+			err := deleteEntriesFrom(db, "eth.receipt_cids")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.receipt_cids"))
+		})
+
+		It("Throws an error if corresponding log IPFS block entry not found", func() {
+			err := deleteEntriesFrom(db, "public.blocks")
+			Expect(err).ToNot(HaveOccurred())
+
+			err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
+		})
+	})
+
+})
+
+func deleteEntriesFrom(db *sqlx.DB, tableName string) error {
+	pgStr := "DELETE FROM %s"
+	_, err := db.Exec(fmt.Sprintf(pgStr, tableName))
+	return err
+}
@@ -20,14 +20,18 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
-	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres"
-	ipldEth "github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-	ethServerShared "github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres"
+	ipldEth "github.com/vulcanize/ipld-eth-server/v4/pkg/eth"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
 )

 var (
 	big8  = big.NewInt(8)
 	big32 = big.NewInt(32)
+
+	ReferentialIntegrityErr = "referential integrity check failed at block %d, entry for %s not found"
+	EntryNotFoundErr        = "entry for %s not found"
 )

 type service struct {
@@ -37,17 +41,19 @@ type service struct {
 	logger          *log.Logger
 	chainCfg        *params.ChainConfig
 	quitChan        chan bool
+	progressChan    chan uint64
 }

-func NewService(db *sqlx.DB, blockNum, trailNum uint64, sleepInterval uint, chainCfg *params.ChainConfig) *service {
+func NewService(cfg *Config, progressChan chan uint64) *service {
 	return &service{
-		db:            db,
-		blockNum:      blockNum,
-		trail:         trailNum,
-		sleepInterval: sleepInterval,
+		db:            cfg.DB,
+		blockNum:      cfg.BlockNum,
+		trail:         cfg.Trail,
+		sleepInterval: cfg.SleepInterval,
 		logger:        log.New(),
-		chainCfg:      chainCfg,
+		chainCfg:      cfg.ChainCfg,
 		quitChan:      make(chan bool),
+		progressChan:  progressChan,
 	}
 }

@@ -96,7 +102,9 @@ func (s *service) Start(ctx context.Context, wg *sync.WaitGroup) {
 		select {
 		case <-s.quitChan:
 			s.logger.Infof("last validated block %v", idxBlockNum-1)
-			s.logger.Info("stopping ipld-eth-db-validator process")
+			if s.progressChan != nil {
+				close(s.progressChan)
+			}
 			return
 		default:
 			idxBlockNum, err = s.Validate(ctx, api, idxBlockNum)
@@ -111,6 +119,7 @@ func (s *service) Start(ctx context.Context, wg *sync.WaitGroup) {

 // Stop is used to gracefully stop the service
 func (s *service) Stop() {
+	s.logger.Info("stopping ipld-eth-db-validator process")
 	close(s.quitChan)
 }

@@ -129,6 +138,10 @@ func (s *service) Validate(ctx context.Context, api *ipldEth.PublicEthAPI, idxBl
 		}

 		s.logger.Infof("state root verified for block %d", idxBlockNum)
+		if s.progressChan != nil {
+			s.progressChan <- idxBlockNum
+		}
+
 		idxBlockNum++
 	} else {
 		// Sleep / wait for head to move ahead
@@ -0,0 +1,13 @@
+package validator_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestValidator(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Validator Suite")
+}
@@ -1,12 +1,14 @@
+#!/bin/bash
+
 set -e
 set -o xtrace

 export PGPASSWORD=password
 export DATABASE_USER=vdbm
-export DATABASE_PORT=8077
+export DATABASE_PORT=8066
 export DATABASE_PASSWORD=password
 export DATABASE_HOSTNAME=127.0.0.1
-export DATABASE_NAME=vulcanize_testing
+export DATABASE_NAME=vulcanize_testing_v4

 # Wait for containers to be up and execute the integration test.
 while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
@@ -1,9 +1,17 @@
+#!/bin/bash
+
+set -e
+
 # Clear up existing docker images and volume.
 docker-compose down --remove-orphans --volumes

-docker-compose -f docker-compose.yml up -d ipld-eth-db
-sleep 10
+# Spin up TimescaleDB
+docker-compose -f docker-compose.yml up -d migrations ipld-eth-db
+sleep 45

+# Run unit tests
+go clean -testcache
 PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test

+# Clean up
 docker-compose down --remove-orphans --volumes
@@ -1,4 +1,4 @@
-# Test Insructions
+# Test Instructions

 ## Setup

@@ -6,18 +6,18 @@

  - Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator) and [go-ethereum](https://github.com/vulcanize/go-ethereum) repositories.

-  - Checkout [v3 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.17-statediff-3.2.1) in go-ethereum repo.
+  - Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-4.0.2-alpha) in go-ethereum repo.

    ```bash
    # In go-ethereum repo.
-    git checkout v1.10.17-statediff-3.2.1
+    git checkout v1.10.18-statediff-4.0.2-alpha
    ```

  - Checkout working commit in stack-orchestrator repo.

    ```bash
    # In stack-orchestrator repo.
-    git checkout 3bb1796a59827fb755410c5ce69fac567a0f832b
+    git checkout 418957a1f745c921b21286c13bb033f922a91ae9
    ```

 ## Run
@@ -55,6 +55,10 @@
      vulcanize_test_contract=~/ipld-eth-db-validator/test/contract

      db_write=true
+      ipld_eth_server_db_dependency=access-node
+      go_ethereum_db_dependency=access-node
+
+      connecting_db_name=vulcanize_testing_v4
      ```

    - Run stack-orchestrator:
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: AGPL-3.0
+pragma solidity ^0.8.0;
+
+contract Test {
+  address payable owner;
+
+  modifier onlyOwner {
+    require(
+      msg.sender == owner,
+      "Only owner can call this function."
+    );
+    _;
+  }
+
+  uint256[100] data;
+
+  constructor() {
+    owner = payable(msg.sender);
+    data = [1];
+  }
+
+  function Put(uint256 addr, uint256 value) public {
+    data[addr] = value;
+  }
+
+  function close() public onlyOwner {
+    selfdestruct(owner);
+  }
+}
@@ -63,6 +63,49 @@ fastify.get('/v1/sendEth', async (req, reply) => {
    }
 });

+fastify.get('/v1/deployTestContract', async (req, reply) => {
+    const testContract = await hre.ethers.getContractFactory("Test");
+    const test = await testContract.deploy();
+    await test.deployed();
+
+    return {
+        address: test.address,
+        txHash: test.deployTransaction.hash,
+        blockNumber: test.deployTransaction.blockNumber,
+        blockHash: test.deployTransaction.blockHash,
+    }
+});
+
+fastify.get('/v1/putTestValue', async (req, reply) => {
+    const addr = req.query.addr;
+    const index = req.query.index;
+    const value = req.query.value;
+
+    const testContract = await hre.ethers.getContractFactory("Test");
+    const test = await testContract.attach(addr);
+
+    const tx = await test.Put(index, value);
+    const receipt = await tx.wait();
+
+    return {
+        blockNumber: receipt.blockNumber,
+    }
+});
+
+fastify.get('/v1/destroyTestContract', async (req, reply) => {
+    const addr = req.query.addr;
+
+    const testContract = await hre.ethers.getContractFactory("Test");
+    const test = await testContract.attach(addr);
+
+    await test.destroy();
+    const blockNum = await hre.ethers.provider.getBlockNumber()
+
+    return {
+        blockNumber: blockNum,
+    }
+})
+
 async function main() {
    try {
        await fastify.listen(3000, '0.0.0.0');
@@ -72,4 +115,4 @@ async function main() {
    }
 }

-main();
+main();
@@ -0,0 +1,54 @@
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
+)
+
+type PutResult struct {
+	BlockNumber int64 `json:"blockNumber"`
+}
+
+const srvUrl = "http://localhost:3000"
+
+func DeployTestContract() (*ethServerIntegration.ContractDeployed, error) {
+	ethServerIntegration.DeployContract()
+	res, err := http.Get(fmt.Sprintf("%s/v1/deployTestContract", srvUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	var contract ethServerIntegration.ContractDeployed
+	decoder := json.NewDecoder(res.Body)
+
+	return &contract, decoder.Decode(&contract)
+}
+
+func PutTestValue(addr string, index, value int) (*PutResult, error) {
+	res, err := http.Get(fmt.Sprintf("%s/v1/putTestValue?addr=%s&index=%d&value=%d", srvUrl, addr, index, value))
+	if err != nil {
+		return nil, err
+	}
+
+	var blockNumber PutResult
+	decoder := json.NewDecoder(res.Body)
+
+	return &blockNumber, decoder.Decode(&blockNumber)
+}
+
+func DestroyTestContract(addr string) (*ethServerIntegration.ContractDestroyed, error) {
+	res, err := http.Get(fmt.Sprintf("%s/v1/destroyTestContract?addr=%s", srvUrl, addr))
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	var data ethServerIntegration.ContractDestroyed
+	decoder := json.NewDecoder(res.Body)
+
+	return &data, decoder.Decode(&data)
+}
@@ -2,45 +2,107 @@ package integration_test

 import (
 	"context"
+	"sync"
 	"time"

 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

 	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
+	integration "github.com/vulcanize/ipld-eth-db-validator/test"

-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
-	integration "github.com/vulcanize/ipld-eth-server/v3/test"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
+	ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
 )

 const (
+	blockNum               = 1
 	trail                  = 0
 	validatorSleepInterval = uint(5)
 )

+var (
+	testAddresses = []string{
+		"0x1111111111111111111111111111111111111112",
+		"0x1ca7c995f8eF0A2989BbcE08D5B7Efe50A584aa1",
+		"0x9a4b666af23a2cdb4e5538e1d222a445aeb82134",
+		"0xF7C7AEaECD2349b129d5d15790241c32eeE4607B",
+		"0x992b6E9BFCA1F7b0797Cee10b0170E536EAd3532",
+		"0x29ed93a7454Bc17a8D4A24D0627009eE0849B990",
+		"0x66E3dCA826b04B5d4988F7a37c91c9b1041e579D",
+		"0x96288939Ac7048c27E0E087b02bDaad3cd61b37b",
+		"0xD354280BCd771541c935b15bc04342c26086FE9B",
+		"0x7f887e25688c274E77b8DeB3286A55129B55AF14",
+	}
+)
+
 var _ = Describe("Integration test", func() {
 	ctx := context.Background()

-	var contract *integration.ContractDeployed
-	var contractErr error
-	sleepInterval := 5 * time.Second
+	var contract *ethServerIntegration.ContractDeployed
+	var err error
+	sleepInterval := 2 * time.Second
+	timeout := 4 * time.Second
+
+	db := shared.SetupDB()
+	cfg := validator.Config{
+		DB:            db,
+		BlockNum:      blockNum,
+		Trail:         trail,
+		SleepInterval: validatorSleepInterval,
+		ChainCfg:      validator.IntegrationTestChainConfig,
+	}
+	validationProgressChan := make(chan uint64)
+	service := validator.NewService(&cfg, validationProgressChan)
+
+	wg := new(sync.WaitGroup)
+
+	It("test init", func() {
+		wg.Add(1)
+		go service.Start(ctx, wg)
+
+		// Deploy a dummy contract as the first contract might get deployed at block number 0
+		_, _ = ethServerIntegration.DeployContract()
+		time.Sleep(sleepInterval)
+	})
+
+	defer It("test teardown", func() {
+		service.Stop()
+		wg.Wait()
+
+		Expect(validationProgressChan).To(BeClosed())
+	})

 	Describe("Validate state", func() {
-		BeforeEach(func() {
-			// Deploy a dummy contract as the first contract might get deployed at block number 0
-			_, _ = integration.DeployContract()
+		It("performs validation on contract deployment", func() {
+			contract, err = integration.DeployTestContract()
+			Expect(err).ToNot(HaveOccurred())
 			time.Sleep(sleepInterval)

-			contract, contractErr = integration.DeployContract()
-			time.Sleep(sleepInterval)
+			Expect(validationProgressChan).ToNot(BeClosed())
+			Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(contract.BlockNumber))))
 		})

-		It("Validate state root", func() {
-			Expect(contractErr).ToNot(HaveOccurred())
+		It("performs validation on contract transactions", func() {
+			for i := 0; i < 10; i++ {
+				res, txErr := integration.PutTestValue(contract.Address, i, i)
+				Expect(txErr).ToNot(HaveOccurred())
+				time.Sleep(sleepInterval)

-			db := shared.SetupDB()
-			srvc := validator.NewService(db, uint64(contract.BlockNumber), trail, validatorSleepInterval, validator.IntegrationTestChainConfig)
-			srvc.Start(ctx, nil)
+				Expect(validationProgressChan).ToNot(BeClosed())
+				Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(res.BlockNumber))))
+			}
+		})
+
+		It("performs validation on eth transfer transactions", func() {
+			for _, address := range testAddresses {
+				tx, txErr := ethServerIntegration.SendEth(address, "0.01")
+				Expect(txErr).ToNot(HaveOccurred())
+				time.Sleep(sleepInterval)
+
+				Expect(validationProgressChan).ToNot(BeClosed())
+				Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(tx.BlockNumber))))
+			}
 		})
 	})
 })
@@ -12,8 +12,8 @@ import (
 	"github.com/jmoiron/sqlx"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
+	"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"

 	"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
 	"github.com/vulcanize/ipld-eth-db-validator/validator_test"
@@ -120,6 +120,9 @@ var _ = Describe("eth state reading tests", func() {
 			for i := uint64(blockHeight); i <= chainLength-trail; i++ {
 				err = validator.ValidateBlock(context.Background(), api, i)
 				Expect(err).ToNot(HaveOccurred())
+
+				err = validator.ValidateReferentialIntegrity(db, i)
+				Expect(err).ToNot(HaveOccurred())
 			}
 		})
 	})
Author	SHA1	Message	Date
Abdul Rabbani prathamesh Ashwin Phatak	d111dd85db	Feature/update go geth sharding (#13 ) * Upgrade Geth, Go and add CICD * update ipld-eth-server version * Track validation progress on a channel * Add integration tests * Setup validator config and update instructions to run tests locally * Update readme and tests * Update test to use v4 Infrastructure * Inlcude the env file * Fix config file write * Update DB configuration Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com> Co-authored-by: Ashwin Phatak <ashwinpphatak@gmail.com>	2022-06-03 09:28:02 -04:00
prathamesh0	cc935dc97b	Add checks to validate referential integrity in the data (#7 ) * Add checks to validate referential integrity * Add unit tests for referential integrity validation * Use EXISTS in referential integrity validation queries	2022-06-01 11:29:53 +05:30
Ashwin Phatak	effbdc3f58	Merge pull request #9 from deep-stack/pm-merge-perpetual Merge changes to make validation process perpetual	2022-05-31 13:44:36 +05:30
prathamesh	31cc17b70f	Merge changes to make validation process perpetual	2022-05-31 13:09:45 +05:30
Ashwin Phatak	e89d64aa72	Merge pull request #5 from deep-stack/pm-v4-changes Upgrade dependencies to v4	2022-05-23 20:01:52 +05:30
prathamesh	a56da1782a	Update instructions to run tests	2022-05-23 19:43:21 +05:30
prathamesh	e102a2b5ac	Upgrade dependencies to v4	2022-05-23 19:08:17 +05:30