interface changes from ethdb.Database and state/storage validation, run db container(s) for go test -v ./... in CICD

.github/workflows/basic-build-test.yml

@@ -21,3 +21,10 @@ jobs:
     - name: Build
       run: go build -v ./...
 
+    - name : Bring up docker DB containers
+      run: docker compose up -d
+    
+    - name: Run test
+      run: |
+        sleep 30;
+        go test -v ./...

cmd/validateTrie.go

@@ -97,9 +97,13 @@ func validateTrie() {
 		if contractAddrStr == "" {
 			logWithCommand.Fatal("must provide a contract address for storage trie validation")
 		}
+		if stateRootStr == "" {
+			logWithCommand.Fatal("must provide a state root for state trie validation")
+		}
 		storageRoot := common.HexToHash(storageRootStr)
 		addr := common.HexToAddress(contractAddrStr)
-		if err = v.ValidateStorageTrie(addr, storageRoot); err != nil {
+		stateRoot := common.HexToHash(stateRootStr)
+		if err = v.ValidateStorageTrie(stateRoot, addr, storageRoot); err != nil {
 			logWithCommand.Fatalf("Storage trie for contract %s and root %s not complete\r\nerr: %v", addr.String(), storageRoot.String(), err)
 		}
 		logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String())

docker-compose.yml

@@ -0,0 +1,27 @@
+version: "3.2"
+
+services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v4.2.3-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
+  ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
+    restart: always
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_testing"
+      POSTGRES_PASSWORD: "password"
+    ports:
+      - "127.0.0.1:8077:5432"
+    volumes:
+      - ./statediff/indexer/database/file:/file_indexer

pkg/validator.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/spf13/viper"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -61,6 +62,17 @@ var (
 	emptyCodeHash         = crypto.Keccak256(nil)
 )
 
+type KVSWithAncient struct {
+	kvs ethdb.KeyValueStore
+	ethdb.Database
+}
+
+func NewKVSDatabaseWithAncient(kvs ethdb.KeyValueStore) ethdb.Database {
+	return &KVSWithAncient{
+		kvs: kvs,
+	}
+}
+
 // NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database
 func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator {
 	kvs := pgipfsethdb.NewKeyValueStore(db, pgipfsethdb.CacheConfig{
@@ -78,7 +90,7 @@ func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator {
 	normalizeParams(&par)
 	return &Validator{
 		kvs:           kvs,
-		trieDB:        trie.NewDatabase(kvs),
+		trieDB:        trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
 		stateDatabase: state.NewDatabase(database),
 		db:            database.(*pgipfsethdb.Database),
 		params:        par,
@@ -96,7 +108,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
 	normalizeParams(&par)
 	return &Validator{
 		kvs:           kvs,
-		trieDB:        trie.NewDatabase(kvs),
+		trieDB:        trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
 		stateDatabase: state.NewDatabase(database),
 		params:        par,
 	}
@@ -108,7 +120,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
 func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
 	return &Validator{
 		kvs:           kvs,
-		trieDB:        trie.NewDatabase(kvs),
+		trieDB:        trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
 		stateDatabase: state.NewDatabase(database),
 	}
 }
@@ -147,10 +159,10 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
 }
 
 // ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete
-func (v *Validator) ValidateStorageTrie(address common.Address, storageRoot common.Hash) error {
+func (v *Validator) ValidateStorageTrie(stateRoot common.Hash, address common.Address, storageRoot common.Hash) error {
 	// Generate the state.NodeIterator for this root
 	addrHash := crypto.Keccak256Hash(address.Bytes())
-	t, err := v.stateDatabase.OpenStorageTrie(addrHash, storageRoot)
+	t, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot)
 	if err != nil {
 		return err
 	}
@@ -184,7 +196,7 @@ func (v *Validator) iterate(it trie.NodeIterator, storage bool) error {
 		if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil {
 			return err
 		}
-		dataTrie, err := v.stateDatabase.OpenStorageTrie(common.BytesToHash(it.LeafKey()), account.Root)
+		dataTrie, err := v.stateDatabase.OpenStorageTrie(common.HexToHash(viper.GetString("validator.stateRoot")), common.BytesToHash(it.LeafKey()), account.Root)
 		if err != nil {
 			return err
 		}

pkg/validator_test.go

@@ -203,7 +203,7 @@ var (
 
 	config = validator.Config{
 		Hostname: "localhost",
-		Name:     "cerc_testing",
+		Name:     "vulcanize_testing",
 		User:     "vdbm",
 		Password: "password",
 		Port:     8077,
@@ -308,19 +308,19 @@ var _ = Describe("PG-IPFS Validator", func() {
 		})
 		It("Returns an error the storage root node is missing", func() {
 			loadTrie(nil, missingRootStorageNodes)
-			err = v.ValidateStorageTrie(contractAddr, storageRoot)
+			err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("missing trie node"))
 		})
 		It("Returns an error if the entire storage trie cannot be validated", func() {
 			loadTrie(nil, missingNodeStorageNodes)
-			err = v.ValidateStorageTrie(contractAddr, storageRoot)
+			err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("missing trie node"))
 		})
 		It("Returns no error if the entire storage trie can be validated", func() {
 			loadTrie(nil, trieStorageNodes)
-			err = v.ValidateStorageTrie(contractAddr, storageRoot)
+			err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})