interface changes from ethdb.Database and state/storage validation, run db container(s) for go test -v ./... in CICD

This commit is contained in:
Michael Shaw 2023-03-10 15:04:05 -05:00
parent 31c3436624
commit 8f5cee5be6
5 changed files with 62 additions and 12 deletions

View File

@ -21,3 +21,10 @@ jobs:
- name: Build - name: Build
run: go build -v ./... run: go build -v ./...
- name : Bring up docker DB containers
run: docker compose up -d
- name: Run test
run: |
sleep 30;
go test -v ./...

View File

@ -97,9 +97,13 @@ func validateTrie() {
if contractAddrStr == "" { if contractAddrStr == "" {
logWithCommand.Fatal("must provide a contract address for storage trie validation") logWithCommand.Fatal("must provide a contract address for storage trie validation")
} }
if stateRootStr == "" {
logWithCommand.Fatal("must provide a state root for state trie validation")
}
storageRoot := common.HexToHash(storageRootStr) storageRoot := common.HexToHash(storageRootStr)
addr := common.HexToAddress(contractAddrStr) addr := common.HexToAddress(contractAddrStr)
if err = v.ValidateStorageTrie(addr, storageRoot); err != nil { stateRoot := common.HexToHash(stateRootStr)
if err = v.ValidateStorageTrie(stateRoot, addr, storageRoot); err != nil {
logWithCommand.Fatalf("Storage trie for contract %s and root %s not complete\r\nerr: %v", addr.String(), storageRoot.String(), err) logWithCommand.Fatalf("Storage trie for contract %s and root %s not complete\r\nerr: %v", addr.String(), storageRoot.String(), err)
} }
logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String()) logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String())

27
docker-compose.yml Normal file
View File

@ -0,0 +1,27 @@
version: "3.2"
services:
migrations:
restart: on-failure
depends_on:
- ipld-eth-db
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v4.2.3-alpha
environment:
DATABASE_USER: "vdbm"
DATABASE_NAME: "vulcanize_testing"
DATABASE_PASSWORD: "password"
DATABASE_HOSTNAME: "ipld-eth-db"
DATABASE_PORT: 5432
ipld-eth-db:
image: timescale/timescaledb:latest-pg14
restart: always
command: ["postgres", "-c", "log_statement=all"]
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_testing"
POSTGRES_PASSWORD: "password"
ports:
- "127.0.0.1:8077:5432"
volumes:
- ./statediff/indexer/database/file:/file_indexer

View File

@ -20,6 +20,7 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"github.com/spf13/viper"
"time" "time"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
@ -61,6 +62,17 @@ var (
emptyCodeHash = crypto.Keccak256(nil) emptyCodeHash = crypto.Keccak256(nil)
) )
type KVSWithAncient struct {
kvs ethdb.KeyValueStore
ethdb.Database
}
func NewKVSDatabaseWithAncient(kvs ethdb.KeyValueStore) ethdb.Database {
return &KVSWithAncient{
kvs: kvs,
}
}
// NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database // NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database
func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator { func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator {
kvs := pgipfsethdb.NewKeyValueStore(db, pgipfsethdb.CacheConfig{ kvs := pgipfsethdb.NewKeyValueStore(db, pgipfsethdb.CacheConfig{
@ -78,7 +90,7 @@ func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator {
normalizeParams(&par) normalizeParams(&par)
return &Validator{ return &Validator{
kvs: kvs, kvs: kvs,
trieDB: trie.NewDatabase(kvs), trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
stateDatabase: state.NewDatabase(database), stateDatabase: state.NewDatabase(database),
db: database.(*pgipfsethdb.Database), db: database.(*pgipfsethdb.Database),
params: par, params: par,
@ -96,7 +108,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
normalizeParams(&par) normalizeParams(&par)
return &Validator{ return &Validator{
kvs: kvs, kvs: kvs,
trieDB: trie.NewDatabase(kvs), trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
stateDatabase: state.NewDatabase(database), stateDatabase: state.NewDatabase(database),
params: par, params: par,
} }
@ -108,7 +120,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator { func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
return &Validator{ return &Validator{
kvs: kvs, kvs: kvs,
trieDB: trie.NewDatabase(kvs), trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
stateDatabase: state.NewDatabase(database), stateDatabase: state.NewDatabase(database),
} }
} }
@ -147,10 +159,10 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
} }
// ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete // ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete
func (v *Validator) ValidateStorageTrie(address common.Address, storageRoot common.Hash) error { func (v *Validator) ValidateStorageTrie(stateRoot common.Hash, address common.Address, storageRoot common.Hash) error {
// Generate the state.NodeIterator for this root // Generate the state.NodeIterator for this root
addrHash := crypto.Keccak256Hash(address.Bytes()) addrHash := crypto.Keccak256Hash(address.Bytes())
t, err := v.stateDatabase.OpenStorageTrie(addrHash, storageRoot) t, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot)
if err != nil { if err != nil {
return err return err
} }
@ -184,7 +196,7 @@ func (v *Validator) iterate(it trie.NodeIterator, storage bool) error {
if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil { if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil {
return err return err
} }
dataTrie, err := v.stateDatabase.OpenStorageTrie(common.BytesToHash(it.LeafKey()), account.Root) dataTrie, err := v.stateDatabase.OpenStorageTrie(common.HexToHash(viper.GetString("validator.stateRoot")), common.BytesToHash(it.LeafKey()), account.Root)
if err != nil { if err != nil {
return err return err
} }

View File

@ -203,7 +203,7 @@ var (
config = validator.Config{ config = validator.Config{
Hostname: "localhost", Hostname: "localhost",
Name: "cerc_testing", Name: "vulcanize_testing",
User: "vdbm", User: "vdbm",
Password: "password", Password: "password",
Port: 8077, Port: 8077,
@ -308,19 +308,19 @@ var _ = Describe("PG-IPFS Validator", func() {
}) })
It("Returns an error the storage root node is missing", func() { It("Returns an error the storage root node is missing", func() {
loadTrie(nil, missingRootStorageNodes) loadTrie(nil, missingRootStorageNodes)
err = v.ValidateStorageTrie(contractAddr, storageRoot) err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node")) Expect(err.Error()).To(ContainSubstring("missing trie node"))
}) })
It("Returns an error if the entire storage trie cannot be validated", func() { It("Returns an error if the entire storage trie cannot be validated", func() {
loadTrie(nil, missingNodeStorageNodes) loadTrie(nil, missingNodeStorageNodes)
err = v.ValidateStorageTrie(contractAddr, storageRoot) err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node")) Expect(err.Error()).To(ContainSubstring("missing trie node"))
}) })
It("Returns no error if the entire storage trie can be validated", func() { It("Returns no error if the entire storage trie can be validated", func() {
loadTrie(nil, trieStorageNodes) loadTrie(nil, trieStorageNodes)
err = v.ValidateStorageTrie(contractAddr, storageRoot) err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
}) })