From 8f5cee5be61a2d34f43638d78a3adb1db0bb5aa7 Mon Sep 17 00:00:00 2001 From: Michael Shaw Date: Fri, 10 Mar 2023 15:04:05 -0500 Subject: [PATCH] interface changes from ethdb.Database and state/storage validation, run db container(s) for go test -v ./... in CICD --- .github/workflows/basic-build-test.yml | 9 ++++++++- cmd/validateTrie.go | 6 +++++- docker-compose.yml | 27 ++++++++++++++++++++++++++ pkg/validator.go | 24 +++++++++++++++++------ pkg/validator_test.go | 8 ++++---- 5 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 docker-compose.yml diff --git a/.github/workflows/basic-build-test.yml b/.github/workflows/basic-build-test.yml index 03620c6..436c2db 100644 --- a/.github/workflows/basic-build-test.yml +++ b/.github/workflows/basic-build-test.yml @@ -20,4 +20,11 @@ jobs: - name: Build run: go build -v ./... - + + - name : Bring up docker DB containers + run: docker compose up -d + + - name: Run test + run: | + sleep 30; + go test -v ./... \ No newline at end of file diff --git a/cmd/validateTrie.go b/cmd/validateTrie.go index 61bdc62..bbd3525 100644 --- a/cmd/validateTrie.go +++ b/cmd/validateTrie.go @@ -97,9 +97,13 @@ func validateTrie() { if contractAddrStr == "" { logWithCommand.Fatal("must provide a contract address for storage trie validation") } + if stateRootStr == "" { + logWithCommand.Fatal("must provide a state root for state trie validation") + } storageRoot := common.HexToHash(storageRootStr) addr := common.HexToAddress(contractAddrStr) - if err = v.ValidateStorageTrie(addr, storageRoot); err != nil { + stateRoot := common.HexToHash(stateRootStr) + if err = v.ValidateStorageTrie(stateRoot, addr, storageRoot); err != nil { logWithCommand.Fatalf("Storage trie for contract %s and root %s not complete\r\nerr: %v", addr.String(), storageRoot.String(), err) } logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String()) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..12947ed --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + migrations: + restart: on-failure + depends_on: + - ipld-eth-db + image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v4.2.3-alpha + environment: + DATABASE_USER: "vdbm" + DATABASE_NAME: "vulcanize_testing" + DATABASE_PASSWORD: "password" + DATABASE_HOSTNAME: "ipld-eth-db" + DATABASE_PORT: 5432 + + ipld-eth-db: + image: timescale/timescaledb:latest-pg14 + restart: always + command: ["postgres", "-c", "log_statement=all"] + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_testing" + POSTGRES_PASSWORD: "password" + ports: + - "127.0.0.1:8077:5432" + volumes: + - ./statediff/indexer/database/file:/file_indexer diff --git a/pkg/validator.go b/pkg/validator.go index fed96c1..485202f 100644 --- a/pkg/validator.go +++ b/pkg/validator.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "fmt" + "github.com/spf13/viper" "time" "github.com/ethereum/go-ethereum/common" @@ -61,6 +62,17 @@ var ( emptyCodeHash = crypto.Keccak256(nil) ) +type KVSWithAncient struct { + kvs ethdb.KeyValueStore + ethdb.Database +} + +func NewKVSDatabaseWithAncient(kvs ethdb.KeyValueStore) ethdb.Database { + return &KVSWithAncient{ + kvs: kvs, + } +} + // NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator { kvs := pgipfsethdb.NewKeyValueStore(db, pgipfsethdb.CacheConfig{ @@ -78,7 +90,7 @@ func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator { normalizeParams(&par) return &Validator{ kvs: kvs, - trieDB: trie.NewDatabase(kvs), + trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)), stateDatabase: state.NewDatabase(database), db: database.(*pgipfsethdb.Database), params: par, @@ -96,7 +108,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator { normalizeParams(&par) return &Validator{ kvs: kvs, - trieDB: trie.NewDatabase(kvs), + trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)), stateDatabase: state.NewDatabase(database), params: par, } @@ -108,7 +120,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator { func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator { return &Validator{ kvs: kvs, - trieDB: trie.NewDatabase(kvs), + trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)), stateDatabase: state.NewDatabase(database), } } @@ -147,10 +159,10 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error { } // ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete -func (v *Validator) ValidateStorageTrie(address common.Address, storageRoot common.Hash) error { +func (v *Validator) ValidateStorageTrie(stateRoot common.Hash, address common.Address, storageRoot common.Hash) error { // Generate the state.NodeIterator for this root addrHash := crypto.Keccak256Hash(address.Bytes()) - t, err := v.stateDatabase.OpenStorageTrie(addrHash, storageRoot) + t, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot) if err != nil { return err } @@ -184,7 +196,7 @@ func (v *Validator) iterate(it trie.NodeIterator, storage bool) error { if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil { return err } - dataTrie, err := v.stateDatabase.OpenStorageTrie(common.BytesToHash(it.LeafKey()), account.Root) + dataTrie, err := v.stateDatabase.OpenStorageTrie(common.HexToHash(viper.GetString("validator.stateRoot")), common.BytesToHash(it.LeafKey()), account.Root) if err != nil { return err } diff --git a/pkg/validator_test.go b/pkg/validator_test.go index a73f38c..4ccb996 100644 --- a/pkg/validator_test.go +++ b/pkg/validator_test.go @@ -203,7 +203,7 @@ var ( config = validator.Config{ Hostname: "localhost", - Name: "cerc_testing", + Name: "vulcanize_testing", User: "vdbm", Password: "password", Port: 8077, @@ -308,19 +308,19 @@ var _ = Describe("PG-IPFS Validator", func() { }) It("Returns an error the storage root node is missing", func() { loadTrie(nil, missingRootStorageNodes) - err = v.ValidateStorageTrie(contractAddr, storageRoot) + err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing trie node")) }) It("Returns an error if the entire storage trie cannot be validated", func() { loadTrie(nil, missingNodeStorageNodes) - err = v.ValidateStorageTrie(contractAddr, storageRoot) + err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing trie node")) }) It("Returns no error if the entire storage trie can be validated", func() { loadTrie(nil, trieStorageNodes) - err = v.ValidateStorageTrie(contractAddr, storageRoot) + err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot) Expect(err).ToNot(HaveOccurred()) }) })