From 961e10bfe8e0152d95183c4c4c529db144812448 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 14 Sep 2020 10:15:32 -0500 Subject: [PATCH] unit tests for ancient interfaces --- postgres/ancient.go | 65 +++-- postgres/ancient_test.go | 232 ++++++++++++++++++ postgres/database.go | 3 +- ...0004_create_eth_ancient_headers_table.sql} | 0 ...00005_create_eth_ancient_hashes_table.sql} | 0 ...00006_create_eth_ancient_bodies_table.sql} | 0 ...007_create_eth_ancient_receipts_table.sql} | 0 ...=> 00008_create_eth_ancient_tds_table.sql} | 0 postgres/test_helpers.go | 38 ++- 9 files changed, 308 insertions(+), 30 deletions(-) create mode 100644 postgres/ancient_test.go rename postgres/db/migrations/{00004_create_ancient_headers_table.sql => 00004_create_eth_ancient_headers_table.sql} (100%) rename postgres/db/migrations/{00005_create_ancient_hashes_table.sql => 00005_create_eth_ancient_hashes_table.sql} (100%) rename postgres/db/migrations/{00006_create_ancient_bodies_table.sql => 00006_create_eth_ancient_bodies_table.sql} (100%) rename postgres/db/migrations/{00007_create_ancient_receipts_table.sql => 00007_create_eth_ancient_receipts_table.sql} (100%) rename postgres/db/migrations/{00008_create_ancient_tds_table.sql => 00008_create_eth_ancient_tds_table.sql} (100%) diff --git a/postgres/ancient.go b/postgres/ancient.go index ef9c358..e6c4454 100644 --- a/postgres/ancient.go +++ b/postgres/ancient.go @@ -17,26 +17,27 @@ package pgipfsethdb import ( + "database/sql" "fmt" "github.com/sirupsen/logrus" ) const ( - // freezerHeaderTable indicates the name of the freezer header table. - freezerHeaderTable = "headers" + // FreezerHeaderTable indicates the name of the freezer header table. + FreezerHeaderTable = "headers" - // freezerHashTable indicates the name of the freezer canonical hash table. - freezerHashTable = "hashes" + // FreezerHashTable indicates the name of the freezer canonical hash table. + FreezerHashTable = "hashes" - // freezerBodiesTable indicates the name of the freezer block body table. - freezerBodiesTable = "bodies" + // FreezerBodiesTable indicates the name of the freezer block body table. + FreezerBodiesTable = "bodies" - // freezerReceiptTable indicates the name of the freezer receipts table. - freezerReceiptTable = "receipts" + // FreezerReceiptTable indicates the name of the freezer receipts table. + FreezerReceiptTable = "receipts" - // freezerDifficultyTable indicates the name of the freezer total difficulty table. - freezerDifficultyTable = "diffs" + // FreezerDifficultyTable indicates the name of the freezer total difficulty table. + FreezerDifficultyTable = "diffs" // ancient append Postgres statements appendAncientHeaderPgStr = "INSERT INTO eth.ancient_headers (block_number, header) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET header = $2" @@ -78,15 +79,15 @@ const ( func (d *Database) HasAncient(kind string, number uint64) (bool, error) { var pgStr string switch kind { - case freezerHeaderTable: + case FreezerHeaderTable: pgStr = hasAncientHeaderPgStr - case freezerHashTable: + case FreezerHashTable: pgStr = hasAncientHashPgStr - case freezerBodiesTable: + case FreezerBodiesTable: pgStr = hasAncientBodyPgStr - case freezerReceiptTable: + case FreezerReceiptTable: pgStr = hasAncientReceiptsPgStr - case freezerDifficultyTable: + case FreezerDifficultyTable: pgStr = hasAncientTDPgStr default: return false, fmt.Errorf("unexpected ancient kind: %s", kind) @@ -100,15 +101,15 @@ func (d *Database) HasAncient(kind string, number uint64) (bool, error) { func (d *Database) Ancient(kind string, number uint64) ([]byte, error) { var pgStr string switch kind { - case freezerHeaderTable: + case FreezerHeaderTable: pgStr = getAncientHeaderPgStr - case freezerHashTable: + case FreezerHashTable: pgStr = getAncientHashPgStr - case freezerBodiesTable: + case FreezerBodiesTable: pgStr = getAncientBodyPgStr - case freezerReceiptTable: + case FreezerReceiptTable: pgStr = getAncientReceiptsPgStr - case freezerDifficultyTable: + case FreezerDifficultyTable: pgStr = getAncientTDPgStr default: return nil, fmt.Errorf("unexpected ancient kind: %s", kind) @@ -121,7 +122,13 @@ func (d *Database) Ancient(kind string, number uint64) ([]byte, error) { // Ancients returns the ancient item numbers in the ancient store func (d *Database) Ancients() (uint64, error) { num := new(uint64) - return *num, d.db.Get(num, ancientsPgStr) + if err := d.db.Get(num, ancientsPgStr); err != nil { + if err == sql.ErrNoRows { + return 0, nil + } + return 0, err + } + return *num, nil } // AncientSize satisfies the ethdb.AncientReader interface @@ -129,15 +136,15 @@ func (d *Database) Ancients() (uint64, error) { func (d *Database) AncientSize(kind string) (uint64, error) { var tableName string switch kind { - case freezerHeaderTable: + case FreezerHeaderTable: tableName = "eth.ancient_headers" - case freezerHashTable: + case FreezerHashTable: tableName = "eth.ancient_hashes" - case freezerBodiesTable: + case FreezerBodiesTable: tableName = "eth.ancient_bodies" - case freezerReceiptTable: + case FreezerReceiptTable: tableName = "eth.ancient_receipts" - case freezerDifficultyTable: + case FreezerDifficultyTable: tableName = "eth.ancient_tds" default: return 0, fmt.Errorf("unexpected ancient kind: %s", kind) @@ -223,5 +230,9 @@ func (d *Database) Sync() error { if d.ancientTx == nil { return nil } - return d.ancientTx.Commit() + if err := d.ancientTx.Commit(); err != nil { + return err + } + d.ancientTx = nil + return nil } diff --git a/postgres/ancient_test.go b/postgres/ancient_test.go new file mode 100644 index 0000000..8cd85ee --- /dev/null +++ b/postgres/ancient_test.go @@ -0,0 +1,232 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pgipfsethdb_test + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/ipfs-ethdb/postgres" +) + +var ( + ancientDB ethdb.Database + testBlockNumber uint64 = 1 + testAncientHeader = types.Header{Number: big.NewInt(2)} + testAncientHeaderRLP, _ = rlp.EncodeToBytes(testHeader2) + testAncientHash = testAncientHeader.Hash().Bytes() + testAncientBodyBytes = make([]byte, 10000) + testAncientReceiptsBytes = make([]byte, 5000) + testAncientTD, _ = new(big.Int).SetString("1000000000000000000000", 10) + testAncientTDBytes = testAncientTD.Bytes() +) + +var _ = Describe("Ancient", func() { + BeforeEach(func() { + db, err = pgipfsethdb.TestDB() + Expect(err).ToNot(HaveOccurred()) + ancientDB = pgipfsethdb.NewDatabase(db) + + }) + AfterEach(func() { + err = pgipfsethdb.ResetTestDB(db) + Expect(err).ToNot(HaveOccurred()) + }) + + Describe("AppendAncient/Sync/Has", func() { + It("adds eth objects to the Ancient database and returns whether or not an ancient record exists", func() { + hasAncient(testBlockNumber, false) + + err = ancientDB.AppendAncient(testBlockNumber, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes) + Expect(err).ToNot(HaveOccurred()) + + hasAncient(testBlockNumber, false) + + err = ancientDB.Sync() + Expect(err).ToNot(HaveOccurred()) + + hasAncient(testBlockNumber, true) + }) + }) + + Describe("AppendAncient/Sync/Ancient", func() { + It("adds the eth objects to the Ancient database and returns the ancient objects on request", func() { + hasAncient(testBlockNumber, false) + + _, err := ancientDB.Ancient(pgipfsethdb.FreezerHeaderTable, testBlockNumber) + Expect(err).To(HaveOccurred()) + _, err = ancientDB.Ancient(pgipfsethdb.FreezerHashTable, testBlockNumber) + Expect(err).To(HaveOccurred()) + _, err = ancientDB.Ancient(pgipfsethdb.FreezerBodiesTable, testBlockNumber) + Expect(err).To(HaveOccurred()) + _, err = ancientDB.Ancient(pgipfsethdb.FreezerReceiptTable, testBlockNumber) + Expect(err).To(HaveOccurred()) + _, err = ancientDB.Ancient(pgipfsethdb.FreezerDifficultyTable, testBlockNumber) + Expect(err).To(HaveOccurred()) + + err = ancientDB.AppendAncient(testBlockNumber, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes) + Expect(err).ToNot(HaveOccurred()) + err = ancientDB.Sync() + Expect(err).ToNot(HaveOccurred()) + + hasAncient(testBlockNumber, true) + + ancientHeader, err := ancientDB.Ancient(pgipfsethdb.FreezerHeaderTable, testBlockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientHeader).To(Equal(testAncientHeaderRLP)) + + ancientHash, err := ancientDB.Ancient(pgipfsethdb.FreezerHashTable, testBlockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientHash).To(Equal(testAncientHash)) + + ancientBody, err := ancientDB.Ancient(pgipfsethdb.FreezerBodiesTable, testBlockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientBody).To(Equal(testAncientBodyBytes)) + + ancientReceipts, err := ancientDB.Ancient(pgipfsethdb.FreezerReceiptTable, testBlockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientReceipts).To(Equal(testAncientReceiptsBytes)) + + ancientTD, err := ancientDB.Ancient(pgipfsethdb.FreezerDifficultyTable, testBlockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientTD).To(Equal(testAncientTDBytes)) + }) + }) + + Describe("AppendAncient/Sync/Ancients", func() { + It("returns the height of the ancient database", func() { + ancients, err := ancientDB.Ancients() + Expect(err).ToNot(HaveOccurred()) + Expect(ancients).To(Equal(uint64(0))) + + for i := uint64(0); i <= 100; i++ { + hasAncient(i, false) + err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes) + Expect(err).ToNot(HaveOccurred()) + } + + err = ancientDB.Sync() + Expect(err).ToNot(HaveOccurred()) + + for i := uint64(0); i <= 100; i++ { + hasAncient(i, true) + } + ancients, err = ancientDB.Ancients() + Expect(err).ToNot(HaveOccurred()) + Expect(ancients).To(Equal(uint64(100))) + }) + }) + + Describe("AppendAncient/Truncate/Sync", func() { + It("truncates the ancient database to the provided height", func() { + for i := uint64(0); i <= 100; i++ { + hasAncient(i, false) + err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes) + Expect(err).ToNot(HaveOccurred()) + } + + err = ancientDB.Sync() + Expect(err).ToNot(HaveOccurred()) + + err = ancientDB.TruncateAncients(50) + Expect(err).ToNot(HaveOccurred()) + + for i := uint64(0); i <= 100; i++ { + hasAncient(i, true) + } + + ancients, err := ancientDB.Ancients() + Expect(err).ToNot(HaveOccurred()) + Expect(ancients).To(Equal(uint64(100))) + + err = ancientDB.Sync() + Expect(err).ToNot(HaveOccurred()) + + for i := uint64(0); i <= 100; i++ { + if i <= 50 { + hasAncient(i, true) + } else { + hasAncient(i, false) + } + } + + ancients, err = ancientDB.Ancients() + Expect(err).ToNot(HaveOccurred()) + Expect(ancients).To(Equal(uint64(50))) + }) + }) + + Describe("AppendAncient/Sync/AncientSize", func() { + It("adds the eth objects to the Ancient database and returns the ancient objects on request", func() { + for i := uint64(0); i <= 100; i++ { + hasAncient(i, false) + err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes) + Expect(err).ToNot(HaveOccurred()) + } + + err = ancientDB.Sync() + Expect(err).ToNot(HaveOccurred()) + + for i := uint64(0); i <= 100; i++ { + hasAncient(i, true) + } + + ancientHeaderSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerHeaderTable) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientHeaderSize).To(Equal(uint64(106496))) + + ancientHashSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerHashTable) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientHashSize).To(Equal(uint64(32768))) + + ancientBodySize, err := ancientDB.AncientSize(pgipfsethdb.FreezerBodiesTable) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientBodySize).To(Equal(uint64(73728))) + + ancientReceiptsSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerReceiptTable) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientReceiptsSize).To(Equal(uint64(65536))) + + ancientTDSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerDifficultyTable) + Expect(err).ToNot(HaveOccurred()) + Expect(ancientTDSize).To(Equal(uint64(32768))) + }) + }) +}) + +func hasAncient(blockNumber uint64, shouldHave bool) { + has, err := ancientDB.HasAncient(pgipfsethdb.FreezerHeaderTable, blockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(Equal(shouldHave)) + has, err = ancientDB.HasAncient(pgipfsethdb.FreezerHashTable, blockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(Equal(shouldHave)) + has, err = ancientDB.HasAncient(pgipfsethdb.FreezerBodiesTable, blockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(Equal(shouldHave)) + has, err = ancientDB.HasAncient(pgipfsethdb.FreezerReceiptTable, blockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(Equal(shouldHave)) + has, err = ancientDB.HasAncient(pgipfsethdb.FreezerDifficultyTable, blockNumber) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(Equal(shouldHave)) +} diff --git a/postgres/database.go b/postgres/database.go index c00c129..3872935 100644 --- a/postgres/database.go +++ b/postgres/database.go @@ -188,7 +188,8 @@ func (d *Database) Stat(property string) (string, error) { // Compact satisfies the ethdb.Compacter interface // Compact flattens the underlying data store for the given key range func (d *Database) Compact(start []byte, limit []byte) error { - return errNotSupported + // this leveldb functionality doesn't translate to Postgres + return nil } // NewBatch satisfies the ethdb.Batcher interface diff --git a/postgres/db/migrations/00004_create_ancient_headers_table.sql b/postgres/db/migrations/00004_create_eth_ancient_headers_table.sql similarity index 100% rename from postgres/db/migrations/00004_create_ancient_headers_table.sql rename to postgres/db/migrations/00004_create_eth_ancient_headers_table.sql diff --git a/postgres/db/migrations/00005_create_ancient_hashes_table.sql b/postgres/db/migrations/00005_create_eth_ancient_hashes_table.sql similarity index 100% rename from postgres/db/migrations/00005_create_ancient_hashes_table.sql rename to postgres/db/migrations/00005_create_eth_ancient_hashes_table.sql diff --git a/postgres/db/migrations/00006_create_ancient_bodies_table.sql b/postgres/db/migrations/00006_create_eth_ancient_bodies_table.sql similarity index 100% rename from postgres/db/migrations/00006_create_ancient_bodies_table.sql rename to postgres/db/migrations/00006_create_eth_ancient_bodies_table.sql diff --git a/postgres/db/migrations/00007_create_ancient_receipts_table.sql b/postgres/db/migrations/00007_create_eth_ancient_receipts_table.sql similarity index 100% rename from postgres/db/migrations/00007_create_ancient_receipts_table.sql rename to postgres/db/migrations/00007_create_eth_ancient_receipts_table.sql diff --git a/postgres/db/migrations/00008_create_ancient_tds_table.sql b/postgres/db/migrations/00008_create_eth_ancient_tds_table.sql similarity index 100% rename from postgres/db/migrations/00008_create_ancient_tds_table.sql rename to postgres/db/migrations/00008_create_eth_ancient_tds_table.sql diff --git a/postgres/test_helpers.go b/postgres/test_helpers.go index 7eb4260..e62f787 100644 --- a/postgres/test_helpers.go +++ b/postgres/test_helpers.go @@ -16,7 +16,9 @@ package pgipfsethdb -import "github.com/jmoiron/sqlx" +import ( + "github.com/jmoiron/sqlx" +) // TestDB connect to the testing database // it assumes the database has the IPFS public.blocks table present @@ -28,6 +30,38 @@ func TestDB() (*sqlx.DB, error) { // ResetTestDB drops all rows in the test db public.blocks table func ResetTestDB(db *sqlx.DB) error { - _, err := db.Exec("TRUNCATE public.blocks CASCADE") + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { + if p := recover(); p != nil { + tx.Rollback() + panic(p) + } else if err != nil { + tx.Rollback() + } else { + err = tx.Commit() + } + }() + if _, err := tx.Exec("TRUNCATE public.blocks CASCADE"); err != nil { + return err + } + if _, err := tx.Exec("TRUNCATE eth.key_preimages CASCADE"); err != nil { + return err + } + if _, err := tx.Exec("TRUNCATE eth.ancient_headers CASCADE"); err != nil { + return err + } + if _, err := tx.Exec("TRUNCATE eth.ancient_hashes CASCADE"); err != nil { + return err + } + if _, err := tx.Exec("TRUNCATE eth.ancient_bodies CASCADE"); err != nil { + return err + } + if _, err := tx.Exec("TRUNCATE eth.ancient_receipts CASCADE"); err != nil { + return err + } + _, err = tx.Exec("TRUNCATE eth.ancient_tds CASCADE") return err }