unit tests for ancient interfaces

postgres/ancient.go

@@ -17,26 +17,27 @@
 package pgipfsethdb
 
 import (
+	"database/sql"
 	"fmt"
 
 	"github.com/sirupsen/logrus"
 )
 
 const (
-	// freezerHeaderTable indicates the name of the freezer header table.
-	freezerHeaderTable = "headers"
+	// FreezerHeaderTable indicates the name of the freezer header table.
+	FreezerHeaderTable = "headers"
 
-	// freezerHashTable indicates the name of the freezer canonical hash table.
-	freezerHashTable = "hashes"
+	// FreezerHashTable indicates the name of the freezer canonical hash table.
+	FreezerHashTable = "hashes"
 
-	// freezerBodiesTable indicates the name of the freezer block body table.
-	freezerBodiesTable = "bodies"
+	// FreezerBodiesTable indicates the name of the freezer block body table.
+	FreezerBodiesTable = "bodies"
 
-	// freezerReceiptTable indicates the name of the freezer receipts table.
-	freezerReceiptTable = "receipts"
+	// FreezerReceiptTable indicates the name of the freezer receipts table.
+	FreezerReceiptTable = "receipts"
 
-	// freezerDifficultyTable indicates the name of the freezer total difficulty table.
-	freezerDifficultyTable = "diffs"
+	// FreezerDifficultyTable indicates the name of the freezer total difficulty table.
+	FreezerDifficultyTable = "diffs"
 
 	// ancient append Postgres statements
 	appendAncientHeaderPgStr   = "INSERT INTO eth.ancient_headers (block_number, header) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET header = $2"
@@ -78,15 +79,15 @@ const (
 func (d *Database) HasAncient(kind string, number uint64) (bool, error) {
 	var pgStr string
 	switch kind {
-	case freezerHeaderTable:
+	case FreezerHeaderTable:
 		pgStr = hasAncientHeaderPgStr
-	case freezerHashTable:
+	case FreezerHashTable:
 		pgStr = hasAncientHashPgStr
-	case freezerBodiesTable:
+	case FreezerBodiesTable:
 		pgStr = hasAncientBodyPgStr
-	case freezerReceiptTable:
+	case FreezerReceiptTable:
 		pgStr = hasAncientReceiptsPgStr
-	case freezerDifficultyTable:
+	case FreezerDifficultyTable:
 		pgStr = hasAncientTDPgStr
 	default:
 		return false, fmt.Errorf("unexpected ancient kind: %s", kind)
@@ -100,15 +101,15 @@ func (d *Database) HasAncient(kind string, number uint64) (bool, error) {
 func (d *Database) Ancient(kind string, number uint64) ([]byte, error) {
 	var pgStr string
 	switch kind {
-	case freezerHeaderTable:
+	case FreezerHeaderTable:
 		pgStr = getAncientHeaderPgStr
-	case freezerHashTable:
+	case FreezerHashTable:
 		pgStr = getAncientHashPgStr
-	case freezerBodiesTable:
+	case FreezerBodiesTable:
 		pgStr = getAncientBodyPgStr
-	case freezerReceiptTable:
+	case FreezerReceiptTable:
 		pgStr = getAncientReceiptsPgStr
-	case freezerDifficultyTable:
+	case FreezerDifficultyTable:
 		pgStr = getAncientTDPgStr
 	default:
 		return nil, fmt.Errorf("unexpected ancient kind: %s", kind)
@@ -121,7 +122,13 @@ func (d *Database) Ancient(kind string, number uint64) ([]byte, error) {
 // Ancients returns the ancient item numbers in the ancient store
 func (d *Database) Ancients() (uint64, error) {
 	num := new(uint64)
-	return *num, d.db.Get(num, ancientsPgStr)
+	if err := d.db.Get(num, ancientsPgStr); err != nil {
+		if err == sql.ErrNoRows {
+			return 0, nil
+		}
+		return 0, err
+	}
+	return *num, nil
 }
 
 // AncientSize satisfies the ethdb.AncientReader interface
@@ -129,15 +136,15 @@ func (d *Database) Ancients() (uint64, error) {
 func (d *Database) AncientSize(kind string) (uint64, error) {
 	var tableName string
 	switch kind {
-	case freezerHeaderTable:
+	case FreezerHeaderTable:
 		tableName = "eth.ancient_headers"
-	case freezerHashTable:
+	case FreezerHashTable:
 		tableName = "eth.ancient_hashes"
-	case freezerBodiesTable:
+	case FreezerBodiesTable:
 		tableName = "eth.ancient_bodies"
-	case freezerReceiptTable:
+	case FreezerReceiptTable:
 		tableName = "eth.ancient_receipts"
-	case freezerDifficultyTable:
+	case FreezerDifficultyTable:
 		tableName = "eth.ancient_tds"
 	default:
 		return 0, fmt.Errorf("unexpected ancient kind: %s", kind)
@@ -223,5 +230,9 @@ func (d *Database) Sync() error {
 	if d.ancientTx == nil {
 		return nil
 	}
-	return d.ancientTx.Commit()
+	if err := d.ancientTx.Commit(); err != nil {
+		return err
+	}
+	d.ancientTx = nil
+	return nil
 }

postgres/ancient_test.go

@@ -0,0 +1,232 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rlp"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipfs-ethdb/postgres"
+)
+
+var (
+	ancientDB                ethdb.Database
+	testBlockNumber          uint64 = 1
+	testAncientHeader               = types.Header{Number: big.NewInt(2)}
+	testAncientHeaderRLP, _         = rlp.EncodeToBytes(testHeader2)
+	testAncientHash                 = testAncientHeader.Hash().Bytes()
+	testAncientBodyBytes            = make([]byte, 10000)
+	testAncientReceiptsBytes        = make([]byte, 5000)
+	testAncientTD, _                = new(big.Int).SetString("1000000000000000000000", 10)
+	testAncientTDBytes              = testAncientTD.Bytes()
+)
+
+var _ = Describe("Ancient", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+		ancientDB = pgipfsethdb.NewDatabase(db)
+
+	})
+	AfterEach(func() {
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("AppendAncient/Sync/Has", func() {
+		It("adds eth objects to the Ancient database and returns whether or not an ancient record exists", func() {
+			hasAncient(testBlockNumber, false)
+
+			err = ancientDB.AppendAncient(testBlockNumber, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+			Expect(err).ToNot(HaveOccurred())
+
+			hasAncient(testBlockNumber, false)
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			hasAncient(testBlockNumber, true)
+		})
+	})
+
+	Describe("AppendAncient/Sync/Ancient", func() {
+		It("adds the eth objects to the Ancient database and returns the ancient objects on request", func() {
+			hasAncient(testBlockNumber, false)
+
+			_, err := ancientDB.Ancient(pgipfsethdb.FreezerHeaderTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+			_, err = ancientDB.Ancient(pgipfsethdb.FreezerHashTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+			_, err = ancientDB.Ancient(pgipfsethdb.FreezerBodiesTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+			_, err = ancientDB.Ancient(pgipfsethdb.FreezerReceiptTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+			_, err = ancientDB.Ancient(pgipfsethdb.FreezerDifficultyTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+
+			err = ancientDB.AppendAncient(testBlockNumber, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+			Expect(err).ToNot(HaveOccurred())
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			hasAncient(testBlockNumber, true)
+
+			ancientHeader, err := ancientDB.Ancient(pgipfsethdb.FreezerHeaderTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientHeader).To(Equal(testAncientHeaderRLP))
+
+			ancientHash, err := ancientDB.Ancient(pgipfsethdb.FreezerHashTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientHash).To(Equal(testAncientHash))
+
+			ancientBody, err := ancientDB.Ancient(pgipfsethdb.FreezerBodiesTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientBody).To(Equal(testAncientBodyBytes))
+
+			ancientReceipts, err := ancientDB.Ancient(pgipfsethdb.FreezerReceiptTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientReceipts).To(Equal(testAncientReceiptsBytes))
+
+			ancientTD, err := ancientDB.Ancient(pgipfsethdb.FreezerDifficultyTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientTD).To(Equal(testAncientTDBytes))
+		})
+	})
+
+	Describe("AppendAncient/Sync/Ancients", func() {
+		It("returns the height of the ancient database", func() {
+			ancients, err := ancientDB.Ancients()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancients).To(Equal(uint64(0)))
+
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, false)
+				err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, true)
+			}
+			ancients, err = ancientDB.Ancients()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancients).To(Equal(uint64(100)))
+		})
+	})
+
+	Describe("AppendAncient/Truncate/Sync", func() {
+		It("truncates the ancient database to the provided height", func() {
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, false)
+				err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ancientDB.TruncateAncients(50)
+			Expect(err).ToNot(HaveOccurred())
+
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, true)
+			}
+
+			ancients, err := ancientDB.Ancients()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancients).To(Equal(uint64(100)))
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			for i := uint64(0); i <= 100; i++ {
+				if i <= 50 {
+					hasAncient(i, true)
+				} else {
+					hasAncient(i, false)
+				}
+			}
+
+			ancients, err = ancientDB.Ancients()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancients).To(Equal(uint64(50)))
+		})
+	})
+
+	Describe("AppendAncient/Sync/AncientSize", func() {
+		It("adds the eth objects to the Ancient database and returns the ancient objects on request", func() {
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, false)
+				err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, true)
+			}
+
+			ancientHeaderSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerHeaderTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientHeaderSize).To(Equal(uint64(106496)))
+
+			ancientHashSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerHashTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientHashSize).To(Equal(uint64(32768)))
+
+			ancientBodySize, err := ancientDB.AncientSize(pgipfsethdb.FreezerBodiesTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientBodySize).To(Equal(uint64(73728)))
+
+			ancientReceiptsSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerReceiptTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientReceiptsSize).To(Equal(uint64(65536)))
+
+			ancientTDSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerDifficultyTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientTDSize).To(Equal(uint64(32768)))
+		})
+	})
+})
+
+func hasAncient(blockNumber uint64, shouldHave bool) {
+	has, err := ancientDB.HasAncient(pgipfsethdb.FreezerHeaderTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+	has, err = ancientDB.HasAncient(pgipfsethdb.FreezerHashTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+	has, err = ancientDB.HasAncient(pgipfsethdb.FreezerBodiesTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+	has, err = ancientDB.HasAncient(pgipfsethdb.FreezerReceiptTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+	has, err = ancientDB.HasAncient(pgipfsethdb.FreezerDifficultyTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+}

postgres/database.go

@@ -188,7 +188,8 @@ func (d *Database) Stat(property string) (string, error) {
 // Compact satisfies the ethdb.Compacter interface
 // Compact flattens the underlying data store for the given key range
 func (d *Database) Compact(start []byte, limit []byte) error {
-	return errNotSupported
+	// this leveldb functionality doesn't translate to Postgres
+	return nil
 }
 
 // NewBatch satisfies the ethdb.Batcher interface

postgres/test_helpers.go

@@ -16,7 +16,9 @@
 
 package pgipfsethdb
 
-import "github.com/jmoiron/sqlx"
+import (
+	"github.com/jmoiron/sqlx"
+)
 
 // TestDB connect to the testing database
 // it assumes the database has the IPFS public.blocks table present
@@ -28,6 +30,38 @@ func TestDB() (*sqlx.DB, error) {
 
 // ResetTestDB drops all rows in the test db public.blocks table
 func ResetTestDB(db *sqlx.DB) error {
-	_, err := db.Exec("TRUNCATE public.blocks CASCADE")
+	tx, err := db.Beginx()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			tx.Rollback()
+			panic(p)
+		} else if err != nil {
+			tx.Rollback()
+		} else {
+			err = tx.Commit()
+		}
+	}()
+	if _, err := tx.Exec("TRUNCATE public.blocks CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.key_preimages CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.ancient_headers CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.ancient_hashes CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.ancient_bodies CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.ancient_receipts CASCADE"); err != nil {
+		return err
+	}
+	_, err = tx.Exec("TRUNCATE eth.ancient_tds CASCADE")
 	return err
 }