database tests for different key prefixes

postgres/batch_test.go

@@ -29,10 +29,10 @@ import (
 )
 
 var (
-	batch         ethdb.Batch
+	batch             ethdb.Batch
-	testHeader2   = types.Header{Number: big.NewInt(2)}
+	testHeader2       = types.Header{Number: big.NewInt(2)}
-	testValue2, _ = rlp.EncodeToBytes(testHeader2)
+	testValue2, _     = rlp.EncodeToBytes(testHeader2)
-	testEthKey2   = testHeader2.Hash().Bytes()
+	testKeccakEthKey2 = testHeader2.Hash().Bytes()
 )
 
 var _ = Describe("Batch", func() {
@@ -49,24 +49,24 @@ var _ = Describe("Batch", func() {
 
 	Describe("Put/Write", func() {
 		It("adds the key-value pair to the batch", func() {
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = database.Get(testEthKey2)
+			_, err = database.Get(testKeccakEthKey2)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 
-			err = batch.Put(testEthKey, testValue)
+			err = batch.Put(testKeccakEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			err = batch.Put(testEthKey2, testValue2)
+			err = batch.Put(testKeccakEthKey2, testValue2)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Write()
 			Expect(err).ToNot(HaveOccurred())
 
-			val, err := database.Get(testEthKey)
+			val, err := database.Get(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).To(Equal(testValue))
-			val2, err := database.Get(testEthKey2)
+			val2, err := database.Get(testKeccakEthKey2)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val2).To(Equal(testValue2))
 		})
@@ -74,25 +74,25 @@ var _ = Describe("Batch", func() {
 
 	Describe("Delete/Reset/Write", func() {
 		It("deletes the key-value pair in the batch", func() {
-			err = batch.Put(testEthKey, testValue)
+			err = batch.Put(testKeccakEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			err = batch.Put(testEthKey2, testValue2)
+			err = batch.Put(testKeccakEthKey2, testValue2)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Write()
 			Expect(err).ToNot(HaveOccurred())
 
 			batch.Reset()
-			err = batch.Delete(testEthKey)
+			err = batch.Delete(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
-			err = batch.Delete(testEthKey2)
+			err = batch.Delete(testKeccakEthKey2)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Write()
 			Expect(err).ToNot(HaveOccurred())
 
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = database.Get(testEthKey2)
+			_, err = database.Get(testKeccakEthKey2)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 		})
@@ -100,9 +100,9 @@ var _ = Describe("Batch", func() {
 
 	Describe("ValueSize/Reset", func() {
 		It("returns the size of data in the batch queued for write", func() {
-			err = batch.Put(testEthKey, testValue)
+			err = batch.Put(testKeccakEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			err = batch.Put(testEthKey2, testValue2)
+			err = batch.Put(testKeccakEthKey2, testValue2)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Write()
 			Expect(err).ToNot(HaveOccurred())

postgres/database_test.go

@@ -19,6 +19,8 @@ package pgipfsethdb_test
 import (
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/common"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -30,13 +32,26 @@ import (
 )
 
 var (
-	database     ethdb.Database
+	database         ethdb.Database
-	db           *sqlx.DB
+	db               *sqlx.DB
-	err          error
+	err              error
-	testHeader   = types.Header{Number: big.NewInt(1337)}
+	testHeader       = types.Header{Number: big.NewInt(1337)}
-	testValue, _ = rlp.EncodeToBytes(testHeader)
+	testValue, _     = rlp.EncodeToBytes(testHeader)
-	testEthKey   = testHeader.Hash().Bytes()
+	testKeccakEthKey = testHeader.Hash().Bytes()
-	testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey)
+	testMhKey, _     = pgipfsethdb.MultihashKeyFromKeccak256(testKeccakEthKey)
+
+	testPrefixedEthKey = append(append([]byte("prefix"), pgipfsethdb.KeyDelineation...), testKeccakEthKey...)
+	testPrefixedDsKey  = common.Bytes2Hex(testPrefixedEthKey)
+
+	testSuffixedEthKey = append(append(testPrefixedEthKey, pgipfsethdb.KeyDelineation...), []byte("suffix")...)
+	testSuffixedDsKey  = common.Bytes2Hex(testSuffixedEthKey)
+
+	testHeaderEthKey = append(append(append(append(pgipfsethdb.HeaderPrefix, pgipfsethdb.KeyDelineation...),
+		[]byte("number")...), pgipfsethdb.NumberDelineation...), testKeccakEthKey...)
+	testHeaderDsKey = testMhKey
+
+	testPreimageEthKey = append(append(pgipfsethdb.PreimagePrefix, pgipfsethdb.KeyDelineation...), testKeccakEthKey...)
+	testPreimageDsKey  = testMhKey
 )
 
 var _ = Describe("Database", func() {
@@ -50,65 +65,321 @@ var _ = Describe("Database", func() {
 		Expect(err).ToNot(HaveOccurred())
 	})
 
-	Describe("Has", func() {
+	Describe("Has - Keccak keys", func() {
 		It("returns false if a key-pair doesn't exist in the db", func() {
-			has, err := database.Has(testEthKey)
+			has, err := database.Has(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(has).ToNot(BeTrue())
 		})
 		It("returns true if a key-pair exists in the db", func() {
 			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testEthKey, testMhKey)
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey)
 			Expect(err).ToNot(HaveOccurred())
-			has, err := database.Has(testEthKey)
+			has, err := database.Has(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(has).To(BeTrue())
 		})
 	})
 
-	Describe("Get", func() {
+	Describe("Has - Prefixed keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Suffixed keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Header keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Preimage keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Get - Keccak keys", func() {
 		It("throws an err if the key-pair doesn't exist in the db", func() {
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 		})
 		It("returns the value associated with the key, if the pair exists", func() {
 			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testEthKey, testMhKey)
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey)
 			Expect(err).ToNot(HaveOccurred())
-			val, err := database.Get(testEthKey)
+			val, err := database.Get(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).To(Equal(testValue))
 		})
 	})
 
-	Describe("Put", func() {
+	Describe("Get - Prefixed keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Suffixed keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Header keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Preimage keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testPreimageEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Keccak keys", func() {
 		It("persists the key-value pair in the database", func() {
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 
-			err = database.Put(testEthKey, testValue)
+			err = database.Put(testKeccakEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			val, err := database.Get(testEthKey)
+			val, err := database.Get(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).To(Equal(testValue))
 		})
 	})
 
-	Describe("Delete", func() {
+	Describe("Put - Prefixed keys", func() {
-		It("removes the key-value pair from the database", func() {
+		It("persists the key-value pair in the database", func() {
-			err = database.Put(testEthKey, testValue)
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testPrefixedEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			val, err := database.Get(testEthKey)
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Suffixed keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testSuffixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Header keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testHeaderEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Preimage keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testPreimageEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testPreimageEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Delete - Keccak keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testKeccakEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).To(Equal(testValue))
 
-			err = database.Delete(testEthKey)
+			err = database.Delete(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Prefixed keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testPrefixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Suffixed keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testSuffixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Header keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testHeaderEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Preimage keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testPreimageEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testPreimageEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 		})

postgres/util.go

@@ -47,6 +47,8 @@ func DatastoreKeyFromGethKey(h []byte) (string, error) {
 	case Preimage:
 		return MultihashKeyFromKeccak256(keyComponents[1])
 	case Prefixed, Suffixed:
+		// This data is not mapped by hash => content by geth, store it using the prefixed/suffixed key directly
+		// I.e. the public.blocks datastore key == the hex representation of the geth key
 		return common.Bytes2Hex(h), nil
 	default:
 		return "", fmt.Errorf("invalid formatting of database key: %x", h)