Merge pull request #15 from vulcanize/iterator_support_1.9.10

Handle different key prefixes

README.md

@@ -71,7 +71,7 @@ func main() {
 
     database := ipfsethdb.NewDatabase(ipfsNode.Blocks)
     stateDatabase := state.NewDatabase(database)
-    stateDB, _ := state.New(common.Hash{}, stateDatabase, nil)
+    stateDB, _ := state.New(common.Hash{}, stateDatabase)
     stateDBNodeIterator := state.NewNodeIterator(stateDB)
     // do stuff with the statedb node iterator
 }

postgres/batch.go

@@ -44,14 +44,14 @@ func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
 // Put inserts the given value into the key-value data store
 // Key is expected to be the keccak256 hash of value
 func (b *Batch) Put(key []byte, value []byte) (err error) {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	dsKey, err := DatastoreKeyFromGethKey(key)
 	if err != nil {
 		return err
 	}
-	if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
+	if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil {
 		return err
 	}
-	if _, err = b.tx.Exec(putPreimagePgStr, key, mhKey); err != nil {
+	if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey); err != nil {
 		return err
 	}
 	b.valueSize += len(value)

postgres/batch_test.go

@@ -29,10 +29,10 @@ import (
 )
 
 var (
-	batch         ethdb.Batch
+	batch             ethdb.Batch
-	testHeader2   = types.Header{Number: big.NewInt(2)}
+	testHeader2       = types.Header{Number: big.NewInt(2)}
-	testValue2, _ = rlp.EncodeToBytes(testHeader2)
+	testValue2, _     = rlp.EncodeToBytes(testHeader2)
-	testEthKey2   = testHeader2.Hash().Bytes()
+	testKeccakEthKey2 = testHeader2.Hash().Bytes()
 )
 
 var _ = Describe("Batch", func() {
@@ -49,24 +49,24 @@ var _ = Describe("Batch", func() {
 
 	Describe("Put/Write", func() {
 		It("adds the key-value pair to the batch", func() {
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = database.Get(testEthKey2)
+			_, err = database.Get(testKeccakEthKey2)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 
-			err = batch.Put(testEthKey, testValue)
+			err = batch.Put(testKeccakEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			err = batch.Put(testEthKey2, testValue2)
+			err = batch.Put(testKeccakEthKey2, testValue2)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Write()
 			Expect(err).ToNot(HaveOccurred())
 
-			val, err := database.Get(testEthKey)
+			val, err := database.Get(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).To(Equal(testValue))
-			val2, err := database.Get(testEthKey2)
+			val2, err := database.Get(testKeccakEthKey2)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val2).To(Equal(testValue2))
 		})
@@ -74,25 +74,25 @@ var _ = Describe("Batch", func() {
 
 	Describe("Delete/Reset/Write", func() {
 		It("deletes the key-value pair in the batch", func() {
-			err = batch.Put(testEthKey, testValue)
+			err = batch.Put(testKeccakEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			err = batch.Put(testEthKey2, testValue2)
+			err = batch.Put(testKeccakEthKey2, testValue2)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Write()
 			Expect(err).ToNot(HaveOccurred())
 
 			batch.Reset()
-			err = batch.Delete(testEthKey)
+			err = batch.Delete(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
-			err = batch.Delete(testEthKey2)
+			err = batch.Delete(testKeccakEthKey2)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Write()
 			Expect(err).ToNot(HaveOccurred())
 
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = database.Get(testEthKey2)
+			_, err = database.Get(testKeccakEthKey2)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 		})
@@ -100,9 +100,9 @@ var _ = Describe("Batch", func() {
 
 	Describe("ValueSize/Reset", func() {
 		It("returns the size of data in the batch queued for write", func() {
-			err = batch.Put(testEthKey, testValue)
+			err = batch.Put(testKeccakEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			err = batch.Put(testEthKey2, testValue2)
+			err = batch.Put(testKeccakEthKey2, testValue2)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Write()
 			Expect(err).ToNot(HaveOccurred())

postgres/database.go

@@ -33,7 +33,7 @@ var (
 	hasPgStr         = "SELECT exists(select 1 from eth.key_preimages WHERE eth_key = $1)"
 	getPgStr         = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1"
 	putPgStr         = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
-	putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO NOTHING"
+	putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO UPDATE SET ipfs_key = $2"
 	deletePgStr      = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1"
 	dbSizePgStr      = "SELECT pg_database_size(current_database())"
 )
@@ -78,7 +78,7 @@ func (d *Database) Get(key []byte) ([]byte, error) {
 // Key is expected to be the keccak256 hash of value
 // Put inserts the keccak256 key into the eth.key_preimages table
 func (d *Database) Put(key []byte, value []byte) error {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	dsKey, err := DatastoreKeyFromGethKey(key)
 	if err != nil {
 		return err
 	}
@@ -95,10 +95,10 @@ func (d *Database) Put(key []byte, value []byte) error {
 			err = tx.Commit()
 		}
 	}()
-	if _, err = tx.Exec(putPgStr, mhKey, value); err != nil {
+	if _, err = tx.Exec(putPgStr, dsKey, value); err != nil {
 		return err
 	}
-	_, err = tx.Exec(putPreimagePgStr, key, mhKey)
+	_, err = tx.Exec(putPreimagePgStr, key, dsKey)
 	return err
 }
 

postgres/database_test.go

@@ -19,6 +19,8 @@ package pgipfsethdb_test
 import (
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/common"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -30,13 +32,26 @@ import (
 )
 
 var (
-	database     ethdb.Database
+	database         ethdb.Database
-	db           *sqlx.DB
+	db               *sqlx.DB
-	err          error
+	err              error
-	testHeader   = types.Header{Number: big.NewInt(1337)}
+	testHeader       = types.Header{Number: big.NewInt(1337)}
-	testValue, _ = rlp.EncodeToBytes(testHeader)
+	testValue, _     = rlp.EncodeToBytes(testHeader)
-	testEthKey   = testHeader.Hash().Bytes()
+	testKeccakEthKey = testHeader.Hash().Bytes()
-	testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey)
+	testMhKey, _     = pgipfsethdb.MultihashKeyFromKeccak256(testKeccakEthKey)
+
+	testPrefixedEthKey = append(append([]byte("prefix"), pgipfsethdb.KeyDelineation...), testKeccakEthKey...)
+	testPrefixedDsKey  = common.Bytes2Hex(testPrefixedEthKey)
+
+	testSuffixedEthKey = append(append(testPrefixedEthKey, pgipfsethdb.KeyDelineation...), []byte("suffix")...)
+	testSuffixedDsKey  = common.Bytes2Hex(testSuffixedEthKey)
+
+	testHeaderEthKey = append(append(append(append(pgipfsethdb.HeaderPrefix, pgipfsethdb.KeyDelineation...),
+		[]byte("number")...), pgipfsethdb.NumberDelineation...), testKeccakEthKey...)
+	testHeaderDsKey = testMhKey
+
+	testPreimageEthKey = append(append(pgipfsethdb.PreimagePrefix, pgipfsethdb.KeyDelineation...), testKeccakEthKey...)
+	testPreimageDsKey  = testMhKey
 )
 
 var _ = Describe("Database", func() {
@@ -50,65 +65,321 @@ var _ = Describe("Database", func() {
 		Expect(err).ToNot(HaveOccurred())
 	})
 
-	Describe("Has", func() {
+	Describe("Has - Keccak keys", func() {
 		It("returns false if a key-pair doesn't exist in the db", func() {
-			has, err := database.Has(testEthKey)
+			has, err := database.Has(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(has).ToNot(BeTrue())
 		})
 		It("returns true if a key-pair exists in the db", func() {
 			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testEthKey, testMhKey)
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey)
 			Expect(err).ToNot(HaveOccurred())
-			has, err := database.Has(testEthKey)
+			has, err := database.Has(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(has).To(BeTrue())
 		})
 	})
 
-	Describe("Get", func() {
+	Describe("Has - Prefixed keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Suffixed keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Header keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Preimage keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Get - Keccak keys", func() {
 		It("throws an err if the key-pair doesn't exist in the db", func() {
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 		})
 		It("returns the value associated with the key, if the pair exists", func() {
 			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testEthKey, testMhKey)
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey)
 			Expect(err).ToNot(HaveOccurred())
-			val, err := database.Get(testEthKey)
+			val, err := database.Get(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).To(Equal(testValue))
 		})
 	})
 
-	Describe("Put", func() {
+	Describe("Get - Prefixed keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Suffixed keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Header keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Preimage keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testPreimageEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Keccak keys", func() {
 		It("persists the key-value pair in the database", func() {
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 
-			err = database.Put(testEthKey, testValue)
+			err = database.Put(testKeccakEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			val, err := database.Get(testEthKey)
+			val, err := database.Get(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).To(Equal(testValue))
 		})
 	})
 
-	Describe("Delete", func() {
+	Describe("Put - Prefixed keys", func() {
-		It("removes the key-value pair from the database", func() {
+		It("persists the key-value pair in the database", func() {
-			err = database.Put(testEthKey, testValue)
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testPrefixedEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
-			val, err := database.Get(testEthKey)
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Suffixed keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testSuffixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Header keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testHeaderEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Preimage keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testPreimageEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testPreimageEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Delete - Keccak keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testKeccakEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(val).To(Equal(testValue))
 
-			err = database.Delete(testEthKey)
+			err = database.Delete(testKeccakEthKey)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = database.Get(testEthKey)
+			_, err = database.Get(testKeccakEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Prefixed keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testPrefixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Suffixed keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testSuffixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Header keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testHeaderEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Preimage keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testPreimageEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testPreimageEthKey)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 		})

postgres/doc.md

@@ -33,7 +33,7 @@ func main() {
 
     database := pgipfsethdb.NewDatabase(db)
     stateDatabase := state.NewDatabase(database)
-    stateDB, _ := state.New(common.Hash{}, stateDatabase, nil)
+    stateDB, _ := state.New(common.Hash{}, stateDatabase)
     stateDBNodeIterator := state.NewNodeIterator(stateDB)
     // do stuff with the statedb node iterator
 }

postgres/key_type.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import "bytes"
+
+type KeyType uint
+
+const (
+	Invalid KeyType = iota
+	Keccak
+	Prefixed
+	Suffixed
+	Header
+	Preimage
+)
+
+var (
+	// keyDelineation is used to delineate the key prefixes and suffixes
+	KeyDelineation = []byte("/")
+
+	// numberDelineation is used to delineate the block number encoded in a key
+	NumberDelineation = []byte(":")
+
+	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
+	HeaderPrefix   = []byte("h")           // headerPrefix + num (uint64 big endian) + hash -> header
+	PreimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage
+)
+
+// ResolveKeyType returns the key type based on the prefix
+func ResolveKeyType(key []byte) (KeyType, [][]byte) {
+	sk := bytes.Split(key, KeyDelineation)
+
+	switch len(sk) {
+	case 1:
+		return Keccak, sk
+	case 2:
+		switch prefix := sk[0]; {
+		case bytes.Equal(prefix, HeaderPrefix):
+			return Header, bytes.Split(sk[1], NumberDelineation)
+		case bytes.Equal(prefix, PreimagePrefix):
+			return Preimage, sk
+		default:
+			return Prefixed, sk
+		}
+	case 3:
+		return Suffixed, sk
+	}
+	return Invalid, sk
+}

postgres/test_helpers.go

@@ -0,0 +1,33 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import "github.com/jmoiron/sqlx"
+
+// TestDB connect to the testing database
+// it assumes the database has the IPFS public.blocks table present
+// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
+func TestDB() (*sqlx.DB, error) {
+	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	return sqlx.Connect("postgres", connectStr)
+}
+
+// ResetTestDB drops all rows in the test db public.blocks table
+func ResetTestDB(db *sqlx.DB) error {
+	_, err := db.Exec("TRUNCATE public.blocks CASCADE")
+	return err
+}

postgres/util.go

@@ -17,9 +17,11 @@
 package pgipfsethdb
 
 import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ipfs/go-ipfs-blockstore"
 	"github.com/ipfs/go-ipfs-ds-help"
-	"github.com/jmoiron/sqlx"
 	_ "github.com/lib/pq" //postgres driver
 	"github.com/multiformats/go-multihash"
 )
@@ -34,16 +36,22 @@ func MultihashKeyFromKeccak256(h []byte) (string, error) {
 	return blockstore.BlockPrefix.String() + dbKey.String(), nil
 }
 
-// TestDB connect to the testing database
+// DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key
-// it assumes the database has the IPFS public.blocks table present
+func DatastoreKeyFromGethKey(h []byte) (string, error) {
-// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
+	keyType, keyComponents := ResolveKeyType(h)
-func TestDB() (*sqlx.DB, error) {
+	switch keyType {
-	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	case Keccak:
-	return sqlx.Connect("postgres", connectStr)
+		return MultihashKeyFromKeccak256(h)
-}
+	case Header:
+		return MultihashKeyFromKeccak256(keyComponents[1])
+	case Preimage:
+		return MultihashKeyFromKeccak256(keyComponents[1])
+	case Prefixed, Suffixed:
+		// This data is not mapped by hash => content by geth, store it using the prefixed/suffixed key directly
+		// I.e. the public.blocks datastore key == the hex representation of the geth key
+		return common.Bytes2Hex(h), nil
+	default:
+		return "", fmt.Errorf("invalid formatting of database key: %x", h)
+	}
 
-// ResetTestDB drops all rows in the test db public.blocks table
-func ResetTestDB(db *sqlx.DB) error {
-	_, err := db.Exec("TRUNCATE public.blocks CASCADE")
-	return err
 }