Merge pull request #16 from vulcanize/iterator_support_1.9.10

Iterator support 1.9.10

postgres/batch.go

@@ -44,14 +44,14 @@ func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
 // Put inserts the given value into the key-value data store
 // Key is expected to be the keccak256 hash of value
 func (b *Batch) Put(key []byte, value []byte) (err error) {
-	dsKey, err := DatastoreKeyFromGethKey(key)
+	dsKey, prefix, err := DatastoreKeyFromGethKey(key)
 	if err != nil {
 		return err
 	}
 	if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil {
 		return err
 	}
-	if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey); err != nil {
+	if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey, prefix); err != nil {
 		return err
 	}
 	b.valueSize += len(value)
@@ -74,6 +74,7 @@ func (b *Batch) ValueSize() int {
 
 // Write satisfies the ethdb.Batch interface
 // Write flushes any accumulated data to disk
+// Reset should be called after every write
 func (b *Batch) Write() error {
 	if b.tx == nil {
 		return nil

postgres/database.go

@@ -29,11 +29,11 @@ import (
 
 var errNotSupported = errors.New("this operation is not supported")
 
-var (
+const (
 	hasPgStr         = "SELECT exists(select 1 from eth.key_preimages WHERE eth_key = $1)"
 	getPgStr         = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1"
 	putPgStr         = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
-	putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO UPDATE SET ipfs_key = $2"
+	putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key, prefix) VALUES ($1, $2, $3) ON CONFLICT (eth_key) DO UPDATE SET (ipfs_key, prefix) = ($2, $3)"
 	deletePgStr      = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1"
 	dbSizePgStr      = "SELECT pg_database_size(current_database())"
 )
@@ -78,7 +78,7 @@ func (d *Database) Get(key []byte) ([]byte, error) {
 // Key is expected to be the keccak256 hash of value
 // Put inserts the keccak256 key into the eth.key_preimages table
 func (d *Database) Put(key []byte, value []byte) error {
-	dsKey, err := DatastoreKeyFromGethKey(key)
+	dsKey, prefix, err := DatastoreKeyFromGethKey(key)
 	if err != nil {
 		return err
 	}
@@ -98,7 +98,7 @@ func (d *Database) Put(key []byte, value []byte) error {
 	if _, err = tx.Exec(putPgStr, dsKey, value); err != nil {
 		return err
 	}
-	_, err = tx.Exec(putPreimagePgStr, key, dsKey)
+	_, err = tx.Exec(putPreimagePgStr, key, dsKey, prefix)
 	return err
 }
 
@@ -200,20 +200,20 @@ func (d *Database) NewBatch() ethdb.Batch {
 // NewIterator creates a binary-alphabetical iterator over the entire keyspace
 // contained within the key-value database.
 func (d *Database) NewIterator() ethdb.Iterator {
-	return NewIterator([]byte{}, []byte{}, d.db)
+	return NewIterator(nil, nil, d.db)
 }
 
 // NewIteratorWithStart creates a binary-alphabetical iterator over a subset of
 // database content starting at a particular initial key (or after, if it does
 // not exist).
 func (d *Database) NewIteratorWithStart(start []byte) ethdb.Iterator {
-	return NewIterator(start, []byte{}, d.db)
+	return NewIterator(start, nil, d.db)
 }
 
 // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
 // of database content with a particular key prefix.
 func (d *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
-	return NewIterator([]byte{}, prefix, d.db)
+	return NewIterator(nil, prefix, d.db)
 }
 
 // Close satisfies the io.Closer interface

postgres/db/migrations/00003_create_eth_key_preimages_table.sql

@@ -1,6 +1,7 @@
 -- +goose Up
 CREATE TABLE IF NOT EXISTS eth.key_preimages (
   eth_key BYTEA UNIQUE NOT NULL,
+  prefix BYTEA,
   ipfs_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
 );
 

postgres/iterator.go

@@ -21,15 +21,31 @@ import (
 	"github.com/jmoiron/sqlx"
 )
 
+const (
+	initPgStr = `SELECT eth_key, data FROM public.blocks
+				INNER JOIN eth.key_preimages ON (ipfs_key = key)
+				WHERE eth_key = $1`
+	nextPgStr = `SELECT eth_key, data FROM public.blocks
+				INNER JOIN eth.key_preimages ON (ipfs_key = key)
+				WHERE eth_key > $1
+				ORDER BY eth_key LIMIT 1`
+	nextPgStrWithPrefix = `SELECT eth_key, data FROM public.blocks
+				INNER JOIN eth.key_preimages ON (ipfs_key = key)
+				WHERE eth_key > $1 AND prefix = $2 
+				ORDER BY eth_key LIMIT 1`
+)
+
+type nextModel struct {
+	Key   []byte `db:"eth_key"`
+	Value []byte `db:"data"`
+}
+
 // Iterator is the type that satisfies the ethdb.Iterator interface for PG-IPFS Ethereum data using a direct Postgres connection
-// Iteratee interface is used in Geth for various tests, trie/sync_bloom.go (for fast sync),
-// rawdb.InspectDatabase, and the new core/state/snapshot features.
-// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed
-// from the ethdb.KeyValueStoreand ethdb.Database interfaces)
 type Iterator struct {
-	db                 *sqlx.DB
+	db                               *sqlx.DB
-	currentKey, prefix []byte
+	currentKey, prefix, currentValue []byte
-	err                error
+	err                              error
+	init                             bool
 }
 
 // NewIterator returns an ethdb.Iterator interface for PG-IPFS
@@ -38,6 +54,7 @@ func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
 		db:         db,
 		prefix:     prefix,
 		currentKey: start,
+		init:       start != nil,
 	}
 }
 
@@ -45,9 +62,26 @@ func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
 // Next moves the iterator to the next key/value pair
 // It returns whether the iterator is exhausted
 func (i *Iterator) Next() bool {
-	// this is complicated by the ipfs db keys not being the keccak256 hashes
+	next := new(nextModel)
-	// go-ethereum usage of this method expects the iteration to occur over keccak256 keys
+	if i.init {
-	panic("implement me: Next")
+		i.init = false
+		if err := i.db.Get(next, initPgStr, i.currentKey); err != nil {
+			i.currentKey, i.currentValue, i.err = nil, nil, err
+			return false
+		}
+	} else if i.prefix != nil {
+		if err := i.db.Get(next, nextPgStrWithPrefix, i.currentKey, i.prefix); err != nil {
+			i.currentKey, i.currentValue, i.err = nil, nil, err
+			return false
+		}
+	} else {
+		if err := i.db.Get(next, nextPgStr, i.currentKey); err != nil {
+			i.currentKey, i.currentValue, i.err = nil, nil, err
+			return false
+		}
+	}
+	i.currentKey, i.currentValue, i.err = next.Key, next.Value, nil
+	return true
 }
 
 // Error satisfies the ethdb.Iterator interface
@@ -70,19 +104,12 @@ func (i *Iterator) Key() []byte {
 // The caller should not modify the contents of the returned slice
 // and its contents may change on the next call to Next
 func (i *Iterator) Value() []byte {
-	mhKey, err := MultihashKeyFromKeccak256(i.currentKey)
+	return i.currentValue
-	if err != nil {
-		i.err = err
-		return nil
-	}
-	var data []byte
-	i.err = i.db.Get(&data, getPgStr, mhKey)
-	return data
 }
 
 // Release satisfies the ethdb.Iterator interface
 // Release releases associated resources
 // Release should always succeed and can be called multiple times without causing error
 func (i *Iterator) Release() {
-	i.db.Close()
+	i.db, i.currentKey, i.currentValue, i.err, i.prefix = nil, nil, nil, nil, nil
 }

postgres/iterator_test.go

@@ -0,0 +1,513 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb_test
+
+import (
+	"database/sql"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipfs-ethdb/postgres"
+)
+
+var (
+	iterator            ethdb.Iterator
+	testPrefix          = []byte("testPrefix")
+	testEthKey1         = []byte{'\x01'}
+	testEthKey2         = []byte{'\x01', '\x01'}
+	testEthKey3         = []byte{'\x01', '\x02'}
+	testEthKey4         = []byte{'\x01', '\x0e'}
+	testEthKey5         = []byte{'\x01', '\x02', '\x01'}
+	testEthKey6         = []byte{'\x01', '\x0e', '\x01'}
+	prefixedTestEthKey1 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey1...)
+	prefixedTestEthKey2 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey2...)
+	prefixedTestEthKey3 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey3...)
+	prefixedTestEthKey4 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey4...)
+	prefixedTestEthKey5 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey5...)
+	prefixedTestEthKey6 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey6...)
+	mockValue1          = []byte{1}
+	mockValue2          = []byte{2}
+	mockValue3          = []byte{3}
+	mockValue4          = []byte{4}
+	mockValue5          = []byte{5}
+	mockValue6          = []byte{6}
+)
+
+var _ = Describe("Iterator", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+		database = pgipfsethdb.NewDatabase(db)
+		// non-prefixed entries
+		err = database.Put(testEthKey1, mockValue1)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey2, mockValue2)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey3, mockValue3)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey4, mockValue4)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey5, mockValue5)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey6, mockValue6)
+		Expect(err).ToNot(HaveOccurred())
+		// prefixed entries
+		err = database.Put(prefixedTestEthKey1, mockValue1)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey2, mockValue2)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey3, mockValue3)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey4, mockValue4)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey5, mockValue5)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey6, mockValue6)
+		Expect(err).ToNot(HaveOccurred())
+	})
+	AfterEach(func() {
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("NewIterator", func() {
+		It("iterates over the entire key-set (prefixed or not)", func() {
+			iterator = database.NewIterator()
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+	})
+
+	Describe("NewIteratorWithPrefix", func() {
+		It("iterates over all db entries that have the provided prefix", func() {
+			iterator = database.NewIteratorWithPrefix(testPrefix)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+
+		It("behaves as no prefix is provided if prefix is nil", func() {
+			iterator = database.NewIteratorWithPrefix(nil)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+
+		It("considers empty but non-nil []byte a valid prefix, which precludes iteration over any other prefixed keys", func() {
+			iterator = database.NewIteratorWithPrefix([]byte{})
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+	})
+
+	Describe("NewIteratorWithStart", func() {
+		It("iterates over the entire key-set (prefixed or not) starting with at the provided path", func() {
+			iterator = database.NewIteratorWithStart(testEthKey2)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+
+		It("iterates over the entire key-set (prefixed or not) starting with at the provided path", func() {
+			iterator = database.NewIteratorWithStart(prefixedTestEthKey3)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+	})
+
+	Describe("Release", func() {
+		It("releases resources associated with the Iterator", func() {
+			iterator = database.NewIteratorWithStart(testEthKey2)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			iterator.Release()
+			iterator.Release() // check that we don't panic if called multiple times
+
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+			Expect(func() { iterator.Next() }).To(Panic()) // check that we panic if we try to use released iterator
+
+			// We can still create a new iterator from the same backing db
+			iterator = database.NewIteratorWithStart(testEthKey2)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+		})
+	})
+})

postgres/util.go

@@ -37,21 +37,25 @@ func MultihashKeyFromKeccak256(h []byte) (string, error) {
 }
 
 // DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key
-func DatastoreKeyFromGethKey(h []byte) (string, error) {
+// It also returns the key's prefix, if it has one
+func DatastoreKeyFromGethKey(h []byte) (string, []byte, error) {
 	keyType, keyComponents := ResolveKeyType(h)
 	switch keyType {
 	case Keccak:
-		return MultihashKeyFromKeccak256(h)
+		mhKey, err := MultihashKeyFromKeccak256(h)
+		return mhKey, nil, err
 	case Header:
-		return MultihashKeyFromKeccak256(keyComponents[1])
+		mhKey, err := MultihashKeyFromKeccak256(keyComponents[1])
+		return mhKey, keyComponents[0], err
 	case Preimage:
-		return MultihashKeyFromKeccak256(keyComponents[1])
+		mhKey, err := MultihashKeyFromKeccak256(keyComponents[1])
+		return mhKey, keyComponents[0], err
 	case Prefixed, Suffixed:
 		// This data is not mapped by hash => content by geth, store it using the prefixed/suffixed key directly
 		// I.e. the public.blocks datastore key == the hex representation of the geth key
-		return common.Bytes2Hex(h), nil
+		// Alternatively, decompose the data and derive the hash
+		return common.Bytes2Hex(h), keyComponents[0], nil
 	default:
-		return "", fmt.Errorf("invalid formatting of database key: %x", h)
+		return "", nil, fmt.Errorf("invalid formatting of database key: %x", h)
 	}
-
 }