diff --git a/postgres/batch.go b/postgres/batch.go index 838f857..8e38292 100644 --- a/postgres/batch.go +++ b/postgres/batch.go @@ -44,14 +44,14 @@ func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch { // Put inserts the given value into the key-value data store // Key is expected to be the keccak256 hash of value func (b *Batch) Put(key []byte, value []byte) (err error) { - dsKey, err := DatastoreKeyFromGethKey(key) + dsKey, prefix, err := DatastoreKeyFromGethKey(key) if err != nil { return err } if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil { return err } - if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey); err != nil { + if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey, prefix); err != nil { return err } b.valueSize += len(value) @@ -74,6 +74,7 @@ func (b *Batch) ValueSize() int { // Write satisfies the ethdb.Batch interface // Write flushes any accumulated data to disk +// Reset should be called after every write func (b *Batch) Write() error { if b.tx == nil { return nil diff --git a/postgres/database.go b/postgres/database.go index b2d9f71..1e5a958 100644 --- a/postgres/database.go +++ b/postgres/database.go @@ -29,11 +29,11 @@ import ( var errNotSupported = errors.New("this operation is not supported") -var ( +const ( hasPgStr = "SELECT exists(select 1 from eth.key_preimages WHERE eth_key = $1)" getPgStr = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1" putPgStr = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING" - putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO UPDATE SET ipfs_key = $2" + putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key, prefix) VALUES ($1, $2, $3) ON CONFLICT (eth_key) DO UPDATE SET (ipfs_key, prefix) = ($2, $3)" deletePgStr = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1" dbSizePgStr = "SELECT pg_database_size(current_database())" ) @@ -78,7 +78,7 @@ func (d *Database) Get(key []byte) ([]byte, error) { // Key is expected to be the keccak256 hash of value // Put inserts the keccak256 key into the eth.key_preimages table func (d *Database) Put(key []byte, value []byte) error { - dsKey, err := DatastoreKeyFromGethKey(key) + dsKey, prefix, err := DatastoreKeyFromGethKey(key) if err != nil { return err } @@ -98,7 +98,7 @@ func (d *Database) Put(key []byte, value []byte) error { if _, err = tx.Exec(putPgStr, dsKey, value); err != nil { return err } - _, err = tx.Exec(putPreimagePgStr, key, dsKey) + _, err = tx.Exec(putPreimagePgStr, key, dsKey, prefix) return err } @@ -200,20 +200,20 @@ func (d *Database) NewBatch() ethdb.Batch { // NewIterator creates a binary-alphabetical iterator over the entire keyspace // contained within the key-value database. func (d *Database) NewIterator() ethdb.Iterator { - return NewIterator([]byte{}, []byte{}, d.db) + return NewIterator(nil, nil, d.db) } // NewIteratorWithStart creates a binary-alphabetical iterator over a subset of // database content starting at a particular initial key (or after, if it does // not exist). func (d *Database) NewIteratorWithStart(start []byte) ethdb.Iterator { - return NewIterator(start, []byte{}, d.db) + return NewIterator(start, nil, d.db) } // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset // of database content with a particular key prefix. func (d *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { - return NewIterator([]byte{}, prefix, d.db) + return NewIterator(nil, prefix, d.db) } // Close satisfies the io.Closer interface diff --git a/postgres/db/migrations/00003_create_eth_key_preimages_table.sql b/postgres/db/migrations/00003_create_eth_key_preimages_table.sql index e949506..52974d5 100644 --- a/postgres/db/migrations/00003_create_eth_key_preimages_table.sql +++ b/postgres/db/migrations/00003_create_eth_key_preimages_table.sql @@ -1,6 +1,7 @@ -- +goose Up CREATE TABLE IF NOT EXISTS eth.key_preimages ( eth_key BYTEA UNIQUE NOT NULL, + prefix BYTEA, ipfs_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED ); diff --git a/postgres/iterator.go b/postgres/iterator.go index f673729..120e8fa 100644 --- a/postgres/iterator.go +++ b/postgres/iterator.go @@ -21,15 +21,31 @@ import ( "github.com/jmoiron/sqlx" ) +const ( + initPgStr = `SELECT eth_key, data FROM public.blocks + INNER JOIN eth.key_preimages ON (ipfs_key = key) + WHERE eth_key = $1` + nextPgStr = `SELECT eth_key, data FROM public.blocks + INNER JOIN eth.key_preimages ON (ipfs_key = key) + WHERE eth_key > $1 + ORDER BY eth_key LIMIT 1` + nextPgStrWithPrefix = `SELECT eth_key, data FROM public.blocks + INNER JOIN eth.key_preimages ON (ipfs_key = key) + WHERE eth_key > $1 AND prefix = $2 + ORDER BY eth_key LIMIT 1` +) + +type nextModel struct { + Key []byte `db:"eth_key"` + Value []byte `db:"data"` +} + // Iterator is the type that satisfies the ethdb.Iterator interface for PG-IPFS Ethereum data using a direct Postgres connection -// Iteratee interface is used in Geth for various tests, trie/sync_bloom.go (for fast sync), -// rawdb.InspectDatabase, and the new core/state/snapshot features. -// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed -// from the ethdb.KeyValueStoreand ethdb.Database interfaces) type Iterator struct { - db *sqlx.DB - currentKey, prefix []byte - err error + db *sqlx.DB + currentKey, prefix, currentValue []byte + err error + init bool } // NewIterator returns an ethdb.Iterator interface for PG-IPFS @@ -38,6 +54,7 @@ func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator { db: db, prefix: prefix, currentKey: start, + init: start != nil, } } @@ -45,9 +62,26 @@ func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator { // Next moves the iterator to the next key/value pair // It returns whether the iterator is exhausted func (i *Iterator) Next() bool { - // this is complicated by the ipfs db keys not being the keccak256 hashes - // go-ethereum usage of this method expects the iteration to occur over keccak256 keys - panic("implement me: Next") + next := new(nextModel) + if i.init { + i.init = false + if err := i.db.Get(next, initPgStr, i.currentKey); err != nil { + i.currentKey, i.currentValue, i.err = nil, nil, err + return false + } + } else if i.prefix != nil { + if err := i.db.Get(next, nextPgStrWithPrefix, i.currentKey, i.prefix); err != nil { + i.currentKey, i.currentValue, i.err = nil, nil, err + return false + } + } else { + if err := i.db.Get(next, nextPgStr, i.currentKey); err != nil { + i.currentKey, i.currentValue, i.err = nil, nil, err + return false + } + } + i.currentKey, i.currentValue, i.err = next.Key, next.Value, nil + return true } // Error satisfies the ethdb.Iterator interface @@ -70,19 +104,12 @@ func (i *Iterator) Key() []byte { // The caller should not modify the contents of the returned slice // and its contents may change on the next call to Next func (i *Iterator) Value() []byte { - mhKey, err := MultihashKeyFromKeccak256(i.currentKey) - if err != nil { - i.err = err - return nil - } - var data []byte - i.err = i.db.Get(&data, getPgStr, mhKey) - return data + return i.currentValue } // Release satisfies the ethdb.Iterator interface // Release releases associated resources // Release should always succeed and can be called multiple times without causing error func (i *Iterator) Release() { - i.db.Close() + i.db, i.currentKey, i.currentValue, i.err, i.prefix = nil, nil, nil, nil, nil } diff --git a/postgres/iterator_test.go b/postgres/iterator_test.go new file mode 100644 index 0000000..7952e74 --- /dev/null +++ b/postgres/iterator_test.go @@ -0,0 +1,513 @@ +// VulcanizeDB +// Copyright © 2020 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pgipfsethdb_test + +import ( + "database/sql" + + "github.com/ethereum/go-ethereum/ethdb" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/ipfs-ethdb/postgres" +) + +var ( + iterator ethdb.Iterator + testPrefix = []byte("testPrefix") + testEthKey1 = []byte{'\x01'} + testEthKey2 = []byte{'\x01', '\x01'} + testEthKey3 = []byte{'\x01', '\x02'} + testEthKey4 = []byte{'\x01', '\x0e'} + testEthKey5 = []byte{'\x01', '\x02', '\x01'} + testEthKey6 = []byte{'\x01', '\x0e', '\x01'} + prefixedTestEthKey1 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey1...) + prefixedTestEthKey2 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey2...) + prefixedTestEthKey3 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey3...) + prefixedTestEthKey4 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey4...) + prefixedTestEthKey5 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey5...) + prefixedTestEthKey6 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey6...) + mockValue1 = []byte{1} + mockValue2 = []byte{2} + mockValue3 = []byte{3} + mockValue4 = []byte{4} + mockValue5 = []byte{5} + mockValue6 = []byte{6} +) + +var _ = Describe("Iterator", func() { + BeforeEach(func() { + db, err = pgipfsethdb.TestDB() + Expect(err).ToNot(HaveOccurred()) + database = pgipfsethdb.NewDatabase(db) + // non-prefixed entries + err = database.Put(testEthKey1, mockValue1) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(testEthKey2, mockValue2) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(testEthKey3, mockValue3) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(testEthKey4, mockValue4) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(testEthKey5, mockValue5) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(testEthKey6, mockValue6) + Expect(err).ToNot(HaveOccurred()) + // prefixed entries + err = database.Put(prefixedTestEthKey1, mockValue1) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(prefixedTestEthKey2, mockValue2) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(prefixedTestEthKey3, mockValue3) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(prefixedTestEthKey4, mockValue4) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(prefixedTestEthKey5, mockValue5) + Expect(err).ToNot(HaveOccurred()) + err = database.Put(prefixedTestEthKey6, mockValue6) + Expect(err).ToNot(HaveOccurred()) + }) + AfterEach(func() { + err = pgipfsethdb.ResetTestDB(db) + Expect(err).ToNot(HaveOccurred()) + }) + + Describe("NewIterator", func() { + It("iterates over the entire key-set (prefixed or not)", func() { + iterator = database.NewIterator() + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(BeNil()) + + more := iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey1)) + Expect(iterator.Value()).To(Equal(mockValue1)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey1)) + Expect(iterator.Value()).To(Equal(mockValue1)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).ToNot(BeTrue()) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(Equal(sql.ErrNoRows)) + }) + }) + + Describe("NewIteratorWithPrefix", func() { + It("iterates over all db entries that have the provided prefix", func() { + iterator = database.NewIteratorWithPrefix(testPrefix) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(BeNil()) + + more := iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey1)) + Expect(iterator.Value()).To(Equal(mockValue1)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).ToNot(BeTrue()) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(Equal(sql.ErrNoRows)) + }) + + It("behaves as no prefix is provided if prefix is nil", func() { + iterator = database.NewIteratorWithPrefix(nil) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(BeNil()) + + more := iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey1)) + Expect(iterator.Value()).To(Equal(mockValue1)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey1)) + Expect(iterator.Value()).To(Equal(mockValue1)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).ToNot(BeTrue()) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(Equal(sql.ErrNoRows)) + }) + + It("considers empty but non-nil []byte a valid prefix, which precludes iteration over any other prefixed keys", func() { + iterator = database.NewIteratorWithPrefix([]byte{}) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(BeNil()) + + more := iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey1)) + Expect(iterator.Value()).To(Equal(mockValue1)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).ToNot(BeTrue()) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(Equal(sql.ErrNoRows)) + }) + }) + + Describe("NewIteratorWithStart", func() { + It("iterates over the entire key-set (prefixed or not) starting with at the provided path", func() { + iterator = database.NewIteratorWithStart(testEthKey2) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Error()).To(BeNil()) + + more := iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey1)) + Expect(iterator.Value()).To(Equal(mockValue1)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).ToNot(BeTrue()) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(Equal(sql.ErrNoRows)) + }) + + It("iterates over the entire key-set (prefixed or not) starting with at the provided path", func() { + iterator = database.NewIteratorWithStart(prefixedTestEthKey3) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey3)) + Expect(iterator.Error()).To(BeNil()) + + more := iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey3)) + Expect(iterator.Value()).To(Equal(mockValue3)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey5)) + Expect(iterator.Value()).To(Equal(mockValue5)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey4)) + Expect(iterator.Value()).To(Equal(mockValue4)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(prefixedTestEthKey6)) + Expect(iterator.Value()).To(Equal(mockValue6)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).ToNot(BeTrue()) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(Equal(sql.ErrNoRows)) + }) + }) + + Describe("Release", func() { + It("releases resources associated with the Iterator", func() { + iterator = database.NewIteratorWithStart(testEthKey2) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Error()).To(BeNil()) + + more := iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + + iterator.Release() + iterator.Release() // check that we don't panic if called multiple times + + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(BeNil()) + Expect(iterator.Error()).To(BeNil()) + Expect(func() { iterator.Next() }).To(Panic()) // check that we panic if we try to use released iterator + + // We can still create a new iterator from the same backing db + iterator = database.NewIteratorWithStart(testEthKey2) + Expect(iterator.Value()).To(BeNil()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Error()).To(BeNil()) + + more = iterator.Next() + Expect(more).To(BeTrue()) + Expect(iterator.Key()).To(Equal(testEthKey2)) + Expect(iterator.Value()).To(Equal(mockValue2)) + Expect(iterator.Error()).To(BeNil()) + }) + }) +}) diff --git a/postgres/util.go b/postgres/util.go index e2a11fe..92838d3 100644 --- a/postgres/util.go +++ b/postgres/util.go @@ -37,21 +37,25 @@ func MultihashKeyFromKeccak256(h []byte) (string, error) { } // DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key -func DatastoreKeyFromGethKey(h []byte) (string, error) { +// It also returns the key's prefix, if it has one +func DatastoreKeyFromGethKey(h []byte) (string, []byte, error) { keyType, keyComponents := ResolveKeyType(h) switch keyType { case Keccak: - return MultihashKeyFromKeccak256(h) + mhKey, err := MultihashKeyFromKeccak256(h) + return mhKey, nil, err case Header: - return MultihashKeyFromKeccak256(keyComponents[1]) + mhKey, err := MultihashKeyFromKeccak256(keyComponents[1]) + return mhKey, keyComponents[0], err case Preimage: - return MultihashKeyFromKeccak256(keyComponents[1]) + mhKey, err := MultihashKeyFromKeccak256(keyComponents[1]) + return mhKey, keyComponents[0], err case Prefixed, Suffixed: // This data is not mapped by hash => content by geth, store it using the prefixed/suffixed key directly // I.e. the public.blocks datastore key == the hex representation of the geth key - return common.Bytes2Hex(h), nil + // Alternatively, decompose the data and derive the hash + return common.Bytes2Hex(h), keyComponents[0], nil default: - return "", fmt.Errorf("invalid formatting of database key: %x", h) + return "", nil, fmt.Errorf("invalid formatting of database key: %x", h) } - }