From dece84dc59145e540ce49aa91e22739806c36117 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Tue, 8 Sep 2020 16:14:27 -0500 Subject: [PATCH 1/3] revert to version that works with 1.9.10 --- README.md | 2 +- postgres/doc.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 952c22c..eb1232a 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ func main() { database := ipfsethdb.NewDatabase(ipfsNode.Blocks) stateDatabase := state.NewDatabase(database) - stateDB, _ := state.New(common.Hash{}, stateDatabase, nil) + stateDB, _ := state.New(common.Hash{}, stateDatabase) stateDBNodeIterator := state.NewNodeIterator(stateDB) // do stuff with the statedb node iterator } diff --git a/postgres/doc.md b/postgres/doc.md index 1e8f1b9..504fcde 100644 --- a/postgres/doc.md +++ b/postgres/doc.md @@ -33,7 +33,7 @@ func main() { database := pgipfsethdb.NewDatabase(db) stateDatabase := state.NewDatabase(database) - stateDB, _ := state.New(common.Hash{}, stateDatabase, nil) + stateDB, _ := state.New(common.Hash{}, stateDatabase) stateDBNodeIterator := state.NewNodeIterator(stateDB) // do stuff with the statedb node iterator } From 05648112e9624d4e228a4db1aa981969b8cc7b9b Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 9 Sep 2020 10:51:10 -0500 Subject: [PATCH 2/3] handle different key prefixes --- postgres/batch.go | 6 ++-- postgres/database.go | 8 ++--- postgres/key_type.go | 64 ++++++++++++++++++++++++++++++++++++++++ postgres/test_helpers.go | 33 +++++++++++++++++++++ postgres/util.go | 30 +++++++++++-------- 5 files changed, 122 insertions(+), 19 deletions(-) create mode 100644 postgres/key_type.go create mode 100644 postgres/test_helpers.go diff --git a/postgres/batch.go b/postgres/batch.go index 74a883d..838f857 100644 --- a/postgres/batch.go +++ b/postgres/batch.go @@ -44,14 +44,14 @@ func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch { // Put inserts the given value into the key-value data store // Key is expected to be the keccak256 hash of value func (b *Batch) Put(key []byte, value []byte) (err error) { - mhKey, err := MultihashKeyFromKeccak256(key) + dsKey, err := DatastoreKeyFromGethKey(key) if err != nil { return err } - if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil { + if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil { return err } - if _, err = b.tx.Exec(putPreimagePgStr, key, mhKey); err != nil { + if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey); err != nil { return err } b.valueSize += len(value) diff --git a/postgres/database.go b/postgres/database.go index 3a357a6..b2d9f71 100644 --- a/postgres/database.go +++ b/postgres/database.go @@ -33,7 +33,7 @@ var ( hasPgStr = "SELECT exists(select 1 from eth.key_preimages WHERE eth_key = $1)" getPgStr = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1" putPgStr = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING" - putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO NOTHING" + putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO UPDATE SET ipfs_key = $2" deletePgStr = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1" dbSizePgStr = "SELECT pg_database_size(current_database())" ) @@ -78,7 +78,7 @@ func (d *Database) Get(key []byte) ([]byte, error) { // Key is expected to be the keccak256 hash of value // Put inserts the keccak256 key into the eth.key_preimages table func (d *Database) Put(key []byte, value []byte) error { - mhKey, err := MultihashKeyFromKeccak256(key) + dsKey, err := DatastoreKeyFromGethKey(key) if err != nil { return err } @@ -95,10 +95,10 @@ func (d *Database) Put(key []byte, value []byte) error { err = tx.Commit() } }() - if _, err = tx.Exec(putPgStr, mhKey, value); err != nil { + if _, err = tx.Exec(putPgStr, dsKey, value); err != nil { return err } - _, err = tx.Exec(putPreimagePgStr, key, mhKey) + _, err = tx.Exec(putPreimagePgStr, key, dsKey) return err } diff --git a/postgres/key_type.go b/postgres/key_type.go new file mode 100644 index 0000000..b92164a --- /dev/null +++ b/postgres/key_type.go @@ -0,0 +1,64 @@ +// VulcanizeDB +// Copyright © 2020 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pgipfsethdb + +import "bytes" + +type KeyType uint + +const ( + Invalid KeyType = iota + Keccak + Prefixed + Suffixed + Header + Preimage +) + +var ( + // keyDelineation is used to delineate the key prefixes and suffixes + KeyDelineation = []byte("/") + + // numberDelineation is used to delineate the block number encoded in a key + NumberDelineation = []byte(":") + + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). + HeaderPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + PreimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage +) + +// ResolveKeyType returns the key type based on the prefix +func ResolveKeyType(key []byte) (KeyType, [][]byte) { + sk := bytes.Split(key, KeyDelineation) + + switch len(sk) { + case 1: + return Keccak, sk + case 2: + switch prefix := sk[0]; { + case bytes.Equal(prefix, HeaderPrefix): + return Header, bytes.Split(sk[1], NumberDelineation) + case bytes.Equal(prefix, PreimagePrefix): + return Preimage, sk + default: + return Prefixed, sk + } + case 3: + return Suffixed, sk + } + return Invalid, sk +} diff --git a/postgres/test_helpers.go b/postgres/test_helpers.go new file mode 100644 index 0000000..7eb4260 --- /dev/null +++ b/postgres/test_helpers.go @@ -0,0 +1,33 @@ +// VulcanizeDB +// Copyright © 2020 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pgipfsethdb + +import "github.com/jmoiron/sqlx" + +// TestDB connect to the testing database +// it assumes the database has the IPFS public.blocks table present +// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table +func TestDB() (*sqlx.DB, error) { + connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable" + return sqlx.Connect("postgres", connectStr) +} + +// ResetTestDB drops all rows in the test db public.blocks table +func ResetTestDB(db *sqlx.DB) error { + _, err := db.Exec("TRUNCATE public.blocks CASCADE") + return err +} diff --git a/postgres/util.go b/postgres/util.go index e12f0cc..4b28199 100644 --- a/postgres/util.go +++ b/postgres/util.go @@ -17,9 +17,11 @@ package pgipfsethdb import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-ds-help" - "github.com/jmoiron/sqlx" _ "github.com/lib/pq" //postgres driver "github.com/multiformats/go-multihash" ) @@ -34,16 +36,20 @@ func MultihashKeyFromKeccak256(h []byte) (string, error) { return blockstore.BlockPrefix.String() + dbKey.String(), nil } -// TestDB connect to the testing database -// it assumes the database has the IPFS public.blocks table present -// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table -func TestDB() (*sqlx.DB, error) { - connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable" - return sqlx.Connect("postgres", connectStr) -} +// DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key +func DatastoreKeyFromGethKey(h []byte) (string, error) { + keyType, keyComponents := ResolveKeyType(h) + switch keyType { + case Keccak: + return MultihashKeyFromKeccak256(h) + case Header: + return MultihashKeyFromKeccak256(keyComponents[1]) + case Preimage: + return MultihashKeyFromKeccak256(keyComponents[1]) + case Prefixed, Suffixed: + return common.Bytes2Hex(h), nil + default: + return "", fmt.Errorf("invalid formatting of database key: %x", h) + } -// ResetTestDB drops all rows in the test db public.blocks table -func ResetTestDB(db *sqlx.DB) error { - _, err := db.Exec("TRUNCATE public.blocks CASCADE") - return err } From 7c6660a5a6432316bd052ce100709af7a285d77f Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 9 Sep 2020 10:51:30 -0500 Subject: [PATCH 3/3] database tests for different key prefixes --- postgres/batch_test.go | 36 ++--- postgres/database_test.go | 321 +++++++++++++++++++++++++++++++++++--- postgres/util.go | 2 + 3 files changed, 316 insertions(+), 43 deletions(-) diff --git a/postgres/batch_test.go b/postgres/batch_test.go index 208d0ce..5d89168 100644 --- a/postgres/batch_test.go +++ b/postgres/batch_test.go @@ -29,10 +29,10 @@ import ( ) var ( - batch ethdb.Batch - testHeader2 = types.Header{Number: big.NewInt(2)} - testValue2, _ = rlp.EncodeToBytes(testHeader2) - testEthKey2 = testHeader2.Hash().Bytes() + batch ethdb.Batch + testHeader2 = types.Header{Number: big.NewInt(2)} + testValue2, _ = rlp.EncodeToBytes(testHeader2) + testKeccakEthKey2 = testHeader2.Hash().Bytes() ) var _ = Describe("Batch", func() { @@ -49,24 +49,24 @@ var _ = Describe("Batch", func() { Describe("Put/Write", func() { It("adds the key-value pair to the batch", func() { - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - _, err = database.Get(testEthKey2) + _, err = database.Get(testKeccakEthKey2) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - err = batch.Put(testEthKey, testValue) + err = batch.Put(testKeccakEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - err = batch.Put(testEthKey2, testValue2) + err = batch.Put(testKeccakEthKey2, testValue2) Expect(err).ToNot(HaveOccurred()) err = batch.Write() Expect(err).ToNot(HaveOccurred()) - val, err := database.Get(testEthKey) + val, err := database.Get(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(val).To(Equal(testValue)) - val2, err := database.Get(testEthKey2) + val2, err := database.Get(testKeccakEthKey2) Expect(err).ToNot(HaveOccurred()) Expect(val2).To(Equal(testValue2)) }) @@ -74,25 +74,25 @@ var _ = Describe("Batch", func() { Describe("Delete/Reset/Write", func() { It("deletes the key-value pair in the batch", func() { - err = batch.Put(testEthKey, testValue) + err = batch.Put(testKeccakEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - err = batch.Put(testEthKey2, testValue2) + err = batch.Put(testKeccakEthKey2, testValue2) Expect(err).ToNot(HaveOccurred()) err = batch.Write() Expect(err).ToNot(HaveOccurred()) batch.Reset() - err = batch.Delete(testEthKey) + err = batch.Delete(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) - err = batch.Delete(testEthKey2) + err = batch.Delete(testKeccakEthKey2) Expect(err).ToNot(HaveOccurred()) err = batch.Write() Expect(err).ToNot(HaveOccurred()) - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - _, err = database.Get(testEthKey2) + _, err = database.Get(testKeccakEthKey2) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) }) @@ -100,9 +100,9 @@ var _ = Describe("Batch", func() { Describe("ValueSize/Reset", func() { It("returns the size of data in the batch queued for write", func() { - err = batch.Put(testEthKey, testValue) + err = batch.Put(testKeccakEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - err = batch.Put(testEthKey2, testValue2) + err = batch.Put(testKeccakEthKey2, testValue2) Expect(err).ToNot(HaveOccurred()) err = batch.Write() Expect(err).ToNot(HaveOccurred()) diff --git a/postgres/database_test.go b/postgres/database_test.go index ca560bf..dc45803 100644 --- a/postgres/database_test.go +++ b/postgres/database_test.go @@ -19,6 +19,8 @@ package pgipfsethdb_test import ( "math/big" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" @@ -30,13 +32,26 @@ import ( ) var ( - database ethdb.Database - db *sqlx.DB - err error - testHeader = types.Header{Number: big.NewInt(1337)} - testValue, _ = rlp.EncodeToBytes(testHeader) - testEthKey = testHeader.Hash().Bytes() - testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey) + database ethdb.Database + db *sqlx.DB + err error + testHeader = types.Header{Number: big.NewInt(1337)} + testValue, _ = rlp.EncodeToBytes(testHeader) + testKeccakEthKey = testHeader.Hash().Bytes() + testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testKeccakEthKey) + + testPrefixedEthKey = append(append([]byte("prefix"), pgipfsethdb.KeyDelineation...), testKeccakEthKey...) + testPrefixedDsKey = common.Bytes2Hex(testPrefixedEthKey) + + testSuffixedEthKey = append(append(testPrefixedEthKey, pgipfsethdb.KeyDelineation...), []byte("suffix")...) + testSuffixedDsKey = common.Bytes2Hex(testSuffixedEthKey) + + testHeaderEthKey = append(append(append(append(pgipfsethdb.HeaderPrefix, pgipfsethdb.KeyDelineation...), + []byte("number")...), pgipfsethdb.NumberDelineation...), testKeccakEthKey...) + testHeaderDsKey = testMhKey + + testPreimageEthKey = append(append(pgipfsethdb.PreimagePrefix, pgipfsethdb.KeyDelineation...), testKeccakEthKey...) + testPreimageDsKey = testMhKey ) var _ = Describe("Database", func() { @@ -50,65 +65,321 @@ var _ = Describe("Database", func() { Expect(err).ToNot(HaveOccurred()) }) - Describe("Has", func() { + Describe("Has - Keccak keys", func() { It("returns false if a key-pair doesn't exist in the db", func() { - has, err := database.Has(testEthKey) + has, err := database.Has(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(has).ToNot(BeTrue()) }) It("returns true if a key-pair exists in the db", func() { _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue) Expect(err).ToNot(HaveOccurred()) - _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testEthKey, testMhKey) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey) Expect(err).ToNot(HaveOccurred()) - has, err := database.Has(testEthKey) + has, err := database.Has(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(has).To(BeTrue()) }) }) - Describe("Get", func() { + Describe("Has - Prefixed keys", func() { + It("returns false if a key-pair doesn't exist in the db", func() { + has, err := database.Has(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).ToNot(BeTrue()) + }) + It("returns true if a key-pair exists in the db", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey) + Expect(err).ToNot(HaveOccurred()) + has, err := database.Has(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(BeTrue()) + }) + }) + + Describe("Has - Suffixed keys", func() { + It("returns false if a key-pair doesn't exist in the db", func() { + has, err := database.Has(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).ToNot(BeTrue()) + }) + It("returns true if a key-pair exists in the db", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey) + Expect(err).ToNot(HaveOccurred()) + has, err := database.Has(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(BeTrue()) + }) + }) + + Describe("Has - Header keys", func() { + It("returns false if a key-pair doesn't exist in the db", func() { + has, err := database.Has(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).ToNot(BeTrue()) + }) + It("returns true if a key-pair exists in the db", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey) + Expect(err).ToNot(HaveOccurred()) + has, err := database.Has(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(BeTrue()) + }) + }) + + Describe("Has - Preimage keys", func() { + It("returns false if a key-pair doesn't exist in the db", func() { + has, err := database.Has(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).ToNot(BeTrue()) + }) + It("returns true if a key-pair exists in the db", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey) + Expect(err).ToNot(HaveOccurred()) + has, err := database.Has(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(BeTrue()) + }) + }) + + Describe("Get - Keccak keys", func() { It("throws an err if the key-pair doesn't exist in the db", func() { - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) }) It("returns the value associated with the key, if the pair exists", func() { _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue) Expect(err).ToNot(HaveOccurred()) - _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testEthKey, testMhKey) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey) Expect(err).ToNot(HaveOccurred()) - val, err := database.Get(testEthKey) + val, err := database.Get(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(val).To(Equal(testValue)) }) }) - Describe("Put", func() { + Describe("Get - Prefixed keys", func() { + It("throws an err if the key-pair doesn't exist in the db", func() { + _, err = database.Get(testPrefixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("returns the value associated with the key, if the pair exists", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Get - Suffixed keys", func() { + It("throws an err if the key-pair doesn't exist in the db", func() { + _, err = database.Get(testSuffixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("returns the value associated with the key, if the pair exists", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Get - Header keys", func() { + It("throws an err if the key-pair doesn't exist in the db", func() { + _, err = database.Get(testHeaderEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("returns the value associated with the key, if the pair exists", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Get - Preimage keys", func() { + It("throws an err if the key-pair doesn't exist in the db", func() { + _, err = database.Get(testPreimageEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("returns the value associated with the key, if the pair exists", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Put - Keccak keys", func() { It("persists the key-value pair in the database", func() { - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - err = database.Put(testEthKey, testValue) + err = database.Put(testKeccakEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - val, err := database.Get(testEthKey) + val, err := database.Get(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(val).To(Equal(testValue)) }) }) - Describe("Delete", func() { - It("removes the key-value pair from the database", func() { - err = database.Put(testEthKey, testValue) + Describe("Put - Prefixed keys", func() { + It("persists the key-value pair in the database", func() { + _, err = database.Get(testPrefixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + err = database.Put(testPrefixedEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - val, err := database.Get(testEthKey) + val, err := database.Get(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Put - Suffixed keys", func() { + It("persists the key-value pair in the database", func() { + _, err = database.Get(testSuffixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + err = database.Put(testSuffixedEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Put - Header keys", func() { + It("persists the key-value pair in the database", func() { + _, err = database.Get(testHeaderEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + err = database.Put(testHeaderEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Put - Preimage keys", func() { + It("persists the key-value pair in the database", func() { + _, err = database.Get(testPreimageEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + err = database.Put(testPreimageEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Delete - Keccak keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testKeccakEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(val).To(Equal(testValue)) - err = database.Delete(testEthKey) + err = database.Delete(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("Delete - Prefixed keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testPrefixedEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + + err = database.Delete(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + _, err = database.Get(testPrefixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("Delete - Suffixed keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testSuffixedEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + + err = database.Delete(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + _, err = database.Get(testSuffixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("Delete - Header keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testHeaderEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + + err = database.Delete(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + _, err = database.Get(testHeaderEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("Delete - Preimage keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testPreimageEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + + err = database.Delete(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + _, err = database.Get(testPreimageEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) }) diff --git a/postgres/util.go b/postgres/util.go index 4b28199..e2a11fe 100644 --- a/postgres/util.go +++ b/postgres/util.go @@ -47,6 +47,8 @@ func DatastoreKeyFromGethKey(h []byte) (string, error) { case Preimage: return MultihashKeyFromKeccak256(keyComponents[1]) case Prefixed, Suffixed: + // This data is not mapped by hash => content by geth, store it using the prefixed/suffixed key directly + // I.e. the public.blocks datastore key == the hex representation of the geth key return common.Bytes2Hex(h), nil default: return "", fmt.Errorf("invalid formatting of database key: %x", h)