From 7c6660a5a6432316bd052ce100709af7a285d77f Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 9 Sep 2020 10:51:30 -0500 Subject: [PATCH] database tests for different key prefixes --- postgres/batch_test.go | 36 ++--- postgres/database_test.go | 321 +++++++++++++++++++++++++++++++++++--- postgres/util.go | 2 + 3 files changed, 316 insertions(+), 43 deletions(-) diff --git a/postgres/batch_test.go b/postgres/batch_test.go index 208d0ce..5d89168 100644 --- a/postgres/batch_test.go +++ b/postgres/batch_test.go @@ -29,10 +29,10 @@ import ( ) var ( - batch ethdb.Batch - testHeader2 = types.Header{Number: big.NewInt(2)} - testValue2, _ = rlp.EncodeToBytes(testHeader2) - testEthKey2 = testHeader2.Hash().Bytes() + batch ethdb.Batch + testHeader2 = types.Header{Number: big.NewInt(2)} + testValue2, _ = rlp.EncodeToBytes(testHeader2) + testKeccakEthKey2 = testHeader2.Hash().Bytes() ) var _ = Describe("Batch", func() { @@ -49,24 +49,24 @@ var _ = Describe("Batch", func() { Describe("Put/Write", func() { It("adds the key-value pair to the batch", func() { - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - _, err = database.Get(testEthKey2) + _, err = database.Get(testKeccakEthKey2) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - err = batch.Put(testEthKey, testValue) + err = batch.Put(testKeccakEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - err = batch.Put(testEthKey2, testValue2) + err = batch.Put(testKeccakEthKey2, testValue2) Expect(err).ToNot(HaveOccurred()) err = batch.Write() Expect(err).ToNot(HaveOccurred()) - val, err := database.Get(testEthKey) + val, err := database.Get(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(val).To(Equal(testValue)) - val2, err := database.Get(testEthKey2) + val2, err := database.Get(testKeccakEthKey2) Expect(err).ToNot(HaveOccurred()) Expect(val2).To(Equal(testValue2)) }) @@ -74,25 +74,25 @@ var _ = Describe("Batch", func() { Describe("Delete/Reset/Write", func() { It("deletes the key-value pair in the batch", func() { - err = batch.Put(testEthKey, testValue) + err = batch.Put(testKeccakEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - err = batch.Put(testEthKey2, testValue2) + err = batch.Put(testKeccakEthKey2, testValue2) Expect(err).ToNot(HaveOccurred()) err = batch.Write() Expect(err).ToNot(HaveOccurred()) batch.Reset() - err = batch.Delete(testEthKey) + err = batch.Delete(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) - err = batch.Delete(testEthKey2) + err = batch.Delete(testKeccakEthKey2) Expect(err).ToNot(HaveOccurred()) err = batch.Write() Expect(err).ToNot(HaveOccurred()) - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - _, err = database.Get(testEthKey2) + _, err = database.Get(testKeccakEthKey2) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) }) @@ -100,9 +100,9 @@ var _ = Describe("Batch", func() { Describe("ValueSize/Reset", func() { It("returns the size of data in the batch queued for write", func() { - err = batch.Put(testEthKey, testValue) + err = batch.Put(testKeccakEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - err = batch.Put(testEthKey2, testValue2) + err = batch.Put(testKeccakEthKey2, testValue2) Expect(err).ToNot(HaveOccurred()) err = batch.Write() Expect(err).ToNot(HaveOccurred()) diff --git a/postgres/database_test.go b/postgres/database_test.go index ca560bf..dc45803 100644 --- a/postgres/database_test.go +++ b/postgres/database_test.go @@ -19,6 +19,8 @@ package pgipfsethdb_test import ( "math/big" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" @@ -30,13 +32,26 @@ import ( ) var ( - database ethdb.Database - db *sqlx.DB - err error - testHeader = types.Header{Number: big.NewInt(1337)} - testValue, _ = rlp.EncodeToBytes(testHeader) - testEthKey = testHeader.Hash().Bytes() - testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey) + database ethdb.Database + db *sqlx.DB + err error + testHeader = types.Header{Number: big.NewInt(1337)} + testValue, _ = rlp.EncodeToBytes(testHeader) + testKeccakEthKey = testHeader.Hash().Bytes() + testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testKeccakEthKey) + + testPrefixedEthKey = append(append([]byte("prefix"), pgipfsethdb.KeyDelineation...), testKeccakEthKey...) + testPrefixedDsKey = common.Bytes2Hex(testPrefixedEthKey) + + testSuffixedEthKey = append(append(testPrefixedEthKey, pgipfsethdb.KeyDelineation...), []byte("suffix")...) + testSuffixedDsKey = common.Bytes2Hex(testSuffixedEthKey) + + testHeaderEthKey = append(append(append(append(pgipfsethdb.HeaderPrefix, pgipfsethdb.KeyDelineation...), + []byte("number")...), pgipfsethdb.NumberDelineation...), testKeccakEthKey...) + testHeaderDsKey = testMhKey + + testPreimageEthKey = append(append(pgipfsethdb.PreimagePrefix, pgipfsethdb.KeyDelineation...), testKeccakEthKey...) + testPreimageDsKey = testMhKey ) var _ = Describe("Database", func() { @@ -50,65 +65,321 @@ var _ = Describe("Database", func() { Expect(err).ToNot(HaveOccurred()) }) - Describe("Has", func() { + Describe("Has - Keccak keys", func() { It("returns false if a key-pair doesn't exist in the db", func() { - has, err := database.Has(testEthKey) + has, err := database.Has(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(has).ToNot(BeTrue()) }) It("returns true if a key-pair exists in the db", func() { _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue) Expect(err).ToNot(HaveOccurred()) - _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testEthKey, testMhKey) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey) Expect(err).ToNot(HaveOccurred()) - has, err := database.Has(testEthKey) + has, err := database.Has(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(has).To(BeTrue()) }) }) - Describe("Get", func() { + Describe("Has - Prefixed keys", func() { + It("returns false if a key-pair doesn't exist in the db", func() { + has, err := database.Has(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).ToNot(BeTrue()) + }) + It("returns true if a key-pair exists in the db", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey) + Expect(err).ToNot(HaveOccurred()) + has, err := database.Has(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(BeTrue()) + }) + }) + + Describe("Has - Suffixed keys", func() { + It("returns false if a key-pair doesn't exist in the db", func() { + has, err := database.Has(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).ToNot(BeTrue()) + }) + It("returns true if a key-pair exists in the db", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey) + Expect(err).ToNot(HaveOccurred()) + has, err := database.Has(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(BeTrue()) + }) + }) + + Describe("Has - Header keys", func() { + It("returns false if a key-pair doesn't exist in the db", func() { + has, err := database.Has(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).ToNot(BeTrue()) + }) + It("returns true if a key-pair exists in the db", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey) + Expect(err).ToNot(HaveOccurred()) + has, err := database.Has(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(BeTrue()) + }) + }) + + Describe("Has - Preimage keys", func() { + It("returns false if a key-pair doesn't exist in the db", func() { + has, err := database.Has(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).ToNot(BeTrue()) + }) + It("returns true if a key-pair exists in the db", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey) + Expect(err).ToNot(HaveOccurred()) + has, err := database.Has(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(has).To(BeTrue()) + }) + }) + + Describe("Get - Keccak keys", func() { It("throws an err if the key-pair doesn't exist in the db", func() { - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) }) It("returns the value associated with the key, if the pair exists", func() { _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue) Expect(err).ToNot(HaveOccurred()) - _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testEthKey, testMhKey) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey) Expect(err).ToNot(HaveOccurred()) - val, err := database.Get(testEthKey) + val, err := database.Get(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(val).To(Equal(testValue)) }) }) - Describe("Put", func() { + Describe("Get - Prefixed keys", func() { + It("throws an err if the key-pair doesn't exist in the db", func() { + _, err = database.Get(testPrefixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("returns the value associated with the key, if the pair exists", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Get - Suffixed keys", func() { + It("throws an err if the key-pair doesn't exist in the db", func() { + _, err = database.Get(testSuffixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("returns the value associated with the key, if the pair exists", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Get - Header keys", func() { + It("throws an err if the key-pair doesn't exist in the db", func() { + _, err = database.Get(testHeaderEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("returns the value associated with the key, if the pair exists", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Get - Preimage keys", func() { + It("throws an err if the key-pair doesn't exist in the db", func() { + _, err = database.Get(testPreimageEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("returns the value associated with the key, if the pair exists", func() { + _, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue) + Expect(err).ToNot(HaveOccurred()) + _, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Put - Keccak keys", func() { It("persists the key-value pair in the database", func() { - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - err = database.Put(testEthKey, testValue) + err = database.Put(testKeccakEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - val, err := database.Get(testEthKey) + val, err := database.Get(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(val).To(Equal(testValue)) }) }) - Describe("Delete", func() { - It("removes the key-value pair from the database", func() { - err = database.Put(testEthKey, testValue) + Describe("Put - Prefixed keys", func() { + It("persists the key-value pair in the database", func() { + _, err = database.Get(testPrefixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + err = database.Put(testPrefixedEthKey, testValue) Expect(err).ToNot(HaveOccurred()) - val, err := database.Get(testEthKey) + val, err := database.Get(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Put - Suffixed keys", func() { + It("persists the key-value pair in the database", func() { + _, err = database.Get(testSuffixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + err = database.Put(testSuffixedEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Put - Header keys", func() { + It("persists the key-value pair in the database", func() { + _, err = database.Get(testHeaderEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + err = database.Put(testHeaderEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Put - Preimage keys", func() { + It("persists the key-value pair in the database", func() { + _, err = database.Get(testPreimageEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + err = database.Put(testPreimageEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + }) + }) + + Describe("Delete - Keccak keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testKeccakEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) Expect(val).To(Equal(testValue)) - err = database.Delete(testEthKey) + err = database.Delete(testKeccakEthKey) Expect(err).ToNot(HaveOccurred()) - _, err = database.Get(testEthKey) + _, err = database.Get(testKeccakEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("Delete - Prefixed keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testPrefixedEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + + err = database.Delete(testPrefixedEthKey) + Expect(err).ToNot(HaveOccurred()) + _, err = database.Get(testPrefixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("Delete - Suffixed keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testSuffixedEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + + err = database.Delete(testSuffixedEthKey) + Expect(err).ToNot(HaveOccurred()) + _, err = database.Get(testSuffixedEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("Delete - Header keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testHeaderEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + + err = database.Delete(testHeaderEthKey) + Expect(err).ToNot(HaveOccurred()) + _, err = database.Get(testHeaderEthKey) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("Delete - Preimage keys", func() { + It("removes the key-value pair from the database", func() { + err = database.Put(testPreimageEthKey, testValue) + Expect(err).ToNot(HaveOccurred()) + val, err := database.Get(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(testValue)) + + err = database.Delete(testPreimageEthKey) + Expect(err).ToNot(HaveOccurred()) + _, err = database.Get(testPreimageEthKey) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) }) diff --git a/postgres/util.go b/postgres/util.go index 4b28199..e2a11fe 100644 --- a/postgres/util.go +++ b/postgres/util.go @@ -47,6 +47,8 @@ func DatastoreKeyFromGethKey(h []byte) (string, error) { case Preimage: return MultihashKeyFromKeccak256(keyComponents[1]) case Prefixed, Suffixed: + // This data is not mapped by hash => content by geth, store it using the prefixed/suffixed key directly + // I.e. the public.blocks datastore key == the hex representation of the geth key return common.Bytes2Hex(h), nil default: return "", fmt.Errorf("invalid formatting of database key: %x", h)