From 05648112e9624d4e228a4db1aa981969b8cc7b9b Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 9 Sep 2020 10:51:10 -0500 Subject: [PATCH] handle different key prefixes --- postgres/batch.go | 6 ++-- postgres/database.go | 8 ++--- postgres/key_type.go | 64 ++++++++++++++++++++++++++++++++++++++++ postgres/test_helpers.go | 33 +++++++++++++++++++++ postgres/util.go | 30 +++++++++++-------- 5 files changed, 122 insertions(+), 19 deletions(-) create mode 100644 postgres/key_type.go create mode 100644 postgres/test_helpers.go diff --git a/postgres/batch.go b/postgres/batch.go index 74a883d..838f857 100644 --- a/postgres/batch.go +++ b/postgres/batch.go @@ -44,14 +44,14 @@ func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch { // Put inserts the given value into the key-value data store // Key is expected to be the keccak256 hash of value func (b *Batch) Put(key []byte, value []byte) (err error) { - mhKey, err := MultihashKeyFromKeccak256(key) + dsKey, err := DatastoreKeyFromGethKey(key) if err != nil { return err } - if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil { + if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil { return err } - if _, err = b.tx.Exec(putPreimagePgStr, key, mhKey); err != nil { + if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey); err != nil { return err } b.valueSize += len(value) diff --git a/postgres/database.go b/postgres/database.go index 3a357a6..b2d9f71 100644 --- a/postgres/database.go +++ b/postgres/database.go @@ -33,7 +33,7 @@ var ( hasPgStr = "SELECT exists(select 1 from eth.key_preimages WHERE eth_key = $1)" getPgStr = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1" putPgStr = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING" - putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO NOTHING" + putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO UPDATE SET ipfs_key = $2" deletePgStr = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1" dbSizePgStr = "SELECT pg_database_size(current_database())" ) @@ -78,7 +78,7 @@ func (d *Database) Get(key []byte) ([]byte, error) { // Key is expected to be the keccak256 hash of value // Put inserts the keccak256 key into the eth.key_preimages table func (d *Database) Put(key []byte, value []byte) error { - mhKey, err := MultihashKeyFromKeccak256(key) + dsKey, err := DatastoreKeyFromGethKey(key) if err != nil { return err } @@ -95,10 +95,10 @@ func (d *Database) Put(key []byte, value []byte) error { err = tx.Commit() } }() - if _, err = tx.Exec(putPgStr, mhKey, value); err != nil { + if _, err = tx.Exec(putPgStr, dsKey, value); err != nil { return err } - _, err = tx.Exec(putPreimagePgStr, key, mhKey) + _, err = tx.Exec(putPreimagePgStr, key, dsKey) return err } diff --git a/postgres/key_type.go b/postgres/key_type.go new file mode 100644 index 0000000..b92164a --- /dev/null +++ b/postgres/key_type.go @@ -0,0 +1,64 @@ +// VulcanizeDB +// Copyright © 2020 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pgipfsethdb + +import "bytes" + +type KeyType uint + +const ( + Invalid KeyType = iota + Keccak + Prefixed + Suffixed + Header + Preimage +) + +var ( + // keyDelineation is used to delineate the key prefixes and suffixes + KeyDelineation = []byte("/") + + // numberDelineation is used to delineate the block number encoded in a key + NumberDelineation = []byte(":") + + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). + HeaderPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + PreimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage +) + +// ResolveKeyType returns the key type based on the prefix +func ResolveKeyType(key []byte) (KeyType, [][]byte) { + sk := bytes.Split(key, KeyDelineation) + + switch len(sk) { + case 1: + return Keccak, sk + case 2: + switch prefix := sk[0]; { + case bytes.Equal(prefix, HeaderPrefix): + return Header, bytes.Split(sk[1], NumberDelineation) + case bytes.Equal(prefix, PreimagePrefix): + return Preimage, sk + default: + return Prefixed, sk + } + case 3: + return Suffixed, sk + } + return Invalid, sk +} diff --git a/postgres/test_helpers.go b/postgres/test_helpers.go new file mode 100644 index 0000000..7eb4260 --- /dev/null +++ b/postgres/test_helpers.go @@ -0,0 +1,33 @@ +// VulcanizeDB +// Copyright © 2020 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pgipfsethdb + +import "github.com/jmoiron/sqlx" + +// TestDB connect to the testing database +// it assumes the database has the IPFS public.blocks table present +// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table +func TestDB() (*sqlx.DB, error) { + connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable" + return sqlx.Connect("postgres", connectStr) +} + +// ResetTestDB drops all rows in the test db public.blocks table +func ResetTestDB(db *sqlx.DB) error { + _, err := db.Exec("TRUNCATE public.blocks CASCADE") + return err +} diff --git a/postgres/util.go b/postgres/util.go index e12f0cc..4b28199 100644 --- a/postgres/util.go +++ b/postgres/util.go @@ -17,9 +17,11 @@ package pgipfsethdb import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-ds-help" - "github.com/jmoiron/sqlx" _ "github.com/lib/pq" //postgres driver "github.com/multiformats/go-multihash" ) @@ -34,16 +36,20 @@ func MultihashKeyFromKeccak256(h []byte) (string, error) { return blockstore.BlockPrefix.String() + dbKey.String(), nil } -// TestDB connect to the testing database -// it assumes the database has the IPFS public.blocks table present -// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table -func TestDB() (*sqlx.DB, error) { - connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable" - return sqlx.Connect("postgres", connectStr) -} +// DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key +func DatastoreKeyFromGethKey(h []byte) (string, error) { + keyType, keyComponents := ResolveKeyType(h) + switch keyType { + case Keccak: + return MultihashKeyFromKeccak256(h) + case Header: + return MultihashKeyFromKeccak256(keyComponents[1]) + case Preimage: + return MultihashKeyFromKeccak256(keyComponents[1]) + case Prefixed, Suffixed: + return common.Bytes2Hex(h), nil + default: + return "", fmt.Errorf("invalid formatting of database key: %x", h) + } -// ResetTestDB drops all rows in the test db public.blocks table -func ResetTestDB(db *sqlx.DB) error { - _, err := db.Exec("TRUNCATE public.blocks CASCADE") - return err }