handle different key prefixes

postgres/batch.go

@@ -44,14 +44,14 @@ func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
 // Put inserts the given value into the key-value data store
 // Key is expected to be the keccak256 hash of value
 func (b *Batch) Put(key []byte, value []byte) (err error) {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	dsKey, err := DatastoreKeyFromGethKey(key)
 	if err != nil {
 		return err
 	}
-	if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
+	if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil {
 		return err
 	}
-	if _, err = b.tx.Exec(putPreimagePgStr, key, mhKey); err != nil {
+	if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey); err != nil {
 		return err
 	}
 	b.valueSize += len(value)

postgres/database.go

@@ -33,7 +33,7 @@ var (
 	hasPgStr         = "SELECT exists(select 1 from eth.key_preimages WHERE eth_key = $1)"
 	getPgStr         = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1"
 	putPgStr         = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
-	putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO NOTHING"
+	putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO UPDATE SET ipfs_key = $2"
 	deletePgStr      = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1"
 	dbSizePgStr      = "SELECT pg_database_size(current_database())"
 )
@@ -78,7 +78,7 @@ func (d *Database) Get(key []byte) ([]byte, error) {
 // Key is expected to be the keccak256 hash of value
 // Put inserts the keccak256 key into the eth.key_preimages table
 func (d *Database) Put(key []byte, value []byte) error {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	dsKey, err := DatastoreKeyFromGethKey(key)
 	if err != nil {
 		return err
 	}
@@ -95,10 +95,10 @@ func (d *Database) Put(key []byte, value []byte) error {
 			err = tx.Commit()
 		}
 	}()
-	if _, err = tx.Exec(putPgStr, mhKey, value); err != nil {
+	if _, err = tx.Exec(putPgStr, dsKey, value); err != nil {
 		return err
 	}
-	_, err = tx.Exec(putPreimagePgStr, key, mhKey)
+	_, err = tx.Exec(putPreimagePgStr, key, dsKey)
 	return err
 }
 

postgres/key_type.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import "bytes"
+
+type KeyType uint
+
+const (
+	Invalid KeyType = iota
+	Keccak
+	Prefixed
+	Suffixed
+	Header
+	Preimage
+)
+
+var (
+	// keyDelineation is used to delineate the key prefixes and suffixes
+	KeyDelineation = []byte("/")
+
+	// numberDelineation is used to delineate the block number encoded in a key
+	NumberDelineation = []byte(":")
+
+	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
+	HeaderPrefix   = []byte("h")           // headerPrefix + num (uint64 big endian) + hash -> header
+	PreimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage
+)
+
+// ResolveKeyType returns the key type based on the prefix
+func ResolveKeyType(key []byte) (KeyType, [][]byte) {
+	sk := bytes.Split(key, KeyDelineation)
+
+	switch len(sk) {
+	case 1:
+		return Keccak, sk
+	case 2:
+		switch prefix := sk[0]; {
+		case bytes.Equal(prefix, HeaderPrefix):
+			return Header, bytes.Split(sk[1], NumberDelineation)
+		case bytes.Equal(prefix, PreimagePrefix):
+			return Preimage, sk
+		default:
+			return Prefixed, sk
+		}
+	case 3:
+		return Suffixed, sk
+	}
+	return Invalid, sk
+}

postgres/test_helpers.go

@@ -0,0 +1,33 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import "github.com/jmoiron/sqlx"
+
+// TestDB connect to the testing database
+// it assumes the database has the IPFS public.blocks table present
+// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
+func TestDB() (*sqlx.DB, error) {
+	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	return sqlx.Connect("postgres", connectStr)
+}
+
+// ResetTestDB drops all rows in the test db public.blocks table
+func ResetTestDB(db *sqlx.DB) error {
+	_, err := db.Exec("TRUNCATE public.blocks CASCADE")
+	return err
+}

postgres/util.go

@@ -17,9 +17,11 @@
 package pgipfsethdb
 
 import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ipfs/go-ipfs-blockstore"
 	"github.com/ipfs/go-ipfs-ds-help"
-	"github.com/jmoiron/sqlx"
 	_ "github.com/lib/pq" //postgres driver
 	"github.com/multiformats/go-multihash"
 )
@@ -34,16 +36,20 @@ func MultihashKeyFromKeccak256(h []byte) (string, error) {
 	return blockstore.BlockPrefix.String() + dbKey.String(), nil
 }
 
-// TestDB connect to the testing database
+// DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key
-// it assumes the database has the IPFS public.blocks table present
+func DatastoreKeyFromGethKey(h []byte) (string, error) {
-// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
+	keyType, keyComponents := ResolveKeyType(h)
-func TestDB() (*sqlx.DB, error) {
+	switch keyType {
-	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	case Keccak:
-	return sqlx.Connect("postgres", connectStr)
+		return MultihashKeyFromKeccak256(h)
-}
+	case Header:
+		return MultihashKeyFromKeccak256(keyComponents[1])
+	case Preimage:
+		return MultihashKeyFromKeccak256(keyComponents[1])
+	case Prefixed, Suffixed:
+		return common.Bytes2Hex(h), nil
+	default:
+		return "", fmt.Errorf("invalid formatting of database key: %x", h)
+	}
 
-// ResetTestDB drops all rows in the test db public.blocks table
-func ResetTestDB(db *sqlx.DB) error {
-	_, err := db.Exec("TRUNCATE public.blocks CASCADE")
-	return err
 }