handle different key prefixes

This commit is contained in:
Ian Norden 2020-09-09 10:51:10 -05:00
parent dece84dc59
commit 05648112e9
5 changed files with 122 additions and 19 deletions

View File

@ -44,14 +44,14 @@ func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
// Put inserts the given value into the key-value data store
// Key is expected to be the keccak256 hash of value
func (b *Batch) Put(key []byte, value []byte) (err error) {
mhKey, err := MultihashKeyFromKeccak256(key)
dsKey, err := DatastoreKeyFromGethKey(key)
if err != nil {
return err
}
if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil {
return err
}
if _, err = b.tx.Exec(putPreimagePgStr, key, mhKey); err != nil {
if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey); err != nil {
return err
}
b.valueSize += len(value)

View File

@ -33,7 +33,7 @@ var (
hasPgStr = "SELECT exists(select 1 from eth.key_preimages WHERE eth_key = $1)"
getPgStr = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1"
putPgStr = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO NOTHING"
putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2) ON CONFLICT (eth_key) DO UPDATE SET ipfs_key = $2"
deletePgStr = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1"
dbSizePgStr = "SELECT pg_database_size(current_database())"
)
@ -78,7 +78,7 @@ func (d *Database) Get(key []byte) ([]byte, error) {
// Key is expected to be the keccak256 hash of value
// Put inserts the keccak256 key into the eth.key_preimages table
func (d *Database) Put(key []byte, value []byte) error {
mhKey, err := MultihashKeyFromKeccak256(key)
dsKey, err := DatastoreKeyFromGethKey(key)
if err != nil {
return err
}
@ -95,10 +95,10 @@ func (d *Database) Put(key []byte, value []byte) error {
err = tx.Commit()
}
}()
if _, err = tx.Exec(putPgStr, mhKey, value); err != nil {
if _, err = tx.Exec(putPgStr, dsKey, value); err != nil {
return err
}
_, err = tx.Exec(putPreimagePgStr, key, mhKey)
_, err = tx.Exec(putPreimagePgStr, key, dsKey)
return err
}

64
postgres/key_type.go Normal file
View File

@ -0,0 +1,64 @@
// VulcanizeDB
// Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pgipfsethdb
import "bytes"
type KeyType uint
const (
Invalid KeyType = iota
Keccak
Prefixed
Suffixed
Header
Preimage
)
var (
// keyDelineation is used to delineate the key prefixes and suffixes
KeyDelineation = []byte("/")
// numberDelineation is used to delineate the block number encoded in a key
NumberDelineation = []byte(":")
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
HeaderPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
PreimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage
)
// ResolveKeyType returns the key type based on the prefix
func ResolveKeyType(key []byte) (KeyType, [][]byte) {
sk := bytes.Split(key, KeyDelineation)
switch len(sk) {
case 1:
return Keccak, sk
case 2:
switch prefix := sk[0]; {
case bytes.Equal(prefix, HeaderPrefix):
return Header, bytes.Split(sk[1], NumberDelineation)
case bytes.Equal(prefix, PreimagePrefix):
return Preimage, sk
default:
return Prefixed, sk
}
case 3:
return Suffixed, sk
}
return Invalid, sk
}

33
postgres/test_helpers.go Normal file
View File

@ -0,0 +1,33 @@
// VulcanizeDB
// Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pgipfsethdb
import "github.com/jmoiron/sqlx"
// TestDB connect to the testing database
// it assumes the database has the IPFS public.blocks table present
// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
func TestDB() (*sqlx.DB, error) {
connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
return sqlx.Connect("postgres", connectStr)
}
// ResetTestDB drops all rows in the test db public.blocks table
func ResetTestDB(db *sqlx.DB) error {
_, err := db.Exec("TRUNCATE public.blocks CASCADE")
return err
}

View File

@ -17,9 +17,11 @@
package pgipfsethdb
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-ds-help"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq" //postgres driver
"github.com/multiformats/go-multihash"
)
@ -34,16 +36,20 @@ func MultihashKeyFromKeccak256(h []byte) (string, error) {
return blockstore.BlockPrefix.String() + dbKey.String(), nil
}
// TestDB connect to the testing database
// it assumes the database has the IPFS public.blocks table present
// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
func TestDB() (*sqlx.DB, error) {
connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
return sqlx.Connect("postgres", connectStr)
}
// DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key
func DatastoreKeyFromGethKey(h []byte) (string, error) {
keyType, keyComponents := ResolveKeyType(h)
switch keyType {
case Keccak:
return MultihashKeyFromKeccak256(h)
case Header:
return MultihashKeyFromKeccak256(keyComponents[1])
case Preimage:
return MultihashKeyFromKeccak256(keyComponents[1])
case Prefixed, Suffixed:
return common.Bytes2Hex(h), nil
default:
return "", fmt.Errorf("invalid formatting of database key: %x", h)
}
// ResetTestDB drops all rows in the test db public.blocks table
func ResetTestDB(db *sqlx.DB) error {
_, err := db.Exec("TRUNCATE public.blocks CASCADE")
return err
}