cmd/geth/main.go

@@ -146,6 +146,15 @@ var (
 		utils.EWASMInterpreterFlag,
 		utils.EVMInterpreterFlag,
 		configFileFlag,
+		utils.PostgresDatastoreFlag,
+		utils.PostgresDatabaseNameFlag,
+		utils.PostgresHostnameFlag,
+		utils.PostgresPortFlag,
+		utils.PostgresUserFlag,
+		utils.PostgresPasswordFlag,
+		utils.PostgresMaxOpenConnectionsFlag,
+		utils.PostgresMaxIdleConnectionsFlag,
+		utils.PostgresMaxConnLifetimeFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/utils/flags.go

@@ -32,6 +32,8 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/ethereum/go-ethereum/ethdb/postgres"
+
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
@@ -749,6 +751,52 @@ var (
 		Usage: "External EVM configuration (default = built-in interpreter)",
 		Value: "",
 	}
+
+	// Postgres flags
+	PostgresDatastoreFlag = cli.BoolFlag{
+		Name:  "postgres",
+		Usage: "Turn on Postgres as the backing datastore",
+	}
+	PostgresHostnameFlag = cli.StringFlag{
+		Name:  "postgres.hostname",
+		Usage: "Hostname for the Postgres database",
+		Value: "localhost",
+	}
+	PostgresPortFlag = cli.IntFlag{
+		Name:  "postgres.port",
+		Usage: "Port for the Postgres database",
+		Value: 5432,
+	}
+	PostgresUserFlag = cli.StringFlag{
+		Name:  "postgres.user",
+		Usage: "User for the Postgres database",
+		Value: "postgres",
+	}
+	PostgresPasswordFlag = cli.StringFlag{
+		Name:  "postgres.password",
+		Usage: "Password for the Postgres database",
+		Value: "",
+	}
+	PostgresDatabaseNameFlag = cli.StringFlag{
+		Name:  "postgres.database",
+		Usage: "Name for the Postgres database",
+		Value: "geth",
+	}
+	PostgresMaxOpenConnectionsFlag = cli.IntFlag{
+		Name:  "postgres.maxopen",
+		Usage: "Max number of open Postgres connections",
+		Value: 1024,
+	}
+	PostgresMaxIdleConnectionsFlag = cli.IntFlag{
+		Name:  "postgres.maxidle",
+		Usage: "Max number of idle Postgres connections",
+		Value: 1,
+	}
+	PostgresMaxConnLifetimeFlag = cli.DurationFlag{
+		Name:  "postgres.maxlifetime",
+		Usage: "Max lifetime for Postgres connections",
+		Value: 0,
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1174,6 +1222,9 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	setNodeUserIdent(ctx, cfg)
 	setDataDir(ctx, cfg)
 	setSmartCard(ctx, cfg)
+	if ctx.GlobalBool(PostgresDatastoreFlag.Name) {
+		setNodePostgres(ctx, cfg)
+	}
 
 	if ctx.GlobalIsSet(ExternalSignerFlag.Name) {
 		cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name)
@@ -1193,6 +1244,43 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	}
 }
 
+func setNodePostgres(ctx *cli.Context, cfg *node.Config) {
+	if !ctx.GlobalIsSet(PostgresDatabaseNameFlag.Name) {
+		log.Info("Node: Postgres database name not provided, defaulting to 'geth'")
+	}
+	if !ctx.GlobalIsSet(PostgresHostnameFlag.Name) {
+		log.Info("Node: Postgres hostname not provided, defaulting to localhost")
+	}
+	if !ctx.GlobalIsSet(PostgresPortFlag.Name) {
+		log.Info("Node: Postgres port not provided, defaulting to 5432")
+	}
+	if !ctx.GlobalIsSet(PostgresUserFlag.Name) {
+		log.Info("Node: Postgres user not provided, defaulting to 'postgres'")
+	}
+	if !ctx.GlobalIsSet(PostgresPasswordFlag.Name) {
+		log.Info("Node: Postgres password not provided")
+	}
+	if !ctx.GlobalIsSet(PostgresMaxOpenConnectionsFlag.Name) {
+		log.Info("Node: Postgres MaxOpenConnections not set, defaulting to 1024")
+	}
+	if !ctx.GlobalIsSet(PostgresMaxIdleConnectionsFlag.Name) {
+		log.Info("Node: Postgres MaxIdleConnections not set, defaulting to 16")
+	}
+	if !ctx.GlobalIsSet(PostgresMaxConnLifetimeFlag.Name) {
+		log.Info("Node: Postgres MaxConnLifetime not set, defaulting to no limit")
+	}
+	cfg.PostgresConfig = &postgres.Config{
+		Database:    ctx.GlobalString(PostgresDatabaseNameFlag.Name),
+		Hostname:    ctx.GlobalString(PostgresHostnameFlag.Name),
+		Port:        ctx.GlobalInt(PostgresPortFlag.Name),
+		User:        ctx.GlobalString(PostgresUserFlag.Name),
+		Password:    ctx.GlobalString(PostgresPasswordFlag.Name),
+		MaxOpen:     ctx.GlobalInt(PostgresMaxOpenConnectionsFlag.Name),
+		MaxIdle:     ctx.GlobalInt(PostgresMaxIdleConnectionsFlag.Name),
+		MaxLifetime: ctx.GlobalDuration(PostgresMaxConnLifetimeFlag.Name),
+	}
+}
+
 func setSmartCard(ctx *cli.Context, cfg *node.Config) {
 	// Skip enabling smartcards if no path is set
 	path := ctx.GlobalString(SmartCardDaemonPathFlag.Name)
@@ -1429,6 +1517,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	setMiner(ctx, &cfg.Miner)
 	setWhitelist(ctx, cfg)
 	setLes(ctx, cfg)
+	if ctx.GlobalBool(PostgresDatastoreFlag.Name) {
+		setEthPostgres(ctx, cfg)
+	}
 
 	if ctx.GlobalIsSet(SyncModeFlag.Name) {
 		cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
@@ -1524,6 +1615,43 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	}
 }
 
+func setEthPostgres(ctx *cli.Context, cfg *eth.Config) {
+	if !ctx.GlobalIsSet(PostgresDatabaseNameFlag.Name) {
+		log.Info("Eth: Postgres database name not provided, defaulting to 'geth'")
+	}
+	if !ctx.GlobalIsSet(PostgresHostnameFlag.Name) {
+		log.Info("Eth: Postgres hostname not provided, defaulting to localhost")
+	}
+	if !ctx.GlobalIsSet(PostgresPortFlag.Name) {
+		log.Info("Eth: Postgres port not provided, defaulting to 5432")
+	}
+	if !ctx.GlobalIsSet(PostgresUserFlag.Name) {
+		log.Info("Eth: Postgres user not provided, defaulting to 'postgres'")
+	}
+	if !ctx.GlobalIsSet(PostgresPasswordFlag.Name) {
+		log.Info("Eth: Postgres password not provided")
+	}
+	if !ctx.GlobalIsSet(PostgresMaxOpenConnectionsFlag.Name) {
+		log.Info("Eth: Postgres MaxOpenConnections not set, defaulting to 1024")
+	}
+	if !ctx.GlobalIsSet(PostgresMaxIdleConnectionsFlag.Name) {
+		log.Info("Eth: Postgres MaxIdleConnections not set, defaulting to 16")
+	}
+	if !ctx.GlobalIsSet(PostgresMaxConnLifetimeFlag.Name) {
+		log.Info("Eth: Postgres MaxConnLifetime not set, defaulting to no limit")
+	}
+	cfg.PostgresConfig = &postgres.Config{
+		Database:    ctx.GlobalString(PostgresDatabaseNameFlag.Name),
+		Hostname:    ctx.GlobalString(PostgresHostnameFlag.Name),
+		Port:        ctx.GlobalInt(PostgresPortFlag.Name),
+		User:        ctx.GlobalString(PostgresUserFlag.Name),
+		Password:    ctx.GlobalString(PostgresPasswordFlag.Name),
+		MaxOpen:     ctx.GlobalInt(PostgresMaxOpenConnectionsFlag.Name),
+		MaxIdle:     ctx.GlobalInt(PostgresMaxIdleConnectionsFlag.Name),
+		MaxLifetime: ctx.GlobalDuration(PostgresMaxConnLifetimeFlag.Name),
+	}
+}
+
 // RegisterEthService adds an Ethereum client to the stack.
 func RegisterEthService(stack *node.Node, cfg *eth.Config) {
 	var err error

core/rawdb/database.go

@@ -266,11 +266,11 @@ func InspectDatabase(db ethdb.Database) error {
 		)
 		total += size
 		switch {
-		case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
+		case bytes.HasPrefix(key, HeaderPrefix) && bytes.HasSuffix(key, headerTDSuffix):
 			tdSize += size
-		case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
+		case bytes.HasPrefix(key, HeaderPrefix) && bytes.HasSuffix(key, headerHashSuffix):
 			numHashPairing += size
-		case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
+		case bytes.HasPrefix(key, HeaderPrefix) && len(key) == (len(HeaderPrefix)+8+common.HashLength):
 			headerSize += size
 		case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
 			hashNumPairing += size
@@ -280,7 +280,7 @@ func InspectDatabase(db ethdb.Database) error {
 			receiptSize += size
 		case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
 			txlookupSize += size
-		case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength):
+		case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength):
 			preimageSize += size
 		case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
 			bloomBitsSize += size

core/rawdb/schema.go

@@ -26,6 +26,12 @@ import (
 
 // The fields below define the low level database schema prefixing.
 var (
+	// KeyDelineation is used to delineate the key prefixes and suffixes
+	KeyDelineation = []byte("-fix-")
+
+	// NumberDelineation is used to delineate the block number encoded in a key
+	NumberDelineation = []byte("-nmb-")
+
 	// databaseVerisionKey tracks the current database version.
 	databaseVerisionKey = []byte("DatabaseVersion")
 
@@ -42,7 +48,7 @@ var (
 	fastTrieProgressKey = []byte("TrieSync")
 
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
-	headerPrefix       = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
+	HeaderPrefix       = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
 	headerTDSuffix     = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td
 	headerHashSuffix   = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash
 	headerNumberPrefix = []byte("H") // headerNumberPrefix + hash -> num (uint64 big endian)
@@ -56,7 +62,7 @@ var (
 	// Optmism specific
 	txMetaPrefix = []byte("x") // txMetaPrefix + hash -> transaction metadata
 
-	preimagePrefix = []byte("secure-key-")      // preimagePrefix + hash -> preimage
+	PreimagePrefix = []byte("secure-key-")      // preimagePrefix + hash -> preimage
 	configPrefix   = []byte("ethereum-config-") // config prefix for the db
 
 	// Chain index prefixes (use `i` + single byte to avoid mixing data types).
@@ -108,67 +114,67 @@ func encodeBlockNumber(number uint64) []byte {
 	return enc
 }
 
-// headerKeyPrefix = headerPrefix + num (uint64 big endian)
+// headerKeyPrefix = headerPrefix + KeyDelineation + num (uint64 big endian) + NumberDelineation
 func headerKeyPrefix(number uint64) []byte {
-	return append(headerPrefix, encodeBlockNumber(number)...)
+	return append(append(append(HeaderPrefix, KeyDelineation...), encodeBlockNumber(number)...), NumberDelineation...)
 }
 
-// headerKey = headerPrefix + num (uint64 big endian) + hash
+// headerKey = HeaderPrefix + KeyDelineation + num (uint64 big endian) + NumberDelineation + hash
 func headerKey(number uint64, hash common.Hash) []byte {
-	return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
+	return append(append(append(append(HeaderPrefix, KeyDelineation...), encodeBlockNumber(number)...), NumberDelineation...), hash.Bytes()...)
 }
 
-// headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix
+// headerTDKey = HeaderPrefix + KeyDelineation + num (uint64 big endian) + NumberDelineation + hash + KeyDelineation + headerTDSuffix
 func headerTDKey(number uint64, hash common.Hash) []byte {
-	return append(headerKey(number, hash), headerTDSuffix...)
+	return append(append(headerKey(number, hash), KeyDelineation...), headerTDSuffix...)
 }
 
-// headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix
+// headerHashKey = HeaderPrefix + KeyDelineation + num (uint64 big endian) + NumberDelineation + KeyDelineation + headerHashSuffix
 func headerHashKey(number uint64) []byte {
-	return append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)
+	return append(append(append(append(append(HeaderPrefix, KeyDelineation...), encodeBlockNumber(number)...), NumberDelineation...), KeyDelineation...), headerHashSuffix...)
 }
 
-// headerNumberKey = headerNumberPrefix + hash
+// headerNumberKey = headerNumberPrefix + KeyDelineation + hash
 func headerNumberKey(hash common.Hash) []byte {
-	return append(headerNumberPrefix, hash.Bytes()...)
+	return append(append(headerNumberPrefix, KeyDelineation...), hash.Bytes()...)
 }
 
-// blockBodyKey = blockBodyPrefix + num (uint64 big endian) + hash
+// blockBodyKey = blockBodyPrefix + KeyDelineation + num (uint64 big endian) + NumberDelineation + hash
 func blockBodyKey(number uint64, hash common.Hash) []byte {
-	return append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
+	return append(append(append(append(blockBodyPrefix, KeyDelineation...), encodeBlockNumber(number)...), NumberDelineation...), hash.Bytes()...)
 }
 
-// blockReceiptsKey = blockReceiptsPrefix + num (uint64 big endian) + hash
+// blockReceiptsKey = blockReceiptsPrefix + KeyDelineation + num (uint64 big endian) + NumberDelineation + hash
 func blockReceiptsKey(number uint64, hash common.Hash) []byte {
-	return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
+	return append(append(append(append(blockReceiptsPrefix, KeyDelineation...), encodeBlockNumber(number)...), NumberDelineation...), hash.Bytes()...)
 }
 
-// txLookupKey = txLookupPrefix + hash
+// txLookupKey = txLookupPrefix + KeyDelineation + hash
 func txLookupKey(hash common.Hash) []byte {
-	return append(txLookupPrefix, hash.Bytes()...)
+	return append(append(txLookupPrefix, KeyDelineation...), hash.Bytes()...)
 }
 
 // txMetaKey = txMetaPrefix + hash
 func txMetaKey(hash common.Hash) []byte {
-	return append(txMetaPrefix, hash.Bytes()...)
+	return append(append(txMetaPrefix, KeyDelineation...), hash.Bytes()...)
 }
 
-// bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash
+// bloomBitsKey = bloomBitsPrefix + KeyDelineation + bit (uint16 big endian) + section (uint64 big endian) + hash
 func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte {
-	key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...)
+	key := append(append(append(bloomBitsPrefix, KeyDelineation...), make([]byte, 10)...), hash.Bytes()...)
 
-	binary.BigEndian.PutUint16(key[1:], uint16(bit))
-	binary.BigEndian.PutUint64(key[3:], section)
+	binary.BigEndian.PutUint16(key[2:], uint16(bit))
+	binary.BigEndian.PutUint64(key[4:], section)
 
 	return key
 }
 
-// preimageKey = preimagePrefix + hash
+// preimageKey = preimagePrefix + KeyDelineation + hash
 func preimageKey(hash common.Hash) []byte {
-	return append(preimagePrefix, hash.Bytes()...)
+	return append(append(PreimagePrefix, KeyDelineation...), hash.Bytes()...)
 }
 
-// configKey = configPrefix + hash
+// configKey = configPrefix + KeyDelineation + hash
 func configKey(hash common.Hash) []byte {
-	return append(configPrefix, hash.Bytes()...)
+	return append(append(configPrefix, KeyDelineation...), hash.Bytes()...)
 }

eth/config.go

@@ -24,6 +24,8 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/ethereum/go-ethereum/ethdb/postgres"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
@@ -160,4 +162,7 @@ type Config struct {
 
 	// MuirGlacier block override (TODO: remove after the fork)
 	OverrideMuirGlacier *big.Int
+
+	// Config params for Postgres database
+	PostgresConfig *postgres.Config
 }

ethdb/postgres/ancient.go

@@ -0,0 +1,238 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// FreezerHeaderTable indicates the name of the freezer header table.
+	FreezerHeaderTable = "headers"
+
+	// FreezerHashTable indicates the name of the freezer canonical hash table.
+	FreezerHashTable = "hashes"
+
+	// FreezerBodiesTable indicates the name of the freezer block body table.
+	FreezerBodiesTable = "bodies"
+
+	// FreezerReceiptTable indicates the name of the freezer receipts table.
+	FreezerReceiptTable = "receipts"
+
+	// FreezerDifficultyTable indicates the name of the freezer total difficulty table.
+	FreezerDifficultyTable = "diffs"
+
+	// ancient append Postgres statements
+	appendAncientHeaderPgStr   = "INSERT INTO eth.ancient_headers (block_number, header) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET header = $2"
+	appendAncientHashPgStr     = "INSERT INTO eth.ancient_hashes (block_number, hash) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET hash = $2"
+	appendAncientBodyPgStr     = "INSERT INTO eth.ancient_bodies (block_number, body) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET body = $2"
+	appendAncientReceiptsPgStr = "INSERT INTO eth.ancient_receipts (block_number, receipts) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET receipts = $2"
+	appendAncientTDPgStr       = "INSERT INTO eth.ancient_tds (block_number, td) VALUES ($1, $2) ON CONFLICT (block_number) DO UPDATE SET td = $2"
+
+	// ancient truncate Postgres statements
+	truncateAncientHeaderPgStr   = "DELETE FROM eth.ancient_headers WHERE block_number > $1"
+	truncateAncientHashPgStr     = "DELETE FROM eth.ancient_hashes WHERE block_number > $1"
+	truncateAncientBodiesPgStr   = "DELETE FROM eth.ancient_bodies WHERE block_number > $1"
+	truncateAncientReceiptsPgStr = "DELETE FROM eth.ancient_receipts WHERE block_number > $1"
+	truncateAncientTDPgStr       = "DELETE FROM eth.ancient_tds WHERE block_number > $1"
+
+	// ancient size Postgres statement
+	ancientSizePgStr = "SELECT pg_total_relation_size($1)"
+
+	// ancients Postgres statement
+	ancientsPgStr = "SELECT block_number FROM eth.ancient_headers ORDER BY block_number DESC LIMIT 1"
+
+	// ancient has Postgres statements
+	hasAncientHeaderPgStr   = "SELECT exists(SELECT 1 FROM eth.ancient_headers WHERE block_number = $1)"
+	hasAncientHashPgStr     = "SELECT exists(SELECT 1 FROM eth.ancient_hashes WHERE block_number = $1)"
+	hasAncientBodyPgStr     = "SELECT exists(SELECT 1 FROM eth.ancient_bodies WHERE block_number = $1)"
+	hasAncientReceiptsPgStr = "SELECT exists(SELECT 1 FROM eth.ancient_receipts WHERE block_number = $1)"
+	hasAncientTDPgStr       = "SELECT exists(SELECT 1 FROM eth.ancient_tds WHERE block_number = $1)"
+
+	// ancient get Postgres statements
+	getAncientHeaderPgStr   = "SELECT header FROM eth.ancient_headers WHERE block_number = $1"
+	getAncientHashPgStr     = "SELECT hash FROM eth.ancient_hashes WHERE block_number = $1"
+	getAncientBodyPgStr     = "SELECT body FROM eth.ancient_bodies WHERE block_number = $1"
+	getAncientReceiptsPgStr = "SELECT receipts FROM eth.ancient_receipts WHERE block_number = $1"
+	getAncientTDPgStr       = "SELECT td FROM eth.ancient_tds WHERE block_number = $1"
+)
+
+// HasAncient satisfies the ethdb.AncientReader interface
+// HasAncient returns an indicator whether the specified data exists in the ancient store
+func (d *Database) HasAncient(kind string, number uint64) (bool, error) {
+	var pgStr string
+	switch kind {
+	case FreezerHeaderTable:
+		pgStr = hasAncientHeaderPgStr
+	case FreezerHashTable:
+		pgStr = hasAncientHashPgStr
+	case FreezerBodiesTable:
+		pgStr = hasAncientBodyPgStr
+	case FreezerReceiptTable:
+		pgStr = hasAncientReceiptsPgStr
+	case FreezerDifficultyTable:
+		pgStr = hasAncientTDPgStr
+	default:
+		return false, fmt.Errorf("unexpected ancient kind: %s", kind)
+	}
+	has := new(bool)
+	return *has, d.db.Get(has, pgStr, number)
+}
+
+// Ancient satisfies the ethdb.AncientReader interface
+// Ancient retrieves an ancient binary blob from the append-only immutable files
+func (d *Database) Ancient(kind string, number uint64) ([]byte, error) {
+	var pgStr string
+	switch kind {
+	case FreezerHeaderTable:
+		pgStr = getAncientHeaderPgStr
+	case FreezerHashTable:
+		pgStr = getAncientHashPgStr
+	case FreezerBodiesTable:
+		pgStr = getAncientBodyPgStr
+	case FreezerReceiptTable:
+		pgStr = getAncientReceiptsPgStr
+	case FreezerDifficultyTable:
+		pgStr = getAncientTDPgStr
+	default:
+		return nil, fmt.Errorf("unexpected ancient kind: %s", kind)
+	}
+	data := new([]byte)
+	return *data, d.db.Get(data, pgStr, number)
+}
+
+// Ancients satisfies the ethdb.AncientReader interface
+// Ancients returns the ancient item numbers in the ancient store
+func (d *Database) Ancients() (uint64, error) {
+	num := new(uint64)
+	if err := d.db.Get(num, ancientsPgStr); err != nil {
+		if err == sql.ErrNoRows {
+			return 0, nil
+		}
+		return 0, err
+	}
+	return *num, nil
+}
+
+// AncientSize satisfies the ethdb.AncientReader interface
+// AncientSize returns the ancient size of the specified category
+func (d *Database) AncientSize(kind string) (uint64, error) {
+	var tableName string
+	switch kind {
+	case FreezerHeaderTable:
+		tableName = "eth.ancient_headers"
+	case FreezerHashTable:
+		tableName = "eth.ancient_hashes"
+	case FreezerBodiesTable:
+		tableName = "eth.ancient_bodies"
+	case FreezerReceiptTable:
+		tableName = "eth.ancient_receipts"
+	case FreezerDifficultyTable:
+		tableName = "eth.ancient_tds"
+	default:
+		return 0, fmt.Errorf("unexpected ancient kind: %s", kind)
+	}
+	size := new(uint64)
+	return *size, d.db.Get(size, ancientSizePgStr, tableName)
+}
+
+// AppendAncient satisfies the ethdb.AncientWriter interface
+// AppendAncient injects all binary blobs belong to block at the end of the append-only immutable table files
+func (d *Database) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error {
+	// append in batch
+	var err error
+	if d.ancientTx == nil {
+		d.ancientTx, err = d.db.Beginx()
+		if err != nil {
+			return err
+		}
+	}
+	defer func() {
+		if err != nil {
+			if err := d.ancientTx.Rollback(); err != nil {
+				logrus.Error(err)
+				d.ancientTx = nil
+			}
+		}
+	}()
+
+	if _, err := d.ancientTx.Exec(appendAncientHashPgStr, number, hash); err != nil {
+		return err
+	}
+	if _, err := d.ancientTx.Exec(appendAncientHeaderPgStr, number, header); err != nil {
+		return err
+	}
+	if _, err := d.ancientTx.Exec(appendAncientBodyPgStr, number, body); err != nil {
+		return err
+	}
+	if _, err := d.ancientTx.Exec(appendAncientReceiptsPgStr, number, receipts); err != nil {
+		return err
+	}
+	_, err = d.ancientTx.Exec(appendAncientTDPgStr, number, td)
+	return err
+}
+
+// TruncateAncients satisfies the ethdb.AncientWriter interface
+// TruncateAncients discards all but the first n ancient data from the ancient store
+func (d *Database) TruncateAncients(n uint64) error {
+	// truncate in batch
+	var err error
+	if d.ancientTx == nil {
+		d.ancientTx, err = d.db.Beginx()
+		if err != nil {
+			return err
+		}
+	}
+	defer func() {
+		if err != nil {
+			if err := d.ancientTx.Rollback(); err != nil {
+				logrus.Error(err)
+				d.ancientTx = nil
+			}
+		}
+	}()
+	if _, err := d.ancientTx.Exec(truncateAncientHeaderPgStr, n); err != nil {
+		return err
+	}
+	if _, err := d.ancientTx.Exec(truncateAncientHashPgStr, n); err != nil {
+		return err
+	}
+	if _, err := d.ancientTx.Exec(truncateAncientBodiesPgStr, n); err != nil {
+		return err
+	}
+	if _, err := d.ancientTx.Exec(truncateAncientReceiptsPgStr, n); err != nil {
+		return err
+	}
+	_, err = d.ancientTx.Exec(truncateAncientTDPgStr, n)
+	return err
+}
+
+// Sync satisfies the ethdb.AncientWriter interface
+// Sync flushes all in-memory ancient store data to disk
+func (d *Database) Sync() error {
+	if d.ancientTx == nil {
+		return nil
+	}
+	if err := d.ancientTx.Commit(); err != nil {
+		return err
+	}
+	d.ancientTx = nil
+	return nil
+}

ethdb/postgres/ancient_test.go

@@ -0,0 +1,231 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	pgipfsethdb "github.com/ethereum/go-ethereum/ethdb/postgres"
+	"github.com/ethereum/go-ethereum/rlp"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	ancientDB                ethdb.Database
+	testBlockNumber          uint64 = 1
+	testAncientHeader               = types.Header{Number: big.NewInt(2)}
+	testAncientHeaderRLP, _         = rlp.EncodeToBytes(testHeader2)
+	testAncientHash                 = testAncientHeader.Hash().Bytes()
+	testAncientBodyBytes            = make([]byte, 10000)
+	testAncientReceiptsBytes        = make([]byte, 5000)
+	testAncientTD, _                = new(big.Int).SetString("1000000000000000000000", 10)
+	testAncientTDBytes              = testAncientTD.Bytes()
+)
+
+var _ = Describe("Ancient", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+		ancientDB = pgipfsethdb.NewDatabase(db)
+
+	})
+	AfterEach(func() {
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("AppendAncient/Sync/Has", func() {
+		It("adds eth objects to the Ancient database and returns whether or not an ancient record exists", func() {
+			hasAncient(testBlockNumber, false)
+
+			err = ancientDB.AppendAncient(testBlockNumber, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+			Expect(err).ToNot(HaveOccurred())
+
+			hasAncient(testBlockNumber, false)
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			hasAncient(testBlockNumber, true)
+		})
+	})
+
+	Describe("AppendAncient/Sync/Ancient", func() {
+		It("adds the eth objects to the Ancient database and returns the ancient objects on request", func() {
+			hasAncient(testBlockNumber, false)
+
+			_, err := ancientDB.Ancient(pgipfsethdb.FreezerHeaderTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+			_, err = ancientDB.Ancient(pgipfsethdb.FreezerHashTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+			_, err = ancientDB.Ancient(pgipfsethdb.FreezerBodiesTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+			_, err = ancientDB.Ancient(pgipfsethdb.FreezerReceiptTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+			_, err = ancientDB.Ancient(pgipfsethdb.FreezerDifficultyTable, testBlockNumber)
+			Expect(err).To(HaveOccurred())
+
+			err = ancientDB.AppendAncient(testBlockNumber, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+			Expect(err).ToNot(HaveOccurred())
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			hasAncient(testBlockNumber, true)
+
+			ancientHeader, err := ancientDB.Ancient(pgipfsethdb.FreezerHeaderTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientHeader).To(Equal(testAncientHeaderRLP))
+
+			ancientHash, err := ancientDB.Ancient(pgipfsethdb.FreezerHashTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientHash).To(Equal(testAncientHash))
+
+			ancientBody, err := ancientDB.Ancient(pgipfsethdb.FreezerBodiesTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientBody).To(Equal(testAncientBodyBytes))
+
+			ancientReceipts, err := ancientDB.Ancient(pgipfsethdb.FreezerReceiptTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientReceipts).To(Equal(testAncientReceiptsBytes))
+
+			ancientTD, err := ancientDB.Ancient(pgipfsethdb.FreezerDifficultyTable, testBlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientTD).To(Equal(testAncientTDBytes))
+		})
+	})
+
+	Describe("AppendAncient/Sync/Ancients", func() {
+		It("returns the height of the ancient database", func() {
+			ancients, err := ancientDB.Ancients()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancients).To(Equal(uint64(0)))
+
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, false)
+				err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, true)
+			}
+			ancients, err = ancientDB.Ancients()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancients).To(Equal(uint64(100)))
+		})
+	})
+
+	Describe("AppendAncient/Truncate/Sync", func() {
+		It("truncates the ancient database to the provided height", func() {
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, false)
+				err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ancientDB.TruncateAncients(50)
+			Expect(err).ToNot(HaveOccurred())
+
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, true)
+			}
+
+			ancients, err := ancientDB.Ancients()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancients).To(Equal(uint64(100)))
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			for i := uint64(0); i <= 100; i++ {
+				if i <= 50 {
+					hasAncient(i, true)
+				} else {
+					hasAncient(i, false)
+				}
+			}
+
+			ancients, err = ancientDB.Ancients()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancients).To(Equal(uint64(50)))
+		})
+	})
+
+	Describe("AppendAncient/Sync/AncientSize", func() {
+		It("adds the eth objects to the Ancient database and returns the ancient objects on request", func() {
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, false)
+				err = ancientDB.AppendAncient(i, testAncientHash, testAncientHeaderRLP, testAncientBodyBytes, testAncientReceiptsBytes, testAncientTDBytes)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = ancientDB.Sync()
+			Expect(err).ToNot(HaveOccurred())
+
+			for i := uint64(0); i <= 100; i++ {
+				hasAncient(i, true)
+			}
+
+			ancientHeaderSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerHeaderTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientHeaderSize).To(Equal(uint64(106496)))
+
+			ancientHashSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerHashTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientHashSize).To(Equal(uint64(32768)))
+
+			ancientBodySize, err := ancientDB.AncientSize(pgipfsethdb.FreezerBodiesTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientBodySize).To(Equal(uint64(73728)))
+
+			ancientReceiptsSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerReceiptTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientReceiptsSize).To(Equal(uint64(65536)))
+
+			ancientTDSize, err := ancientDB.AncientSize(pgipfsethdb.FreezerDifficultyTable)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ancientTDSize).To(Equal(uint64(32768)))
+		})
+	})
+})
+
+func hasAncient(blockNumber uint64, shouldHave bool) {
+	has, err := ancientDB.HasAncient(pgipfsethdb.FreezerHeaderTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+	has, err = ancientDB.HasAncient(pgipfsethdb.FreezerHashTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+	has, err = ancientDB.HasAncient(pgipfsethdb.FreezerBodiesTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+	has, err = ancientDB.HasAncient(pgipfsethdb.FreezerReceiptTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+	has, err = ancientDB.HasAncient(pgipfsethdb.FreezerDifficultyTable, blockNumber)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(has).To(Equal(shouldHave))
+}

ethdb/postgres/batch.go

@@ -0,0 +1,124 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+)
+
+// Batch is the type that satisfies the ethdb.Batch interface for PG-IPFS Ethereum data using a direct Postgres connection
+type Batch struct {
+	db          *sqlx.DB
+	tx          *sqlx.Tx
+	valueSize   int
+	replayCache map[string][]byte
+}
+
+// NewBatch returns a ethdb.Batch interface for PG-IPFS
+func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
+	b := &Batch{
+		db:          db,
+		tx:          tx,
+		replayCache: make(map[string][]byte),
+	}
+	if tx == nil {
+		b.Reset()
+	}
+	return b
+}
+
+// Put satisfies the ethdb.Batch interface
+// Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
+func (b *Batch) Put(key []byte, value []byte) (err error) {
+	dsKey, prefix, err := DatastoreKeyFromGethKey(key)
+	if err != nil {
+		return err
+	}
+	if _, err = b.tx.Exec(putPgStr, dsKey, value); err != nil {
+		return err
+	}
+	if _, err = b.tx.Exec(putPreimagePgStr, key, dsKey, prefix); err != nil {
+		return err
+	}
+	b.valueSize += len(value)
+	b.replayCache[common.Bytes2Hex(key)] = value
+	return nil
+}
+
+// Delete satisfies the ethdb.Batch interface
+// Delete removes the key from the key-value data store
+func (b *Batch) Delete(key []byte) (err error) {
+	_, err = b.tx.Exec(deletePgStr, key)
+	if err != nil {
+		return err
+	}
+	delete(b.replayCache, common.Bytes2Hex(key))
+	return nil
+}
+
+// ValueSize satisfies the ethdb.Batch interface
+// ValueSize retrieves the amount of data queued up for writing
+// The returned value is the total byte length of all data queued to write
+func (b *Batch) ValueSize() int {
+	return b.valueSize
+}
+
+// Write satisfies the ethdb.Batch interface
+// Write flushes any accumulated data to disk
+// Reset should be called after every write
+func (b *Batch) Write() error {
+	if b.tx == nil {
+		return nil
+	}
+	if err := b.tx.Commit(); err != nil {
+		return err
+	}
+	b.replayCache = nil
+	return nil
+}
+
+// Replay satisfies the ethdb.Batch interface
+// Replay replays the batch contents
+func (b *Batch) Replay(w ethdb.KeyValueWriter) error {
+	if b.tx != nil {
+		b.tx.Rollback()
+		b.tx = nil
+	}
+	for key, value := range b.replayCache {
+		if err := w.Put(common.Hex2Bytes(key), value); err != nil {
+			return err
+		}
+	}
+	b.replayCache = nil
+	return nil
+}
+
+// Reset satisfies the ethdb.Batch interface
+// Reset resets the batch for reuse
+// This should be called after every write
+func (b *Batch) Reset() {
+	var err error
+	b.tx, err = b.db.Beginx()
+	if err != nil {
+		panic(err)
+	}
+	b.replayCache = make(map[string][]byte)
+	b.valueSize = 0
+}

ethdb/postgres/batch_test.go

@@ -0,0 +1,143 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	pgipfsethdb "github.com/ethereum/go-ethereum/ethdb/postgres"
+	"github.com/ethereum/go-ethereum/rlp"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	batch             ethdb.Batch
+	testHeader2       = types.Header{Number: big.NewInt(2)}
+	testValue2, _     = rlp.EncodeToBytes(testHeader2)
+	testKeccakEthKey2 = testHeader2.Hash().Bytes()
+)
+
+var _ = Describe("Batch", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+		database = pgipfsethdb.NewDatabase(db)
+		batch = database.NewBatch()
+	})
+	AfterEach(func() {
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("Put/Write", func() {
+		It("adds the key-value pair to the batch", func() {
+			_, err = database.Get(testKeccakEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = database.Get(testKeccakEthKey2)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = batch.Put(testKeccakEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testKeccakEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			val, err := database.Get(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+			val2, err := database.Get(testKeccakEthKey2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val2).To(Equal(testValue2))
+		})
+	})
+
+	Describe("Delete/Reset/Write", func() {
+		It("deletes the key-value pair in the batch", func() {
+			err = batch.Put(testKeccakEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testKeccakEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			batch.Reset()
+			err = batch.Delete(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Delete(testKeccakEthKey2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = database.Get(testKeccakEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = database.Get(testKeccakEthKey2)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("ValueSize/Reset", func() {
+		It("returns the size of data in the batch queued for write", func() {
+			err = batch.Put(testKeccakEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testKeccakEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			size := batch.ValueSize()
+			Expect(size).To(Equal(len(testValue) + len(testValue2)))
+
+			batch.Reset()
+			size = batch.ValueSize()
+			Expect(size).To(Equal(0))
+		})
+	})
+
+	Describe("Replay", func() {
+		It("returns the size of data in the batch queued for write", func() {
+			err = batch.Put(testKeccakEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testKeccakEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = database.Get(testKeccakEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = database.Get(testKeccakEthKey2)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = batch.Replay(database)
+			Expect(err).ToNot(HaveOccurred())
+
+			val, err := database.Get(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+			val2, err := database.Get(testKeccakEthKey2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val2).To(Equal(testValue2))
+		})
+	})
+})

ethdb/postgres/config.go

@@ -0,0 +1,91 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+)
+
+var (
+	defaultMaxDBConnections   = 1024
+	defaultMaxIdleConnections = 16
+)
+
+// Config holds Postgres connection pool configuration params
+type Config struct {
+	Database string
+	Hostname string
+	Port     int
+	User     string
+	Password string
+
+	// Optimization parameters
+	MaxOpen     int
+	MaxIdle     int
+	MaxLifetime time.Duration
+}
+
+// NewConfig returns a new config struct from provided params
+func NewConfig(database, hostname, password, user string, port, maxOpen, maxIdle int, maxLifetime time.Duration) *Config {
+	return &Config{
+		Database:    database,
+		Hostname:    hostname,
+		Port:        port,
+		User:        user,
+		Password:    password,
+		MaxOpen:     maxOpen,
+		MaxLifetime: maxLifetime,
+		MaxIdle:     maxIdle,
+	}
+}
+
+// DbConnectionString resolves Postgres config params to a connection string
+func DbConnectionString(config *Config) string {
+	if len(config.User) > 0 && len(config.Password) > 0 {
+		return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+			config.User, config.Password, config.Hostname, config.Port, config.Database)
+	}
+	if len(config.User) > 0 && len(config.Password) == 0 {
+		return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
+			config.User, config.Hostname, config.Port, config.Database)
+	}
+	return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", config.Hostname, config.Port, config.Database)
+}
+
+// NewDB opens and returns a new Postgres connection pool using the provided config
+func NewDB(c *Config) (*sqlx.DB, error) {
+	connectStr := DbConnectionString(c)
+	db, err := sqlx.Connect("postgres", connectStr)
+	if err != nil {
+		return nil, err
+	}
+	if c.MaxIdle > 0 {
+		db.SetMaxIdleConns(c.MaxIdle)
+	} else {
+		db.SetMaxIdleConns(defaultMaxIdleConnections)
+	}
+	if c.MaxOpen > 0 {
+		db.SetMaxOpenConns(c.MaxOpen)
+	} else {
+		db.SetMaxOpenConns(defaultMaxDBConnections)
+	}
+	db.SetConnMaxLifetime(c.MaxLifetime)
+	return db, nil
+}

ethdb/postgres/database.go

@@ -0,0 +1,221 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+)
+
+const (
+	hasPgStr         = "SELECT exists(SELECT 1 FROM eth.key_preimages WHERE eth_key = $1)"
+	getPgStr         = "SELECT data FROM public.blocks INNER JOIN eth.key_preimages ON (ipfs_key = blocks.key) WHERE eth_key = $1"
+	putPgStr         = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
+	putPreimagePgStr = "INSERT INTO eth.key_preimages (eth_key, ipfs_key, prefix) VALUES ($1, $2, $3) ON CONFLICT (eth_key) DO UPDATE SET (ipfs_key, prefix) = ($2, $3)"
+	deletePgStr      = "DELETE FROM public.blocks USING eth.key_preimages WHERE ipfs_key = blocks.key AND eth_key = $1"
+	dbSizePgStr      = "SELECT pg_database_size(current_database())"
+)
+
+// Database is the type that satisfies the ethdb.Database and ethdb.KeyValueStore interfaces for PG-IPFS Ethereum data using a direct Postgres connection
+type Database struct {
+	db        *sqlx.DB
+	ancientTx *sqlx.Tx
+}
+
+// NewKeyValueStore returns a ethdb.KeyValueStore interface for PG-IPFS
+func NewKeyValueStore(db *sqlx.DB) ethdb.KeyValueStore {
+	return &Database{
+		db: db,
+	}
+}
+
+// NewDatabase returns a ethdb.Database interface for PG-IPFS
+func NewDatabase(db *sqlx.DB) ethdb.Database {
+	return &Database{
+		db: db,
+	}
+}
+
+// Has satisfies the ethdb.KeyValueReader interface
+// Has retrieves if a key is present in the key-value data store
+// Has uses the eth.key_preimages table
+func (d *Database) Has(key []byte) (bool, error) {
+	var exists bool
+	return exists, d.db.Get(&exists, hasPgStr, key)
+}
+
+// Get satisfies the ethdb.KeyValueReader interface
+// Get retrieves the given key if it's present in the key-value data store
+// Get uses the eth.key_preimages table
+func (d *Database) Get(key []byte) ([]byte, error) {
+	var data []byte
+	return data, d.db.Get(&data, getPgStr, key)
+}
+
+// Put satisfies the ethdb.KeyValueWriter interface
+// Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
+// Put inserts the keccak256 key into the eth.key_preimages table
+func (d *Database) Put(key []byte, value []byte) error {
+	dsKey, prefix, err := DatastoreKeyFromGethKey(key)
+	if err != nil {
+		return err
+	}
+	tx, err := d.db.Beginx()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			if err := tx.Rollback(); err != nil {
+				logrus.Error(err)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+	if _, err = tx.Exec(putPgStr, dsKey, value); err != nil {
+		return err
+	}
+	_, err = tx.Exec(putPreimagePgStr, key, dsKey, prefix)
+	return err
+}
+
+// Delete satisfies the ethdb.KeyValueWriter interface
+// Delete removes the key from the key-value data store
+// Delete uses the eth.key_preimages table
+func (d *Database) Delete(key []byte) error {
+	_, err := d.db.Exec(deletePgStr, key)
+	return err
+}
+
+// DatabaseProperty enum type
+type DatabaseProperty int
+
+const (
+	Unknown DatabaseProperty = iota
+	Size
+	Idle
+	InUse
+	MaxIdleClosed
+	MaxLifetimeClosed
+	MaxOpenConnections
+	OpenConnections
+	WaitCount
+	WaitDuration
+)
+
+// DatabasePropertyFromString helper function
+func DatabasePropertyFromString(property string) (DatabaseProperty, error) {
+	switch strings.ToLower(property) {
+	case "size":
+		return Size, nil
+	case "idle":
+		return Idle, nil
+	case "inuse":
+		return InUse, nil
+	case "maxidleclosed":
+		return MaxIdleClosed, nil
+	case "maxlifetimeclosed":
+		return MaxLifetimeClosed, nil
+	case "maxopenconnections":
+		return MaxOpenConnections, nil
+	case "openconnections":
+		return OpenConnections, nil
+	case "waitcount":
+		return WaitCount, nil
+	case "waitduration":
+		return WaitDuration, nil
+	default:
+		return Unknown, fmt.Errorf("unknown database property")
+	}
+}
+
+// Stat satisfies the ethdb.Stater interface
+// Stat returns a particular internal stat of the database
+func (d *Database) Stat(property string) (string, error) {
+	prop, err := DatabasePropertyFromString(property)
+	if err != nil {
+		return "", err
+	}
+	switch prop {
+	case Size:
+		var byteSize string
+		return byteSize, d.db.Get(&byteSize, dbSizePgStr)
+	case Idle:
+		return string(d.db.Stats().Idle), nil
+	case InUse:
+		return string(d.db.Stats().InUse), nil
+	case MaxIdleClosed:
+		return string(d.db.Stats().MaxIdleClosed), nil
+	case MaxLifetimeClosed:
+		return string(d.db.Stats().MaxLifetimeClosed), nil
+	case MaxOpenConnections:
+		return string(d.db.Stats().MaxOpenConnections), nil
+	case OpenConnections:
+		return string(d.db.Stats().OpenConnections), nil
+	case WaitCount:
+		return string(d.db.Stats().WaitCount), nil
+	case WaitDuration:
+		return d.db.Stats().WaitDuration.String(), nil
+	default:
+		return "", fmt.Errorf("unhandled database property")
+	}
+}
+
+// Compact satisfies the ethdb.Compacter interface
+// Compact flattens the underlying data store for the given key range
+func (d *Database) Compact(start []byte, limit []byte) error {
+	return nil
+}
+
+// NewBatch satisfies the ethdb.Batcher interface
+// NewBatch creates a write-only database that buffers changes to its host db
+// until a final write is called
+func (d *Database) NewBatch() ethdb.Batch {
+	return NewBatch(d.db, nil)
+}
+
+// NewIterator creates a binary-alphabetical iterator over the entire keyspace
+// contained within the key-value database.
+func (d *Database) NewIterator() ethdb.Iterator {
+	return NewIterator(nil, nil, d.db)
+}
+
+// NewIteratorWithStart creates a binary-alphabetical iterator over a subset of
+// database content starting at a particular initial key (or after, if it does
+// not exist).
+func (d *Database) NewIteratorWithStart(start []byte) ethdb.Iterator {
+	return NewIterator(start, nil, d.db)
+}
+
+// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
+// of database content with a particular key prefix.
+func (d *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
+	return NewIterator(nil, prefix, d.db)
+}
+
+// Close satisfies the io.Closer interface
+// Close closes the db connection
+func (d *Database) Close() error {
+	return d.db.DB.Close()
+}

ethdb/postgres/database_test.go

@@ -0,0 +1,385 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	pgipfsethdb "github.com/ethereum/go-ethereum/ethdb/postgres"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	database         ethdb.Database
+	db               *sqlx.DB
+	err              error
+	testHeader       = types.Header{Number: big.NewInt(1337)}
+	testValue, _     = rlp.EncodeToBytes(testHeader)
+	testKeccakEthKey = testHeader.Hash().Bytes()
+	testMhKey, _     = pgipfsethdb.MultihashKeyFromKeccak256(testKeccakEthKey)
+
+	testPrefixedEthKey = append(append([]byte("prefix"), pgipfsethdb.KeyDelineation...), testKeccakEthKey...)
+	testPrefixedDsKey  = common.Bytes2Hex(testPrefixedEthKey)
+
+	testSuffixedEthKey = append(append(testPrefixedEthKey, pgipfsethdb.KeyDelineation...), []byte("suffix")...)
+	testSuffixedDsKey  = common.Bytes2Hex(testSuffixedEthKey)
+
+	testHeaderEthKey = append(append(append(append(pgipfsethdb.HeaderPrefix, pgipfsethdb.KeyDelineation...),
+		[]byte("number")...), pgipfsethdb.NumberDelineation...), testKeccakEthKey...)
+	testHeaderDsKey = testMhKey
+
+	testPreimageEthKey = append(append(pgipfsethdb.PreimagePrefix, pgipfsethdb.KeyDelineation...), testKeccakEthKey...)
+	testPreimageDsKey  = testMhKey
+)
+
+var _ = Describe("Database", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+		database = pgipfsethdb.NewDatabase(db)
+	})
+	AfterEach(func() {
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("Has - Keccak keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Prefixed keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Suffixed keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Header keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Has - Preimage keys", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Get - Keccak keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testKeccakEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testKeccakEthKey, testMhKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Prefixed keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPrefixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPrefixedEthKey, testPrefixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Suffixed keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testSuffixedDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testSuffixedEthKey, testSuffixedDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Header keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testHeaderDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testHeaderEthKey, testHeaderDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Get - Preimage keys", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testPreimageEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testPreimageDsKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = db.Exec("INSERT into eth.key_preimages (eth_key, ipfs_key) VALUES ($1, $2)", testPreimageEthKey, testPreimageDsKey)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Keccak keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testKeccakEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testKeccakEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Prefixed keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testPrefixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Suffixed keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testSuffixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Header keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testHeaderEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put - Preimage keys", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testPreimageEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testPreimageEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Delete - Keccak keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testKeccakEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testKeccakEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testKeccakEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Prefixed keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testPrefixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testPrefixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testPrefixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Suffixed keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testSuffixedEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testSuffixedEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testSuffixedEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Header keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testHeaderEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testHeaderEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testHeaderEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("Delete - Preimage keys", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testPreimageEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testPreimageEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testPreimageEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+})

ethdb/postgres/iterator.go

@@ -0,0 +1,115 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+)
+
+const (
+	initPgStr = `SELECT eth_key, data FROM public.blocks
+				INNER JOIN eth.key_preimages ON (ipfs_key = key)
+				WHERE eth_key = $1`
+	nextPgStr = `SELECT eth_key, data FROM public.blocks
+				INNER JOIN eth.key_preimages ON (ipfs_key = key)
+				WHERE eth_key > $1
+				ORDER BY eth_key LIMIT 1`
+	nextPgStrWithPrefix = `SELECT eth_key, data FROM public.blocks
+				INNER JOIN eth.key_preimages ON (ipfs_key = key)
+				WHERE eth_key > $1 AND prefix = $2 
+				ORDER BY eth_key LIMIT 1`
+)
+
+type nextModel struct {
+	Key   []byte `db:"eth_key"`
+	Value []byte `db:"data"`
+}
+
+// Iterator is the type that satisfies the ethdb.Iterator interface for PG-IPFS Ethereum data using a direct Postgres connection
+type Iterator struct {
+	db                               *sqlx.DB
+	currentKey, prefix, currentValue []byte
+	err                              error
+	init                             bool
+}
+
+// NewIterator returns an ethdb.Iterator interface for PG-IPFS
+func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
+	return &Iterator{
+		db:         db,
+		prefix:     prefix,
+		currentKey: start,
+		init:       start != nil,
+	}
+}
+
+// Next satisfies the ethdb.Iterator interface
+// Next moves the iterator to the next key/value pair
+// It returns whether the iterator is exhausted
+func (i *Iterator) Next() bool {
+	next := new(nextModel)
+	if i.init {
+		i.init = false
+		if err := i.db.Get(next, initPgStr, i.currentKey); err != nil {
+			i.currentKey, i.currentValue, i.err = nil, nil, err
+			return false
+		}
+	} else if i.prefix != nil {
+		if err := i.db.Get(next, nextPgStrWithPrefix, i.currentKey, i.prefix); err != nil {
+			i.currentKey, i.currentValue, i.err = nil, nil, err
+			return false
+		}
+	} else {
+		if err := i.db.Get(next, nextPgStr, i.currentKey); err != nil {
+			i.currentKey, i.currentValue, i.err = nil, nil, err
+			return false
+		}
+	}
+	i.currentKey, i.currentValue, i.err = next.Key, next.Value, nil
+	return true
+}
+
+// Error satisfies the ethdb.Iterator interface
+// Error returns any accumulated error
+// Exhausting all the key/value pairs is not considered to be an error
+func (i *Iterator) Error() error {
+	return i.err
+}
+
+// Key satisfies the ethdb.Iterator interface
+// Key returns the key of the current key/value pair, or nil if done
+// The caller should not modify the contents of the returned slice
+// and its contents may change on the next call to Next
+func (i *Iterator) Key() []byte {
+	return i.currentKey
+}
+
+// Value satisfies the ethdb.Iterator interface
+// Value returns the value of the current key/value pair, or nil if done
+// The caller should not modify the contents of the returned slice
+// and its contents may change on the next call to Next
+func (i *Iterator) Value() []byte {
+	return i.currentValue
+}
+
+// Release satisfies the ethdb.Iterator interface
+// Release releases associated resources
+// Release should always succeed and can be called multiple times without causing error
+func (i *Iterator) Release() {
+	i.db, i.currentKey, i.currentValue, i.err, i.prefix = nil, nil, nil, nil, nil
+}

ethdb/postgres/iterator_test.go

@@ -0,0 +1,512 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"database/sql"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	pgipfsethdb "github.com/ethereum/go-ethereum/ethdb/postgres"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	iterator            ethdb.Iterator
+	testPrefix          = []byte("testPrefix")
+	testEthKey1         = []byte{'\x01'}
+	testEthKey2         = []byte{'\x01', '\x01'}
+	testEthKey3         = []byte{'\x01', '\x02'}
+	testEthKey4         = []byte{'\x01', '\x0e'}
+	testEthKey5         = []byte{'\x01', '\x02', '\x01'}
+	testEthKey6         = []byte{'\x01', '\x0e', '\x01'}
+	prefixedTestEthKey1 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey1...)
+	prefixedTestEthKey2 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey2...)
+	prefixedTestEthKey3 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey3...)
+	prefixedTestEthKey4 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey4...)
+	prefixedTestEthKey5 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey5...)
+	prefixedTestEthKey6 = append(append(testPrefix, pgipfsethdb.KeyDelineation...), testEthKey6...)
+	mockValue1          = []byte{1}
+	mockValue2          = []byte{2}
+	mockValue3          = []byte{3}
+	mockValue4          = []byte{4}
+	mockValue5          = []byte{5}
+	mockValue6          = []byte{6}
+)
+
+var _ = Describe("Iterator", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+		database = pgipfsethdb.NewDatabase(db)
+		// non-prefixed entries
+		err = database.Put(testEthKey1, mockValue1)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey2, mockValue2)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey3, mockValue3)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey4, mockValue4)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey5, mockValue5)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(testEthKey6, mockValue6)
+		Expect(err).ToNot(HaveOccurred())
+		// prefixed entries
+		err = database.Put(prefixedTestEthKey1, mockValue1)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey2, mockValue2)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey3, mockValue3)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey4, mockValue4)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey5, mockValue5)
+		Expect(err).ToNot(HaveOccurred())
+		err = database.Put(prefixedTestEthKey6, mockValue6)
+		Expect(err).ToNot(HaveOccurred())
+	})
+	AfterEach(func() {
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("NewIterator", func() {
+		It("iterates over the entire key-set (prefixed or not)", func() {
+			iterator = database.NewIterator()
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+	})
+
+	Describe("NewIteratorWithPrefix", func() {
+		It("iterates over all db entries that have the provided prefix", func() {
+			iterator = database.NewIteratorWithPrefix(testPrefix)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+
+		It("behaves as no prefix is provided if prefix is nil", func() {
+			iterator = database.NewIteratorWithPrefix(nil)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+
+		It("considers empty but non-nil []byte a valid prefix, which precludes iteration over any other prefixed keys", func() {
+			iterator = database.NewIteratorWithPrefix([]byte{})
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+	})
+
+	Describe("NewIteratorWithStart", func() {
+		It("iterates over the entire key-set (prefixed or not) starting with at the provided path", func() {
+			iterator = database.NewIteratorWithStart(testEthKey2)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey1))
+			Expect(iterator.Value()).To(Equal(mockValue1))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+
+		It("iterates over the entire key-set (prefixed or not) starting with at the provided path", func() {
+			iterator = database.NewIteratorWithStart(prefixedTestEthKey3)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey3))
+			Expect(iterator.Value()).To(Equal(mockValue3))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey5))
+			Expect(iterator.Value()).To(Equal(mockValue5))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey4))
+			Expect(iterator.Value()).To(Equal(mockValue4))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(prefixedTestEthKey6))
+			Expect(iterator.Value()).To(Equal(mockValue6))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).ToNot(BeTrue())
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(Equal(sql.ErrNoRows))
+		})
+	})
+
+	Describe("Release", func() {
+		It("releases resources associated with the Iterator", func() {
+			iterator = database.NewIteratorWithStart(testEthKey2)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more := iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+
+			iterator.Release()
+			iterator.Release() // check that we don't panic if called multiple times
+
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(BeNil())
+			Expect(iterator.Error()).To(BeNil())
+			Expect(func() { iterator.Next() }).To(Panic()) // check that we panic if we try to use released iterator
+
+			// We can still create a new iterator from the same backing db
+			iterator = database.NewIteratorWithStart(testEthKey2)
+			Expect(iterator.Value()).To(BeNil())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Error()).To(BeNil())
+
+			more = iterator.Next()
+			Expect(more).To(BeTrue())
+			Expect(iterator.Key()).To(Equal(testEthKey2))
+			Expect(iterator.Value()).To(Equal(mockValue2))
+			Expect(iterator.Error()).To(BeNil())
+		})
+	})
+})

ethdb/postgres/key_type.go

@@ -0,0 +1,59 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"bytes"
+
+	"github.com/ethereum/go-ethereum/core/rawdb"
+)
+
+type KeyType uint
+
+const (
+	Invalid KeyType = iota
+	Static
+	Keccak
+	Prefixed
+	Suffixed
+	Header
+	Preimage
+)
+
+// ResolveKeyType returns the key type based on the prefix
+func ResolveKeyType(key []byte) (KeyType, [][]byte) {
+	sk := bytes.Split(key, rawdb.KeyDelineation)
+	switch len(sk) {
+	case 1:
+		if len(sk[0]) < 32 {
+			return Static, sk
+		}
+		return Keccak, sk
+	case 2:
+		switch prefix := sk[0]; {
+		case bytes.Equal(prefix, rawdb.HeaderPrefix):
+			return Header, bytes.Split(sk[1], rawdb.NumberDelineation)
+		case bytes.Equal(prefix, rawdb.PreimagePrefix):
+			return Preimage, sk
+		default:
+			return Prefixed, sk
+		}
+	case 3:
+		return Suffixed, sk
+	}
+	return Invalid, sk
+}

ethdb/postgres/postgres_suite_test.go

@@ -0,0 +1,29 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestPGIPFSETHDB(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "PG-IPFS ethdb test")
+}

ethdb/postgres/test_helpers.go

@@ -0,0 +1,65 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import "github.com/jmoiron/sqlx"
+
+// TestDB connect to the testing database
+// it assumes the database has the IPFS public.blocks table present
+// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
+func TestDB() (*sqlx.DB, error) {
+	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	return sqlx.Connect("postgres", connectStr)
+}
+
+// ResetTestDB drops all rows in the test db public.blocks table
+func ResetTestDB(db *sqlx.DB) error {
+	tx, err := db.Beginx()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			tx.Rollback()
+			panic(p)
+		} else if err != nil {
+			tx.Rollback()
+		} else {
+			err = tx.Commit()
+		}
+	}()
+	if _, err := tx.Exec("TRUNCATE public.blocks CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.key_preimages CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.ancient_headers CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.ancient_hashes CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.ancient_bodies CASCADE"); err != nil {
+		return err
+	}
+	if _, err := tx.Exec("TRUNCATE eth.ancient_receipts CASCADE"); err != nil {
+		return err
+	}
+	_, err = tx.Exec("TRUNCATE eth.ancient_tds CASCADE")
+	return err
+}

ethdb/postgres/util.go

@@ -0,0 +1,63 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package postgres
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipfs/go-ipfs-ds-help"
+	_ "github.com/lib/pq" //postgres driver
+	"github.com/multiformats/go-multihash"
+)
+
+// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string
+func MultihashKeyFromKeccak256(h []byte) (string, error) {
+	mh, err := multihash.Encode(h, multihash.KECCAK_256)
+	if err != nil {
+		return "", err
+	}
+	dbKey := dshelp.MultihashToDsKey(mh)
+	return blockstore.BlockPrefix.String() + dbKey.String(), nil
+}
+
+// DatastoreKeyFromGethKey returns the public.blocks key from the provided geth key
+// It also returns the key's prefix, if it has one
+func DatastoreKeyFromGethKey(h []byte) (string, []byte, error) {
+	keyType, keyComponents := ResolveKeyType(h)
+	switch keyType {
+	case Keccak:
+		mhKey, err := MultihashKeyFromKeccak256(h)
+		return mhKey, nil, err
+	case Header:
+		mhKey, err := MultihashKeyFromKeccak256(keyComponents[1])
+		return mhKey, keyComponents[0], err
+	case Preimage:
+		mhKey, err := MultihashKeyFromKeccak256(keyComponents[1])
+		return mhKey, keyComponents[0], err
+	case Prefixed, Suffixed:
+		// This data is not mapped by hash => content by geth, store it using the prefixed/suffixed key directly
+		// I.e. the public.blocks datastore key == the hex representation of the geth key
+		// Alternatively, decompose the data and derive the hash
+		return common.Bytes2Hex(h), keyComponents[0], nil
+	case Static:
+		return common.Bytes2Hex(h), nil, nil
+	default:
+		return "", nil, fmt.Errorf("invalid formatting of database key: %x", h)
+	}
+}

go.mod

@@ -34,13 +34,17 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/huin/goupnp v1.0.0
 	github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883
+	github.com/ipfs/go-ipfs-blockstore v1.0.1
+	github.com/ipfs/go-ipfs-ds-help v1.0.0
 	github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
+	github.com/jmoiron/sqlx v1.2.0
 	github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21
 	github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/mattn/go-colorable v0.1.0
-	github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035
+	github.com/lib/pq v1.8.0
+	github.com/mattn/go-colorable v0.1.1
+	github.com/mattn/go-isatty v0.0.5
+	github.com/multiformats/go-multihash v0.0.14
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c
@@ -51,6 +55,7 @@ require (
 	github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff
 	github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00
 	github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect
+	github.com/sirupsen/logrus v1.6.0
 	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4
 	github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570
 	github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
@@ -60,7 +65,7 @@ require (
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208
 	golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4
 	golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 // indirect
-	golang.org/x/sync v0.0.0-20181108010431-42b317875d0f
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
 	golang.org/x/text v0.3.2
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4

go.sum

@@ -82,9 +82,12 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c h1:zqAKixg3cTcIasAMJV+EcfVbWwLpOZ7LeoWJvcuD/5Q=
 github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -92,10 +95,14 @@ github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54=
 github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKxgp2ojts0FDgUA8dl62bmH0LxKanMoBr6MDTDM=
 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
+github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad h1:eMxs9EL0PvIGS9TTtxg4R+JxuPGav82J8rA+GFnY7po=
 github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
@@ -109,14 +116,43 @@ github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
 github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 h1:FSeK4fZCo8u40n2JMnyAsd6x7+SbvoOMHvQOU/n10P4=
 github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
+github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
+github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
+github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
+github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
+github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
+github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
+github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
+github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.2 h1:h8/n7WPzhp239kkLws+epN3Ic7YtcBPgcaXfEfdVDWM=
+github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-ipfs-blockstore v1.0.1 h1:fnuVj4XdZp4yExhd0CnUwAiMNJHiPnfInhiuwz4lW1w=
+github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE=
+github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
+github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g=
+github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE=
+github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50=
+github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
+github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
+github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
+github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
+github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw=
+github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 h1:F/iKcka0K2LgnKy/fgSBf235AETtm1n1TvBzqu40LE0=
 github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw=
 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -126,17 +162,46 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
+github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o=
 github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw=
 github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4=
 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
+github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
+github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
+github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
+github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
+github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
+github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
+github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
+github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
+github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0=
@@ -150,6 +215,7 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA=
@@ -179,9 +245,13 @@ github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM
 github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9 h1:5Cp3cVwpQP4aCQ6jx6dNLP3IarbYiuStmIzYu+BjQwY=
 github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg=
 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE=
@@ -198,17 +268,27 @@ github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJ
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4=
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
+go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA=
 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 h1:Jcxah/M+oLZ/R4/z5RzfPzGbPXnVDPkEDtf2JnuxN+U=
@@ -216,12 +296,17 @@ golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
@@ -231,12 +316,17 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=

les/clientpool.go

@@ -25,6 +25,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/core/rawdb"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/mclock"
 	"github.com/ethereum/go-ethereum/common/prque"
@@ -654,16 +656,16 @@ const (
 )
 
 var (
-	positiveBalancePrefix    = []byte("pb:")             // dbVersion(uint16 big endian) + positiveBalancePrefix + id -> balance
-	negativeBalancePrefix    = []byte("nb:")             // dbVersion(uint16 big endian) + negativeBalancePrefix + ip -> balance
-	cumulativeRunningTimeKey = []byte("cumulativeTime:") // dbVersion(uint16 big endian) + cumulativeRunningTimeKey -> cumulativeTime
+	positiveBalancePrefix    = []byte("pb")             // dbVersion(uint16 big endian) + positiveBalancePrefix + id -> balance
+	negativeBalancePrefix    = []byte("nb")             // dbVersion(uint16 big endian) + negativeBalancePrefix + ip -> balance
+	cumulativeRunningTimeKey = []byte("cumulativeTime") // dbVersion(uint16 big endian) + cumulativeRunningTimeKey -> cumulativeTime
 )
 
 type nodeDB struct {
 	db              ethdb.Database
 	pcache          *lru.Cache
 	ncache          *lru.Cache
-	auxbuf          []byte                                // 37-byte auxiliary buffer for key encoding
+	auxbuf          []byte                                // 41-byte auxiliary buffer for key encoding
 	verbuf          [2]byte                               // 2-byte auxiliary buffer for db version
 	nbEvictCallBack func(mclock.AbsTime, negBalance) bool // Callback to determine whether the negative balance can be evicted.
 	clock           mclock.Clock
@@ -692,9 +694,9 @@ func (db *nodeDB) close() {
 }
 
 func (db *nodeDB) key(id []byte, neg bool) []byte {
-	prefix := positiveBalancePrefix
+	prefix := append(positiveBalancePrefix, rawdb.KeyDelineation...)
 	if neg {
-		prefix = negativeBalancePrefix
+		prefix = append(negativeBalancePrefix, rawdb.KeyDelineation...)
 	}
 	if len(prefix)+len(db.verbuf)+len(id) > len(db.auxbuf) {
 		db.auxbuf = append(db.auxbuf, make([]byte, len(prefix)+len(db.verbuf)+len(id)-len(db.auxbuf))...)

node/config.go

@@ -26,6 +26,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/ethereum/go-ethereum/ethdb/postgres"
+
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/external"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -191,6 +193,9 @@ type Config struct {
 	staticNodesWarning     bool
 	trustedNodesWarning    bool
 	oldGethResourceWarning bool
+
+	// Config params for using Postgres database
+	PostgresConfig *postgres.Config
 }
 
 // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into

node/node.go

@@ -26,6 +26,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/ethereum/go-ethereum/ethdb/postgres"
+
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -607,6 +609,9 @@ func (n *Node) EventMux() *event.TypeMux {
 // previous can be found) from within the node's instance directory. If the node is
 // ephemeral, a memory database is returned.
 func (n *Node) OpenDatabase(name string, cache, handles int, namespace string) (ethdb.Database, error) {
+	if n.config.PostgresConfig != nil {
+		return n.openPostgresDatabase()
+	}
 	if n.config.DataDir == "" {
 		return rawdb.NewMemoryDatabase(), nil
 	}
@@ -619,6 +624,9 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string) (
 // database to immutable append-only files. If the node is an ephemeral one, a
 // memory database is returned.
 func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string) (ethdb.Database, error) {
+	if n.config.PostgresConfig != nil {
+		return n.openPostgresDatabase()
+	}
 	if n.config.DataDir == "" {
 		return rawdb.NewMemoryDatabase(), nil
 	}
@@ -633,6 +641,14 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer,
 	return rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace)
 }
 
+func (n *Node) openPostgresDatabase() (ethdb.Database, error) {
+	db, err := postgres.NewDB(n.config.PostgresConfig)
+	if err != nil {
+		return nil, err
+	}
+	return postgres.NewDatabase(db), nil
+}
+
 // ResolvePath returns the absolute path of a resource in the instance directory.
 func (n *Node) ResolvePath(x string) string {
 	return n.config.ResolvePath(x)

node/service.go

@@ -20,6 +20,8 @@ import (
 	"path/filepath"
 	"reflect"
 
+	"github.com/ethereum/go-ethereum/ethdb/postgres"
+
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -42,6 +44,9 @@ type ServiceContext struct {
 // if no previous can be found) from within the node's data directory. If the
 // node is an ephemeral one, a memory database is returned.
 func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int, namespace string) (ethdb.Database, error) {
+	if ctx.config.PostgresConfig != nil {
+		return ctx.openPostgresDatabase()
+	}
 	if ctx.config.DataDir == "" {
 		return rawdb.NewMemoryDatabase(), nil
 	}
@@ -54,6 +59,9 @@ func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int, nam
 // database to immutable append-only files. If the node is an ephemeral one, a
 // memory database is returned.
 func (ctx *ServiceContext) OpenDatabaseWithFreezer(name string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) {
+	if ctx.config.PostgresConfig != nil {
+		return ctx.openPostgresDatabase()
+	}
 	if ctx.config.DataDir == "" {
 		return rawdb.NewMemoryDatabase(), nil
 	}
@@ -68,6 +76,14 @@ func (ctx *ServiceContext) OpenDatabaseWithFreezer(name string, cache int, handl
 	return rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace)
 }
 
+func (ctx *ServiceContext) openPostgresDatabase() (ethdb.Database, error) {
+	db, err := postgres.NewDB(ctx.config.PostgresConfig)
+	if err != nil {
+		return nil, err
+	}
+	return postgres.NewDatabase(db), nil
+}
+
 // ResolvePath resolves a user path into the data directory if that was relative
 // and if the user actually uses persistent storage. It will return an empty string
 // for emphemeral storage and the user's own input for absolute paths.

trie/database.go

@@ -59,8 +59,11 @@ var (
 // secureKeyPrefix is the database key prefix used to store trie node preimages.
 var secureKeyPrefix = []byte("secure-key-")
 
-// secureKeyLength is the length of the above prefix + 32byte hash.
-const secureKeyLength = 11 + 32
+// keyDelineation delineates a prefix and/or suffix from the rest of the key
+var keyDelineation = []byte("-fix-")
+
+// secureKeyLength is the length of the above prefix + KeyDelineator + 32byte hash
+const secureKeyLength = 11 + 5 + 32
 
 // Database is an intermediate write layer between the trie data structures and
 // the disk database. The aim is to accumulate trie writes in-memory and only
@@ -452,7 +455,8 @@ func (db *Database) preimage(hash common.Hash) ([]byte, error) {
 // buffer. The caller must not hold onto the return value because it will become
 // invalid on the next call.
 func (db *Database) secureKey(key []byte) []byte {
-	buf := append(db.seckeybuf[:0], secureKeyPrefix...)
+	delineatedPrefix := append(secureKeyPrefix, keyDelineation...)
+	buf := append(db.seckeybuf[:0], delineatedPrefix...)
 	buf = append(buf, key...)
 	return buf
 }