Merge pull request #31 from deep-stack/pm-upgrade-geth

Upgrade geth to v1.10.17

README.md

@@ -1,8 +1,8 @@
-## pg-ipfs-ethdb
+## ipfs-ethdb
 
-[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/pg-ipfs-ethdb)](https://goreportcard.com/report/github.com/vulcanize/pg-ipfs-ethdb)
+[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipfs-ethdb)](https://goreportcard.com/report/github.com/vulcanize/ipfs-ethdb)
 
-> go-ethereum ethdb interfaces for Ethereum state data stored in Postgres-backed IPFS
+> go-ethereum ethdb interfaces for Ethereum state data stored in IPFS
 
 ## Background
 
@@ -10,10 +10,9 @@ Go-ethereum defines a number of interfaces in the [ethdb package](https://github
 interfacing with a state database. These interfaces are used to build higher-level types such as the [trie.Database](https://github.com/ethereum/go-ethereum/blob/master/trie/database.go#L77)
 which are used to perform the bulk of state related needs.
 
-Ethereum data can be stored on IPFS, standard codecs for Etheruem data are defined in the [go-cid](https://github.com/ipfs/go-cid) library. Here at Vulcanize we
-have [extended IPFS](https://github.com/vulcanize/go-ipfs/releases/tag/v0.4.22-alpha) to [use Postgres](https://github.com/vulcanize/go-ipfs-config/releases/tag/v0.0.8-alpha) as a backing database.
-Additionally, [we have extended go-ethereum](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.2) to enable the efficient export of state data in the form of state diff objects.
-Together, this allows us to store all Ethereum data on Postgres-backed IPFS.
+Ethereum data can be stored on IPFS, standard codecs for Etheruem data are defined in the [go-cid](https://github.com/ipfs/go-cid) library.
+Using our [statediffing geth client](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.2) it is feasible to extract every single
+state and storage node and publish it to IPFS.
 
 Geth stores state data in leveldb as key-value pairs between the keccak256 hash of the rlp-encoded object and the rlp-encoded object.
 Ethereum data on IPFS is also stored as key-value pairs with the value being the rlp-encoded byte value for the object,
@@ -22,40 +21,63 @@ ethdb interfaces for Ethereum data on IPFS by handling the conversion of a kecca
 
 
 ## Usage
-To use this module simply import it and build the desired interface around an instance of [sqlx.DB](https://github.com/jmoiron/sqlx), you can then
-employ it as you would the usual [leveldb](https://github.com/ethereum/go-ethereum/tree/master/ethdb/leveldb) or [memorydb](https://github.com/ethereum/go-ethereum/tree/master/ethdb/memorydb) interfaces
-with a few exceptions:
+To use this module import it and build an ethdb interface around an instance of a [go ipfs blockservice](https://github.com/ipfs/go-blockservice), you can then
+employ it as you would the usual [leveldb](https://github.com/ethereum/go-ethereum/tree/master/ethdb/leveldb) or [memorydb](https://github.com/ethereum/go-ethereum/tree/master/ethdb/memorydb) ethdbs
+with some exceptions: the AncientReader, AncientWriter, Compacter, and Iteratee/Iterator interfaces are not functionally complete.
+
+Ancient data does not currently have a representation on IPFS, and recapitulation of the database key iterator and compacter is complicated since go-ethereum
+types that use this interface expect the iterator and compacter to operate over keccak256 hash key ranges, whereas the keys for Ethereum data on IPFS are derived from that hash but not the hash itself.
+
+Iteratee interface is used in Geth for various tests, in trie/sync_bloom.go (for fast sync), rawdb.InspectDatabase, and the new (1.9.15) core/state/snapshot features;
+Ancient interfaces are used for Ancient/frozen data operations (e.g. rawdb/table.go); and Compacter is used in core/state/snapshot, rawdb/table.go, chaincmd.go, and the private debug api.
+
+Outside of these primarily auxiliary capabilities, this package satisfies the interfaces required for many state operations using Ethereum data on IPFS.
+
+e.g.
+ 
+go-ethereum trie.NodeIterator and state.NodeIterator can be constructed from the ethdb.KeyValueStore and ethdb.Database interfaces, respectively:
 
 ```go
 package main
 
 import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/trie"
-	"github.com/jmoiron/sqlx"
-	"github.com/vulcanize/pg-ipfs-ethdb"
+    "context"
+	
+    "github.com/ethereum/go-ethereum/common"
+    "github.com/ethereum/go-ethereum/core/state"
+    "github.com/ethereum/go-ethereum/trie"
+    "github.com/ipfs/go-blockservice"
+    "github.com/ipfs/go-ipfs/core"
+    "github.com/ipfs/go-ipfs/repo/fsrepo"
+    "github.com/jmoiron/sqlx"
+    "github.com/vulcanize/ipfs-ethdb"
 )
 
 func main() {
-    connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
-    db, _ := sqlx.Connect("postgres", connectStr)
-
-    kvs := ipfsethdb.NewKeyValueStore(db)
+    // errors are ignored for brevity
+    ipfsPath := "~/.ipfs"
+    r, _ := fsrepo.Open(ipfsPath)
+    ctx := context.Background()
+    cfg := &core.BuildCfg{
+        Online: false,
+        Repo:   r,
+    }
+    ipfsNode, _ := core.NewNode(ctx, cfg)
+    kvs := ipfsethdb.NewKeyValueStore(ipfsNode.Blocks)
     trieDB := trie.NewDatabase(kvs)
-    trie, _ := trie.New(common.Hash{}, trieDB)
-    // do stuff with trie or trieDB
+    t, _ := trie.New(common.Hash{}, trieDB)
+    trieNodeIterator := t.NodeIterator([]byte{})
+    // do stuff with trie node iterator
 
-    database := ipfsethdb.NewDatabase(db)
-    statedb := state.NewDatabase(database)
-    // do stuff with the state database
+    database := ipfsethdb.NewDatabase(ipfsNode.Blocks)
+    stateDatabase := state.NewDatabase(database)
+    stateDB, _ := state.New(common.Hash{}, stateDatabase, nil)
+    stateDBNodeIterator := state.NewNodeIterator(stateDB)
+    // do stuff with the statedb node iterator
 }
 ```
 
-EXCEPTIONS: AncientReader, AncientWriter, and Iteratee interfaces are not functionally complete.
-
-Ancient data does not currently have a representation on IPFS, and recapitulation of the database key iterator is complicated since go-ethereum
-types that use this interface expect the iterator to iterate over keccak256 hash keys, whereas the keys for Ethereum data on IPFS are derived from that hash but not the hash itself.
+[Types are also available](./postgres/doc.md) for interfacing directly with the data stored in an IPFS-backing Postgres database
 
 ## Maintainers
 @vulcanize

batch.go

@@ -17,85 +17,130 @@
 package ipfsethdb
 
 import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/jmoiron/sqlx"
+	lru "github.com/hashicorp/golang-lru"
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
 )
 
-// Batch is the type that satisfies the ethdb.Batch interface for PG-IPFS Ethereum data
+var (
+	EvictionWarningErr = errors.New("warn: batch has exceeded capacity, data has been evicted")
+)
+
+var _ ethdb.Batch = &Batch{}
+
+// Batch is the type that satisfies the ethdb.Batch interface for IPFS Ethereum data using the ipfs blockservice interface
+// This is ipfs-backing-datastore agnostic but must operate through a configured ipfs node (and so is subject to lockfile contention with e.g. an ipfs daemon)
+// If blockservice block exchange is configured the blockservice can fetch data that are missing locally from IPFS peers
 type Batch struct {
-	db   *sqlx.DB
-	tx   *sqlx.Tx
-	size int
+	blockService          blockservice.BlockService
+	putCache, deleteCache *lru.Cache
+	valueSize             int
 }
 
-// NewBatch returns a ethdb.Batch interface for PG-IPFS
-func NewBatch(db *sqlx.DB) ethdb.Batch {
-	return &Batch{
-		db: db,
+// NewBatch returns a ethdb.Batch interface for IPFS
+func NewBatch(bs blockservice.BlockService, capacity int) (ethdb.Batch, error) {
+	putCache, err := lru.New(capacity)
+	if err != nil {
+		return nil, err
 	}
+	deleteCache, err := lru.New(capacity)
+	if err != nil {
+		return nil, err
+	}
+	return &Batch{
+		blockService: bs,
+		putCache:     putCache,
+		deleteCache:  deleteCache,
+	}, nil
 }
 
 // Put satisfies the ethdb.Batch interface
 // Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
+// Returns an error when batch capacity has been exceeded and data was evicted
+// It is up to ensure they do not exceed capacity
+// The alternative is to check the cache len vs its capacity before inserting
+// but this adds additional overhead to every Put/Delete
 func (b *Batch) Put(key []byte, value []byte) (err error) {
-	if b.tx == nil {
-		b.Reset()
+	b.valueSize += len(value)
+	strKey := common.Bytes2Hex(key)
+	if b.putCache.Add(strKey, value) {
+		return EvictionWarningErr
 	}
-	mhKey, err := MultihashKeyFromKeccak256(key)
-	if err != nil {
-		return err
-	}
-	if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
-		return err
-	}
-	b.size += len(value)
 	return nil
 }
 
 // Delete satisfies the ethdb.Batch interface
 // Delete removes the key from the key-value data store
 func (b *Batch) Delete(key []byte) (err error) {
-	if b.tx == nil {
-		b.Reset()
+	strKey := common.Bytes2Hex(key)
+	if b.deleteCache.Add(strKey, true) {
+		return EvictionWarningErr
 	}
-	mhKey, err := MultihashKeyFromKeccak256(key)
-	if err != nil {
-		return err
-	}
-	_, err = b.tx.Exec(deletePgStr, mhKey)
-	return err
+	return nil
 }
 
 // ValueSize satisfies the ethdb.Batch interface
 // ValueSize retrieves the amount of data queued up for writing
 // The returned value is the total byte length of all data queued to write
 func (b *Batch) ValueSize() int {
-	return b.size
+	return b.valueSize
 }
 
 // Write satisfies the ethdb.Batch interface
 // Write flushes any accumulated data to disk
 func (b *Batch) Write() error {
-	if b.tx == nil {
-		return nil
+	puts := make([]blocks.Block, b.putCache.Len())
+	for i, key := range b.putCache.Keys() {
+		val, _ := b.putCache.Get(key) // don't need to check "ok"s, the key is known and val is always []byte
+		b, err := NewBlock(common.Hex2Bytes(key.(string)), val.([]byte))
+		if err != nil {
+			return err
+		}
+		puts[i] = b
 	}
-	return b.tx.Commit()
+	if err := b.blockService.AddBlocks(puts); err != nil {
+		return err
+	}
+	for _, key := range b.deleteCache.Keys() {
+		// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+		c, err := Keccak256ToCid(common.Hex2Bytes(key.(string)), stateTrieCodec)
+		if err != nil {
+			return err
+		}
+		if err := b.blockService.DeleteBlock(c); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Replay satisfies the ethdb.Batch interface
 // Replay replays the batch contents
 func (b *Batch) Replay(w ethdb.KeyValueWriter) error {
-	return errNotSupported
+	for _, key := range b.putCache.Keys() {
+		val, _ := b.putCache.Get(key)
+		if err := w.Put(key.([]byte), val.([]byte)); err != nil {
+			return err
+		}
+	}
+	for _, key := range b.deleteCache.Keys() {
+		if err := w.Delete(key.([]byte)); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Reset satisfies the ethdb.Batch interface
 // Reset resets the batch for reuse
 // This should be called after every write
 func (b *Batch) Reset() {
-	var err error
-	b.tx, err = b.db.Beginx()
-	if err != nil {
-		panic(err)
-	}
-	b.size = 0
+	b.deleteCache.Purge()
+	b.putCache.Purge()
+	b.valueSize = 0
 }

batch_test.go

@@ -25,7 +25,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/pg-ipfs-ethdb"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb"
 )
 
 var (
@@ -37,24 +37,20 @@ var (
 
 var _ = Describe("Batch", func() {
 	BeforeEach(func() {
-		db, err = ipfsethdb.TestDB()
-		Expect(err).ToNot(HaveOccurred())
-		database = ipfsethdb.NewDatabase(db)
-		batch = database.NewBatch()
-	})
-	AfterEach(func() {
-		err = ipfsethdb.ResetTestDB(db)
+		blockService = ipfsethdb.NewMockBlockservice()
+		batch, err = ipfsethdb.NewBatch(blockService, 1024)
 		Expect(err).ToNot(HaveOccurred())
+		database = ipfsethdb.NewDatabase(blockService)
 	})
 
 	Describe("Put/Write", func() {
 		It("adds the key-value pair to the batch", func() {
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 			_, err = database.Get(testEthKey2)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 
 			err = batch.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
@@ -74,7 +70,6 @@ var _ = Describe("Batch", func() {
 
 	Describe("Delete/Reset/Write", func() {
 		It("deletes the key-value pair in the batch", func() {
-			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
 			err = batch.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Put(testEthKey2, testValue2)
@@ -92,10 +87,10 @@ var _ = Describe("Batch", func() {
 
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 			_, err = database.Get(testEthKey2)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 		})
 	})
 

database.go

@@ -17,85 +17,96 @@
 package ipfsethdb
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/jmoiron/sqlx"
+	"github.com/ipfs/go-blockservice"
 )
 
-var errNotSupported = errors.New("this operation is not supported")
-
 var (
-	hasPgStr    = "SELECT exists(select 1 from public.blocks WHERE key = $1)"
-	getPgStr    = "SELECT data FROM public.blocks WHERE key = $1"
-	putPgStr    = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
-	deletePgStr = "DELETE FROM public.blocks WHERE key = $1"
-	dbSizePgStr = "SELECT pg_database_size(current_database())"
+	stateTrieCodec       uint64 = 0x96
+	defaultBatchCapacity        = 1024
+	errNotSupported             = errors.New("this operation is not supported")
 )
 
-// Database is the type that satisfies the ethdb.Database and ethdb.KeyValueStore interfaces for PG-IPFS Ethereum data
+var _ ethdb.Database = &Database{}
+
+// Database is the type that satisfies the ethdb.Database and ethdb.KeyValueStore interfaces for IPFS Ethereum data
+// This is ipfs-backing-datastore agnostic but must operate through a configured ipfs node (and so is subject to lockfile contention with e.g. an ipfs daemon)
+// If blockservice block exchange is configured the blockservice can fetch data that are missing locally from IPFS peers
 type Database struct {
-	db *sqlx.DB
+	blockService blockservice.BlockService
 }
 
-// NewKeyValueStore returns a ethdb.KeyValueStore interface for PG-IPFS
-func NewKeyValueStore(db *sqlx.DB) ethdb.KeyValueStore {
+// NewKeyValueStore returns a ethdb.KeyValueStore interface for IPFS
+func NewKeyValueStore(bs blockservice.BlockService) ethdb.KeyValueStore {
 	return &Database{
-		db: db,
+		blockService: bs,
 	}
 }
 
-// NewDatabase returns a ethdb.Database interface for PG-IPFS
-func NewDatabase(db *sqlx.DB) ethdb.Database {
+// NewDatabase returns a ethdb.Database interface for IPFS
+func NewDatabase(bs blockservice.BlockService) ethdb.Database {
 	return &Database{
-		db: db,
+		blockService: bs,
 	}
 }
 
+func (d *Database) ModifyAncients(f func(ethdb.AncientWriteOp) error) (int64, error) {
+	return 0, errNotSupported
+}
+
 // Has satisfies the ethdb.KeyValueReader interface
 // Has retrieves if a key is present in the key-value data store
+// This only operates on the local blockstore not through the exchange
 func (d *Database) Has(key []byte) (bool, error) {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(key, stateTrieCodec)
 	if err != nil {
 		return false, err
 	}
-	var exists bool
-	return exists, d.db.Get(&exists, hasPgStr, mhKey)
+	return d.blockService.Blockstore().Has(c)
 }
 
 // Get satisfies the ethdb.KeyValueReader interface
 // Get retrieves the given key if it's present in the key-value data store
 func (d *Database) Get(key []byte) ([]byte, error) {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(key, stateTrieCodec)
 	if err != nil {
 		return nil, err
 	}
-	var data []byte
-	return data, d.db.Get(&data, getPgStr, mhKey)
+	block, err := d.blockService.GetBlock(context.Background(), c)
+	if err != nil {
+		return nil, err
+	}
+	return block.RawData(), nil
 }
 
 // Put satisfies the ethdb.KeyValueWriter interface
 // Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
 func (d *Database) Put(key []byte, value []byte) error {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	b, err := NewBlock(key, value)
 	if err != nil {
 		return err
 	}
-	_, err = d.db.Exec(putPgStr, mhKey, value)
-	return err
+	return d.blockService.AddBlock(b)
 }
 
 // Delete satisfies the ethdb.KeyValueWriter interface
 // Delete removes the key from the key-value data store
 func (d *Database) Delete(key []byte) error {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(key, stateTrieCodec)
 	if err != nil {
 		return err
 	}
-	_, err = d.db.Exec(deletePgStr, mhKey)
-	return err
+	return d.blockService.DeleteBlock(c)
 }
 
 // DatabaseProperty enum type
@@ -103,38 +114,14 @@ type DatabaseProperty int
 
 const (
 	Unknown DatabaseProperty = iota
-	Size
-	Idle
-	InUse
-	MaxIdleClosed
-	MaxLifetimeClosed
-	MaxOpenConnections
-	OpenConnections
-	WaitCount
-	WaitDuration
+	ExchangeOnline
 )
 
 // DatabasePropertyFromString helper function
 func DatabasePropertyFromString(property string) (DatabaseProperty, error) {
 	switch strings.ToLower(property) {
-	case "size":
-		return Size, nil
-	case "idle":
-		return Idle, nil
-	case "inuse":
-		return InUse, nil
-	case "maxidleclosed":
-		return MaxIdleClosed, nil
-	case "maxlifetimeclosed":
-		return MaxLifetimeClosed, nil
-	case "maxopenconnections":
-		return MaxOpenConnections, nil
-	case "openconnections":
-		return OpenConnections, nil
-	case "waitcount":
-		return WaitCount, nil
-	case "waitduration":
-		return WaitDuration, nil
+	case "exchange", "online":
+		return ExchangeOnline, nil
 	default:
 		return Unknown, fmt.Errorf("unknown database property")
 	}
@@ -148,25 +135,9 @@ func (d *Database) Stat(property string) (string, error) {
 		return "", err
 	}
 	switch prop {
-	case Size:
-		var byteSize string
-		return byteSize, d.db.Get(&byteSize, dbSizePgStr)
-	case Idle:
-		return string(d.db.Stats().Idle), nil
-	case InUse:
-		return string(d.db.Stats().InUse), nil
-	case MaxIdleClosed:
-		return string(d.db.Stats().MaxIdleClosed), nil
-	case MaxLifetimeClosed:
-		return string(d.db.Stats().MaxLifetimeClosed), nil
-	case MaxOpenConnections:
-		return string(d.db.Stats().MaxOpenConnections), nil
-	case OpenConnections:
-		return string(d.db.Stats().OpenConnections), nil
-	case WaitCount:
-		return string(d.db.Stats().WaitCount), nil
-	case WaitDuration:
-		return d.db.Stats().WaitDuration.String(), nil
+	case ExchangeOnline:
+		online := d.blockService.Exchange().IsOnline()
+		return strconv.FormatBool(online), nil
 	default:
 		return "", fmt.Errorf("unhandled database property")
 	}
@@ -182,7 +153,21 @@ func (d *Database) Compact(start []byte, limit []byte) error {
 // NewBatch creates a write-only database that buffers changes to its host db
 // until a final write is called
 func (d *Database) NewBatch() ethdb.Batch {
-	return NewBatch(d.db)
+	b, err := NewBatch(d.blockService, defaultBatchCapacity)
+	if err != nil {
+		panic(err)
+	}
+	return b
+}
+
+// NewBatchWithSize satisfies the ethdb.Batcher interface.
+// NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
+func (d *Database) NewBatchWithSize(size int) ethdb.Batch {
+	b, err := NewBatch(d.blockService, size)
+	if err != nil {
+		panic(err)
+	}
+	return b
 }
 
 // NewIterator satisfies the ethdb.Iteratee interface
@@ -193,13 +178,13 @@ func (d *Database) NewBatch() ethdb.Batch {
 // Note: This method assumes that the prefix is NOT part of the start, so there's
 // no need for the caller to prepend the prefix to the start
 func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
-	return NewIterator(start, prefix, d.db)
+	return NewIterator(start, prefix, d.blockService)
 }
 
 // Close satisfies the io.Closer interface
 // Close closes the db connection
 func (d *Database) Close() error {
-	return d.db.DB.Close()
+	return d.blockService.Close()
 }
 
 // HasAncient satisfies the ethdb.AncientReader interface
@@ -220,21 +205,41 @@ func (d *Database) Ancients() (uint64, error) {
 	return 0, errNotSupported
 }
 
+// Tail satisfies the ethdb.AncientReader interface.
+// Tail returns the number of first stored item in the freezer.
+func (d *Database) Tail() (uint64, error) {
+	return 0, errNotSupported
+}
+
 // AncientSize satisfies the ethdb.AncientReader interface
 // AncientSize returns the ancient size of the specified category
 func (d *Database) AncientSize(kind string) (uint64, error) {
 	return 0, errNotSupported
 }
 
-// AppendAncient satisfies the ethdb.AncientWriter interface
-// AppendAncient injects all binary blobs belong to block at the end of the append-only immutable table files
-func (d *Database) AppendAncient(number uint64, hash, header, body, receipt, td []byte) error {
+// AncientRange retrieves all the items in a range, starting from the index 'start'.
+// It will return
+//  - at most 'count' items,
+//  - at least 1 item (even if exceeding the maxBytes), but will otherwise
+//   return as many items as fit into maxBytes.
+func (d *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
+	return nil, errNotSupported
+}
+
+// ReadAncients applies the provided AncientReader function
+func (d *Database) ReadAncients(fn func(ethdb.AncientReader) error) (err error) {
 	return errNotSupported
 }
 
-// TruncateAncients satisfies the ethdb.AncientWriter interface
-// TruncateAncients discards all but the first n ancient data from the ancient store
-func (d *Database) TruncateAncients(n uint64) error {
+// TruncateHead satisfies the ethdb.AncientWriter interface.
+// TruncateHead discards all but the first n ancient data from the ancient store.
+func (d *Database) TruncateHead(n uint64) error {
+	return errNotSupported
+}
+
+// TruncateTail satisfies the ethdb.AncientWriter interface.
+// TruncateTail discards the first n ancient data from the ancient store.
+func (d *Database) TruncateTail(n uint64) error {
 	return errNotSupported
 }
 
@@ -243,3 +248,15 @@ func (d *Database) TruncateAncients(n uint64) error {
 func (d *Database) Sync() error {
 	return errNotSupported
 }
+
+// MigrateTable satisfies the ethdb.AncientWriter interface.
+// MigrateTable processes and migrates entries of a given table to a new format.
+func (d *Database) MigrateTable(string, func([]byte) ([]byte, error)) error {
+	return errNotSupported
+}
+
+// NewSnapshot satisfies the ethdb.Snapshotter interface.
+// NewSnapshot creates a database snapshot based on the current state.
+func (d *Database) NewSnapshot() (ethdb.Snapshot, error) {
+	return nil, errNotSupported
+}

database_test.go

@@ -22,32 +22,26 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/jmoiron/sqlx"
+	"github.com/ipfs/go-blockservice"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/pg-ipfs-ethdb"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb"
 )
 
 var (
 	database     ethdb.Database
-	db           *sqlx.DB
+	blockService blockservice.BlockService
 	err          error
 	testHeader   = types.Header{Number: big.NewInt(1337)}
 	testValue, _ = rlp.EncodeToBytes(testHeader)
 	testEthKey   = testHeader.Hash().Bytes()
-	testMhKey, _ = ipfsethdb.MultihashKeyFromKeccak256(testEthKey)
 )
 
 var _ = Describe("Database", func() {
 	BeforeEach(func() {
-		db, err = ipfsethdb.TestDB()
-		Expect(err).ToNot(HaveOccurred())
-		database = ipfsethdb.NewDatabase(db)
-	})
-	AfterEach(func() {
-		err = ipfsethdb.ResetTestDB(db)
-		Expect(err).ToNot(HaveOccurred())
+		blockService = ipfsethdb.NewMockBlockservice()
+		database = ipfsethdb.NewDatabase(blockService)
 	})
 
 	Describe("Has", func() {
@@ -57,7 +51,7 @@ var _ = Describe("Database", func() {
 			Expect(has).ToNot(BeTrue())
 		})
 		It("returns true if a key-pair exists in the db", func() {
-			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			err := database.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
 			has, err := database.Has(testEthKey)
 			Expect(err).ToNot(HaveOccurred())
@@ -69,10 +63,10 @@ var _ = Describe("Database", func() {
 		It("throws an err if the key-pair doesn't exist in the db", func() {
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 		})
 		It("returns the value associated with the key, if the pair exists", func() {
-			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			err := database.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
 			val, err := database.Get(testEthKey)
 			Expect(err).ToNot(HaveOccurred())
@@ -84,7 +78,7 @@ var _ = Describe("Database", func() {
 		It("persists the key-value pair in the database", func() {
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 
 			err = database.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
@@ -106,7 +100,7 @@ var _ = Describe("Database", func() {
 			Expect(err).ToNot(HaveOccurred())
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 		})
 	})
 })

go.mod

@@ -1,14 +1,22 @@
-module github.com/vulcanize/pg-ipfs-ethdb
+module github.com/vulcanize/ipfs-ethdb
 
-go 1.13
+go 1.15
 
 require (
-	github.com/ethereum/go-ethereum v1.9.14
-	github.com/ipfs/go-ipfs-blockstore v1.0.0
+	github.com/btcsuite/btcd v0.22.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/ethereum/go-ethereum v1.10.17
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
+	github.com/ipfs/go-block-format v0.0.3
+	github.com/ipfs/go-blockservice v0.1.3
+	github.com/ipfs/go-cid v0.0.7
+	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
-	github.com/jmoiron/sqlx v1.2.0
-	github.com/lib/pq v1.0.0
-	github.com/multiformats/go-multihash v0.0.13
-	github.com/onsi/ginkgo v1.7.0
-	github.com/onsi/gomega v1.4.3
+	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/lib/pq v1.10.5
+	github.com/mailgun/groupcache/v2 v2.3.0
+	github.com/multiformats/go-multihash v0.1.0
+	github.com/onsi/ginkgo v1.16.5
+	github.com/onsi/gomega v1.19.0
 )

iterator.go

@@ -17,27 +17,31 @@
 package ipfsethdb
 
 import (
+	"context"
+
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/jmoiron/sqlx"
+	"github.com/ipfs/go-blockservice"
 )
 
-// Iterator is the type that satisfies the ethdb.Iterator interface for PG-IPFS Ethereum data
-// Iteratee interface is only used in Geth for various tests, trie/sync_bloom.go (for fast sync), and rawdb.InspectDatabase
-// Don't need this interface for the majority of state operations
-// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed from the ethdb.KeyValueStore and ethdb.Database interfaces)
-// ethdb.KeyValueStore => trie.Database => trie.Trie => trie.NodeIterator
+var _ ethdb.Iterator = &Iterator{}
+
+// Iterator is the type that satisfies the ethdb.Iterator interface for IPFS Ethereum data
+// Iteratee interface is used in Geth for various tests, trie/sync_bloom.go (for fast sync),
+// rawdb.InspectDatabase, and the new core/state/snapshot features.
+// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed
+// from the ethdb.KeyValueStoreand ethdb.Database interfaces)
 type Iterator struct {
-	db                 *sqlx.DB
+	blockService       blockservice.BlockService
 	currentKey, prefix []byte
 	err                error
 }
 
-// NewIterator returns a ethdb.Iterator interface for PG-IPFS
-func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
+// NewIterator returns an ethdb.Iterator interface for PG-IPFS
+func NewIterator(start, prefix []byte, bs blockservice.BlockService) ethdb.Iterator {
 	return &Iterator{
-		db:         db,
-		prefix:     prefix,
-		currentKey: start,
+		blockService: bs,
+		prefix:       prefix,
+		currentKey:   start,
 	}
 }
 
@@ -70,19 +74,23 @@ func (i *Iterator) Key() []byte {
 // The caller should not modify the contents of the returned slice
 // and its contents may change on the next call to Next
 func (i *Iterator) Value() []byte {
-	mhKey, err := MultihashKeyFromKeccak256(i.currentKey)
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(i.currentKey, stateTrieCodec)
 	if err != nil {
 		i.err = err
 		return nil
 	}
-	var data []byte
-	i.err = i.db.Get(&data, getPgStr, mhKey)
-	return data
+	block, err := i.blockService.GetBlock(context.Background(), c)
+	if err != nil {
+		i.err = err
+		return nil
+	}
+	return block.RawData()
 }
 
 // Release satisfies the ethdb.Iterator interface
 // Release releases associated resources
 // Release should always succeed and can be called multiple times without causing error
 func (i *Iterator) Release() {
-	i.db.Close()
+	i.blockService.Close()
 }

mock_blockservice.go

@@ -0,0 +1,144 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipfsethdb
+
+import (
+	"context"
+	"errors"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	exchange "github.com/ipfs/go-ipfs-exchange-interface"
+)
+
+var (
+	blockNotFoundErr = errors.New("block not found")
+)
+
+type MockBlockservice struct {
+	blockStore *MockBlockstore
+	err        error
+}
+
+func NewMockBlockservice() blockservice.BlockService {
+	return &MockBlockservice{
+		blockStore: &MockBlockstore{
+			blocks: make(map[string]blocks.Block),
+		},
+	}
+}
+
+func (mbs *MockBlockservice) Blockstore() blockstore.Blockstore {
+	return mbs.blockStore
+}
+
+func (mbs *MockBlockservice) Exchange() exchange.Interface {
+	panic("Exchange: implement me")
+}
+
+func (mbs *MockBlockservice) AddBlock(b blocks.Block) error {
+	return mbs.blockStore.Put(b)
+}
+
+func (mbs *MockBlockservice) AddBlocks(bs []blocks.Block) error {
+	return mbs.blockStore.PutMany(bs)
+}
+
+func (mbs *MockBlockservice) DeleteBlock(c cid.Cid) error {
+	return mbs.blockStore.DeleteBlock(c)
+}
+
+func (mbs *MockBlockservice) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	return mbs.blockStore.Get(c)
+}
+
+func (mbs *MockBlockservice) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
+	blockChan := make(chan blocks.Block)
+	go func() {
+		for _, c := range cs {
+			if b, err := mbs.blockStore.Get(c); err == nil {
+				blockChan <- b
+			}
+		}
+	}()
+	return blockChan
+}
+
+func (mbs *MockBlockservice) Close() error {
+	return mbs.err
+}
+
+func (mbs *MockBlockservice) SetError(err error) {
+	mbs.err = err
+}
+
+type MockBlockstore struct {
+	blocks map[string]blocks.Block
+	err    error
+}
+
+func (mbs *MockBlockstore) DeleteBlock(c cid.Cid) error {
+	delete(mbs.blocks, c.String())
+	return mbs.err
+}
+
+func (mbs *MockBlockstore) Has(c cid.Cid) (bool, error) {
+	_, ok := mbs.blocks[c.String()]
+	return ok, mbs.err
+}
+
+func (mbs *MockBlockstore) Get(c cid.Cid) (blocks.Block, error) {
+	obj, ok := mbs.blocks[c.String()]
+	if !ok {
+		return nil, blockNotFoundErr
+	}
+	return obj, mbs.err
+}
+
+func (mbs *MockBlockstore) GetSize(c cid.Cid) (int, error) {
+	obj, ok := mbs.blocks[c.String()]
+	if !ok {
+		return 0, blockNotFoundErr
+	}
+	return len(obj.RawData()), mbs.err
+}
+
+func (mbs *MockBlockstore) Put(b blocks.Block) error {
+	mbs.blocks[b.Cid().String()] = b
+	return mbs.err
+}
+
+func (mbs *MockBlockstore) PutMany(bs []blocks.Block) error {
+	for _, b := range bs {
+		mbs.blocks[b.Cid().String()] = b
+	}
+	return mbs.err
+}
+
+func (mbs *MockBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	panic("AllKeysChan: implement me")
+}
+
+func (mbs *MockBlockstore) HashOnRead(enabled bool) {
+	panic("HasOnRead: implement me")
+}
+
+func (mbs *MockBlockstore) SetError(err error) {
+	mbs.err = err
+}

postgres/batch.go

@@ -0,0 +1,103 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+)
+
+var _ ethdb.Batch = &Batch{}
+
+// Batch is the type that satisfies the ethdb.Batch interface for PG-IPFS Ethereum data using a direct Postgres connection
+type Batch struct {
+	db        *sqlx.DB
+	tx        *sqlx.Tx
+	valueSize int
+}
+
+// NewBatch returns a ethdb.Batch interface for PG-IPFS
+func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
+	b := &Batch{
+		db: db,
+		tx: tx,
+	}
+	if tx == nil {
+		b.Reset()
+	}
+	return b
+}
+
+// Put satisfies the ethdb.Batch interface
+// Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
+func (b *Batch) Put(key []byte, value []byte) (err error) {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return err
+	}
+	if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
+		return err
+	}
+	b.valueSize += len(value)
+	return nil
+}
+
+// Delete satisfies the ethdb.Batch interface
+// Delete removes the key from the key-value data store
+func (b *Batch) Delete(key []byte) (err error) {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return err
+	}
+	_, err = b.tx.Exec(deletePgStr, mhKey)
+	return err
+}
+
+// ValueSize satisfies the ethdb.Batch interface
+// ValueSize retrieves the amount of data queued up for writing
+// The returned value is the total byte length of all data queued to write
+func (b *Batch) ValueSize() int {
+	return b.valueSize
+}
+
+// Write satisfies the ethdb.Batch interface
+// Write flushes any accumulated data to disk
+func (b *Batch) Write() error {
+	if b.tx == nil {
+		return nil
+	}
+	return b.tx.Commit()
+}
+
+// Replay satisfies the ethdb.Batch interface
+// Replay replays the batch contents
+func (b *Batch) Replay(w ethdb.KeyValueWriter) error {
+	return errNotSupported
+}
+
+// Reset satisfies the ethdb.Batch interface
+// Reset resets the batch for reuse
+// This should be called after every write
+func (b *Batch) Reset() {
+	var err error
+	b.tx, err = b.db.Beginx()
+	if err != nil {
+		panic(err)
+	}
+	b.valueSize = 0
+}

postgres/batch_test.go

@@ -0,0 +1,128 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb_test
+
+import (
+	"math/big"
+	"time"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/mailgun/groupcache/v2"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
+)
+
+var (
+	batch         ethdb.Batch
+	testHeader2   = types.Header{Number: big.NewInt(2)}
+	testValue2, _ = rlp.EncodeToBytes(testHeader2)
+	testEthKey2   = testHeader2.Hash().Bytes()
+)
+
+var _ = Describe("Batch", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+
+		cacheConfig := pgipfsethdb.CacheConfig{
+			Name:           "db",
+			Size:           3000000, // 3MB
+			ExpiryDuration: time.Hour,
+		}
+
+		database = pgipfsethdb.NewDatabase(db, cacheConfig)
+		batch = database.NewBatch()
+	})
+	AfterEach(func() {
+		groupcache.DeregisterGroup("db")
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("Put/Write", func() {
+		It("adds the key-value pair to the batch", func() {
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = database.Get(testEthKey2)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = batch.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			val, err := database.Get(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+			val2, err := database.Get(testEthKey2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val2).To(Equal(testValue2))
+		})
+	})
+
+	Describe("Delete/Reset/Write", func() {
+		It("deletes the key-value pair in the batch", func() {
+			err = batch.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			batch.Reset()
+			err = batch.Delete(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Delete(testEthKey2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = database.Get(testEthKey2)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("ValueSize/Reset", func() {
+		It("returns the size of data in the batch queued for write", func() {
+			err = batch.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			size := batch.ValueSize()
+			Expect(size).To(Equal(len(testValue) + len(testValue2)))
+
+			batch.Reset()
+			size = batch.ValueSize()
+			Expect(size).To(Equal(0))
+		})
+	})
+})

postgres/database.go

@@ -0,0 +1,346 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+	"github.com/mailgun/groupcache/v2"
+)
+
+var errNotSupported = errors.New("this operation is not supported")
+
+var (
+	hasPgStr    = "SELECT exists(select 1 from public.blocks WHERE key = $1)"
+	getPgStr    = "SELECT data FROM public.blocks WHERE key = $1"
+	putPgStr    = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
+	deletePgStr = "DELETE FROM public.blocks WHERE key = $1"
+	dbSizePgStr = "SELECT pg_database_size(current_database())"
+)
+
+var _ ethdb.Database = &Database{}
+
+// Database is the type that satisfies the ethdb.Database and ethdb.KeyValueStore interfaces for PG-IPFS Ethereum data using a direct Postgres connection
+type Database struct {
+	db    *sqlx.DB
+	cache *groupcache.Group
+}
+
+func (d *Database) ModifyAncients(f func(ethdb.AncientWriteOp) error) (int64, error) {
+	return 0, errNotSupported
+}
+
+type CacheConfig struct {
+	Name           string
+	Size           int
+	ExpiryDuration time.Duration
+}
+
+// NewKeyValueStore returns a ethdb.KeyValueStore interface for PG-IPFS
+func NewKeyValueStore(db *sqlx.DB, cacheConfig CacheConfig) ethdb.KeyValueStore {
+	database := Database{db: db}
+	database.InitCache(cacheConfig)
+
+	return &database
+}
+
+// NewDatabase returns a ethdb.Database interface for PG-IPFS
+func NewDatabase(db *sqlx.DB, cacheConfig CacheConfig) ethdb.Database {
+	database := Database{db: db}
+	database.InitCache(cacheConfig)
+
+	return &database
+}
+
+func (d *Database) InitCache(cacheConfig CacheConfig) {
+	d.cache = groupcache.NewGroup(cacheConfig.Name, int64(cacheConfig.Size), groupcache.GetterFunc(
+		func(_ context.Context, id string, dest groupcache.Sink) error {
+			val, err := d.dbGet(id)
+
+			if err != nil {
+				return err
+			}
+
+			// Set the value in the groupcache, with expiry
+			if err := dest.SetBytes(val, time.Now().Add(cacheConfig.ExpiryDuration)); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	))
+}
+
+func (d *Database) GetCacheStats() groupcache.Stats {
+	return d.cache.Stats
+}
+
+// Has satisfies the ethdb.KeyValueReader interface
+// Has retrieves if a key is present in the key-value data store
+func (d *Database) Has(key []byte) (bool, error) {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return false, err
+	}
+	var exists bool
+	return exists, d.db.Get(&exists, hasPgStr, mhKey)
+}
+
+// Get retrieves the given key if it's present in the key-value data store
+func (d *Database) dbGet(key string) ([]byte, error) {
+	var data []byte
+	return data, d.db.Get(&data, getPgStr, key)
+}
+
+// Get satisfies the ethdb.KeyValueReader interface
+// Get retrieves the given key if it's present in the key-value data store
+func (d *Database) Get(key []byte) ([]byte, error) {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500)
+	defer cancel()
+
+	var data []byte
+	return data, d.cache.Get(ctx, mhKey, groupcache.AllocatingByteSliceSink(&data))
+}
+
+// Put satisfies the ethdb.KeyValueWriter interface
+// Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
+func (d *Database) Put(key []byte, value []byte) error {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return err
+	}
+	_, err = d.db.Exec(putPgStr, mhKey, value)
+	return err
+}
+
+// Delete satisfies the ethdb.KeyValueWriter interface
+// Delete removes the key from the key-value data store
+func (d *Database) Delete(key []byte) error {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return err
+	}
+
+	_, err = d.db.Exec(deletePgStr, mhKey)
+	if err != nil {
+		return err
+	}
+
+	// Remove from cache.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500)
+	defer cancel()
+	err = d.cache.Remove(ctx, mhKey)
+
+	return err
+}
+
+// DatabaseProperty enum type
+type DatabaseProperty int
+
+const (
+	Unknown DatabaseProperty = iota
+	Size
+	Idle
+	InUse
+	MaxIdleClosed
+	MaxLifetimeClosed
+	MaxOpenConnections
+	OpenConnections
+	WaitCount
+	WaitDuration
+)
+
+// DatabasePropertyFromString helper function
+func DatabasePropertyFromString(property string) (DatabaseProperty, error) {
+	switch strings.ToLower(property) {
+	case "size":
+		return Size, nil
+	case "idle":
+		return Idle, nil
+	case "inuse":
+		return InUse, nil
+	case "maxidleclosed":
+		return MaxIdleClosed, nil
+	case "maxlifetimeclosed":
+		return MaxLifetimeClosed, nil
+	case "maxopenconnections":
+		return MaxOpenConnections, nil
+	case "openconnections":
+		return OpenConnections, nil
+	case "waitcount":
+		return WaitCount, nil
+	case "waitduration":
+		return WaitDuration, nil
+	default:
+		return Unknown, fmt.Errorf("unknown database property")
+	}
+}
+
+// Stat satisfies the ethdb.Stater interface
+// Stat returns a particular internal stat of the database
+func (d *Database) Stat(property string) (string, error) {
+	prop, err := DatabasePropertyFromString(property)
+	if err != nil {
+		return "", err
+	}
+	switch prop {
+	case Size:
+		var byteSize string
+		return byteSize, d.db.Get(&byteSize, dbSizePgStr)
+	case Idle:
+		return strconv.Itoa(d.db.Stats().Idle), nil
+	case InUse:
+		return strconv.Itoa(d.db.Stats().InUse), nil
+	case MaxIdleClosed:
+		return strconv.FormatInt(d.db.Stats().MaxIdleClosed, 10), nil
+	case MaxLifetimeClosed:
+		return strconv.FormatInt(d.db.Stats().MaxLifetimeClosed, 10), nil
+	case MaxOpenConnections:
+		return strconv.Itoa(d.db.Stats().MaxOpenConnections), nil
+	case OpenConnections:
+		return strconv.Itoa(d.db.Stats().OpenConnections), nil
+	case WaitCount:
+		return strconv.FormatInt(d.db.Stats().WaitCount, 10), nil
+	case WaitDuration:
+		return d.db.Stats().WaitDuration.String(), nil
+	default:
+		return "", fmt.Errorf("unhandled database property")
+	}
+}
+
+// Compact satisfies the ethdb.Compacter interface
+// Compact flattens the underlying data store for the given key range
+func (d *Database) Compact(start []byte, limit []byte) error {
+	return errNotSupported
+}
+
+// NewBatch satisfies the ethdb.Batcher interface
+// NewBatch creates a write-only database that buffers changes to its host db
+// until a final write is called
+func (d *Database) NewBatch() ethdb.Batch {
+	return NewBatch(d.db, nil)
+}
+
+// NewBatchWithSize satisfies the ethdb.Batcher interface.
+// NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
+func (d *Database) NewBatchWithSize(size int) ethdb.Batch {
+	return NewBatch(d.db, nil)
+}
+
+// NewIterator satisfies the ethdb.Iteratee interface
+// it creates a binary-alphabetical iterator over a subset
+// of database content with a particular key prefix, starting at a particular
+// initial key (or after, if it does not exist).
+//
+// Note: This method assumes that the prefix is NOT part of the start, so there's
+// no need for the caller to prepend the prefix to the start
+func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
+	return NewIterator(start, prefix, d.db)
+}
+
+// Close satisfies the io.Closer interface
+// Close closes the db connection
+func (d *Database) Close() error {
+	return d.db.DB.Close()
+}
+
+// HasAncient satisfies the ethdb.AncientReader interface
+// HasAncient returns an indicator whether the specified data exists in the ancient store
+func (d *Database) HasAncient(kind string, number uint64) (bool, error) {
+	return false, errNotSupported
+}
+
+// Ancient satisfies the ethdb.AncientReader interface
+// Ancient retrieves an ancient binary blob from the append-only immutable files
+func (d *Database) Ancient(kind string, number uint64) ([]byte, error) {
+	return nil, errNotSupported
+}
+
+// Ancients satisfies the ethdb.AncientReader interface
+// Ancients returns the ancient item numbers in the ancient store
+func (d *Database) Ancients() (uint64, error) {
+	return 0, errNotSupported
+}
+
+// Tail satisfies the ethdb.AncientReader interface.
+// Tail returns the number of first stored item in the freezer.
+func (d *Database) Tail() (uint64, error) {
+	return 0, errNotSupported
+}
+
+// AncientSize satisfies the ethdb.AncientReader interface
+// AncientSize returns the ancient size of the specified category
+func (d *Database) AncientSize(kind string) (uint64, error) {
+	return 0, errNotSupported
+}
+
+// AncientRange retrieves all the items in a range, starting from the index 'start'.
+// It will return
+//  - at most 'count' items,
+//  - at least 1 item (even if exceeding the maxBytes), but will otherwise
+//   return as many items as fit into maxBytes.
+func (d *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
+	return nil, errNotSupported
+}
+
+// ReadAncients applies the provided AncientReader function
+func (d *Database) ReadAncients(fn func(ethdb.AncientReader) error) (err error) {
+	return errNotSupported
+}
+
+// TruncateHead satisfies the ethdb.AncientWriter interface.
+// TruncateHead discards all but the first n ancient data from the ancient store.
+func (d *Database) TruncateHead(n uint64) error {
+	return errNotSupported
+}
+
+// TruncateTail satisfies the ethdb.AncientWriter interface.
+// TruncateTail discards the first n ancient data from the ancient store.
+func (d *Database) TruncateTail(n uint64) error {
+	return errNotSupported
+}
+
+// Sync satisfies the ethdb.AncientWriter interface
+// Sync flushes all in-memory ancient store data to disk
+func (d *Database) Sync() error {
+	return errNotSupported
+}
+
+// MigrateTable satisfies the ethdb.AncientWriter interface.
+// MigrateTable processes and migrates entries of a given table to a new format.
+func (d *Database) MigrateTable(string, func([]byte) ([]byte, error)) error {
+	return errNotSupported
+}
+
+// NewSnapshot satisfies the ethdb.Snapshotter interface.
+// NewSnapshot creates a database snapshot based on the current state.
+func (d *Database) NewSnapshot() (ethdb.Snapshot, error) {
+	return nil, errNotSupported
+}

postgres/database_test.go

@@ -0,0 +1,122 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb_test
+
+import (
+	"math/big"
+	"time"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/jmoiron/sqlx"
+	"github.com/mailgun/groupcache/v2"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
+)
+
+var (
+	database     ethdb.Database
+	db           *sqlx.DB
+	err          error
+	testHeader   = types.Header{Number: big.NewInt(1337)}
+	testValue, _ = rlp.EncodeToBytes(testHeader)
+	testEthKey   = testHeader.Hash().Bytes()
+	testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey)
+)
+
+var _ = Describe("Database", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+
+		cacheConfig := pgipfsethdb.CacheConfig{
+			Name:           "db",
+			Size:           3000000, // 3MB
+			ExpiryDuration: time.Hour,
+		}
+
+		database = pgipfsethdb.NewDatabase(db, cacheConfig)
+	})
+	AfterEach(func() {
+		groupcache.DeregisterGroup("db")
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("Has", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Get", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Delete", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+})

postgres/doc.md

@@ -0,0 +1,40 @@
+## ipfs-ethdb
+
+IPFS has been [extended](https://github.com/vulcanize/go-ipfs/releases/tag/v0.4.22-alpha) to [use Postgres](https://github.com/vulcanize/go-ipfs-config/releases/tag/v0.0.8-alpha) as a backing [datastore](https://github.com/ipfs/go-ds-sql/tree/master/postgres).
+Interfacing directly with the IPFS-backing Postgres database has some advantages over using the blockservice interface.
+Namely, batching of IPFS writes with other Postgres writes and avoiding lock contention on the ipfs repository (lockfile located at the `IPFS_PATH`).
+The downside is that we forgo the block-exchange capabilities of the blockservice, and are only able to fetch data contained in the local datastore.
+
+
+## Usage
+To use this module import it and build an ethdb interface around an instance of [sqlx.DB](https://github.com/jmoiron/sqlx), you can then
+employ it as you would the blockservice-based ethdbs.
+
+```go
+package main
+
+import (
+    "github.com/ethereum/go-ethereum/common"
+    "github.com/ethereum/go-ethereum/core/state"
+    "github.com/ethereum/go-ethereum/trie"
+    "github.com/jmoiron/sqlx"
+    "github.com/vulcanize/ipfs-ethdb/postgres"
+)
+
+func main() {
+    connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+    db, _ := sqlx.Connect("postgres", connectStr)
+
+    kvs := pgipfsethdb.NewKeyValueStore(db)
+    trieDB := trie.NewDatabase(kvs)
+    t, _ := trie.New(common.Hash{}, trieDB)
+    trieNodeIterator := t.NodeIterator([]byte{})
+    // do stuff with trie node iterator
+
+    database := pgipfsethdb.NewDatabase(db)
+    stateDatabase := state.NewDatabase(database)
+    stateDB, _ := state.New(common.Hash{}, stateDatabase, nil)
+    stateDBNodeIterator := state.NewNodeIterator(stateDB)
+    // do stuff with the statedb node iterator
+}
+```

postgres/iterator.go

@@ -0,0 +1,90 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+)
+
+var _ ethdb.Iterator = &Iterator{}
+
+// Iterator is the type that satisfies the ethdb.Iterator interface for PG-IPFS Ethereum data using a direct Postgres connection
+// Iteratee interface is used in Geth for various tests, trie/sync_bloom.go (for fast sync),
+// rawdb.InspectDatabase, and the new core/state/snapshot features.
+// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed
+// from the ethdb.KeyValueStoreand ethdb.Database interfaces)
+type Iterator struct {
+	db                 *sqlx.DB
+	currentKey, prefix []byte
+	err                error
+}
+
+// NewIterator returns an ethdb.Iterator interface for PG-IPFS
+func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
+	return &Iterator{
+		db:         db,
+		prefix:     prefix,
+		currentKey: start,
+	}
+}
+
+// Next satisfies the ethdb.Iterator interface
+// Next moves the iterator to the next key/value pair
+// It returns whether the iterator is exhausted
+func (i *Iterator) Next() bool {
+	// this is complicated by the ipfs db keys not being the keccak256 hashes
+	// go-ethereum usage of this method expects the iteration to occur over keccak256 keys
+	panic("implement me: Next")
+}
+
+// Error satisfies the ethdb.Iterator interface
+// Error returns any accumulated error
+// Exhausting all the key/value pairs is not considered to be an error
+func (i *Iterator) Error() error {
+	return i.err
+}
+
+// Key satisfies the ethdb.Iterator interface
+// Key returns the key of the current key/value pair, or nil if done
+// The caller should not modify the contents of the returned slice
+// and its contents may change on the next call to Next
+func (i *Iterator) Key() []byte {
+	return i.currentKey
+}
+
+// Value satisfies the ethdb.Iterator interface
+// Value returns the value of the current key/value pair, or nil if done
+// The caller should not modify the contents of the returned slice
+// and its contents may change on the next call to Next
+func (i *Iterator) Value() []byte {
+	mhKey, err := MultihashKeyFromKeccak256(i.currentKey)
+	if err != nil {
+		i.err = err
+		return nil
+	}
+	var data []byte
+	i.err = i.db.Get(&data, getPgStr, mhKey)
+	return data
+}
+
+// Release satisfies the ethdb.Iterator interface
+// Release releases associated resources
+// Release should always succeed and can be called multiple times without causing error
+func (i *Iterator) Release() {
+	i.db.Close()
+}

postgres/postgres_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestPGIPFSETHDB(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "PG-IPFS ethdb test")
+}

postgres/util.go

@@ -0,0 +1,49 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import (
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	dshelp "github.com/ipfs/go-ipfs-ds-help"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq" //postgres driver
+	"github.com/multiformats/go-multihash"
+)
+
+// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string
+func MultihashKeyFromKeccak256(h []byte) (string, error) {
+	mh, err := multihash.Encode(h, multihash.KECCAK_256)
+	if err != nil {
+		return "", err
+	}
+	dbKey := dshelp.MultihashToDsKey(mh)
+	return blockstore.BlockPrefix.String() + dbKey.String(), nil
+}
+
+// TestDB connect to the testing database
+// it assumes the database has the IPFS public.blocks table present
+// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
+func TestDB() (*sqlx.DB, error) {
+	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	return sqlx.Connect("postgres", connectStr)
+}
+
+// ResetTestDB drops all rows in the test db public.blocks table
+func ResetTestDB(db *sqlx.DB) error {
+	_, err := db.Exec("TRUNCATE public.blocks CASCADE")
+	return err
+}

suite_test.go

@@ -1,5 +1,5 @@
 // VulcanizeDB
-// Copyright © 2019 Vulcanize
+// Copyright © 2020 Vulcanize
 
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfsethdb_test
+package ipfsethdb
 
 import (
 	"testing"
@@ -23,7 +23,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestIPFSPGIPFSETHDB(t *testing.T) {
+func TestIPFSETHDB(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "PG-IPFS ethdb test")
+	RunSpecs(t, "IPFS ethdb test")
 }

util.go

@@ -17,33 +17,27 @@
 package ipfsethdb
 
 import (
-	"github.com/ipfs/go-ipfs-blockstore"
-	"github.com/ipfs/go-ipfs-ds-help"
-	"github.com/jmoiron/sqlx"
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
 	_ "github.com/lib/pq" //postgres driver
 	"github.com/multiformats/go-multihash"
 )
 
-// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string
-func MultihashKeyFromKeccak256(b []byte) (string, error) {
-	mh, err := multihash.Encode(b, multihash.KECCAK_256)
+// Keccak256ToCid takes a keccak256 hash and returns its cid v1 using the provided codec.
+func Keccak256ToCid(h []byte, codec uint64) (cid.Cid, error) {
+	buf, err := multihash.Encode(h, multihash.KECCAK_256)
 	if err != nil {
-		return "", err
+		return cid.Cid{}, err
 	}
-	dbKey := dshelp.MultihashToDsKey(mh)
-	return blockstore.BlockPrefix.String() + dbKey.String(), nil
+	return cid.NewCidV1(codec, multihash.Multihash(buf)), nil
 }
 
-// TestDB connect to the testing database
-// it assumes the database has the IPFS public.blocks table present
-// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
-func TestDB() (*sqlx.DB, error) {
-	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
-	return sqlx.Connect("postgres", connectStr)
-}
-
-// ResetTestDB drops all rows in the test db public.blocks table
-func ResetTestDB(db *sqlx.DB) error {
-	_, err := db.Exec("TRUNCATE public.blocks")
-	return err
+// NewBlock takes a keccak256 hash key and the rlp []byte value it was derived from and creates an ipfs block object
+func NewBlock(key, value []byte) (blocks.Block, error) {
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(key, cid.EthStateTrie)
+	if err != nil {
+		return nil, err
+	}
+	return blocks.NewBlockWithCid(value, c)
 }