Merge pull request #8 from vulcanize/1.9.15

WIP - IPFS ethdb type using blockservice

README.md

@@ -1,8 +1,8 @@
-## pg-ipfs-ethdb
+## ipfs-ethdb
 
-[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/pg-ipfs-ethdb)](https://goreportcard.com/report/github.com/vulcanize/pg-ipfs-ethdb)
+[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipfs-ethdb)](https://goreportcard.com/report/github.com/vulcanize/ipfs-ethdb)
 
-> go-ethereum ethdb interfaces for Ethereum state data stored in Postgres-backed IPFS
+> go-ethereum ethdb interfaces for Ethereum state data stored in IPFS
 
 ## Background
 
@@ -10,10 +10,9 @@ Go-ethereum defines a number of interfaces in the [ethdb package](https://github
 interfacing with a state database. These interfaces are used to build higher-level types such as the [trie.Database](https://github.com/ethereum/go-ethereum/blob/master/trie/database.go#L77)
 which are used to perform the bulk of state related needs.
 
-Ethereum data can be stored on IPFS, standard codecs for Etheruem data are defined in the [go-cid](https://github.com/ipfs/go-cid) library. Here at Vulcanize we
+Ethereum data can be stored on IPFS, standard codecs for Etheruem data are defined in the [go-cid](https://github.com/ipfs/go-cid) library.
-have [extended IPFS](https://github.com/vulcanize/go-ipfs/releases/tag/v0.4.22-alpha) to [use Postgres](https://github.com/vulcanize/go-ipfs-config/releases/tag/v0.0.8-alpha) as a backing database.
+Using our [statediffing geth client](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.2) it is feasible to extract every single
-Additionally, [we have extended go-ethereum](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.2) to enable the efficient export of state data in the form of state diff objects.
+state and storage node and publish it to IPFS.
-Together, this allows us to store all Ethereum data on Postgres-backed IPFS.
 
 Geth stores state data in leveldb as key-value pairs between the keccak256 hash of the rlp-encoded object and the rlp-encoded object.
 Ethereum data on IPFS is also stored as key-value pairs with the value being the rlp-encoded byte value for the object,
@@ -22,73 +21,57 @@ ethdb interfaces for Ethereum data on IPFS by handling the conversion of a kecca
 
 
 ## Usage
-To use this module simply import it and build the desired interface around an instance of [sqlx.DB](https://github.com/jmoiron/sqlx), you can then
+To use this module import it and build an ethdb interface around an instance of a [go ipfs blockservice](https://github.com/ipfs/go-blockservice), you can then
-employ it as you would the usual [leveldb](https://github.com/ethereum/go-ethereum/tree/master/ethdb/leveldb) or [memorydb](https://github.com/ethereum/go-ethereum/tree/master/ethdb/memorydb) interfaces
+employ it as you would the usual [leveldb](https://github.com/ethereum/go-ethereum/tree/master/ethdb/leveldb) or [memorydb](https://github.com/ethereum/go-ethereum/tree/master/ethdb/memorydb) ethdbs
-with a few exceptions:
+with some exceptions: the AncientReader, AncientWriter, Compacter, and Iteratee/Iterator interfaces are not functionally complete.
+
+Ancient data does not currently have a representation on IPFS, and recapitulation of the database key iterator and compacter is complicated since go-ethereum
+types that use this interface expect the iterator and compacter to operate over keccak256 hash key ranges, whereas the keys for Ethereum data on IPFS are derived from that hash but not the hash itself.
+
+Iteratee interface is used in Geth for various tests, in trie/sync_bloom.go (for fast sync), rawdb.InspectDatabase, and the new (1.9.15) core/state/snapshot features;
+Ancient interfaces are used for Ancient/frozen data operations (e.g. rawdb/table.go); and Compacter is used in core/state/snapshot, rawdb/table.go, chaincmd.go, and the private debug api.
+
+Outside of these primarily auxiliary capabilities, this package satisfies the interfaces required for many state operations using Ethereum data on IPFS.
+
+e.g.
+ 
+go-ethereum trie.NodeIterator and state.NodeIterator can be constructed from the ethdb.KeyValueStore and ethdb.Database interfaces, respectively:
 
 ```go
 package main
 
 import (
+	"context"
+	
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ipfs/go-blockservice"
+    "github.com/ipfs/go-ipfs/core"
+    "github.com/ipfs/go-ipfs/repo/fsrepo"
 	"github.com/jmoiron/sqlx"
-	"github.com/vulcanize/pg-ipfs-ethdb"
+	"github.com/vulcanize/ipfs-ethdb"
 )
 
 func main() {
-    connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	// errors are ignored for brevity
-    db, _ := sqlx.Connect("postgres", connectStr)
+	ipfsPath := "~/.ipfs"
-
+    r, _ := fsrepo.Open(ipfsPath)
-    kvs := ipfsethdb.NewKeyValueStore(db)
+    ctx := context.Background()
-    trieDB := trie.NewDatabase(kvs)
+    cfg := &core.BuildCfg{
-    t, _ := trie.New(common.Hash{}, trieDB)
+        Online: false,
-    // do stuff with trie or trieDB
+        Repo:   r,
-
-    database := ipfsethdb.NewDatabase(db)
-    stateDatabase := state.NewDatabase(database)
-    // do stuff with the state database
     }
-```
+    ipfsNode, _ := core.NewNode(ctx, cfg)
-
+    kvs := ipfsethdb.NewKeyValueStore(ipfsNode.Blocks)
-EXCEPTIONS: AncientReader, AncientWriter, and Iteratee interfaces are not functionally complete.
-
-Ancient data does not currently have a representation on IPFS, and recapitulation of the database key iterator is complicated since go-ethereum
-types that use this interface expect the iterator to iterate over keccak256 hash keys, whereas the keys for Ethereum data on IPFS are derived from that hash but not the hash itself.
-
-Iteratee interface is only used in Geth for various tests, in trie/sync_bloom.go (for fast sync), and for rawdb.InspectDatabase while the Ancient interfaces are only used for Ancient data operations,
-so we don't need these interfaces for the majority of state operations.
-
-The ethdb.Iteratee/ethdb.Iterator interfaces should not be confused with the trie.NodeIterator or state.NodeIterator.
-These can be constructed from the ethdb.KeyValueStore and ethdb.Database interfaces, respectively:
-
-```go
-package main
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/state/snapshot"
-	"github.com/ethereum/go-ethereum/trie"
-	"github.com/jmoiron/sqlx"
-	"github.com/vulcanize/pg-ipfs-ethdb"
-)
-
-func main() {
-    connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
-    db, _ := sqlx.Connect("postgres", connectStr)
-
-    kvs := ipfsethdb.NewKeyValueStore(db)
     trieDB := trie.NewDatabase(kvs)
     t, _ := trie.New(common.Hash{}, trieDB)
     trieNodeIterator := t.NodeIterator([]byte{})
     // do stuff with trie node iterator
 
-    database := ipfsethdb.NewDatabase(db)
+    database := ipfsethdb.NewDatabase(ipfsNode.Blocks)
     stateDatabase := state.NewDatabase(database)
-    snapshotTree := snapshot.New(kvs, trieDB, 1, common.Hash{}, false)
+    stateDB, _ := state.New(common.Hash{}, stateDatabase, nil)
-    stateDB, _ := state.New(common.Hash{}, stateDatabase, snapshotTree)
     stateDBNodeIterator := state.NewNodeIterator(stateDB)
     // do stuff with the statedb node iterator
 }

batch.go

@@ -17,85 +17,129 @@
 package ipfsethdb
 
 import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/jmoiron/sqlx"
+	"github.com/hashicorp/golang-lru"
+	"github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid/_rsrch/cidiface"
 )
 
-// Batch is the type that satisfies the ethdb.Batch interface for PG-IPFS Ethereum data
+var (
+	EvictionWarningErr = errors.New("warn: batch has exceeded capacity, data has been evicted")
+)
+
+// Batch is the type that satisfies the ethdb.Batch interface for IPFS Ethereum data using the ipfs blockservice interface
+// This is ipfs-backing-datastore agnostic but must operate through a configured ipfs node (and so is subject to lockfile contention with e.g. an ipfs daemon)
+// If blockservice block exchange is configured the blockservice can fetch data that are missing locally from IPFS peers
 type Batch struct {
-	db   *sqlx.DB
+	blockService          blockservice.BlockService
-	tx   *sqlx.Tx
+	putCache, deleteCache *lru.Cache
-	size int
+	valueSize             int
 }
 
-// NewBatch returns a ethdb.Batch interface for PG-IPFS
+// NewBatch returns a ethdb.Batch interface for IPFS
-func NewBatch(db *sqlx.DB) ethdb.Batch {
+func NewBatch(bs blockservice.BlockService, capacity int) (ethdb.Batch, error) {
-	return &Batch{
+	putCache, err := lru.New(capacity)
-		db: db,
+	if err != nil {
+		return nil, err
 	}
+	deleteCache, err := lru.New(capacity)
+	if err != nil {
+		return nil, err
+	}
+	return &Batch{
+		blockService: bs,
+		putCache:     putCache,
+		deleteCache:  deleteCache,
+	}, nil
 }
 
 // Put satisfies the ethdb.Batch interface
 // Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
+// Returns an error when batch capacity has been exceeded and data was evicted
+// It is up to ensure they do not exceed capacity
+// The alternative is to check the cache len vs its capacity before inserting
+// but this adds additional overhead to every Put/Delete
 func (b *Batch) Put(key []byte, value []byte) (err error) {
-	if b.tx == nil {
+	b.valueSize += len(value)
-		b.Reset()
+	strKey := common.Bytes2Hex(key)
+	if b.putCache.Add(strKey, value) {
+		return EvictionWarningErr
 	}
-	mhKey, err := MultihashKeyFromKeccak256(key)
-	if err != nil {
-		return err
-	}
-	if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
-		return err
-	}
-	b.size += len(value)
 	return nil
 }
 
 // Delete satisfies the ethdb.Batch interface
 // Delete removes the key from the key-value data store
 func (b *Batch) Delete(key []byte) (err error) {
-	if b.tx == nil {
+	strKey := common.Bytes2Hex(key)
-		b.Reset()
+	if b.deleteCache.Add(strKey, true) {
+		return EvictionWarningErr
 	}
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	return nil
-	if err != nil {
-		return err
-	}
-	_, err = b.tx.Exec(deletePgStr, mhKey)
-	return err
 }
 
 // ValueSize satisfies the ethdb.Batch interface
 // ValueSize retrieves the amount of data queued up for writing
 // The returned value is the total byte length of all data queued to write
 func (b *Batch) ValueSize() int {
-	return b.size
+	return b.valueSize
 }
 
 // Write satisfies the ethdb.Batch interface
 // Write flushes any accumulated data to disk
 func (b *Batch) Write() error {
-	if b.tx == nil {
+	puts := make([]blocks.Block, b.putCache.Len())
-		return nil
+	for i, key := range b.putCache.Keys() {
+		val, _ := b.putCache.Get(key) // don't need to check "ok"s, the key is known and val is always []byte
+		b, err := NewBlock(common.Hex2Bytes(key.(string)), val.([]byte))
+		if err != nil {
+			return err
 		}
-	return b.tx.Commit()
+		puts[i] = b
+	}
+	if err := b.blockService.AddBlocks(puts); err != nil {
+		return err
+	}
+	for _, key := range b.deleteCache.Keys() {
+		// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+		c, err := Keccak256ToCid(common.Hex2Bytes(key.(string)), cid.EthStateTrie)
+		if err != nil {
+			return err
+		}
+		if err := b.blockService.DeleteBlock(c); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Replay satisfies the ethdb.Batch interface
 // Replay replays the batch contents
 func (b *Batch) Replay(w ethdb.KeyValueWriter) error {
-	return errNotSupported
+	for _, key := range b.putCache.Keys() {
+		val, _ := b.putCache.Get(key)
+		if err := w.Put(key.([]byte), val.([]byte)); err != nil {
+			return err
+		}
+	}
+	for _, key := range b.deleteCache.Keys() {
+		if err := w.Delete(key.([]byte)); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Reset satisfies the ethdb.Batch interface
 // Reset resets the batch for reuse
 // This should be called after every write
 func (b *Batch) Reset() {
-	var err error
+	b.deleteCache.Purge()
-	b.tx, err = b.db.Beginx()
+	b.putCache.Purge()
-	if err != nil {
+	b.valueSize = 0
-		panic(err)
-	}
-	b.size = 0
 }

batch_test.go

@@ -25,7 +25,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/pg-ipfs-ethdb"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb"
 )
 
 var (
@@ -37,24 +37,20 @@ var (
 
 var _ = Describe("Batch", func() {
 	BeforeEach(func() {
-		db, err = ipfsethdb.TestDB()
+		blockService = ipfsethdb.NewMockBlockservice()
-		Expect(err).ToNot(HaveOccurred())
+		batch, err = ipfsethdb.NewBatch(blockService, 1024)
-		database = ipfsethdb.NewDatabase(db)
-		batch = database.NewBatch()
-	})
-	AfterEach(func() {
-		err = ipfsethdb.ResetTestDB(db)
 		Expect(err).ToNot(HaveOccurred())
+		database = ipfsethdb.NewDatabase(blockService)
 	})
 
 	Describe("Put/Write", func() {
 		It("adds the key-value pair to the batch", func() {
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 			_, err = database.Get(testEthKey2)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 
 			err = batch.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
@@ -74,7 +70,6 @@ var _ = Describe("Batch", func() {
 
 	Describe("Delete/Reset/Write", func() {
 		It("deletes the key-value pair in the batch", func() {
-			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
 			err = batch.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
 			err = batch.Put(testEthKey2, testValue2)
@@ -92,10 +87,10 @@ var _ = Describe("Batch", func() {
 
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 			_, err = database.Get(testEthKey2)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 		})
 	})
 

database.go

@@ -17,85 +17,90 @@
 package ipfsethdb
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"github.com/ipfs/go-cid/_rsrch/cidiface"
+	"strconv"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/jmoiron/sqlx"
+	"github.com/ipfs/go-blockservice"
 )
 
-var errNotSupported = errors.New("this operation is not supported")
-
 var (
-	hasPgStr    = "SELECT exists(select 1 from public.blocks WHERE key = $1)"
+	defaultBatchCapacity = 1024
-	getPgStr    = "SELECT data FROM public.blocks WHERE key = $1"
+	errNotSupported      = errors.New("this operation is not supported")
-	putPgStr    = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
-	deletePgStr = "DELETE FROM public.blocks WHERE key = $1"
-	dbSizePgStr = "SELECT pg_database_size(current_database())"
 )
 
-// Database is the type that satisfies the ethdb.Database and ethdb.KeyValueStore interfaces for PG-IPFS Ethereum data
+// Database is the type that satisfies the ethdb.Database and ethdb.KeyValueStore interfaces for IPFS Ethereum data
+// This is ipfs-backing-datastore agnostic but must operate through a configured ipfs node (and so is subject to lockfile contention with e.g. an ipfs daemon)
+// If blockservice block exchange is configured the blockservice can fetch data that are missing locally from IPFS peers
 type Database struct {
-	db *sqlx.DB
+	blockService blockservice.BlockService
 }
 
-// NewKeyValueStore returns a ethdb.KeyValueStore interface for PG-IPFS
+// NewKeyValueStore returns a ethdb.KeyValueStore interface for IPFS
-func NewKeyValueStore(db *sqlx.DB) ethdb.KeyValueStore {
+func NewKeyValueStore(bs blockservice.BlockService) ethdb.KeyValueStore {
 	return &Database{
-		db: db,
+		blockService: bs,
 	}
 }
 
-// NewDatabase returns a ethdb.Database interface for PG-IPFS
+// NewDatabase returns a ethdb.Database interface for IPFS
-func NewDatabase(db *sqlx.DB) ethdb.Database {
+func NewDatabase(bs blockservice.BlockService) ethdb.Database {
 	return &Database{
-		db: db,
+		blockService: bs,
 	}
 }
 
 // Has satisfies the ethdb.KeyValueReader interface
 // Has retrieves if a key is present in the key-value data store
+// This only operates on the local blockstore not through the exchange
 func (d *Database) Has(key []byte) (bool, error) {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(key, cid.EthStateTrie)
 	if err != nil {
 		return false, err
 	}
-	var exists bool
+	return d.blockService.Blockstore().Has(c)
-	return exists, d.db.Get(&exists, hasPgStr, mhKey)
 }
 
 // Get satisfies the ethdb.KeyValueReader interface
 // Get retrieves the given key if it's present in the key-value data store
 func (d *Database) Get(key []byte) ([]byte, error) {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(key, cid.EthStateTrie)
 	if err != nil {
 		return nil, err
 	}
-	var data []byte
+	block, err := d.blockService.GetBlock(context.Background(), c)
-	return data, d.db.Get(&data, getPgStr, mhKey)
+	if err != nil {
+		return nil, err
+	}
+	return block.RawData(), nil
 }
 
 // Put satisfies the ethdb.KeyValueWriter interface
 // Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
 func (d *Database) Put(key []byte, value []byte) error {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	b, err := NewBlock(key, value)
 	if err != nil {
 		return err
 	}
-	_, err = d.db.Exec(putPgStr, mhKey, value)
+	return d.blockService.AddBlock(b)
-	return err
 }
 
 // Delete satisfies the ethdb.KeyValueWriter interface
 // Delete removes the key from the key-value data store
 func (d *Database) Delete(key []byte) error {
-	mhKey, err := MultihashKeyFromKeccak256(key)
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(key, cid.EthStateTrie)
 	if err != nil {
 		return err
 	}
-	_, err = d.db.Exec(deletePgStr, mhKey)
+	return d.blockService.DeleteBlock(c)
-	return err
 }
 
 // DatabaseProperty enum type
@@ -103,38 +108,14 @@ type DatabaseProperty int
 
 const (
 	Unknown DatabaseProperty = iota
-	Size
+	ExchangeOnline
-	Idle
-	InUse
-	MaxIdleClosed
-	MaxLifetimeClosed
-	MaxOpenConnections
-	OpenConnections
-	WaitCount
-	WaitDuration
 )
 
 // DatabasePropertyFromString helper function
 func DatabasePropertyFromString(property string) (DatabaseProperty, error) {
 	switch strings.ToLower(property) {
-	case "size":
+	case "exchange", "online":
-		return Size, nil
+		return ExchangeOnline, nil
-	case "idle":
-		return Idle, nil
-	case "inuse":
-		return InUse, nil
-	case "maxidleclosed":
-		return MaxIdleClosed, nil
-	case "maxlifetimeclosed":
-		return MaxLifetimeClosed, nil
-	case "maxopenconnections":
-		return MaxOpenConnections, nil
-	case "openconnections":
-		return OpenConnections, nil
-	case "waitcount":
-		return WaitCount, nil
-	case "waitduration":
-		return WaitDuration, nil
 	default:
 		return Unknown, fmt.Errorf("unknown database property")
 	}
@@ -148,25 +129,9 @@ func (d *Database) Stat(property string) (string, error) {
 		return "", err
 	}
 	switch prop {
-	case Size:
+	case ExchangeOnline:
-		var byteSize string
+		online := d.blockService.Exchange().IsOnline()
-		return byteSize, d.db.Get(&byteSize, dbSizePgStr)
+		return strconv.FormatBool(online), nil
-	case Idle:
-		return string(d.db.Stats().Idle), nil
-	case InUse:
-		return string(d.db.Stats().InUse), nil
-	case MaxIdleClosed:
-		return string(d.db.Stats().MaxIdleClosed), nil
-	case MaxLifetimeClosed:
-		return string(d.db.Stats().MaxLifetimeClosed), nil
-	case MaxOpenConnections:
-		return string(d.db.Stats().MaxOpenConnections), nil
-	case OpenConnections:
-		return string(d.db.Stats().OpenConnections), nil
-	case WaitCount:
-		return string(d.db.Stats().WaitCount), nil
-	case WaitDuration:
-		return d.db.Stats().WaitDuration.String(), nil
 	default:
 		return "", fmt.Errorf("unhandled database property")
 	}
@@ -182,7 +147,11 @@ func (d *Database) Compact(start []byte, limit []byte) error {
 // NewBatch creates a write-only database that buffers changes to its host db
 // until a final write is called
 func (d *Database) NewBatch() ethdb.Batch {
-	return NewBatch(d.db)
+	b, err := NewBatch(d.blockService, defaultBatchCapacity)
+	if err != nil {
+		panic(err)
+	}
+	return b
 }
 
 // NewIterator satisfies the ethdb.Iteratee interface
@@ -193,13 +162,13 @@ func (d *Database) NewBatch() ethdb.Batch {
 // Note: This method assumes that the prefix is NOT part of the start, so there's
 // no need for the caller to prepend the prefix to the start
 func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
-	return NewIterator(start, prefix, d.db)
+	return NewIterator(start, prefix, d.blockService)
 }
 
 // Close satisfies the io.Closer interface
 // Close closes the db connection
 func (d *Database) Close() error {
-	return d.db.DB.Close()
+	return d.blockService.Close()
 }
 
 // HasAncient satisfies the ethdb.AncientReader interface

database_test.go

@@ -22,32 +22,26 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/jmoiron/sqlx"
+	"github.com/ipfs/go-blockservice"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/pg-ipfs-ethdb"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb"
 )
 
 var (
 	database     ethdb.Database
-	db           *sqlx.DB
+	blockService blockservice.BlockService
 	err          error
 	testHeader   = types.Header{Number: big.NewInt(1337)}
 	testValue, _ = rlp.EncodeToBytes(testHeader)
 	testEthKey   = testHeader.Hash().Bytes()
-	testMhKey, _ = ipfsethdb.MultihashKeyFromKeccak256(testEthKey)
 )
 
 var _ = Describe("Database", func() {
 	BeforeEach(func() {
-		db, err = ipfsethdb.TestDB()
+		blockService = ipfsethdb.NewMockBlockservice()
-		Expect(err).ToNot(HaveOccurred())
+		database = ipfsethdb.NewDatabase(blockService)
-		database = ipfsethdb.NewDatabase(db)
-	})
-	AfterEach(func() {
-		err = ipfsethdb.ResetTestDB(db)
-		Expect(err).ToNot(HaveOccurred())
 	})
 
 	Describe("Has", func() {
@@ -57,7 +51,7 @@ var _ = Describe("Database", func() {
 			Expect(has).ToNot(BeTrue())
 		})
 		It("returns true if a key-pair exists in the db", func() {
-			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			err := database.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
 			has, err := database.Has(testEthKey)
 			Expect(err).ToNot(HaveOccurred())
@@ -69,10 +63,10 @@ var _ = Describe("Database", func() {
 		It("throws an err if the key-pair doesn't exist in the db", func() {
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 		})
 		It("returns the value associated with the key, if the pair exists", func() {
-			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			err := database.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
 			val, err := database.Get(testEthKey)
 			Expect(err).ToNot(HaveOccurred())
@@ -84,7 +78,7 @@ var _ = Describe("Database", func() {
 		It("persists the key-value pair in the database", func() {
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 
 			err = database.Put(testEthKey, testValue)
 			Expect(err).ToNot(HaveOccurred())
@@ -106,7 +100,7 @@ var _ = Describe("Database", func() {
 			Expect(err).ToNot(HaveOccurred())
 			_, err = database.Get(testEthKey)
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			Expect(err.Error()).To(ContainSubstring("block not found"))
 		})
 	})
 })

go.mod

@@ -1,14 +1,19 @@
-module github.com/vulcanize/pg-ipfs-ethdb
+module github.com/vulcanize/ipfs-ethdb
 
 go 1.13
 
 require (
 	github.com/ethereum/go-ethereum v1.9.15
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/ipfs/go-block-format v0.0.2
+	github.com/ipfs/go-blockservice v0.1.3
+	github.com/ipfs/go-cid v0.0.5
 	github.com/ipfs/go-ipfs-blockstore v1.0.0
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
+	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/lib/pq v1.0.0
 	github.com/multiformats/go-multihash v0.0.13
-	github.com/onsi/ginkgo v1.7.0
+	github.com/onsi/ginkgo v1.8.0
-	github.com/onsi/gomega v1.4.3
+	github.com/onsi/gomega v1.5.0
 )

go.sum

@@ -1,3 +1,4 @@
+github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
 github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4=
@@ -12,10 +13,12 @@ github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
 github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
+github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
@@ -24,21 +27,38 @@ github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:
 github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
+github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
+github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
+github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
+github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
+github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
+github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
+github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
+github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
+github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
+github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
+github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
+github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/ethereum/go-ethereum v1.9.15 h1:wrWl+QrtutRUJ9LZXdUqBoGoo2b1tOCYRDrAOQhCY3A=
 github.com/ethereum/go-ethereum v1.9.15/go.mod h1:slT8bPPRhXsyNTwHQxrOnjuTZ1sDXRajW11EkJ84QJ0=
@@ -46,6 +66,7 @@ github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
@@ -57,16 +78,21 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
 github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -74,36 +100,74 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
 github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
+github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
+github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM=
+github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
 github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
+github.com/ipfs/go-blockservice v0.1.3 h1:9XgsPMwwWJSC9uVr2pMDsW2qFTBSkxpGMhmna8mIjPM=
+github.com/ipfs/go-blockservice v0.1.3/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
 github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU=
 github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
-github.com/ipfs/go-datastore v0.4.1 h1:W4ZfzyhNi3xmuU5dQhjfuRn/wFuqEE1KnOmmQiOevEY=
+github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
+github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
+github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
 github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8=
+github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
+github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
+github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
+github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
+github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
 github.com/ipfs/go-ipfs-blockstore v1.0.0 h1:pmFp5sFYsYVvMOp9X01AK3s85usVcLvkBTRsN6SnfUA=
 github.com/ipfs/go-ipfs-blockstore v1.0.0/go.mod h1:knLVdhVU9L7CC4T+T4nvGdeUIPAXlnd9zmXfp+9MIjU=
+github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
+github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
+github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo=
+github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs=
 github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g=
 github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE=
+github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM=
+github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM=
+github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
+github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
+github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=
 github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50=
 github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
 github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
+github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
+github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
+github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
+github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
+github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
-github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw=
+github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
+github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
+github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs=
 github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
+github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10=
+github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
+github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -111,33 +175,94 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
+github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
+github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
+github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc=
+github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8=
+github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8=
+github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8=
+github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro=
+github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8=
+github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco=
+github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco=
+github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE=
+github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI=
+github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g=
+github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90=
+github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo=
+github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE=
+github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY=
+github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
+github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY=
+github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY=
+github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
+github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
+github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4=
+github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
+github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
+github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
+github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA=
+github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8=
+github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI=
+github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q=
+github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0=
+github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU=
+github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
+github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
+github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI=
+github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
+github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
+github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
+github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc=
+github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc=
+github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo=
+github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
+github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
 github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
 github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
+github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
+github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
+github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU=
 github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA=
 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
+github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
+github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
 github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
@@ -146,10 +271,12 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -170,6 +297,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible h1:+gAR1bMhuoQnZMTWFIvp7ukynULPsteLzG+siZKLtD8=
 github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
+github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -182,14 +311,25 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
+github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8=
+github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=
+github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
+github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
+golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA=
 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -197,6 +337,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 h1:Jcxah/M+oLZ/R4/z5RzfPzGbPXnVDPkEDtf2JnuxN+U=
 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -206,7 +347,9 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -216,6 +359,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

iterator.go

@@ -17,25 +17,28 @@
 package ipfsethdb
 
 import (
+	"context"
+	"github.com/ipfs/go-cid/_rsrch/cidiface"
+
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/jmoiron/sqlx"
+	"github.com/ipfs/go-blockservice"
 )
 
-// Iterator is the type that satisfies the ethdb.Iterator interface for PG-IPFS Ethereum data
+// Iterator is the type that satisfies the ethdb.Iterator interface for IPFS Ethereum data
-// Iteratee interface is only used in Geth for various tests, trie/sync_bloom.go (for fast sync), and rawdb.InspectDatabase
+// Iteratee interface is used in Geth for various tests, trie/sync_bloom.go (for fast sync),
-// Don't need this interface for the majority of state operations
+// rawdb.InspectDatabase, and the new core/state/snapshot features.
-// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed from the ethdb.KeyValueStore and ethdb.Database interfaces)
+// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed
-// ethdb.KeyValueStore => trie.Database => trie.Trie => trie.NodeIterator
+// from the ethdb.KeyValueStoreand ethdb.Database interfaces)
 type Iterator struct {
-	db                 *sqlx.DB
+	blockService       blockservice.BlockService
 	currentKey, prefix []byte
 	err                error
 }
 
-// NewIterator returns a ethdb.Iterator interface for PG-IPFS
+// NewIterator returns an ethdb.Iterator interface for PG-IPFS
-func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
+func NewIterator(start, prefix []byte, bs blockservice.BlockService) ethdb.Iterator {
 	return &Iterator{
-		db:         db,
+		blockService: bs,
 		prefix:       prefix,
 		currentKey:   start,
 	}
@@ -70,19 +73,23 @@ func (i *Iterator) Key() []byte {
 // The caller should not modify the contents of the returned slice
 // and its contents may change on the next call to Next
 func (i *Iterator) Value() []byte {
-	mhKey, err := MultihashKeyFromKeccak256(i.currentKey)
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
+	c, err := Keccak256ToCid(i.currentKey, cid.EthStateTrie)
 	if err != nil {
 		i.err = err
 		return nil
 	}
-	var data []byte
+	block, err := i.blockService.GetBlock(context.Background(), c)
-	i.err = i.db.Get(&data, getPgStr, mhKey)
+	if err != nil {
-	return data
+		i.err = err
+		return nil
+	}
+	return block.RawData()
 }
 
 // Release satisfies the ethdb.Iterator interface
 // Release releases associated resources
 // Release should always succeed and can be called multiple times without causing error
 func (i *Iterator) Release() {
-	i.db.Close()
+	i.blockService.Close()
 }

mock_blockservice.go

@@ -0,0 +1,144 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipfsethdb
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipfs/go-ipfs-exchange-interface"
+)
+
+var (
+	blockNotFoundErr = errors.New("block not found")
+)
+
+type MockBlockservice struct {
+	blockStore *MockBlockstore
+	err        error
+}
+
+func NewMockBlockservice() blockservice.BlockService {
+	return &MockBlockservice{
+		blockStore: &MockBlockstore{
+			blocks: make(map[string]blocks.Block),
+		},
+	}
+}
+
+func (mbs *MockBlockservice) Blockstore() blockstore.Blockstore {
+	return mbs.blockStore
+}
+
+func (mbs *MockBlockservice) Exchange() exchange.Interface {
+	panic("Exchange: implement me")
+}
+
+func (mbs *MockBlockservice) AddBlock(b blocks.Block) error {
+	return mbs.blockStore.Put(b)
+}
+
+func (mbs *MockBlockservice) AddBlocks(bs []blocks.Block) error {
+	return mbs.blockStore.PutMany(bs)
+}
+
+func (mbs *MockBlockservice) DeleteBlock(c cid.Cid) error {
+	return mbs.blockStore.DeleteBlock(c)
+}
+
+func (mbs *MockBlockservice) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	return mbs.blockStore.Get(c)
+}
+
+func (mbs *MockBlockservice) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
+	blockChan := make(chan blocks.Block)
+	go func() {
+		for _, c := range cs {
+			if b, err := mbs.blockStore.Get(c); err == nil {
+				blockChan <- b
+			}
+		}
+	}()
+	return blockChan
+}
+
+func (mbs *MockBlockservice) Close() error {
+	return mbs.err
+}
+
+func (mbs *MockBlockservice) SetError(err error) {
+	mbs.err = err
+}
+
+type MockBlockstore struct {
+	blocks map[string]blocks.Block
+	err    error
+}
+
+func (mbs *MockBlockstore) DeleteBlock(c cid.Cid) error {
+	delete(mbs.blocks, c.String())
+	return mbs.err
+}
+
+func (mbs *MockBlockstore) Has(c cid.Cid) (bool, error) {
+	_, ok := mbs.blocks[c.String()]
+	return ok, mbs.err
+}
+
+func (mbs *MockBlockstore) Get(c cid.Cid) (blocks.Block, error) {
+	obj, ok := mbs.blocks[c.String()]
+	if !ok {
+		return nil, blockNotFoundErr
+	}
+	return obj, mbs.err
+}
+
+func (mbs *MockBlockstore) GetSize(c cid.Cid) (int, error) {
+	obj, ok := mbs.blocks[c.String()]
+	if !ok {
+		return 0, blockNotFoundErr
+	}
+	return len(obj.RawData()), mbs.err
+}
+
+func (mbs *MockBlockstore) Put(b blocks.Block) error {
+	mbs.blocks[b.Cid().String()] = b
+	return mbs.err
+}
+
+func (mbs *MockBlockstore) PutMany(bs []blocks.Block) error {
+	for _, b := range bs {
+		mbs.blocks[b.Cid().String()] = b
+	}
+	return mbs.err
+}
+
+func (mbs *MockBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	panic("AllKeysChan: implement me")
+}
+
+func (mbs *MockBlockstore) HashOnRead(enabled bool) {
+	panic("HasOnRead: implement me")
+}
+
+func (mbs *MockBlockstore) SetError(err error) {
+	mbs.err = err
+}

postgres/batch.go

@@ -0,0 +1,101 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+)
+
+// Batch is the type that satisfies the ethdb.Batch interface for PG-IPFS Ethereum data using a direct Postgres connection
+type Batch struct {
+	db        *sqlx.DB
+	tx        *sqlx.Tx
+	valueSize int
+}
+
+// NewBatch returns a ethdb.Batch interface for PG-IPFS
+func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
+	b := &Batch{
+		db: db,
+		tx: tx,
+	}
+	if tx == nil {
+		b.Reset()
+	}
+	return b
+}
+
+// Put satisfies the ethdb.Batch interface
+// Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
+func (b *Batch) Put(key []byte, value []byte) (err error) {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return err
+	}
+	if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
+		return err
+	}
+	b.valueSize += len(value)
+	return nil
+}
+
+// Delete satisfies the ethdb.Batch interface
+// Delete removes the key from the key-value data store
+func (b *Batch) Delete(key []byte) (err error) {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return err
+	}
+	_, err = b.tx.Exec(deletePgStr, mhKey)
+	return err
+}
+
+// ValueSize satisfies the ethdb.Batch interface
+// ValueSize retrieves the amount of data queued up for writing
+// The returned value is the total byte length of all data queued to write
+func (b *Batch) ValueSize() int {
+	return b.valueSize
+}
+
+// Write satisfies the ethdb.Batch interface
+// Write flushes any accumulated data to disk
+func (b *Batch) Write() error {
+	if b.tx == nil {
+		return nil
+	}
+	return b.tx.Commit()
+}
+
+// Replay satisfies the ethdb.Batch interface
+// Replay replays the batch contents
+func (b *Batch) Replay(w ethdb.KeyValueWriter) error {
+	return errNotSupported
+}
+
+// Reset satisfies the ethdb.Batch interface
+// Reset resets the batch for reuse
+// This should be called after every write
+func (b *Batch) Reset() {
+	var err error
+	b.tx, err = b.db.Beginx()
+	if err != nil {
+		panic(err)
+	}
+	b.valueSize = 0
+}

postgres/batch_test.go

@@ -0,0 +1,118 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rlp"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
+)
+
+var (
+	batch         ethdb.Batch
+	testHeader2   = types.Header{Number: big.NewInt(2)}
+	testValue2, _ = rlp.EncodeToBytes(testHeader2)
+	testEthKey2   = testHeader2.Hash().Bytes()
+)
+
+var _ = Describe("Batch", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+		database = pgipfsethdb.NewDatabase(db)
+		batch = database.NewBatch()
+	})
+	AfterEach(func() {
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("Put/Write", func() {
+		It("adds the key-value pair to the batch", func() {
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = database.Get(testEthKey2)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = batch.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			val, err := database.Get(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+			val2, err := database.Get(testEthKey2)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val2).To(Equal(testValue2))
+		})
+	})
+
+	Describe("Delete/Reset/Write", func() {
+		It("deletes the key-value pair in the batch", func() {
+			err = batch.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			batch.Reset()
+			err = batch.Delete(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Delete(testEthKey2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = database.Get(testEthKey2)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("ValueSize/Reset", func() {
+		It("returns the size of data in the batch queued for write", func() {
+			err = batch.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Put(testEthKey2, testValue2)
+			Expect(err).ToNot(HaveOccurred())
+			err = batch.Write()
+			Expect(err).ToNot(HaveOccurred())
+
+			size := batch.ValueSize()
+			Expect(size).To(Equal(len(testValue) + len(testValue2)))
+
+			batch.Reset()
+			size = batch.ValueSize()
+			Expect(size).To(Equal(0))
+		})
+	})
+})

postgres/database.go

@@ -0,0 +1,246 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+)
+
+var errNotSupported = errors.New("this operation is not supported")
+
+var (
+	hasPgStr    = "SELECT exists(select 1 from public.blocks WHERE key = $1)"
+	getPgStr    = "SELECT data FROM public.blocks WHERE key = $1"
+	putPgStr    = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
+	deletePgStr = "DELETE FROM public.blocks WHERE key = $1"
+	dbSizePgStr = "SELECT pg_database_size(current_database())"
+)
+
+// Database is the type that satisfies the ethdb.Database and ethdb.KeyValueStore interfaces for PG-IPFS Ethereum data using a direct Postgres connection
+type Database struct {
+	db *sqlx.DB
+}
+
+// NewKeyValueStore returns a ethdb.KeyValueStore interface for PG-IPFS
+func NewKeyValueStore(db *sqlx.DB) ethdb.KeyValueStore {
+	return &Database{
+		db: db,
+	}
+}
+
+// NewDatabase returns a ethdb.Database interface for PG-IPFS
+func NewDatabase(db *sqlx.DB) ethdb.Database {
+	return &Database{
+		db: db,
+	}
+}
+
+// Has satisfies the ethdb.KeyValueReader interface
+// Has retrieves if a key is present in the key-value data store
+func (d *Database) Has(key []byte) (bool, error) {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return false, err
+	}
+	var exists bool
+	return exists, d.db.Get(&exists, hasPgStr, mhKey)
+}
+
+// Get satisfies the ethdb.KeyValueReader interface
+// Get retrieves the given key if it's present in the key-value data store
+func (d *Database) Get(key []byte) ([]byte, error) {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return nil, err
+	}
+	var data []byte
+	return data, d.db.Get(&data, getPgStr, mhKey)
+}
+
+// Put satisfies the ethdb.KeyValueWriter interface
+// Put inserts the given value into the key-value data store
+// Key is expected to be the keccak256 hash of value
+func (d *Database) Put(key []byte, value []byte) error {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return err
+	}
+	_, err = d.db.Exec(putPgStr, mhKey, value)
+	return err
+}
+
+// Delete satisfies the ethdb.KeyValueWriter interface
+// Delete removes the key from the key-value data store
+func (d *Database) Delete(key []byte) error {
+	mhKey, err := MultihashKeyFromKeccak256(key)
+	if err != nil {
+		return err
+	}
+	_, err = d.db.Exec(deletePgStr, mhKey)
+	return err
+}
+
+// DatabaseProperty enum type
+type DatabaseProperty int
+
+const (
+	Unknown DatabaseProperty = iota
+	Size
+	Idle
+	InUse
+	MaxIdleClosed
+	MaxLifetimeClosed
+	MaxOpenConnections
+	OpenConnections
+	WaitCount
+	WaitDuration
+)
+
+// DatabasePropertyFromString helper function
+func DatabasePropertyFromString(property string) (DatabaseProperty, error) {
+	switch strings.ToLower(property) {
+	case "size":
+		return Size, nil
+	case "idle":
+		return Idle, nil
+	case "inuse":
+		return InUse, nil
+	case "maxidleclosed":
+		return MaxIdleClosed, nil
+	case "maxlifetimeclosed":
+		return MaxLifetimeClosed, nil
+	case "maxopenconnections":
+		return MaxOpenConnections, nil
+	case "openconnections":
+		return OpenConnections, nil
+	case "waitcount":
+		return WaitCount, nil
+	case "waitduration":
+		return WaitDuration, nil
+	default:
+		return Unknown, fmt.Errorf("unknown database property")
+	}
+}
+
+// Stat satisfies the ethdb.Stater interface
+// Stat returns a particular internal stat of the database
+func (d *Database) Stat(property string) (string, error) {
+	prop, err := DatabasePropertyFromString(property)
+	if err != nil {
+		return "", err
+	}
+	switch prop {
+	case Size:
+		var byteSize string
+		return byteSize, d.db.Get(&byteSize, dbSizePgStr)
+	case Idle:
+		return string(d.db.Stats().Idle), nil
+	case InUse:
+		return string(d.db.Stats().InUse), nil
+	case MaxIdleClosed:
+		return string(d.db.Stats().MaxIdleClosed), nil
+	case MaxLifetimeClosed:
+		return string(d.db.Stats().MaxLifetimeClosed), nil
+	case MaxOpenConnections:
+		return string(d.db.Stats().MaxOpenConnections), nil
+	case OpenConnections:
+		return string(d.db.Stats().OpenConnections), nil
+	case WaitCount:
+		return string(d.db.Stats().WaitCount), nil
+	case WaitDuration:
+		return d.db.Stats().WaitDuration.String(), nil
+	default:
+		return "", fmt.Errorf("unhandled database property")
+	}
+}
+
+// Compact satisfies the ethdb.Compacter interface
+// Compact flattens the underlying data store for the given key range
+func (d *Database) Compact(start []byte, limit []byte) error {
+	return errNotSupported
+}
+
+// NewBatch satisfies the ethdb.Batcher interface
+// NewBatch creates a write-only database that buffers changes to its host db
+// until a final write is called
+func (d *Database) NewBatch() ethdb.Batch {
+	return NewBatch(d.db, nil)
+}
+
+// NewIterator satisfies the ethdb.Iteratee interface
+// it creates a binary-alphabetical iterator over a subset
+// of database content with a particular key prefix, starting at a particular
+// initial key (or after, if it does not exist).
+//
+// Note: This method assumes that the prefix is NOT part of the start, so there's
+// no need for the caller to prepend the prefix to the start
+func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
+	return NewIterator(start, prefix, d.db)
+}
+
+// Close satisfies the io.Closer interface
+// Close closes the db connection
+func (d *Database) Close() error {
+	return d.db.DB.Close()
+}
+
+// HasAncient satisfies the ethdb.AncientReader interface
+// HasAncient returns an indicator whether the specified data exists in the ancient store
+func (d *Database) HasAncient(kind string, number uint64) (bool, error) {
+	return false, errNotSupported
+}
+
+// Ancient satisfies the ethdb.AncientReader interface
+// Ancient retrieves an ancient binary blob from the append-only immutable files
+func (d *Database) Ancient(kind string, number uint64) ([]byte, error) {
+	return nil, errNotSupported
+}
+
+// Ancients satisfies the ethdb.AncientReader interface
+// Ancients returns the ancient item numbers in the ancient store
+func (d *Database) Ancients() (uint64, error) {
+	return 0, errNotSupported
+}
+
+// AncientSize satisfies the ethdb.AncientReader interface
+// AncientSize returns the ancient size of the specified category
+func (d *Database) AncientSize(kind string) (uint64, error) {
+	return 0, errNotSupported
+}
+
+// AppendAncient satisfies the ethdb.AncientWriter interface
+// AppendAncient injects all binary blobs belong to block at the end of the append-only immutable table files
+func (d *Database) AppendAncient(number uint64, hash, header, body, receipt, td []byte) error {
+	return errNotSupported
+}
+
+// TruncateAncients satisfies the ethdb.AncientWriter interface
+// TruncateAncients discards all but the first n ancient data from the ancient store
+func (d *Database) TruncateAncients(n uint64) error {
+	return errNotSupported
+}
+
+// Sync satisfies the ethdb.AncientWriter interface
+// Sync flushes all in-memory ancient store data to disk
+func (d *Database) Sync() error {
+	return errNotSupported
+}

postgres/database_test.go

@@ -0,0 +1,112 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
+)
+
+var (
+	database     ethdb.Database
+	db           *sqlx.DB
+	err          error
+	testHeader   = types.Header{Number: big.NewInt(1337)}
+	testValue, _ = rlp.EncodeToBytes(testHeader)
+	testEthKey   = testHeader.Hash().Bytes()
+	testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey)
+)
+
+var _ = Describe("Database", func() {
+	BeforeEach(func() {
+		db, err = pgipfsethdb.TestDB()
+		Expect(err).ToNot(HaveOccurred())
+		database = pgipfsethdb.NewDatabase(db)
+	})
+	AfterEach(func() {
+		err = pgipfsethdb.ResetTestDB(db)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("Has", func() {
+		It("returns false if a key-pair doesn't exist in the db", func() {
+			has, err := database.Has(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).ToNot(BeTrue())
+		})
+		It("returns true if a key-pair exists in the db", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			has, err := database.Has(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(has).To(BeTrue())
+		})
+	})
+
+	Describe("Get", func() {
+		It("throws an err if the key-pair doesn't exist in the db", func() {
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("returns the value associated with the key, if the pair exists", func() {
+			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Put", func() {
+		It("persists the key-value pair in the database", func() {
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			err = database.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+		})
+	})
+
+	Describe("Delete", func() {
+		It("removes the key-value pair from the database", func() {
+			err = database.Put(testEthKey, testValue)
+			Expect(err).ToNot(HaveOccurred())
+			val, err := database.Get(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(testValue))
+
+			err = database.Delete(testEthKey)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = database.Get(testEthKey)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+})

postgres/doc.md

@@ -0,0 +1,41 @@
+## ipfs-ethdb
+
+IPFS has been [extended](https://github.com/vulcanize/go-ipfs/releases/tag/v0.4.22-alpha) to [use Postgres](https://github.com/vulcanize/go-ipfs-config/releases/tag/v0.0.8-alpha) as a backing [datastore](https://github.com/ipfs/go-ds-sql/tree/master/postgres).
+Interfacing directly with the IPFS-backing Postgres database has some advantages over using the blockservice interface.
+Namely, batching of IPFS writes with other Postgres writes and avoiding lock contention on the ipfs repository (lockfile located at the `IPFS_PATH`).
+The downside is that we forgo the block-exchange capabilities of the blockservice, and are only able to fetch data contained in the local datastore.
+
+
+## Usage
+To use this module import it and build an ethdb interface around an instance of [sqlx.DB](https://github.com/jmoiron/sqlx), you can then
+employ it as you would the blockservice-based ethdbs.
+
+```go
+package main
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/jmoiron/sqlx"
+	"github.com/vulcanize/ipfs-ethdb/postgres"
+)
+
+func main() {
+    connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+    db, _ := sqlx.Connect("postgres", connectStr)
+
+    kvs := pgipfsethdb.NewKeyValueStore(db)
+    trieDB := trie.NewDatabase(kvs)
+    t, _ := trie.New(common.Hash{}, trieDB)
+    trieNodeIterator := t.NodeIterator([]byte{})
+    // do stuff with trie node iterator
+
+    database := pgipfsethdb.NewDatabase(db)
+    stateDatabase := state.NewDatabase(database)
+    stateDB, _ := state.New(common.Hash{}, stateDatabase, nil)
+    stateDBNodeIterator := state.NewNodeIterator(stateDB)
+    // do stuff with the statedb node iterator
+}
+```

postgres/iterator.go

@@ -0,0 +1,88 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/jmoiron/sqlx"
+)
+
+// Iterator is the type that satisfies the ethdb.Iterator interface for PG-IPFS Ethereum data using a direct Postgres connection
+// Iteratee interface is used in Geth for various tests, trie/sync_bloom.go (for fast sync),
+// rawdb.InspectDatabase, and the new core/state/snapshot features.
+// This should not be confused with trie.NodeIterator or state.NodeIteraor (which can be constructed
+// from the ethdb.KeyValueStoreand ethdb.Database interfaces)
+type Iterator struct {
+	db                 *sqlx.DB
+	currentKey, prefix []byte
+	err                error
+}
+
+// NewIterator returns an ethdb.Iterator interface for PG-IPFS
+func NewIterator(start, prefix []byte, db *sqlx.DB) ethdb.Iterator {
+	return &Iterator{
+		db:         db,
+		prefix:     prefix,
+		currentKey: start,
+	}
+}
+
+// Next satisfies the ethdb.Iterator interface
+// Next moves the iterator to the next key/value pair
+// It returns whether the iterator is exhausted
+func (i *Iterator) Next() bool {
+	// this is complicated by the ipfs db keys not being the keccak256 hashes
+	// go-ethereum usage of this method expects the iteration to occur over keccak256 keys
+	panic("implement me: Next")
+}
+
+// Error satisfies the ethdb.Iterator interface
+// Error returns any accumulated error
+// Exhausting all the key/value pairs is not considered to be an error
+func (i *Iterator) Error() error {
+	return i.err
+}
+
+// Key satisfies the ethdb.Iterator interface
+// Key returns the key of the current key/value pair, or nil if done
+// The caller should not modify the contents of the returned slice
+// and its contents may change on the next call to Next
+func (i *Iterator) Key() []byte {
+	return i.currentKey
+}
+
+// Value satisfies the ethdb.Iterator interface
+// Value returns the value of the current key/value pair, or nil if done
+// The caller should not modify the contents of the returned slice
+// and its contents may change on the next call to Next
+func (i *Iterator) Value() []byte {
+	mhKey, err := MultihashKeyFromKeccak256(i.currentKey)
+	if err != nil {
+		i.err = err
+		return nil
+	}
+	var data []byte
+	i.err = i.db.Get(&data, getPgStr, mhKey)
+	return data
+}
+
+// Release satisfies the ethdb.Iterator interface
+// Release releases associated resources
+// Release should always succeed and can be called multiple times without causing error
+func (i *Iterator) Release() {
+	i.db.Close()
+}

postgres/postgres_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestPGIPFSETHDB(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "PG-IPFS ethdb test")
+}

postgres/util.go

@@ -0,0 +1,49 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgipfsethdb
+
+import (
+	"github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipfs/go-ipfs-ds-help"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq" //postgres driver
+	"github.com/multiformats/go-multihash"
+)
+
+// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string
+func MultihashKeyFromKeccak256(h []byte) (string, error) {
+	mh, err := multihash.Encode(h, multihash.KECCAK_256)
+	if err != nil {
+		return "", err
+	}
+	dbKey := dshelp.MultihashToDsKey(mh)
+	return blockstore.BlockPrefix.String() + dbKey.String(), nil
+}
+
+// TestDB connect to the testing database
+// it assumes the database has the IPFS public.blocks table present
+// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
+func TestDB() (*sqlx.DB, error) {
+	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	return sqlx.Connect("postgres", connectStr)
+}
+
+// ResetTestDB drops all rows in the test db public.blocks table
+func ResetTestDB(db *sqlx.DB) error {
+	_, err := db.Exec("TRUNCATE public.blocks")
+	return err
+}

suite_test.go

@@ -1,5 +1,5 @@
 // VulcanizeDB
-// Copyright © 2019 Vulcanize
+// Copyright © 2020 Vulcanize
 
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfsethdb_test
+package ipfsethdb
 
 import (
 	"testing"
@@ -23,7 +23,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestIPFSPGIPFSETHDB(t *testing.T) {
+func TestIPFSETHDB(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "PG-IPFS ethdb test")
+	RunSpecs(t, "IPFS ethdb test")
 }

util.go

@@ -17,33 +17,27 @@
 package ipfsethdb
 
 import (
-	"github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-ipfs-ds-help"
+	"github.com/ipfs/go-cid"
-	"github.com/jmoiron/sqlx"
 	_ "github.com/lib/pq" //postgres driver
 	"github.com/multiformats/go-multihash"
 )
 
-// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string
+// Keccak256ToCid takes a keccak256 hash and returns its cid v1 using the provided codec.
-func MultihashKeyFromKeccak256(b []byte) (string, error) {
+func Keccak256ToCid(h []byte, codec uint64) (cid.Cid, error) {
-	mh, err := multihash.Encode(b, multihash.KECCAK_256)
+	buf, err := multihash.Encode(h, multihash.KECCAK_256)
 	if err != nil {
-		return "", err
+		return cid.Cid{}, err
 	}
-	dbKey := dshelp.MultihashToDsKey(mh)
+	return cid.NewCidV1(codec ,multihash.Multihash(buf)), nil
-	return blockstore.BlockPrefix.String() + dbKey.String(), nil
 }
 
-// TestDB connect to the testing database
+// NewBlock takes a keccak256 hash key and the rlp []byte value it was derived from and creates an ipfs block object
-// it assumes the database has the IPFS public.blocks table present
+func NewBlock(key, value []byte) (blocks.Block, error) {
-// DO NOT use a production db for the test db, as it will remove all contents of the public.blocks table
+	// we are using state codec because we don't know the codec and at this level the codec doesn't matter, the datastore key is multihash-only derived
-func TestDB() (*sqlx.DB, error) {
+	c, err := Keccak256ToCid(key, cid.EthStateTrie)
-	connectStr := "postgresql://localhost:5432/vulcanize_testing?sslmode=disable"
+	if err != nil {
-	return sqlx.Connect("postgres", connectStr)
+		return nil, err
 	}
-
+	return blocks.NewBlockWithCid(value, c)
-// ResetTestDB drops all rows in the test db public.blocks table
-func ResetTestDB(db *sqlx.DB) error {
-	_, err := db.Exec("TRUNCATE public.blocks")
-	return err
 }