Merge pull request #57 from cerc-io/ian/v4_dev

[v4] use stock v1.11.5 geth

README.md

@@ -34,7 +34,7 @@ Ancient interfaces are used for Ancient/frozen data operations (e.g. rawdb/table
 Outside of these primarily auxiliary capabilities, this package satisfies the interfaces required for many state operations using Ethereum data on IPFS.
 
 e.g.
- 
+
 go-ethereum trie.NodeIterator and state.NodeIterator can be constructed from the ethdb.KeyValueStore and ethdb.Database interfaces, respectively:
 
 ```go
@@ -42,7 +42,7 @@ package main
 
 import (
     "context"
-	
+
     "github.com/ethereum/go-ethereum/common"
     "github.com/ethereum/go-ethereum/core/state"
     "github.com/ethereum/go-ethereum/trie"
@@ -50,7 +50,7 @@ import (
     "github.com/ipfs/go-ipfs/core"
     "github.com/ipfs/go-ipfs/repo/fsrepo"
     "github.com/jmoiron/sqlx"
-    "github.com/vulcanize/ipfs-ethdb"
+    "github.com/vulcanize/ipfs-ethdb/v4"
 )
 
 func main() {

batch.go

@@ -17,6 +17,7 @@
 package ipfsethdb
 
 import (
+	"context"
 	"errors"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -103,7 +104,7 @@ func (b *Batch) Write() error {
 		}
 		puts[i] = b
 	}
-	if err := b.blockService.AddBlocks(puts); err != nil {
+	if err := b.blockService.AddBlocks(context.Background(), puts); err != nil {
 		return err
 	}
 	for _, key := range b.deleteCache.Keys() {
@@ -112,7 +113,7 @@ func (b *Batch) Write() error {
 		if err != nil {
 			return err
 		}
-		if err := b.blockService.DeleteBlock(c); err != nil {
+		if err := b.blockService.DeleteBlock(context.Background(), c); err != nil {
 			return err
 		}
 	}

batch_test.go

@@ -25,7 +25,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	ipfsethdb "github.com/vulcanize/ipfs-ethdb"
+	ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4"
 )
 
 var (

database.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strconv"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -69,7 +68,7 @@ func (d *Database) Has(key []byte) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	return d.blockService.Blockstore().Has(c)
+	return d.blockService.Blockstore().Has(context.Background(), c)
 }
 
 // Get satisfies the ethdb.KeyValueReader interface
@@ -95,7 +94,7 @@ func (d *Database) Put(key []byte, value []byte) error {
 	if err != nil {
 		return err
 	}
-	return d.blockService.AddBlock(b)
+	return d.blockService.AddBlock(context.Background(), b)
 }
 
 // Delete satisfies the ethdb.KeyValueWriter interface
@@ -106,7 +105,7 @@ func (d *Database) Delete(key []byte) error {
 	if err != nil {
 		return err
 	}
-	return d.blockService.DeleteBlock(c)
+	return d.blockService.DeleteBlock(context.Background(), c)
 }
 
 // DatabaseProperty enum type
@@ -135,9 +134,6 @@ func (d *Database) Stat(property string) (string, error) {
 		return "", err
 	}
 	switch prop {
-	case ExchangeOnline:
-		online := d.blockService.Exchange().IsOnline()
-		return strconv.FormatBool(online), nil
 	default:
 		return "", fmt.Errorf("unhandled database property")
 	}
@@ -227,7 +223,7 @@ func (d *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][
 }
 
 // ReadAncients applies the provided AncientReader function
-func (d *Database) ReadAncients(fn func(ethdb.AncientReader) error) (err error) {
+func (d *Database) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
 	return errNotSupported
 }
 
@@ -260,3 +256,8 @@ func (d *Database) MigrateTable(string, func([]byte) ([]byte, error)) error {
 func (d *Database) NewSnapshot() (ethdb.Snapshot, error) {
 	return nil, errNotSupported
 }
+
+// AncientDatadir returns an error as we don't have a backing chain freezer.
+func (d *Database) AncientDatadir() (string, error) {
+	return "", errNotSupported
+}

database_test.go

@@ -26,7 +26,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	ipfsethdb "github.com/vulcanize/ipfs-ethdb"
+	ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4"
 )
 
 var (

go.mod

@@ -1,22 +1,71 @@
-module github.com/vulcanize/ipfs-ethdb
+module github.com/cerc-io/ipfs-ethdb/v4
 
-go 1.15
+go 1.19
 
 require (
-	github.com/btcsuite/btcd v0.22.1 // indirect
-	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/ethereum/go-ethereum v1.10.17
+	github.com/ethereum/go-ethereum v1.11.5
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
 	github.com/ipfs/go-block-format v0.0.3
-	github.com/ipfs/go-blockservice v0.1.3
-	github.com/ipfs/go-cid v0.0.7
-	github.com/ipfs/go-ipfs-blockstore v1.0.1
-	github.com/ipfs/go-ipfs-ds-help v1.0.0
-	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
+	github.com/ipfs/go-blockservice v0.4.0
+	github.com/ipfs/go-cid v0.2.0
+	github.com/ipfs/go-ipfs-blockstore v1.2.0
+	github.com/ipfs/go-ipfs-ds-help v1.1.0
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.0
 	github.com/jmoiron/sqlx v1.3.5
-	github.com/lib/pq v1.10.5
+	github.com/lib/pq v1.10.7
 	github.com/mailgun/groupcache/v2 v2.3.0
 	github.com/multiformats/go-multihash v0.1.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
+	github.com/sirupsen/logrus v1.6.0
+)
+
+require github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+
+require (
+	github.com/btcsuite/btcd v0.22.0-beta // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-datastore v0.5.0 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-format v0.4.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.3.0 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.1 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.0.3 // indirect
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.0.3 // indirect
+	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/segmentio/fasthash v1.0.3 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	go.opentelemetry.io/otel v1.7.0 // indirect
+	go.opentelemetry.io/otel/trace v1.7.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.16.0 // indirect
+	golang.org/x/crypto v0.6.0 // indirect
+	golang.org/x/net v0.6.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	lukechampine.com/blake3 v1.1.6 // indirect
 )

mock_blockservice.go

@@ -52,27 +52,27 @@ func (mbs *MockBlockservice) Exchange() exchange.Interface {
 	panic("Exchange: implement me")
 }
 
-func (mbs *MockBlockservice) AddBlock(b blocks.Block) error {
-	return mbs.blockStore.Put(b)
+func (mbs *MockBlockservice) AddBlock(ctx context.Context, b blocks.Block) error {
+	return mbs.blockStore.Put(ctx, b)
 }
 
-func (mbs *MockBlockservice) AddBlocks(bs []blocks.Block) error {
-	return mbs.blockStore.PutMany(bs)
+func (mbs *MockBlockservice) AddBlocks(ctx context.Context, bs []blocks.Block) error {
+	return mbs.blockStore.PutMany(ctx, bs)
 }
 
-func (mbs *MockBlockservice) DeleteBlock(c cid.Cid) error {
-	return mbs.blockStore.DeleteBlock(c)
+func (mbs *MockBlockservice) DeleteBlock(ctx context.Context, c cid.Cid) error {
+	return mbs.blockStore.DeleteBlock(ctx, c)
 }
 
 func (mbs *MockBlockservice) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
-	return mbs.blockStore.Get(c)
+	return mbs.blockStore.Get(ctx, c)
 }
 
 func (mbs *MockBlockservice) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
 	blockChan := make(chan blocks.Block)
 	go func() {
 		for _, c := range cs {
-			if b, err := mbs.blockStore.Get(c); err == nil {
+			if b, err := mbs.blockStore.Get(ctx, c); err == nil {
 				blockChan <- b
 			}
 		}
@@ -93,17 +93,17 @@ type MockBlockstore struct {
 	err    error
 }
 
-func (mbs *MockBlockstore) DeleteBlock(c cid.Cid) error {
+func (mbs *MockBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error {
 	delete(mbs.blocks, c.String())
 	return mbs.err
 }
 
-func (mbs *MockBlockstore) Has(c cid.Cid) (bool, error) {
+func (mbs *MockBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
 	_, ok := mbs.blocks[c.String()]
 	return ok, mbs.err
 }
 
-func (mbs *MockBlockstore) Get(c cid.Cid) (blocks.Block, error) {
+func (mbs *MockBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
 	obj, ok := mbs.blocks[c.String()]
 	if !ok {
 		return nil, blockNotFoundErr
@@ -111,7 +111,7 @@ func (mbs *MockBlockstore) Get(c cid.Cid) (blocks.Block, error) {
 	return obj, mbs.err
 }
 
-func (mbs *MockBlockstore) GetSize(c cid.Cid) (int, error) {
+func (mbs *MockBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	obj, ok := mbs.blocks[c.String()]
 	if !ok {
 		return 0, blockNotFoundErr
@@ -119,12 +119,12 @@ func (mbs *MockBlockstore) GetSize(c cid.Cid) (int, error) {
 	return len(obj.RawData()), mbs.err
 }
 
-func (mbs *MockBlockstore) Put(b blocks.Block) error {
+func (mbs *MockBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	mbs.blocks[b.Cid().String()] = b
 	return mbs.err
 }
 
-func (mbs *MockBlockstore) PutMany(bs []blocks.Block) error {
+func (mbs *MockBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 	for _, b := range bs {
 		mbs.blocks[b.Cid().String()] = b
 	}

postgres/batch.go

@@ -17,6 +17,8 @@
 package pgipfsethdb
 
 import (
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/jmoiron/sqlx"
 )
@@ -28,13 +30,16 @@ type Batch struct {
 	db        *sqlx.DB
 	tx        *sqlx.Tx
 	valueSize int
+
+	blockNumber *big.Int
 }
 
 // NewBatch returns a ethdb.Batch interface for PG-IPFS
-func NewBatch(db *sqlx.DB, tx *sqlx.Tx) ethdb.Batch {
+func NewBatch(db *sqlx.DB, tx *sqlx.Tx, blockNumber *big.Int) ethdb.Batch {
 	b := &Batch{
-		db: db,
-		tx: tx,
+		db:          db,
+		tx:          tx,
+		blockNumber: blockNumber,
 	}
 	if tx == nil {
 		b.Reset()
@@ -50,7 +55,7 @@ func (b *Batch) Put(key []byte, value []byte) (err error) {
 	if err != nil {
 		return err
 	}
-	if _, err = b.tx.Exec(putPgStr, mhKey, value); err != nil {
+	if _, err = b.tx.Exec(putPgStr, mhKey, value, b.blockNumber.Uint64()); err != nil {
 		return err
 	}
 	b.valueSize += len(value)

postgres/batch_test.go

@@ -27,7 +27,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
+	pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres"
 )
 
 var (
@@ -49,6 +49,11 @@ var _ = Describe("Batch", func() {
 		}
 
 		database = pgipfsethdb.NewDatabase(db, cacheConfig)
+
+		databaseWithBlock, ok := database.(*pgipfsethdb.Database)
+		Expect(ok).To(BeTrue())
+		(*databaseWithBlock).BlockNumber = testBlockNumber
+
 		batch = database.NewBatch()
 	})
 	AfterEach(func() {

postgres/database.go

@@ -18,8 +18,10 @@ package pgipfsethdb
 
 import (
 	"context"
+	"database/sql"
 	"errors"
 	"fmt"
+	"math/big"
 	"strconv"
 	"strings"
 	"time"
@@ -27,14 +29,15 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/jmoiron/sqlx"
 	"github.com/mailgun/groupcache/v2"
+	log "github.com/sirupsen/logrus"
 )
 
 var errNotSupported = errors.New("this operation is not supported")
 
 var (
-	hasPgStr    = "SELECT exists(select 1 from public.blocks WHERE key = $1)"
-	getPgStr    = "SELECT data FROM public.blocks WHERE key = $1"
-	putPgStr    = "INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING"
+	hasPgStr    = "SELECT exists(select 1 from public.blocks WHERE key = $1 LIMIT 1)"
+	getPgStr    = "SELECT data FROM public.blocks WHERE key = $1 LIMIT 1"
+	putPgStr    = "INSERT INTO public.blocks (key, data, block_number) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING"
 	deletePgStr = "DELETE FROM public.blocks WHERE key = $1"
 	dbSizePgStr = "SELECT pg_database_size(current_database())"
 )
@@ -45,6 +48,8 @@ var _ ethdb.Database = &Database{}
 type Database struct {
 	db    *sqlx.DB
 	cache *groupcache.Group
+
+	BlockNumber *big.Int
 }
 
 func (d *Database) ModifyAncients(f func(ethdb.AncientWriteOp) error) (int64, error) {
@@ -110,7 +115,12 @@ func (d *Database) Has(key []byte) (bool, error) {
 // Get retrieves the given key if it's present in the key-value data store
 func (d *Database) dbGet(key string) ([]byte, error) {
 	var data []byte
-	return data, d.db.Get(&data, getPgStr, key)
+	err := d.db.Get(&data, getPgStr, key)
+	if err == sql.ErrNoRows {
+		log.Warn("Database miss for key", key)
+	}
+
+	return data, err
 }
 
 // Get satisfies the ethdb.KeyValueReader interface
@@ -136,7 +146,7 @@ func (d *Database) Put(key []byte, value []byte) error {
 	if err != nil {
 		return err
 	}
-	_, err = d.db.Exec(putPgStr, mhKey, value)
+	_, err = d.db.Exec(putPgStr, mhKey, value, d.BlockNumber.Uint64())
 	return err
 }
 
@@ -245,13 +255,13 @@ func (d *Database) Compact(start []byte, limit []byte) error {
 // NewBatch creates a write-only database that buffers changes to its host db
 // until a final write is called
 func (d *Database) NewBatch() ethdb.Batch {
-	return NewBatch(d.db, nil)
+	return NewBatch(d.db, nil, d.BlockNumber)
 }
 
 // NewBatchWithSize satisfies the ethdb.Batcher interface.
 // NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
 func (d *Database) NewBatchWithSize(size int) ethdb.Batch {
-	return NewBatch(d.db, nil)
+	return NewBatch(d.db, nil, d.BlockNumber)
 }
 
 // NewIterator satisfies the ethdb.Iteratee interface
@@ -311,7 +321,7 @@ func (d *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][
 }
 
 // ReadAncients applies the provided AncientReader function
-func (d *Database) ReadAncients(fn func(ethdb.AncientReader) error) (err error) {
+func (d *Database) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
 	return errNotSupported
 }
 
@@ -344,3 +354,8 @@ func (d *Database) MigrateTable(string, func([]byte) ([]byte, error)) error {
 func (d *Database) NewSnapshot() (ethdb.Snapshot, error) {
 	return nil, errNotSupported
 }
+
+// AncientDatadir returns an error as we don't have a backing chain freezer.
+func (d *Database) AncientDatadir() (string, error) {
+	return "", errNotSupported
+}

postgres/database_test.go

@@ -28,17 +28,18 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
+	pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres"
 )
 
 var (
-	database     ethdb.Database
-	db           *sqlx.DB
-	err          error
-	testHeader   = types.Header{Number: big.NewInt(1337)}
-	testValue, _ = rlp.EncodeToBytes(testHeader)
-	testEthKey   = testHeader.Hash().Bytes()
-	testMhKey, _ = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey)
+	database        ethdb.Database
+	db              *sqlx.DB
+	err             error
+	testBlockNumber = big.NewInt(1337)
+	testHeader      = types.Header{Number: testBlockNumber}
+	testValue, _    = rlp.EncodeToBytes(testHeader)
+	testEthKey      = testHeader.Hash().Bytes()
+	testMhKey, _    = pgipfsethdb.MultihashKeyFromKeccak256(testEthKey)
 )
 
 var _ = Describe("Database", func() {
@@ -53,6 +54,10 @@ var _ = Describe("Database", func() {
 		}
 
 		database = pgipfsethdb.NewDatabase(db, cacheConfig)
+
+		databaseWithBlock, ok := database.(*pgipfsethdb.Database)
+		Expect(ok).To(BeTrue())
+		(*databaseWithBlock).BlockNumber = testBlockNumber
 	})
 	AfterEach(func() {
 		groupcache.DeregisterGroup("db")
@@ -67,7 +72,7 @@ var _ = Describe("Database", func() {
 			Expect(has).ToNot(BeTrue())
 		})
 		It("returns true if a key-pair exists in the db", func() {
-			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			_, err = db.Exec("INSERT into public.blocks (key, data, block_number) VALUES ($1, $2, $3)", testMhKey, testValue, testBlockNumber.Uint64())
 			Expect(err).ToNot(HaveOccurred())
 			has, err := database.Has(testEthKey)
 			Expect(err).ToNot(HaveOccurred())
@@ -82,7 +87,7 @@ var _ = Describe("Database", func() {
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 		})
 		It("returns the value associated with the key, if the pair exists", func() {
-			_, err = db.Exec("INSERT into public.blocks (key, data) VALUES ($1, $2)", testMhKey, testValue)
+			_, err = db.Exec("INSERT into public.blocks (key, data, block_number) VALUES ($1, $2, $3)", testMhKey, testValue, testBlockNumber.Uint64())
 			Expect(err).ToNot(HaveOccurred())
 			val, err := database.Get(testEthKey)
 			Expect(err).ToNot(HaveOccurred())

postgres/doc.md

@@ -18,7 +18,7 @@ import (
     "github.com/ethereum/go-ethereum/core/state"
     "github.com/ethereum/go-ethereum/trie"
     "github.com/jmoiron/sqlx"
-    "github.com/vulcanize/ipfs-ethdb/postgres"
+    "github.com/vulcanize/ipfs-ethdb/v4/postgres"
 )
 
 func main() {