fix backfill operations and dependency issue; hopefully travis will work now

Gopkg.lock

@@ -129,19 +129,18 @@
   version = "v1.7.1"
 
 [[projects]]
-  digest = "1:5f5090f05382959db941fa45acbeb7f4c5241aa8ac0f8f4393dec696e5953f53"
+  digest = "1:98436a9785f711ffee66f06265296dd36afd6e3dc532410c98a9051b5fd4c2f1"
   name = "github.com/dgraph-io/badger"
   packages = [
     ".",
     "options",
-    "protos",
+    "pb",
     "skl",
     "table",
     "y",
   ]
   pruneopts = "UT"
-  revision = "99233d725dbdd26d156c61b2f42ae1671b794656"
-  version = "gx/v1.5.4"
+  revision = "0ce1d2e26af1ba8b8a72ea864145a3e1e3b382cd"
 
 [[projects]]
   branch = "master"
@@ -2289,6 +2288,7 @@
     "github.com/ethereum/go-ethereum/accounts/abi/bind",
     "github.com/ethereum/go-ethereum/common",
     "github.com/ethereum/go-ethereum/common/hexutil",
+    "github.com/ethereum/go-ethereum/core",
     "github.com/ethereum/go-ethereum/core/rawdb",
     "github.com/ethereum/go-ethereum/core/state",
     "github.com/ethereum/go-ethereum/core/types",

Gopkg.toml

@@ -63,8 +63,8 @@
   version = "0.4.20"
 
 [[override]]
-  name = "github.com/ipfs/go-ds-badger"
-  version = "0.0.3"
+  name = "github.com/dgraph-io/badger"
+  revision = "0ce1d2e26af1ba8b8a72ea864145a3e1e3b382cd"
 
 [prune]
   go-tests = true

cmd/streamSubscribe.go

@@ -17,7 +17,9 @@ package cmd
 
 import (
 	"bytes"
+	"fmt"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -48,6 +50,7 @@ func init() {
 }
 
 func streamSubscribe() {
+	log.SetLevel(log.DebugLevel)
 	// Prep the subscription config/filters to be sent to the server
 	subscriptionConfig()
 
@@ -61,27 +64,26 @@ func streamSubscribe() {
 	// Subscribe to the seed node service with the given config/filter parameters
 	sub, err := str.Stream(payloadChan, subConfig)
 	if err != nil {
-		println(err.Error())
 		log.Fatal(err)
 	}
-
+	log.Info("awaiting payloads")
 	// Receive response payloads and print out the results
 	for {
 		select {
 		case payload := <-payloadChan:
-			if payload.Err != nil {
-				log.Error(payload.Err)
+			if payload.ErrMsg != "" {
+				log.Error(payload.ErrMsg)
+				continue
 			}
 			for _, headerRlp := range payload.HeadersRlp {
 				var header types.Header
 				err = rlp.Decode(bytes.NewBuffer(headerRlp), &header)
 				if err != nil {
-					println(err.Error())
 					log.Error(err)
+					continue
 				}
-				println("Header")
-				println(header.Hash().Hex())
-				println(header.Number.Int64())
+				fmt.Printf("Header number %d, hash %s\n", header.Number.Int64(), header.Hash().Hex())
+				fmt.Printf("header: %v\n", header)
 			}
 			for _, trxRlp := range payload.TransactionsRlp {
 				var trx types.Transaction
@@ -89,11 +91,11 @@ func streamSubscribe() {
 				stream := rlp.NewStream(buff, 0)
 				err := trx.DecodeRLP(stream)
 				if err != nil {
-					println(err.Error())
 					log.Error(err)
+					continue
 				}
-				println("Trx")
-				println(trx.Hash().Hex())
+				fmt.Printf("Transaction with hash %s\n", trx.Hash().Hex())
+				fmt.Printf("trx: %v", trx)
 			}
 			for _, rctRlp := range payload.ReceiptsRlp {
 				var rct types.Receipt
@@ -101,52 +103,59 @@ func streamSubscribe() {
 				stream := rlp.NewStream(buff, 0)
 				err = rct.DecodeRLP(stream)
 				if err != nil {
-					println(err.Error())
 					log.Error(err)
+					continue
 				}
-				println("Rct")
+				fmt.Printf("Receipt with block hash %s, trx hash %s\n", rct.BlockHash.Hex(), rct.TxHash.Hex())
+				fmt.Printf("rct: %v", rct)
 				for _, l := range rct.Logs {
-					println("log")
-					println(l.BlockHash.Hex())
-					println(l.TxHash.Hex())
-					println(l.Address.Hex())
+					if len(l.Topics) < 1 {
+						log.Error(fmt.Sprintf("log only has %d topics", len(l.Topics)))
+						continue
+					}
+					fmt.Printf("Log for block hash %s, trx hash %s, address %s, and with topic0 %s\n",
+						l.BlockHash.Hex(), l.TxHash.Hex(), l.Address.Hex(), l.Topics[0].Hex())
+					fmt.Printf("log: %v\n", l)
 				}
 			}
+			// This assumes leafs only
 			for key, stateRlp := range payload.StateNodesRlp {
 				var acct state.Account
 				err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct)
 				if err != nil {
-					println(err.Error())
 					log.Error(err)
+					continue
 				}
-				println("State")
-				print("key: ")
-				println(key.Hex())
-				print("root: ")
-				println(acct.Root.Hex())
-				print("balance: ")
-				println(acct.Balance.Int64())
+				fmt.Printf("Account for key %s, and root %s, with balance %d\n",
+					key.Hex(), acct.Root.Hex(), acct.Balance.Int64())
+				fmt.Printf("state account: %v\n", acct)
 			}
 			for stateKey, mappedRlp := range payload.StorageNodesRlp {
-				println("Storage")
-				print("state key: ")
-				println(stateKey.Hex())
+				fmt.Printf("Storage for state key %s ", stateKey.Hex())
 				for storageKey, storageRlp := range mappedRlp {
-					println("Storage")
-					print("key: ")
-					println(storageKey.Hex())
+					fmt.Printf("with storage key %s\n", storageKey.Hex())
 					var i []interface{}
 					err := rlp.DecodeBytes(storageRlp, i)
 					if err != nil {
-						println(err.Error())
 						log.Error(err)
+						continue
+					}
+					// if a leaf node
+					if len(i) == 2 {
+						keyBytes, ok := i[0].([]byte)
+						if !ok {
+							continue
+						}
+						valueBytes, ok := i[0].([]byte)
+						if !ok {
+							continue
+						}
+						fmt.Printf("Storage leaf key: %s, and value hash: %s\n",
+							common.BytesToHash(keyBytes).Hex(), common.BytesToHash(valueBytes).Hex())
 					}
-					print("bytes: ")
-					println(storageRlp)
 				}
 			}
 		case err = <-sub.Err():
-			println(err.Error())
 			log.Fatal(err)
 		}
 	}

cmd/syncPublishScreenAndServe.go

@@ -47,6 +47,7 @@ func init() {
 }
 
 func syncPublishScreenAndServe() {
+	log.SetLevel(log.DebugLevel)
 	blockChain, ethClient, rpcClient := getBlockChainAndClients()
 
 	db := utils.LoadPostgres(databaseConfig, blockChain.Node())

pkg/ipfs/api.go

@@ -64,8 +64,13 @@ func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters config.S
 		for {
 			select {
 			case packet := <-payloadChannel:
-				if err := notifier.Notify(rpcSub.ID, packet); err != nil {
-					log.Error("Failed to send state diff packet", "err", err)
+				if notifyErr := notifier.Notify(rpcSub.ID, packet); notifyErr != nil {
+					log.Error("Failed to send state diff packet", "err", notifyErr)
+					unSubErr := api.snp.Unsubscribe(rpcSub.ID)
+					if unSubErr != nil {
+						log.Error("Failed to unsubscribe from the state diff service", unSubErr)
+					}
+					return
 				}
 			case <-rpcSub.Err():
 				err := api.snp.Unsubscribe(rpcSub.ID)

pkg/ipfs/converter_test.go

@@ -20,19 +20,18 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
 )
 
 var _ = Describe("Converter", func() {
 	Describe("Convert", func() {
 		It("Converts StatediffPayloads into IPLDPayloads", func() {
 			mockConverter := mocks.PayloadConverter{}
-			mockConverter.ReturnIPLDPayload = &helpers.MockIPLDPayload
-			ipldPayload, err := mockConverter.Convert(helpers.MockStatediffPayload)
+			mockConverter.ReturnIPLDPayload = &mocks.MockIPLDPayload
+			ipldPayload, err := mockConverter.Convert(mocks.MockStatediffPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(ipldPayload).To(Equal(&helpers.MockIPLDPayload))
-			Expect(mockConverter.PassedStatediffPayload).To(Equal(helpers.MockStatediffPayload))
+			Expect(ipldPayload).To(Equal(&mocks.MockIPLDPayload))
+			Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStatediffPayload))
 		})
 	})
 })

pkg/ipfs/fetcher.go

@@ -20,7 +20,6 @@ import (
 	"context"
 
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
@@ -50,6 +49,7 @@ func NewIPLDFetcher(ipfsPath string) (*EthIPLDFetcher, error) {
 
 // FetchCIDs is the exported method for fetching and returning all the cids passed in a CidWrapper
 func (f *EthIPLDFetcher) FetchCIDs(cids CidWrapper) (*IpldWrapper, error) {
+	log.Debug("fetching iplds")
 	blocks := &IpldWrapper{
 		Headers:      make([]blocks.Block, 0),
 		Transactions: make([]blocks.Block, 0),
@@ -85,6 +85,7 @@ func (f *EthIPLDFetcher) FetchCIDs(cids CidWrapper) (*IpldWrapper, error) {
 // fetchHeaders fetches headers
 // It uses the f.fetchBatch method
 func (f *EthIPLDFetcher) fetchHeaders(cids CidWrapper, blocks *IpldWrapper) error {
+	log.Debug("fetching header iplds")
 	headerCids := make([]cid.Cid, 0, len(cids.Headers))
 	for _, c := range cids.Headers {
 		dc, err := cid.Decode(c)
@@ -103,6 +104,7 @@ func (f *EthIPLDFetcher) fetchHeaders(cids CidWrapper, blocks *IpldWrapper) erro
 // fetchTrxs fetches transactions
 // It uses the f.fetchBatch method
 func (f *EthIPLDFetcher) fetchTrxs(cids CidWrapper, blocks *IpldWrapper) error {
+	log.Debug("fetching transaction iplds")
 	trxCids := make([]cid.Cid, 0, len(cids.Transactions))
 	for _, c := range cids.Transactions {
 		dc, err := cid.Decode(c)
@@ -121,6 +123,7 @@ func (f *EthIPLDFetcher) fetchTrxs(cids CidWrapper, blocks *IpldWrapper) error {
 // fetchRcts fetches receipts
 // It uses the f.fetchBatch method
 func (f *EthIPLDFetcher) fetchRcts(cids CidWrapper, blocks *IpldWrapper) error {
+	log.Debug("fetching receipt iplds")
 	rctCids := make([]cid.Cid, 0, len(cids.Receipts))
 	for _, c := range cids.Receipts {
 		dc, err := cid.Decode(c)
@@ -140,6 +143,7 @@ func (f *EthIPLDFetcher) fetchRcts(cids CidWrapper, blocks *IpldWrapper) error {
 // It uses the single f.fetch method instead of the batch fetch, because it
 // needs to maintain the data's relation to state keys
 func (f *EthIPLDFetcher) fetchState(cids CidWrapper, blocks *IpldWrapper) error {
+	log.Debug("fetching state iplds")
 	for _, stateNode := range cids.StateNodes {
 		if stateNode.CID == "" || stateNode.Key == "" {
 			continue
@@ -161,6 +165,7 @@ func (f *EthIPLDFetcher) fetchState(cids CidWrapper, blocks *IpldWrapper) error
 // It uses the single f.fetch method instead of the batch fetch, because it
 // needs to maintain the data's relation to state and storage keys
 func (f *EthIPLDFetcher) fetchStorage(cids CidWrapper, blocks *IpldWrapper) error {
+	log.Debug("fetching storage iplds")
 	for _, storageNode := range cids.StorageNodes {
 		if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" {
 			continue

pkg/ipfs/mocks/test_data.go

@@ -14,20 +14,20 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package helpers
+package mocks
 
 import (
 	"errors"
 	"math/big"
 	"math/rand"
 
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
-
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 )
 
 // Test variables

pkg/ipfs/publisher_test.go

@@ -20,19 +20,18 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
 )
 
 var _ = Describe("Publisher", func() {
 	Describe("Publish", func() {
 		It("Publishes IPLDPayload to IPFS", func() {
 			mockPublisher := mocks.IPLDPublisher{}
-			mockPublisher.ReturnCIDPayload = &helpers.MockCIDPayload
-			cidPayload, err := mockPublisher.Publish(&helpers.MockIPLDPayload)
+			mockPublisher.ReturnCIDPayload = &mocks.MockCIDPayload
+			cidPayload, err := mockPublisher.Publish(&mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(cidPayload).To(Equal(&helpers.MockCIDPayload))
-			Expect(mockPublisher.PassedIPLDPayload).To(Equal(&helpers.MockIPLDPayload))
+			Expect(cidPayload).To(Equal(&mocks.MockCIDPayload))
+			Expect(mockPublisher.PassedIPLDPayload).To(Equal(&mocks.MockIPLDPayload))
 		})
 	})
 })

pkg/ipfs/repository_test.go

@@ -20,17 +20,16 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
 )
 
 var _ = Describe("Repository", func() {
 	Describe("Index", func() {
 		It("Indexes CIDs against their metadata", func() {
 			mockRepo := mocks.CIDRepository{}
-			err := mockRepo.Index(&helpers.MockCIDPayload)
+			err := mockRepo.Index(&mocks.MockCIDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(mockRepo.PassedCIDPayload).To(Equal(&helpers.MockCIDPayload))
+			Expect(mockRepo.PassedCIDPayload).To(Equal(&mocks.MockCIDPayload))
 		})
 	})
 })

pkg/ipfs/resolver.go

@@ -18,7 +18,6 @@ package ipfs
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/ipfs/go-block-format"
 )
 

pkg/ipfs/retreiver.go

@@ -19,6 +19,8 @@ package ipfs
 import (
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
+	log "github.com/sirupsen/logrus"
+
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
@@ -49,6 +51,7 @@ func (ecr *EthCIDRetriever) GetLastBlockNumber() (int64, error) {
 
 // RetrieveCIDs is used to retrieve all of the CIDs which conform to the passed StreamFilters
 func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription) ([]CidWrapper, error) {
+	log.Debug("retrieving cids")
 	var endingBlock int64
 	var err error
 	if streamFilters.EndingBlock <= 0 || streamFilters.EndingBlock <= streamFilters.StartingBlock {
@@ -63,70 +66,68 @@ func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription) ([]C
 		return nil, err
 	}
 	for i := streamFilters.StartingBlock; i <= endingBlock; i++ {
-		cw := &CidWrapper{
-			BlockNumber:  i,
-			Headers:      make([]string, 0),
-			Transactions: make([]string, 0),
-			Receipts:     make([]string, 0),
-			StateNodes:   make([]StateNodeCID, 0),
-			StorageNodes: make([]StorageNodeCID, 0),
-		}
+		cw := CidWrapper{}
 		if !streamFilters.HeaderFilter.Off {
-			err = ecr.retrieveHeaderCIDs(tx, streamFilters, cw, i)
+			cw.Headers, err = ecr.retrieveHeaderCIDs(tx, streamFilters, i)
 			if err != nil {
 				tx.Rollback()
+				log.Error("header cid retrieval error")
 				return nil, err
 			}
 		}
 		var trxIds []int64
 		if !streamFilters.TrxFilter.Off {
-			trxIds, err = ecr.retrieveTrxCIDs(tx, streamFilters, cw, i)
+			cw.Transactions, trxIds, err = ecr.retrieveTrxCIDs(tx, streamFilters, i)
 			if err != nil {
 				tx.Rollback()
+				log.Error("transaction cid retrieval error")
 				return nil, err
 			}
 		}
 		if !streamFilters.ReceiptFilter.Off {
-			err = ecr.retrieveRctCIDs(tx, streamFilters, cw, i, trxIds)
+			cw.Receipts, err = ecr.retrieveRctCIDs(tx, streamFilters, i, trxIds)
 			if err != nil {
 				tx.Rollback()
+				log.Error("receipt cid retrieval error")
 				return nil, err
 			}
 		}
 		if !streamFilters.StateFilter.Off {
-			err = ecr.retrieveStateCIDs(tx, streamFilters, cw, i)
+			cw.StateNodes, err = ecr.retrieveStateCIDs(tx, streamFilters, i)
 			if err != nil {
 				tx.Rollback()
+				log.Error("state cid retrieval error")
 				return nil, err
 			}
 		}
 		if !streamFilters.StorageFilter.Off {
-			err = ecr.retrieveStorageCIDs(tx, streamFilters, cw, i)
+			cw.StorageNodes, err = ecr.retrieveStorageCIDs(tx, streamFilters, i)
 			if err != nil {
 				tx.Rollback()
+				log.Error("storage cid retrieval error")
 				return nil, err
 			}
 		}
-		cids = append(cids, *cw)
+		cids = append(cids, cw)
 	}
 
-	return cids, err
+	return cids, tx.Commit()
 }
 
-func (ecr *EthCIDRetriever) retrieveHeaderCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error {
-	var pgStr string
-	if streamFilters.HeaderFilter.FinalOnly {
-		pgStr = `SELECT cid FROM header_cids
-				WHERE block_number = $1
-				AND final IS TRUE`
-	} else {
-		pgStr = `SELECT cid FROM header_cids
+func (ecr *EthCIDRetriever) retrieveHeaderCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]string, error) {
+	log.Debug("retrieving header cids")
+	headers := make([]string, 0)
+	pgStr := `SELECT cid FROM header_cids
 				WHERE block_number = $1`
+	if streamFilters.HeaderFilter.FinalOnly {
+		pgStr += ` AND final IS TRUE`
 	}
-	return tx.Select(cids.Headers, pgStr, blockNumber)
+	err := tx.Select(&headers, pgStr, blockNumber)
+	return headers, err
 }
 
-func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) ([]int64, error) {
+func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]string, []int64, error) {
+	log.Debug("retrieving transaction cids")
 	args := make([]interface{}, 0, 3)
 	type result struct {
 		ID  int64  `db:"id"`
@@ -144,19 +145,21 @@ func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters config.Su
 		pgStr += ` AND transaction_cids.src = ANY($3::VARCHAR(66)[])`
 		args = append(args, pq.Array(streamFilters.TrxFilter.Src))
 	}
-	err := tx.Select(results, pgStr, args...)
+	err := tx.Select(&results, pgStr, args...)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	ids := make([]int64, 0)
+	ids := make([]int64, 0, len(results))
+	cids := make([]string, 0, len(results))
 	for _, res := range results {
-		cids.Transactions = append(cids.Transactions, res.Cid)
+		cids = append(cids, res.Cid)
 		ids = append(ids, res.ID)
 	}
-	return ids, nil
+	return cids, ids, nil
 }
 
-func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64, trxIds []int64) error {
+func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64, trxIds []int64) ([]string, error) {
+	log.Debug("retrieving receipt cids")
 	args := make([]interface{}, 0, 2)
 	pgStr := `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids
 			WHERE receipt_cids.tx_id = transaction_cids.id 
@@ -173,10 +176,13 @@ func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Su
 	} else {
 		pgStr += `)`
 	}
-	return tx.Select(cids.Receipts, pgStr, args...)
+	receiptCids := make([]string, 0)
+	err := tx.Select(&receiptCids, pgStr, args...)
+	return receiptCids, err
 }
 
-func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error {
+func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]StateNodeCID, error) {
+	log.Debug("retrieving state cids")
 	args := make([]interface{}, 0, 2)
 	pgStr := `SELECT state_cids.cid, state_cids.state_key FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
 			WHERE header_cids.block_number = $1`
@@ -190,10 +196,16 @@ func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.
 		pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
 		args = append(args, pq.Array(keys))
 	}
-	return tx.Select(cids.StateNodes, pgStr, args...)
+	if !streamFilters.StorageFilter.IntermediateNodes {
+		pgStr += ` AND state_cids.leaf = TRUE`
+	}
+	stateNodeCIDs := make([]StateNodeCID, 0)
+	err := tx.Select(&stateNodeCIDs, pgStr, args...)
+	return stateNodeCIDs, err
 }
 
-func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error {
+func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]StorageNodeCID, error) {
+	log.Debug("retrieving storage cids")
 	args := make([]interface{}, 0, 3)
 	pgStr := `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key FROM storage_cids, state_cids, header_cids
 			WHERE storage_cids.state_id = state_cids.id 
@@ -213,7 +225,10 @@ func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters confi
 		pgStr += ` AND storage_cids.storage_key = ANY($3::VARCHAR(66)[])`
 		args = append(args, pq.Array(streamFilters.StorageFilter.StorageKeys))
 	}
-	return tx.Select(cids.StorageNodes, pgStr, args...)
+	if !streamFilters.StorageFilter.IntermediateNodes {
+		pgStr += ` AND storage_cids.leaf = TRUE`
+	}
+	storageNodeCIDs := make([]StorageNodeCID, 0)
+	err := tx.Select(&storageNodeCIDs, pgStr, args...)
+	return storageNodeCIDs, err
 }
-
-// ADD IF LEAF ONLY!!

pkg/ipfs/screener.go

@@ -19,12 +19,11 @@ package ipfs
 import (
 	"bytes"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-
-	"github.com/ethereum/go-ethereum/core/types"
-
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
+
+	"github.com/vulcanize/vulcanizedb/pkg/config"
 )
 
 // ResponseScreener is the inteface used to screen eth data and package appropriate data into a response payload

pkg/ipfs/service.go

@@ -121,7 +121,8 @@ func (sap *Service) APIs() []rpc.API {
 }
 
 // SyncAndPublish is the backend processing loop which streams data from geth, converts it to iplds, publishes them to ipfs, and indexes their cids
-// It then forwards the data to the Serve() loop which filters and sends relevant data to client subscriptions
+// This continues on no matter if or how many subscribers there are, it then forwards the data to the ScreenAndServe() loop
+// which filters and sends relevant data to client subscriptions, if there are any
 func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<- IPLDPayload, forwardQuitchan chan<- bool) error {
 	sub, err := sap.Streamer.Stream(sap.PayloadChan)
 	if err != nil {
@@ -173,7 +174,8 @@ func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<-
 	return nil
 }
 
-// ScreenAndServe is the processing loop used to screen data streamed from the state diffing eth node and send the appropriate data to a requesting client subscription
+// ScreenAndServe is the loop used to screen data streamed from the state diffing eth node
+// and send the appropriate portions of it to a requesting client subscription, according to their subscription configuration
 func (sap *Service) ScreenAndServe(wg *sync.WaitGroup, receivePayloadChan <-chan IPLDPayload, receiveQuitchan <-chan bool) {
 	wg.Add(1)
 	go func() {
@@ -206,32 +208,32 @@ func (sap *Service) processResponse(payload IPLDPayload) error {
 
 // Subscribe is used by the API to subscribe to the service loop
 func (sap *Service) Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters *config.Subscription) {
-	log.Info("Subscribing to the statediff service")
+	log.Info("Subscribing to the seed node service")
 	sap.Lock()
 	sap.Subscriptions[id] = Subscription{
 		PayloadChan:   sub,
 		QuitChan:      quitChan,
 		StreamFilters: streamFilters,
 	}
-	sap.Unlock()
 	// If the subscription requests a backfill, use the Postgres index to lookup and retrieve historical data
 	// Otherwise we only filter new data as it is streamed in from the state diffing geth node
-	if streamFilters.BackFill {
+	if streamFilters.BackFill || streamFilters.BackFillOnly {
+		log.Debug("back-filling data for id", id)
 		// Retrieve cached CIDs relevant to this subscriber
-		cids, err := sap.Retriever.RetrieveCIDs(*streamFilters)
+		cidWrappers, err := sap.Retriever.RetrieveCIDs(*streamFilters)
 		if err != nil {
 			log.Error(err)
 			sap.serve(id, ResponsePayload{
-				Err: err,
+				ErrMsg: "CID retrieval error: " + err.Error(),
 			})
 			return
 		}
-		for _, cid := range cids {
-			blocksWrapper, err := sap.Fetcher.FetchCIDs(cid)
+		for _, cidWrapper := range cidWrappers {
+			blocksWrapper, err := sap.Fetcher.FetchCIDs(cidWrapper)
 			if err != nil {
 				log.Error(err)
 				sap.serve(id, ResponsePayload{
-					Err: err,
+					ErrMsg: "IPLD fetching error: " + err.Error(),
 				})
 				return
 			}
@@ -239,18 +241,22 @@ func (sap *Service) Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan ch
 			if err != nil {
 				log.Error(err)
 				sap.serve(id, ResponsePayload{
-					Err: err,
+					ErrMsg: "IPLD resolving error: " + err.Error(),
 				})
 				return
 			}
 			sap.serve(id, *backFillIplds)
 		}
+		if streamFilters.BackFillOnly {
+			delete(sap.Subscriptions, id)
+		}
 	}
+	sap.Unlock()
 }
 
 // Unsubscribe is used to unsubscribe to the StateDiffingService loop
 func (sap *Service) Unsubscribe(id rpc.ID) error {
-	log.Info("Unsubscribing from the statediff service")
+	log.Info("Unsubscribing from the seed node service")
 	sap.Lock()
 	_, ok := sap.Subscriptions[id]
 	if !ok {
@@ -263,7 +269,7 @@ func (sap *Service) Unsubscribe(id rpc.ID) error {
 
 // Start is used to begin the service
 func (sap *Service) Start(*p2p.Server) error {
-	log.Info("Starting statediff service")
+	log.Info("Starting seed node service")
 	wg := new(sync.WaitGroup)
 	payloadChan := make(chan IPLDPayload, payloadChanBufferSize)
 	quitChan := make(chan bool, 1)
@@ -276,7 +282,7 @@ func (sap *Service) Start(*p2p.Server) error {
 
 // Stop is used to close down the service
 func (sap *Service) Stop() error {
-	log.Info("Stopping statediff service")
+	log.Info("Stopping seed node service")
 	close(sap.QuitChan)
 	return nil
 }

pkg/ipfs/service_test.go

@@ -26,8 +26,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
 )
 
 var _ = Describe("Service", func() {
@@ -41,18 +40,18 @@ var _ = Describe("Service", func() {
 				ReturnErr: nil,
 			}
 			mockPublisher := &mocks.IPLDPublisher{
-				ReturnCIDPayload: &helpers.MockCIDPayload,
+				ReturnCIDPayload: &mocks.MockCIDPayload,
 				ReturnErr:        nil,
 			}
 			mockStreamer := &mocks.StateDiffStreamer{
 				ReturnSub: &rpc.ClientSubscription{},
 				StreamPayloads: []statediff.Payload{
-					helpers.MockStatediffPayload,
+					mocks.MockStatediffPayload,
 				},
 				ReturnErr: nil,
 			}
 			mockConverter := &mocks.PayloadConverter{
-				ReturnIPLDPayload: &helpers.MockIPLDPayload,
+				ReturnIPLDPayload: &mocks.MockIPLDPayload,
 				ReturnErr:         nil,
 			}
 			processor := &ipfs.Service{
@@ -68,9 +67,9 @@ var _ = Describe("Service", func() {
 			time.Sleep(2 * time.Second)
 			quitChan <- true
 			wg.Wait()
-			Expect(mockConverter.PassedStatediffPayload).To(Equal(helpers.MockStatediffPayload))
-			Expect(mockCidRepo.PassedCIDPayload).To(Equal(&helpers.MockCIDPayload))
-			Expect(mockPublisher.PassedIPLDPayload).To(Equal(&helpers.MockIPLDPayload))
+			Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStatediffPayload))
+			Expect(mockCidRepo.PassedCIDPayload).To(Equal(&mocks.MockCIDPayload))
+			Expect(mockPublisher.PassedIPLDPayload).To(Equal(&mocks.MockIPLDPayload))
 			Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
 		})
 	})

pkg/ipfs/streamer_test.go

@@ -22,8 +22,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
 )
 
 var _ = Describe("Streamer", func() {
@@ -32,7 +31,7 @@ var _ = Describe("Streamer", func() {
 			mockStreamer := mocks.StateDiffStreamer{}
 			mockStreamer.ReturnSub = &rpc.ClientSubscription{}
 			mockStreamer.StreamPayloads = []statediff.Payload{
-				helpers.MockStatediffPayload,
+				mocks.MockStatediffPayload,
 			}
 			payloadChan := make(chan statediff.Payload, 1)
 			sub, err := mockStreamer.Stream(payloadChan)
@@ -40,7 +39,7 @@ var _ = Describe("Streamer", func() {
 			Expect(sub).To(Equal(&rpc.ClientSubscription{}))
 			Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
 			streamedPayload := <-payloadChan
-			Expect(streamedPayload).To(Equal(helpers.MockStatediffPayload))
+			Expect(streamedPayload).To(Equal(mocks.MockStatediffPayload))
 		})
 	})
 })

pkg/ipfs/types.go

@@ -43,7 +43,7 @@ type ResponsePayload struct {
 	ReceiptsRlp     [][]byte                               `json:"receiptsRlp"`
 	StateNodesRlp   map[common.Hash][]byte                 `json:"stateNodesRlp"`
 	StorageNodesRlp map[common.Hash]map[common.Hash][]byte `json:"storageNodesRlp"`
-	Err             error                                  `json:"error"`
+	ErrMsg          string                                 `json:"errMsg"`
 
 	encoded []byte
 	err     error

vendor/github.com/dgraph-io/badger/.gitignore

@@ -0,0 +1 @@
+p/

vendor/github.com/dgraph-io/badger/.golangci.yml

@@ -0,0 +1,20 @@
+run:
+  tests: false
+
+linters-settings:
+  lll:
+    line-length: 100
+
+linters:
+  disable-all: true
+  enable:
+    - errcheck
+    - ineffassign
+    - gas
+    - gofmt
+    - golint
+    - gosimple
+    - govet
+    - lll
+    - varcheck
+    - unused

vendor/github.com/dgraph-io/badger/.travis.yml

@@ -0,0 +1,25 @@
+language: go
+
+go:
+    - "1.9"
+    - "1.10"
+    - "1.11"
+
+matrix:
+  include:
+    - os: osx
+notifications:
+  email: false
+  slack:
+    secure: X7uBLWYbuUhf8QFE16CoS5z7WvFR8EN9j6cEectMW6mKZ3vwXGwVXRIPsgUq/606DsQdCCx34MR8MRWYGlu6TBolbSe9y0EP0i46yipPz22YtuT7umcVUbGEyx8MZKgG0v1u/zA0O4aCsOBpGAA3gxz8h3JlEHDt+hv6U8xRsSllVLzLSNb5lwxDtcfEDxVVqP47GMEgjLPM28Pyt5qwjk7o5a4YSVzkfdxBXxd3gWzFUWzJ5E3cTacli50dK4GVfiLcQY2aQYoYO7AAvDnvP+TPfjDkBlUEE4MUz5CDIN51Xb+WW33sX7g+r3Bj7V5IRcF973RiYkpEh+3eoiPnyWyxhDZBYilty3b+Hysp6d4Ov/3I3ll7Bcny5+cYjakjkMH3l9w3gs6Y82GlpSLSJshKWS8vPRsxFe0Pstj6QSJXTd9EBaFr+l1ScXjJv/Sya9j8N9FfTuOTESWuaL1auX4Y7zEEVHlA8SCNOO8K0eTfxGZnC/YcIHsR8rePEAcFxfOYQppkyLF/XvAtnb/LMUuu0g4y2qNdme6Oelvyar1tFEMRtbl4mRCdu/krXBFtkrsfUaVY6WTPdvXAGotsFJ0wuA53zGVhlcd3+xAlSlR3c1QX95HIMeivJKb5L4nTjP+xnrmQNtnVk+tG4LSH2ltuwcZSSczModtcBmRefrk=
+
+env:
+  global:
+  - secure: CRkV2+/jlO0gXzzS50XGxfMS117FNwiVjxNY/LeWq06RKD+dDCPxTJl3JCNe3l0cYEPAglV2uMMYukDiTqJ7e+HI4nh4N4mv6lwx39N8dAvJe1x5ITS2T4qk4kTjuQb1Q1vw/ZOxoQqmvNKj2uRmBdJ/HHmysbRJ1OzCWML3OXdUwJf0AYlJzTjpMfkOKr7sTtE4rwyyQtd4tKH1fGdurgI9ZuFd9qvYxK2qcJhsQ6CNqMXt+7FkVkN1rIPmofjjBTNryzUr4COFXuWH95aDAif19DeBW4lbNgo1+FpDsrgmqtuhl6NAuptI8q/imow2KXBYJ8JPXsxW8DVFj0IIp0RCd3GjaEnwBEbxAyiIHLfW7AudyTS/dJOvZffPqXnuJ8xj3OPIdNe4xY0hWl8Ju2HhKfLOAHq7VadHZWd3IHLil70EiL4/JLD1rNbMImUZisFaA8pyrcIvYYebjOnk4TscwKFLedClRSX1XsMjWWd0oykQtrdkHM2IxknnBpaLu7mFnfE07f6dkG0nlpyu4SCLey7hr5FdcEmljA0nIxTSYDg6035fQkBEAbe7hlESOekkVNT9IZPwG+lmt3vU4ofi6NqNbJecOuSB+h36IiZ9s4YQtxYNnLgW14zjuFGGyT5smc3IjBT7qngDjKIgyrSVoRkY/8udy9qbUgvBeW8=
+
+before_script:
+- go get github.com/mattn/goveralls
+script:
+- bash contrib/cover.sh $HOME/build coverage.out || travis_terminate 1
+- goveralls -service=travis-ci -coverprofile=coverage.out || true
+- goveralls -coverprofile=coverage.out -service=travis-ci

vendor/github.com/dgraph-io/badger/README.md

@@ -58,7 +58,7 @@ To start using Badger, install Go 1.8 or above and run `go get`:
 $ go get github.com/dgraph-io/badger/...
 ```
 
-This will retrieve the library and install the `badger_info` command line
+This will retrieve the library and install the `badger` command line
 utility into your `$GOBIN` path.
 
 
@@ -144,7 +144,7 @@ txn := db.NewTransaction(true)
 for k,v := range updates {
   if err := txn.Set([]byte(k),[]byte(v)); err == ErrTxnTooBig {
     _ = txn.Commit()
-    txn = db.NewTransaction(..)
+    txn = db.NewTransaction(true)
     _ = txn.Set([]byte(k),[]byte(v))
   }
 }
@@ -673,6 +673,9 @@ Below is a list of known projects that use Badger:
 * [Babble](https://github.com/mosaicnetworks/babble) - BFT Consensus platform for distributed applications.
 * [Tormenta](https://github.com/jpincas/tormenta) - Embedded object-persistence layer / simple JSON database for Go projects.
 * [BadgerHold](https://github.com/timshannon/badgerhold) - An embeddable NoSQL store for querying Go types built on Badger
+* [Goblero](https://github.com/didil/goblero) - Pure Go embedded persistent job queue backed by BadgerDB
+* [Surfline](https://www.surfline.com) - Serving global wave and weather forecast data with Badger.
+* [Cete](https://github.com/mosuka/cete) - Simple and highly available distributed key-value store built on Badger. Makes it easy bringing up a cluster of Badger with Raft consensus algorithm by hashicorp/raft. 
 
 If you are using Badger in a project please send a pull request to add it to the list.
 

vendor/github.com/dgraph-io/badger/backup.go

@@ -22,10 +22,9 @@ import (
 	"context"
 	"encoding/binary"
 	"io"
-	"sync"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
+	"github.com/dgraph-io/badger/pb"
+	"github.com/dgraph-io/badger/y"
 )
 
 // Backup is a wrapper function over Stream.Backup to generate full and incremental backups of the
@@ -106,11 +105,8 @@ func (stream *Stream) Backup(w io.Writer, since uint64) (uint64, error) {
 			if maxVersion < kv.Version {
 				maxVersion = kv.Version
 			}
-			if err := writeTo(kv, w); err != nil {
-				return err
-			}
 		}
-		return nil
+		return writeTo(list, w)
 	}
 
 	if err := stream.Orchestrate(context.Background()); err != nil {
@@ -119,11 +115,11 @@ func (stream *Stream) Backup(w io.Writer, since uint64) (uint64, error) {
 	return maxVersion, nil
 }
 
-func writeTo(entry *pb.KV, w io.Writer) error {
-	if err := binary.Write(w, binary.LittleEndian, uint64(entry.Size())); err != nil {
+func writeTo(list *pb.KVList, w io.Writer) error {
+	if err := binary.Write(w, binary.LittleEndian, uint64(list.Size())); err != nil {
 		return err
 	}
-	buf, err := entry.Marshal()
+	buf, err := list.Marshal()
 	if err != nil {
 		return err
 	}
@@ -131,38 +127,73 @@ func writeTo(entry *pb.KV, w io.Writer) error {
 	return err
 }
 
+type loader struct {
+	db       *DB
+	throttle *y.Throttle
+	entries  []*Entry
+}
+
+func (db *DB) newLoader(maxPendingWrites int) *loader {
+	return &loader{
+		db:       db,
+		throttle: y.NewThrottle(maxPendingWrites),
+	}
+}
+
+func (l *loader) set(kv *pb.KV) error {
+	var userMeta, meta byte
+	if len(kv.UserMeta) > 0 {
+		userMeta = kv.UserMeta[0]
+	}
+	if len(kv.Meta) > 0 {
+		meta = kv.Meta[0]
+	}
+
+	l.entries = append(l.entries, &Entry{
+		Key:       y.KeyWithTs(kv.Key, kv.Version),
+		Value:     kv.Value,
+		UserMeta:  userMeta,
+		ExpiresAt: kv.ExpiresAt,
+		meta:      meta,
+	})
+	if len(l.entries) >= 1000 {
+		return l.send()
+	}
+	return nil
+}
+
+func (l *loader) send() error {
+	if err := l.throttle.Do(); err != nil {
+		return err
+	}
+	l.db.batchSetAsync(l.entries, func(err error) {
+		l.throttle.Done(err)
+	})
+
+	l.entries = make([]*Entry, 0, 1000)
+	return nil
+}
+
+func (l *loader) finish() error {
+	if len(l.entries) > 0 {
+		if err := l.send(); err != nil {
+			return err
+		}
+	}
+	return l.throttle.Finish()
+}
+
 // Load reads a protobuf-encoded list of all entries from a reader and writes
 // them to the database. This can be used to restore the database from a backup
 // made by calling DB.Backup().
 //
 // DB.Load() should be called on a database that is not running any other
 // concurrent transactions while it is running.
-func (db *DB) Load(r io.Reader) error {
+func (db *DB) Load(r io.Reader, maxPendingWrites int) error {
 	br := bufio.NewReaderSize(r, 16<<10)
 	unmarshalBuf := make([]byte, 1<<10)
-	var entries []*Entry
-	var wg sync.WaitGroup
-	errChan := make(chan error, 1)
-
-	// func to check for pending error before sending off a batch for writing
-	batchSetAsyncIfNoErr := func(entries []*Entry) error {
-		select {
-		case err := <-errChan:
-			return err
-		default:
-			wg.Add(1)
-			return db.batchSetAsync(entries, func(err error) {
-				defer wg.Done()
-				if err != nil {
-					select {
-					case errChan <- err:
-					default:
-					}
-				}
-			})
-		}
-	}
 
+	ldr := db.newLoader(maxPendingWrites)
 	for {
 		var sz uint64
 		err := binary.Read(br, binary.LittleEndian, &sz)
@@ -176,51 +207,31 @@ func (db *DB) Load(r io.Reader) error {
 			unmarshalBuf = make([]byte, sz)
 		}
 
-		e := &pb.KV{}
 		if _, err = io.ReadFull(br, unmarshalBuf[:sz]); err != nil {
 			return err
 		}
-		if err = e.Unmarshal(unmarshalBuf[:sz]); err != nil {
+
+		list := &pb.KVList{}
+		if err := list.Unmarshal(unmarshalBuf[:sz]); err != nil {
 			return err
 		}
-		var userMeta byte
-		if len(e.UserMeta) > 0 {
-			userMeta = e.UserMeta[0]
-		}
-		entries = append(entries, &Entry{
-			Key:       y.KeyWithTs(e.Key, e.Version),
-			Value:     e.Value,
-			UserMeta:  userMeta,
-			ExpiresAt: e.ExpiresAt,
-			meta:      e.Meta[0],
-		})
-		// Update nextTxnTs, memtable stores this timestamp in badger head
-		// when flushed.
-		if e.Version >= db.orc.nextTxnTs {
-			db.orc.nextTxnTs = e.Version + 1
-		}
 
-		if len(entries) == 1000 {
-			if err := batchSetAsyncIfNoErr(entries); err != nil {
+		for _, kv := range list.Kv {
+			if err := ldr.set(kv); err != nil {
 				return err
 			}
-			entries = make([]*Entry, 0, 1000)
+
+			// Update nextTxnTs, memtable stores this
+			// timestamp in badger head when flushed.
+			if kv.Version >= db.orc.nextTxnTs {
+				db.orc.nextTxnTs = kv.Version + 1
+			}
 		}
 	}
 
-	if len(entries) > 0 {
-		if err := batchSetAsyncIfNoErr(entries); err != nil {
-			return err
-		}
-	}
-	wg.Wait()
-
-	select {
-	case err := <-errChan:
+	if err := ldr.finish(); err != nil {
 		return err
-	default:
-		// Mark all versions done up until nextTxnTs.
-		db.orc.txnMark.Done(db.orc.nextTxnTs - 1)
-		return nil
 	}
+	db.orc.txnMark.Done(db.orc.nextTxnTs - 1)
+	return nil
 }

vendor/github.com/dgraph-io/badger/backup_test.go

@@ -1,519 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"reflect"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb"
-)
-
-func TestBackupRestore1(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	db, err := Open(getTestOptions(dir))
-	require.NoError(t, err)
-
-	// Write some stuff
-	entries := []struct {
-		key      []byte
-		val      []byte
-		userMeta byte
-		version  uint64
-	}{
-		{key: []byte("answer1"), val: []byte("42"), version: 1},
-		{key: []byte("answer2"), val: []byte("43"), userMeta: 1, version: 2},
-	}
-
-	err = db.Update(func(txn *Txn) error {
-		e := entries[0]
-		err := txn.SetWithMeta(e.key, e.val, e.userMeta)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	require.NoError(t, err)
-
-	err = db.Update(func(txn *Txn) error {
-		e := entries[1]
-		err := txn.SetWithMeta(e.key, e.val, e.userMeta)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	require.NoError(t, err)
-
-	// Use different directory.
-	dir, err = ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	bak, err := ioutil.TempFile(dir, "badgerbak")
-	require.NoError(t, err)
-	ts, err := db.Backup(bak, 0)
-	t.Logf("New ts: %d\n", ts)
-	require.NoError(t, err)
-	require.NoError(t, bak.Close())
-	require.NoError(t, db.Close())
-
-	db, err = Open(getTestOptions(dir))
-	require.NoError(t, err)
-	defer db.Close()
-	bak, err = os.Open(bak.Name())
-	require.NoError(t, err)
-	defer bak.Close()
-
-	require.NoError(t, db.Load(bak))
-
-	err = db.View(func(txn *Txn) error {
-		opts := DefaultIteratorOptions
-		opts.AllVersions = true
-		it := txn.NewIterator(opts)
-		defer it.Close()
-		var count int
-		for it.Rewind(); it.Valid(); it.Next() {
-			item := it.Item()
-			val, err := item.ValueCopy(nil)
-			if err != nil {
-				return err
-			}
-			require.Equal(t, entries[count].key, item.Key())
-			require.Equal(t, entries[count].val, val)
-			require.Equal(t, entries[count].version, item.Version())
-			require.Equal(t, entries[count].userMeta, item.UserMeta())
-			count++
-		}
-		require.Equal(t, count, 2)
-		return nil
-	})
-	require.NoError(t, err)
-}
-
-func TestBackupRestore2(t *testing.T) {
-	tmpdir, err := ioutil.TempDir("", "badger-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer func() {
-		os.RemoveAll(tmpdir)
-	}()
-
-	s1Path := filepath.Join(tmpdir, "test1")
-	s2Path := filepath.Join(tmpdir, "test2")
-	s3Path := filepath.Join(tmpdir, "test3")
-
-	opts := DefaultOptions
-	opts.Dir = s1Path
-	opts.ValueDir = s1Path
-	db1, err := Open(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	key1 := []byte("key1")
-	key2 := []byte("key2")
-	rawValue := []byte("NotLongValue")
-	N := byte(251)
-	err = db1.Update(func(tx *Txn) error {
-		if err := tx.Set(key1, rawValue); err != nil {
-			return err
-		}
-		return tx.Set(key2, rawValue)
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	for i := byte(1); i < N; i++ {
-		err = db1.Update(func(tx *Txn) error {
-			if err := tx.Set(append(key1, i), rawValue); err != nil {
-				return err
-			}
-			return tx.Set(append(key2, i), rawValue)
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	var backup bytes.Buffer
-	_, err = db1.Backup(&backup, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fmt.Println("backup1 length:", backup.Len())
-
-	opts = DefaultOptions
-	opts.Dir = s2Path
-	opts.ValueDir = s2Path
-	db2, err := Open(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = db2.Load(&backup)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := byte(1); i < N; i++ {
-		err = db2.View(func(tx *Txn) error {
-			k := append(key1, i)
-			item, err := tx.Get(k)
-			if err != nil {
-				if err == ErrKeyNotFound {
-					return fmt.Errorf("Key %q has been not found, but was set\n", k)
-				}
-				return err
-			}
-			v, err := item.ValueCopy(nil)
-			if err != nil {
-				return err
-			}
-			if !reflect.DeepEqual(v, rawValue) {
-				return fmt.Errorf("Values not match, got %v, expected %v", v, rawValue)
-			}
-			return nil
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	for i := byte(1); i < N; i++ {
-		err = db2.Update(func(tx *Txn) error {
-			if err := tx.Set(append(key1, i), rawValue); err != nil {
-				return err
-			}
-			return tx.Set(append(key2, i), rawValue)
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	backup.Reset()
-	_, err = db2.Backup(&backup, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fmt.Println("backup2 length:", backup.Len())
-	opts = DefaultOptions
-	opts.Dir = s3Path
-	opts.ValueDir = s3Path
-	db3, err := Open(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db3.Load(&backup)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := byte(1); i < N; i++ {
-		err = db3.View(func(tx *Txn) error {
-			k := append(key1, i)
-			item, err := tx.Get(k)
-			if err != nil {
-				if err == ErrKeyNotFound {
-					return fmt.Errorf("Key %q has been not found, but was set\n", k)
-				}
-				return err
-			}
-			v, err := item.ValueCopy(nil)
-			if err != nil {
-				return err
-			}
-			if !reflect.DeepEqual(v, rawValue) {
-				return fmt.Errorf("Values not match, got %v, expected %v", v, rawValue)
-			}
-			return nil
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-}
-
-var randSrc = rand.NewSource(time.Now().UnixNano())
-
-func createEntries(n int) []*pb.KV {
-	entries := make([]*pb.KV, n)
-	for i := 0; i < n; i++ {
-		entries[i] = &pb.KV{
-			Key:      []byte(fmt.Sprint("key", i)),
-			Value:    []byte{1},
-			UserMeta: []byte{0},
-			Meta:     []byte{0},
-		}
-	}
-	return entries
-}
-
-func populateEntries(db *DB, entries []*pb.KV) error {
-	return db.Update(func(txn *Txn) error {
-		var err error
-		for i, e := range entries {
-			if err = txn.Set(e.Key, e.Value); err != nil {
-				return err
-			}
-			entries[i].Version = 1
-		}
-		return nil
-	})
-}
-
-func TestBackup(t *testing.T) {
-	var bb bytes.Buffer
-
-	tmpdir, err := ioutil.TempDir("", "badger-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpdir)
-
-	opts := DefaultOptions
-	opts.Dir = filepath.Join(tmpdir, "backup0")
-	opts.ValueDir = opts.Dir
-	db1, err := Open(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	N := 1000
-	entries := createEntries(N)
-	require.NoError(t, populateEntries(db1, entries))
-
-	_, err = db1.Backup(&bb, 0)
-	require.NoError(t, err)
-
-	err = db1.View(func(txn *Txn) error {
-		opts := DefaultIteratorOptions
-		it := txn.NewIterator(opts)
-		defer it.Close()
-		var count int
-		for it.Rewind(); it.Valid(); it.Next() {
-			item := it.Item()
-			idx, err := strconv.Atoi(string(item.Key())[3:])
-			if err != nil {
-				return err
-			}
-			if idx > N || !bytes.Equal(entries[idx].Key, item.Key()) {
-				return fmt.Errorf("%s: %s", string(item.Key()), ErrKeyNotFound)
-			}
-			count++
-		}
-		if N != count {
-			return fmt.Errorf("wrong number of items: %d expected, %d actual", N, count)
-		}
-		return nil
-	})
-	require.NoError(t, err)
-}
-
-func TestBackupRestore3(t *testing.T) {
-	var bb bytes.Buffer
-
-	tmpdir, err := ioutil.TempDir("", "badger-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpdir)
-
-	opts := DefaultOptions
-	N := 1000
-	entries := createEntries(N)
-
-	// backup
-	{
-		opts.Dir = filepath.Join(tmpdir, "backup1")
-		opts.ValueDir = opts.Dir
-		db1, err := Open(opts)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		require.NoError(t, populateEntries(db1, entries))
-
-		_, err = db1.Backup(&bb, 0)
-		require.NoError(t, err)
-		require.NoError(t, db1.Close())
-	}
-	require.True(t, len(entries) == N)
-	require.True(t, bb.Len() > 0)
-
-	// restore
-	opts.Dir = filepath.Join(tmpdir, "restore1")
-	opts.ValueDir = opts.Dir
-	db2, err := Open(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	require.NoError(t, db2.Load(&bb))
-
-	// verify
-	err = db2.View(func(txn *Txn) error {
-		opts := DefaultIteratorOptions
-		it := txn.NewIterator(opts)
-		defer it.Close()
-		var count int
-		for it.Rewind(); it.Valid(); it.Next() {
-			item := it.Item()
-			idx, err := strconv.Atoi(string(item.Key())[3:])
-			if err != nil {
-				return err
-			}
-			if idx > N || !bytes.Equal(entries[idx].Key, item.Key()) {
-				return fmt.Errorf("%s: %s", string(item.Key()), ErrKeyNotFound)
-			}
-			count++
-		}
-		if N != count {
-			return fmt.Errorf("wrong number of items: %d expected, %d actual", N, count)
-		}
-		return nil
-	})
-	require.NoError(t, err)
-}
-
-func TestBackupLoadIncremental(t *testing.T) {
-	tmpdir, err := ioutil.TempDir("", "badger-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpdir)
-
-	opts := DefaultOptions
-	N := 100
-	entries := createEntries(N)
-	updates := make(map[int]byte)
-	var bb bytes.Buffer
-
-	// backup
-	{
-		opts.Dir = filepath.Join(tmpdir, "backup2")
-		opts.ValueDir = opts.Dir
-		db1, err := Open(opts)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		require.NoError(t, populateEntries(db1, entries))
-		since, err := db1.Backup(&bb, 0)
-		require.NoError(t, err)
-
-		ints := rand.New(randSrc).Perm(N)
-
-		// pick 10 items to mark as deleted.
-		err = db1.Update(func(txn *Txn) error {
-			for _, i := range ints[:10] {
-				if err := txn.Delete(entries[i].Key); err != nil {
-					return err
-				}
-				updates[i] = bitDelete
-			}
-			return nil
-		})
-		require.NoError(t, err)
-		since, err = db1.Backup(&bb, since)
-		require.NoError(t, err)
-
-		// pick 5 items to mark as expired.
-		err = db1.Update(func(txn *Txn) error {
-			for _, i := range (ints)[10:15] {
-				if err := txn.SetWithTTL(
-					entries[i].Key, entries[i].Value, -time.Hour); err != nil {
-					return err
-				}
-				updates[i] = bitDelete // expired
-			}
-			return nil
-		})
-		require.NoError(t, err)
-		since, err = db1.Backup(&bb, since)
-		require.NoError(t, err)
-
-		// pick 5 items to mark as discard.
-		err = db1.Update(func(txn *Txn) error {
-			for _, i := range ints[15:20] {
-				if err := txn.SetWithDiscard(entries[i].Key, entries[i].Value, 0); err != nil {
-					return err
-				}
-				updates[i] = bitDiscardEarlierVersions
-			}
-			return nil
-		})
-		require.NoError(t, err)
-		_, err = db1.Backup(&bb, since)
-		require.NoError(t, err)
-		require.NoError(t, db1.Close())
-	}
-	require.True(t, len(entries) == N)
-	require.True(t, bb.Len() > 0)
-
-	// restore
-	opts.Dir = filepath.Join(tmpdir, "restore2")
-	opts.ValueDir = opts.Dir
-	db2, err := Open(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	require.NoError(t, db2.Load(&bb))
-
-	// verify
-	actual := make(map[int]byte)
-	err = db2.View(func(txn *Txn) error {
-		opts := DefaultIteratorOptions
-		opts.AllVersions = true
-		it := txn.NewIterator(opts)
-		defer it.Close()
-		var count int
-		for it.Rewind(); it.Valid(); it.Next() {
-			item := it.Item()
-			idx, err := strconv.Atoi(string(item.Key())[3:])
-			if err != nil {
-				return err
-			}
-			if item.IsDeletedOrExpired() {
-				_, ok := updates[idx]
-				if !ok {
-					return fmt.Errorf("%s: not expected to be updated but it is",
-						string(item.Key()))
-				}
-				actual[idx] = item.meta
-				count++
-				continue
-			}
-		}
-		if len(updates) != count {
-			return fmt.Errorf("mismatched updated items: %d expected, %d actual",
-				len(updates), count)
-		}
-		return nil
-	})
-	require.NoError(t, err, "%v %v", updates, actual)
-}

vendor/github.com/dgraph-io/badger/badger/.gitignore

@@ -1 +0,0 @@
-/badger

vendor/github.com/dgraph-io/badger/badger/cmd/backup.go

@@ -1,72 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"os"
-
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger"
-	"gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra"
-)
-
-var backupFile string
-var truncate bool
-
-// backupCmd represents the backup command
-var backupCmd = &cobra.Command{
-	Use:   "backup",
-	Short: "Backup Badger database.",
-	Long: `Backup Badger database to a file in a version-agnostic manner.
-
-Iterates over each key-value pair, encodes it along with its metadata and
-version in protocol buffers and writes them to a file. This file can later be
-used by the restore command to create an identical copy of the
-database.`,
-	RunE: doBackup,
-}
-
-func init() {
-	RootCmd.AddCommand(backupCmd)
-	backupCmd.Flags().StringVarP(&backupFile, "backup-file", "f",
-		"badger.bak", "File to backup to")
-	backupCmd.Flags().BoolVarP(&truncate, "truncate", "t",
-		false, "Allow value log truncation if required.")
-}
-
-func doBackup(cmd *cobra.Command, args []string) error {
-	// Open DB
-	opts := badger.DefaultOptions
-	opts.Dir = sstDir
-	opts.ValueDir = vlogDir
-	opts.Truncate = truncate
-	db, err := badger.Open(opts)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	// Create File
-	f, err := os.Create(backupFile)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	// Run Backup
-	_, err = db.Backup(f, 0)
-	return err
-}

vendor/github.com/dgraph-io/badger/badger/cmd/bank.go

@@ -1,451 +0,0 @@
-/*
- * Copyright 2018 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"log"
-	"math"
-	"math/rand"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra"
-)
-
-var testCmd = &cobra.Command{
-	Use:   "bank",
-	Short: "Run bank test on Badger.",
-	Long: `
-This command runs bank test on Badger, inspired by Jepsen. It creates many
-accounts and moves money among them transactionally. It also reads the sum total
-of all the accounts, to ensure that the total never changes.
-`,
-}
-
-var bankTest = &cobra.Command{
-	Use:   "test",
-	Short: "Execute bank test on Badger.",
-	RunE:  runTest,
-}
-
-var bankDisect = &cobra.Command{
-	Use:   "disect",
-	Short: "Disect the bank output.",
-	Long: `
-Disect the bank output BadgerDB to find the first transaction which causes
-failure of the total invariant.
-`,
-	RunE: runDisect,
-}
-
-var numGoroutines, numAccounts, numPrevious int
-var duration string
-var stopAll int32
-var mmap bool
-
-const keyPrefix = "account:"
-
-const initialBal uint64 = 100
-
-func init() {
-	RootCmd.AddCommand(testCmd)
-	testCmd.AddCommand(bankTest)
-	testCmd.AddCommand(bankDisect)
-
-	testCmd.Flags().IntVarP(
-		&numAccounts, "accounts", "a", 10000, "Number of accounts in the bank.")
-	bankTest.Flags().IntVarP(
-		&numGoroutines, "conc", "c", 16, "Number of concurrent transactions to run.")
-	bankTest.Flags().StringVarP(&duration, "duration", "d", "3m", "How long to run the test.")
-	bankTest.Flags().BoolVarP(&mmap, "mmap", "m", false, "If true, mmap LSM tree. Default is RAM.")
-	bankDisect.Flags().IntVarP(&numPrevious, "previous", "p", 12,
-		"Starting from the violation txn, how many previous versions to retrieve.")
-}
-
-func key(account int) []byte {
-	return []byte(fmt.Sprintf("%s%s", keyPrefix, strconv.Itoa(account)))
-}
-
-func toAccount(key []byte) int {
-	i, err := strconv.Atoi(string(key[len(keyPrefix):]))
-	y.Check(err)
-	return i
-}
-
-func toUint64(val []byte) uint64 {
-	u, err := strconv.ParseUint(string(val), 10, 64)
-	y.Check(err)
-	return uint64(u)
-}
-
-func toSlice(bal uint64) []byte {
-	return []byte(strconv.FormatUint(bal, 10))
-}
-
-func getBalance(txn *badger.Txn, account int) (uint64, error) {
-	item, err := txn.Get(key(account))
-	if err != nil {
-		return 0, err
-	}
-
-	var bal uint64
-	err = item.Value(func(v []byte) error {
-		bal = toUint64(v)
-		return nil
-	})
-	return bal, err
-}
-
-func putBalance(txn *badger.Txn, account int, bal uint64) error {
-	return txn.Set(key(account), toSlice(bal))
-}
-
-func min(a, b uint64) uint64 {
-	if a < b {
-		return a
-	}
-	return b
-}
-
-var errAbandoned = errors.New("Transaction abandonded due to insufficient balance")
-
-func moveMoney(db *badger.DB, from, to int) error {
-	return db.Update(func(txn *badger.Txn) error {
-		balf, err := getBalance(txn, from)
-		if err != nil {
-			return err
-		}
-		balt, err := getBalance(txn, to)
-		if err != nil {
-			return err
-		}
-
-		floor := min(balf, balt)
-		if floor < 5 {
-			return errAbandoned
-		}
-		// Move the money.
-		balf -= 5
-		balt += 5
-
-		if err = putBalance(txn, from, balf); err != nil {
-			return err
-		}
-		return putBalance(txn, to, balt)
-	})
-}
-
-type account struct {
-	Id  int
-	Bal uint64
-}
-
-func diff(a, b []account) string {
-	var buf bytes.Buffer
-	y.AssertTruef(len(a) == len(b), "len(a)=%d. len(b)=%d\n", len(a), len(b))
-	for i := range a {
-		ai := a[i]
-		bi := b[i]
-		if ai.Id != bi.Id || ai.Bal != bi.Bal {
-			buf.WriteString(fmt.Sprintf("Index: %d. Account [%+v] -> [%+v]\n", i, ai, bi))
-		}
-	}
-	return buf.String()
-}
-
-var errFailure = errors.New("Found an balance mismatch. Test failed.")
-
-// seekTotal retrives the total of all accounts by seeking for each account key.
-func seekTotal(txn *badger.Txn) ([]account, error) {
-	expected := uint64(numAccounts) * uint64(initialBal)
-	var accounts []account
-
-	var total uint64
-	for i := 0; i < numAccounts; i++ {
-		item, err := txn.Get(key(i))
-		if err != nil {
-			log.Printf("Error for account: %d. err=%v. key=%q\n", i, err, key(i))
-			return accounts, err
-		}
-		val, err := item.ValueCopy(nil)
-		if err != nil {
-			return accounts, err
-		}
-		acc := account{
-			Id:  i,
-			Bal: toUint64(val),
-		}
-		accounts = append(accounts, acc)
-		total += acc.Bal
-	}
-	if total != expected {
-		log.Printf("Balance did NOT match up. Expected: %d. Received: %d",
-			expected, total)
-		atomic.AddInt32(&stopAll, 1)
-		return accounts, errFailure
-	}
-	return accounts, nil
-}
-
-// Range is [lowTs, highTs).
-func findFirstInvalidTxn(db *badger.DB, lowTs, highTs uint64) uint64 {
-	checkAt := func(ts uint64) error {
-		txn := db.NewTransactionAt(ts, false)
-		_, err := seekTotal(txn)
-		txn.Discard()
-		return err
-	}
-
-	if highTs-lowTs < 1 {
-		log.Printf("Checking at lowTs: %d\n", lowTs)
-		err := checkAt(lowTs)
-		if err == errFailure {
-			fmt.Printf("Violation at ts: %d\n", lowTs)
-			return lowTs
-		} else if err != nil {
-			log.Printf("Error at lowTs: %d. Err=%v\n", lowTs, err)
-			return 0
-		}
-		fmt.Printf("No violation found at ts: %d\n", lowTs)
-		return 0
-	}
-
-	midTs := (lowTs + highTs) / 2
-	log.Println()
-	log.Printf("Checking. low=%d. high=%d. mid=%d\n", lowTs, highTs, midTs)
-	err := checkAt(midTs)
-	if err == badger.ErrKeyNotFound || err == nil {
-		// If no failure, move to higher ts.
-		return findFirstInvalidTxn(db, midTs+1, highTs)
-	}
-	// Found an error.
-	return findFirstInvalidTxn(db, lowTs, midTs)
-}
-
-func compareTwo(db *badger.DB, before, after uint64) {
-	fmt.Printf("Comparing @ts=%d with @ts=%d\n", before, after)
-	txn := db.NewTransactionAt(before, false)
-	prev, err := seekTotal(txn)
-	if err == errFailure {
-		// pass
-	} else {
-		y.Check(err)
-	}
-	txn.Discard()
-
-	txn = db.NewTransactionAt(after, false)
-	now, err := seekTotal(txn)
-	if err == errFailure {
-		// pass
-	} else {
-		y.Check(err)
-	}
-	txn.Discard()
-
-	fmt.Println(diff(prev, now))
-}
-
-func runDisect(cmd *cobra.Command, args []string) error {
-	opts := badger.DefaultOptions
-	opts.Dir = sstDir
-	opts.ValueDir = vlogDir
-	opts.ReadOnly = true
-
-	// The total did not match up. So, let's disect the DB to find the
-	// transction which caused the total mismatch.
-	db, err := badger.OpenManaged(opts)
-	if err != nil {
-		return err
-	}
-	fmt.Println("opened db")
-
-	var min, max uint64 = math.MaxUint64, 0
-	{
-		txn := db.NewTransactionAt(uint64(math.MaxUint32), false)
-		iopt := badger.DefaultIteratorOptions
-		iopt.AllVersions = true
-		itr := txn.NewIterator(iopt)
-		for itr.Rewind(); itr.Valid(); itr.Next() {
-			item := itr.Item()
-			if min > item.Version() {
-				min = item.Version()
-			}
-			if max < item.Version() {
-				max = item.Version()
-			}
-		}
-		itr.Close()
-		txn.Discard()
-	}
-
-	log.Printf("min=%d. max=%d\n", min, max)
-	ts := findFirstInvalidTxn(db, min, max)
-	fmt.Println()
-	if ts == 0 {
-		fmt.Println("Nothing found. Exiting.")
-		return nil
-	}
-
-	for i := 0; i < numPrevious; i++ {
-		compareTwo(db, ts-1-uint64(i), ts-uint64(i))
-	}
-	return nil
-}
-
-func runTest(cmd *cobra.Command, args []string) error {
-	rand.Seed(time.Now().UnixNano())
-
-	// Open DB
-	opts := badger.DefaultOptions
-	opts.Dir = sstDir
-	opts.ValueDir = vlogDir
-	opts.MaxTableSize = 4 << 20 // Force more compactions.
-	opts.NumLevelZeroTables = 2
-	opts.NumMemtables = 2
-	// Do not GC any versions, because we need them for the disect.
-	opts.NumVersionsToKeep = int(math.MaxInt32)
-	opts.ValueThreshold = 1 // Make all values go to value log.
-	if mmap {
-		opts.TableLoadingMode = options.MemoryMap
-	}
-	log.Printf("Opening DB with options: %+v\n", opts)
-
-	db, err := badger.Open(opts)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	wb := db.NewWriteBatch()
-	for i := 0; i < numAccounts; i++ {
-		y.Check(wb.Set(key(i), toSlice(initialBal), 0))
-	}
-	log.Println("Waiting for writes to be done...")
-	y.Check(wb.Flush())
-
-	log.Println("Bank initialization OK. Commencing test.")
-	log.Printf("Running with %d accounts, and %d goroutines.\n", numAccounts, numGoroutines)
-	log.Printf("Using keyPrefix: %s\n", keyPrefix)
-
-	dur, err := time.ParseDuration(duration)
-	y.Check(err)
-
-	// startTs := time.Now()
-	endTs := time.Now().Add(dur)
-	var total, errors, reads uint64
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		ticker := time.NewTicker(time.Second)
-		defer ticker.Stop()
-
-		for range ticker.C {
-			if atomic.LoadInt32(&stopAll) > 0 {
-				// Do not proceed.
-				return
-			}
-			// log.Printf("[%6s] Total: %d. Errors: %d Reads: %d.\n",
-			// 	time.Since(startTs).Round(time.Second).String(),
-			// 	atomic.LoadUint64(&total),
-			// 	atomic.LoadUint64(&errors),
-			// 	atomic.LoadUint64(&reads))
-			if time.Now().After(endTs) {
-				return
-			}
-		}
-	}()
-
-	// RW goroutines.
-	for i := 0; i < numGoroutines; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			ticker := time.NewTicker(10 * time.Microsecond)
-			defer ticker.Stop()
-
-			for range ticker.C {
-				if atomic.LoadInt32(&stopAll) > 0 {
-					// Do not proceed.
-					return
-				}
-				if time.Now().After(endTs) {
-					return
-				}
-				from := rand.Intn(numAccounts)
-				to := rand.Intn(numAccounts)
-				if from == to {
-					continue
-				}
-				err := moveMoney(db, from, to)
-				atomic.AddUint64(&total, 1)
-				if err == nil {
-					log.Printf("Moved $5. %d -> %d\n", from, to)
-				} else {
-					atomic.AddUint64(&errors, 1)
-				}
-			}
-		}()
-	}
-
-	// RO goroutine.
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-
-		ticker := time.NewTicker(10 * time.Microsecond)
-		defer ticker.Stop()
-
-		for range ticker.C {
-			if atomic.LoadInt32(&stopAll) > 0 {
-				// Do not proceed.
-				return
-			}
-			if time.Now().After(endTs) {
-				return
-			}
-
-			y.Check(db.View(func(txn *badger.Txn) error {
-				_, err := seekTotal(txn)
-				if err != nil {
-					log.Printf("Error while calculating total: %v", err)
-				} else {
-					atomic.AddUint64(&reads, 1)
-				}
-				return nil
-			}))
-		}
-	}()
-	wg.Wait()
-
-	if atomic.LoadInt32(&stopAll) == 0 {
-		log.Println("Test OK")
-		return nil
-	}
-	log.Println("Test FAILED")
-	return fmt.Errorf("Test FAILED")
-}

vendor/github.com/dgraph-io/badger/badger/cmd/fill.go

@@ -1,93 +0,0 @@
-/*
- * Copyright 2018 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/rand"
-	"time"
-
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra"
-)
-
-var fillCmd = &cobra.Command{
-	Use:   "fill",
-	Short: "Fill Badger with random data.",
-	Long: `
-This command would fill Badger with random data. Useful for testing and performance analysis.
-`,
-	RunE: fill,
-}
-
-var keySz, valSz int
-var numKeys float64
-var force bool
-
-const mil float64 = 1e6
-
-func init() {
-	RootCmd.AddCommand(fillCmd)
-	fillCmd.Flags().IntVarP(&keySz, "key-size", "k", 32, "Size of key")
-	fillCmd.Flags().IntVarP(&valSz, "val-size", "v", 128, "Size of value")
-	fillCmd.Flags().Float64VarP(&numKeys, "keys-mil", "m", 10.0,
-		"Number of keys to add in millions")
-	fillCmd.Flags().BoolVarP(&force, "force-compact", "f", true, "Force compact level 0 on close.")
-}
-
-func fill(cmd *cobra.Command, args []string) error {
-	opts := badger.DefaultOptions
-	opts.Dir = sstDir
-	opts.ValueDir = vlogDir
-	opts.Truncate = truncate
-	opts.SyncWrites = false
-	opts.CompactL0OnClose = force
-	opts.TableLoadingMode = options.FileIO
-	opts.ValueLogLoadingMode = options.FileIO
-
-	db, err := badger.Open(opts)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		start := time.Now()
-		err := db.Close()
-		opts.Infof("DB.Close. Error: %v. Time taken: %s", err, time.Since(start))
-	}()
-
-	start := time.Now()
-	batch := db.NewWriteBatch()
-	num := int64(numKeys * mil)
-	for i := int64(1); i <= num; i++ {
-		k := make([]byte, keySz)
-		v := make([]byte, valSz)
-		y.Check2(rand.Read(k))
-		y.Check2(rand.Read(v))
-		if err := batch.Set(k, v, 0); err != nil {
-			return err
-		}
-		if i%1e5 == 0 {
-			opts.Infof("Written keys: %d\n", i)
-		}
-	}
-	if err := batch.Flush(); err != nil {
-		return err
-	}
-	opts.Infof("%d keys written. Time taken: %s\n", num, time.Since(start))
-	return nil
-}

vendor/github.com/dgraph-io/badger/badger/cmd/flatten.go

@@ -1,56 +0,0 @@
-/*
- * Copyright 2018 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger"
-	"gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra"
-)
-
-var flattenCmd = &cobra.Command{
-	Use:   "flatten",
-	Short: "Flatten the LSM tree.",
-	Long: `
-This command would compact all the LSM tables into one level.
-`,
-	RunE: flatten,
-}
-
-var numWorkers int
-
-func init() {
-	RootCmd.AddCommand(flattenCmd)
-	flattenCmd.Flags().IntVarP(&numWorkers, "num-workers", "w", 1,
-		"Number of concurrent compactors to run. More compactors would use more"+
-			" server resources to potentially achieve faster compactions.")
-}
-
-func flatten(cmd *cobra.Command, args []string) error {
-	opts := badger.DefaultOptions
-	opts.Dir = sstDir
-	opts.ValueDir = vlogDir
-	opts.Truncate = truncate
-	opts.NumCompactors = 0
-
-	db, err := badger.Open(opts)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	return db.Flatten(numWorkers)
-}

vendor/github.com/dgraph-io/badger/badger/cmd/info.go

@@ -1,294 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-
-	humanize "gx/ipfs/QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy/go-humanize"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra"
-)
-
-var infoCmd = &cobra.Command{
-	Use:   "info",
-	Short: "Health info about Badger database.",
-	Long: `
-This command prints information about the badger key-value store.  It reads MANIFEST and prints its
-info. It also prints info about missing/extra files, and general information about the value log
-files (which are not referenced by the manifest).  Use this tool to report any issues about Badger
-to the Dgraph team.
-`,
-	Run: func(cmd *cobra.Command, args []string) {
-		err := printInfo(sstDir, vlogDir)
-		if err != nil {
-			fmt.Println("Error:", err.Error())
-			os.Exit(1)
-		}
-		if !showTables {
-			return
-		}
-		err = tableInfo(sstDir, vlogDir)
-		if err != nil {
-			fmt.Println("Error:", err.Error())
-			os.Exit(1)
-		}
-	},
-}
-
-var showTables bool
-
-func init() {
-	RootCmd.AddCommand(infoCmd)
-	infoCmd.Flags().BoolVarP(&showTables, "show-tables", "s", false,
-		"If set to true, show tables as well.")
-}
-
-func hbytes(sz int64) string {
-	return humanize.Bytes(uint64(sz))
-}
-
-func dur(src, dst time.Time) string {
-	return humanize.RelTime(dst, src, "earlier", "later")
-}
-
-func tableInfo(dir, valueDir string) error {
-	// Open DB
-	opts := badger.DefaultOptions
-	opts.TableLoadingMode = options.MemoryMap
-	opts.Dir = sstDir
-	opts.ValueDir = vlogDir
-	opts.ReadOnly = true
-
-	db, err := badger.Open(opts)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	tables := db.Tables()
-	for _, t := range tables {
-		lk, lv := y.ParseKey(t.Left), y.ParseTs(t.Left)
-		rk, rv := y.ParseKey(t.Right), y.ParseTs(t.Right)
-		fmt.Printf("SSTable [L%d, %03d] [%20X, v%-10d -> %20X, v%-10d]\n",
-			t.Level, t.ID, lk, lv, rk, rv)
-	}
-	return nil
-}
-
-func printInfo(dir, valueDir string) error {
-	if dir == "" {
-		return fmt.Errorf("--dir not supplied")
-	}
-	if valueDir == "" {
-		valueDir = dir
-	}
-	fp, err := os.Open(filepath.Join(dir, badger.ManifestFilename))
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if fp != nil {
-			fp.Close()
-		}
-	}()
-	manifest, truncOffset, err := badger.ReplayManifestFile(fp)
-	if err != nil {
-		return err
-	}
-	fp.Close()
-	fp = nil
-
-	fileinfos, err := ioutil.ReadDir(dir)
-	if err != nil {
-		return err
-	}
-	fileinfoByName := make(map[string]os.FileInfo)
-	fileinfoMarked := make(map[string]bool)
-	for _, info := range fileinfos {
-		fileinfoByName[info.Name()] = info
-		fileinfoMarked[info.Name()] = false
-	}
-
-	fmt.Println()
-	var baseTime time.Time
-	// fmt.Print("\n[Manifest]\n")
-	manifestTruncated := false
-	manifestInfo, ok := fileinfoByName[badger.ManifestFilename]
-	if ok {
-		fileinfoMarked[badger.ManifestFilename] = true
-		truncatedString := ""
-		if truncOffset != manifestInfo.Size() {
-			truncatedString = fmt.Sprintf(" [TRUNCATED to %d]", truncOffset)
-			manifestTruncated = true
-		}
-
-		baseTime = manifestInfo.ModTime()
-		fmt.Printf("[%25s] %-12s %6s MA%s\n", manifestInfo.ModTime().Format(time.RFC3339),
-			manifestInfo.Name(), hbytes(manifestInfo.Size()), truncatedString)
-	} else {
-		fmt.Printf("%s [MISSING]\n", manifestInfo.Name())
-	}
-
-	numMissing := 0
-	numEmpty := 0
-
-	levelSizes := make([]int64, len(manifest.Levels))
-	for level, lm := range manifest.Levels {
-		// fmt.Printf("\n[Level %d]\n", level)
-		// We create a sorted list of table ID's so that output is in consistent order.
-		tableIDs := make([]uint64, 0, len(lm.Tables))
-		for id := range lm.Tables {
-			tableIDs = append(tableIDs, id)
-		}
-		sort.Slice(tableIDs, func(i, j int) bool {
-			return tableIDs[i] < tableIDs[j]
-		})
-		for _, tableID := range tableIDs {
-			tableFile := table.IDToFilename(tableID)
-			tm, ok1 := manifest.Tables[tableID]
-			file, ok2 := fileinfoByName[tableFile]
-			if ok1 && ok2 {
-				fileinfoMarked[tableFile] = true
-				emptyString := ""
-				fileSize := file.Size()
-				if fileSize == 0 {
-					emptyString = " [EMPTY]"
-					numEmpty++
-				}
-				levelSizes[level] += fileSize
-				// (Put level on every line to make easier to process with sed/perl.)
-				fmt.Printf("[%25s] %-12s %6s L%d %x%s\n", dur(baseTime, file.ModTime()),
-					tableFile, hbytes(fileSize), level, tm.Checksum, emptyString)
-			} else {
-				fmt.Printf("%s [MISSING]\n", tableFile)
-				numMissing++
-			}
-		}
-	}
-
-	valueDirFileinfos := fileinfos
-	if valueDir != dir {
-		valueDirFileinfos, err = ioutil.ReadDir(valueDir)
-		if err != nil {
-			return err
-		}
-	}
-
-	// If valueDir is different from dir, holds extra files in the value dir.
-	valueDirExtras := []os.FileInfo{}
-
-	valueLogSize := int64(0)
-	// fmt.Print("\n[Value Log]\n")
-	for _, file := range valueDirFileinfos {
-		if !strings.HasSuffix(file.Name(), ".vlog") {
-			if valueDir != dir {
-				valueDirExtras = append(valueDirExtras, file)
-			}
-			continue
-		}
-
-		fileSize := file.Size()
-		emptyString := ""
-		if fileSize == 0 {
-			emptyString = " [EMPTY]"
-			numEmpty++
-		}
-		valueLogSize += fileSize
-		fmt.Printf("[%25s] %-12s %6s VL%s\n", dur(baseTime, file.ModTime()), file.Name(),
-			hbytes(fileSize), emptyString)
-
-		fileinfoMarked[file.Name()] = true
-	}
-
-	numExtra := 0
-	for _, file := range fileinfos {
-		if fileinfoMarked[file.Name()] {
-			continue
-		}
-		if numExtra == 0 {
-			fmt.Print("\n[EXTRA]\n")
-		}
-		fmt.Printf("[%s] %-12s %6s\n", file.ModTime().Format(time.RFC3339),
-			file.Name(), hbytes(file.Size()))
-		numExtra++
-	}
-
-	numValueDirExtra := 0
-	for _, file := range valueDirExtras {
-		if numValueDirExtra == 0 {
-			fmt.Print("\n[ValueDir EXTRA]\n")
-		}
-		fmt.Printf("[%s] %-12s %6s\n", file.ModTime().Format(time.RFC3339),
-			file.Name(), hbytes(file.Size()))
-		numValueDirExtra++
-	}
-
-	fmt.Print("\n[Summary]\n")
-	totalIndexSize := int64(0)
-	for i, sz := range levelSizes {
-		fmt.Printf("Level %d size: %12s\n", i, hbytes(sz))
-		totalIndexSize += sz
-	}
-
-	fmt.Printf("Total index size: %8s\n", hbytes(totalIndexSize))
-	fmt.Printf("Value log size: %10s\n", hbytes(valueLogSize))
-	fmt.Println()
-	totalExtra := numExtra + numValueDirExtra
-	if totalExtra == 0 && numMissing == 0 && numEmpty == 0 && !manifestTruncated {
-		fmt.Println("Abnormalities: None.")
-	} else {
-		fmt.Println("Abnormalities:")
-	}
-	fmt.Printf("%d extra %s.\n", totalExtra, pluralFiles(totalExtra))
-	fmt.Printf("%d missing %s.\n", numMissing, pluralFiles(numMissing))
-	fmt.Printf("%d empty %s.\n", numEmpty, pluralFiles(numEmpty))
-	fmt.Printf("%d truncated %s.\n", boolToNum(manifestTruncated),
-		pluralManifest(manifestTruncated))
-
-	return nil
-}
-
-func boolToNum(x bool) int {
-	if x {
-		return 1
-	}
-	return 0
-}
-
-func pluralManifest(manifestTruncated bool) string {
-	if manifestTruncated {
-		return "manifest"
-	}
-	return "manifests"
-}
-
-func pluralFiles(count int) string {
-	if count == 1 {
-		return "file"
-	}
-	return "files"
-}

vendor/github.com/dgraph-io/badger/badger/cmd/restore.go

@@ -1,81 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"errors"
-	"os"
-	"path"
-
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger"
-	"gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra"
-)
-
-var restoreFile string
-
-// restoreCmd represents the restore command
-var restoreCmd = &cobra.Command{
-	Use:   "restore",
-	Short: "Restore Badger database.",
-	Long: `Restore Badger database from a file.
-
-It reads a file generated using the backup command (or by calling the
-DB.Backup() API method) and writes each key-value pair found in the file to
-the Badger database.
-
-Restore creates a new database, and currently does not work on an already
-existing database.`,
-	RunE: doRestore,
-}
-
-func init() {
-	RootCmd.AddCommand(restoreCmd)
-	restoreCmd.Flags().StringVarP(&restoreFile, "backup-file", "f",
-		"badger.bak", "File to restore from")
-}
-
-func doRestore(cmd *cobra.Command, args []string) error {
-	// Check if the DB already exists
-	manifestFile := path.Join(sstDir, badger.ManifestFilename)
-	if _, err := os.Stat(manifestFile); err == nil { // No error. File already exists.
-		return errors.New("Cannot restore to an already existing database")
-	} else if os.IsNotExist(err) {
-		// pass
-	} else { // Return an error if anything other than the error above
-		return err
-	}
-
-	// Open DB
-	opts := badger.DefaultOptions
-	opts.Dir = sstDir
-	opts.ValueDir = vlogDir
-	db, err := badger.Open(opts)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	// Open File
-	f, err := os.Open(restoreFile)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	// Run restore
-	return db.Load(f)
-}

vendor/github.com/dgraph-io/badger/badger/cmd/root.go

@@ -1,65 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"strings"
-
-	"gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra"
-)
-
-var sstDir, vlogDir string
-
-// RootCmd represents the base command when called without any subcommands
-var RootCmd = &cobra.Command{
-	Use:               "badger",
-	Short:             "Tools to manage Badger database.",
-	PersistentPreRunE: validateRootCmdArgs,
-}
-
-// Execute adds all child commands to the root command and sets flags appropriately.
-// This is called by main.main(). It only needs to happen once to the rootCmd.
-func Execute() {
-	if err := RootCmd.Execute(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}
-
-func init() {
-	RootCmd.PersistentFlags().StringVar(&sstDir, "dir", "",
-		"Directory where the LSM tree files are located. (required)")
-
-	RootCmd.PersistentFlags().StringVar(&vlogDir, "vlog-dir", "",
-		"Directory where the value log files are located, if different from --dir")
-}
-
-func validateRootCmdArgs(cmd *cobra.Command, args []string) error {
-	if strings.HasPrefix(cmd.Use, "help ") { // No need to validate if it is help
-		return nil
-	}
-	if sstDir == "" {
-		return errors.New("--dir not specified")
-	}
-	if vlogDir == "" {
-		vlogDir = sstDir
-	}
-	return nil
-}

vendor/github.com/dgraph-io/badger/badger/main.go

@@ -1,42 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"fmt"
-	"net/http"
-	_ "net/http/pprof"
-	"runtime"
-
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/badger/cmd"
-)
-
-func main() {
-	go func() {
-		for i := 8080; i < 9080; i++ {
-			fmt.Printf("Listening for /debug HTTP requests at port: %d\n", i)
-			if err := http.ListenAndServe(fmt.Sprintf("localhost:%d", i), nil); err != nil {
-				fmt.Println("Port busy. Trying another one...")
-				continue
-
-			}
-		}
-	}()
-	runtime.SetBlockProfileRate(100)
-	runtime.GOMAXPROCS(128)
-	cmd.Execute()
-}

vendor/github.com/dgraph-io/badger/batch.go

@@ -19,15 +19,17 @@ package badger
 import (
 	"sync"
 	"time"
+
+	"github.com/dgraph-io/badger/y"
 )
 
 // WriteBatch holds the necessary info to perform batched writes.
 type WriteBatch struct {
 	sync.Mutex
-	txn *Txn
-	db  *DB
-	wg  sync.WaitGroup
-	err error
+	txn      *Txn
+	db       *DB
+	throttle *y.Throttle
+	err      error
 }
 
 // NewWriteBatch creates a new WriteBatch. This provides a way to conveniently do a lot of writes,
@@ -36,7 +38,18 @@ type WriteBatch struct {
 // creating and committing transactions. Due to the nature of SSI guaratees provided by Badger,
 // blind writes can never encounter transaction conflicts (ErrConflict).
 func (db *DB) NewWriteBatch() *WriteBatch {
-	return &WriteBatch{db: db, txn: db.newTransaction(true, true)}
+	return &WriteBatch{
+		db:       db,
+		txn:      db.newTransaction(true, true),
+		throttle: y.NewThrottle(16),
+	}
+}
+
+// SetMaxPendingTxns sets a limit on maximum number of pending transactions while writing batches.
+// This function should be called before using WriteBatch. Default value of MaxPendingTxns is
+// 16 to minimise memory usage.
+func (wb *WriteBatch) SetMaxPendingTxns(max int) {
+	wb.throttle = y.NewThrottle(max)
 }
 
 // Cancel function must be called if there's a chance that Flush might not get
@@ -47,13 +60,15 @@ func (db *DB) NewWriteBatch() *WriteBatch {
 //
 // Note that any committed writes would still go through despite calling Cancel.
 func (wb *WriteBatch) Cancel() {
-	wb.wg.Wait()
+	if err := wb.throttle.Finish(); err != nil {
+		wb.db.opt.Errorf("WatchBatch.Cancel error while finishing: %v", err)
+	}
 	wb.txn.Discard()
 }
 
 func (wb *WriteBatch) callback(err error) {
 	// sync.WaitGroup is thread-safe, so it doesn't need to be run inside wb.Lock.
-	defer wb.wg.Done()
+	defer wb.throttle.Done(err)
 	if err == nil {
 		return
 	}
@@ -123,9 +138,9 @@ func (wb *WriteBatch) commit() error {
 	if wb.err != nil {
 		return wb.err
 	}
-	// Get a new txn before we commit this one. So, the new txn doesn't need
-	// to wait for this one to commit.
-	wb.wg.Add(1)
+	if err := wb.throttle.Do(); err != nil {
+		return err
+	}
 	wb.txn.CommitWith(wb.callback)
 	wb.txn = wb.db.newTransaction(true, true)
 	wb.txn.readTs = 0 // We're not reading anything.
@@ -140,8 +155,10 @@ func (wb *WriteBatch) Flush() error {
 	wb.txn.Discard()
 	wb.Unlock()
 
-	wb.wg.Wait()
-	// Safe to access error without any synchronization here.
+	if err := wb.throttle.Finish(); err != nil {
+		return err
+	}
+
 	return wb.err
 }
 

vendor/github.com/dgraph-io/badger/batch_test.go

@@ -1,69 +0,0 @@
-/*
- * Copyright 2018 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestWriteBatch(t *testing.T) {
-	key := func(i int) []byte {
-		return []byte(fmt.Sprintf("%10d", i))
-	}
-	val := func(i int) []byte {
-		return []byte(fmt.Sprintf("%128d", i))
-	}
-
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		wb := db.NewWriteBatch()
-		defer wb.Cancel()
-
-		N, M := 50000, 1000
-		start := time.Now()
-
-		for i := 0; i < N; i++ {
-			require.NoError(t, wb.Set(key(i), val(i), 0))
-		}
-		for i := 0; i < M; i++ {
-			require.NoError(t, wb.Delete(key(i)))
-		}
-		require.NoError(t, wb.Flush())
-		t.Logf("Time taken for %d writes (w/ test options): %s\n", N+M, time.Since(start))
-
-		err := db.View(func(txn *Txn) error {
-			itr := txn.NewIterator(DefaultIteratorOptions)
-			defer itr.Close()
-
-			i := M
-			for itr.Rewind(); itr.Valid(); itr.Next() {
-				item := itr.Item()
-				require.Equal(t, string(key(i)), string(item.Key()))
-				valcopy, err := item.ValueCopy(nil)
-				require.NoError(t, err)
-				require.Equal(t, val(i), valcopy)
-				i++
-			}
-			require.Equal(t, N, i)
-			return nil
-		})
-		require.NoError(t, err)
-	})
-}

vendor/github.com/dgraph-io/badger/compaction.go

@@ -23,10 +23,10 @@ import (
 	"math"
 	"sync"
 
-	"gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace"
+	"golang.org/x/net/trace"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
+	"github.com/dgraph-io/badger/table"
+	"github.com/dgraph-io/badger/y"
 )
 
 type keyRange struct {
@@ -65,7 +65,9 @@ func (r keyRange) overlapsWith(dst keyRange) bool {
 }
 
 func getKeyRange(tables []*table.Table) keyRange {
-	y.AssertTrue(len(tables) > 0)
+	if len(tables) == 0 {
+		return keyRange{}
+	}
 	smallest := tables[0].Smallest()
 	biggest := tables[0].Biggest()
 	for i := 1; i < len(tables); i++ {
@@ -129,7 +131,7 @@ func (cs *compactStatus) toLog(tr trace.Trace) {
 
 	tr.LazyPrintf("Compaction status:")
 	for i, l := range cs.levels {
-		if len(l.debug()) == 0 {
+		if l.debug() == "" {
 			continue
 		}
 		tr.LazyPrintf("[%d] %s", i, l.debug())

vendor/github.com/dgraph-io/badger/contrib/cover.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-SRC="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."
-TMP=$(mktemp /tmp/badger-coverage-XXXXX.txt)
-
-BUILD=$1
-OUT=$2
-
-set -e
-
-pushd $SRC &> /dev/null
-
-# create coverage output
-echo 'mode: atomic' > $OUT
-for PKG in $(go list ./...|grep -v -E 'vendor'); do
-  go test -covermode=atomic -coverprofile=$TMP $PKG
-  tail -n +2 $TMP >> $OUT
-done
-
-# Another round of tests after turning off mmap
-go test -v -vlog_mmap=false github.com/dgraph-io/badger
-
-popd &> /dev/null

vendor/github.com/dgraph-io/badger/db.go

@@ -18,8 +18,11 @@ package badger
 
 import (
 	"bytes"
+	"context"
 	"encoding/binary"
+	"encoding/hex"
 	"expvar"
+	"io"
 	"math"
 	"os"
 	"path/filepath"
@@ -29,20 +32,22 @@ import (
 	"sync/atomic"
 	"time"
 
-	humanize "gx/ipfs/QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy/go-humanize"
-	"gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/skl"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/options"
+	"github.com/dgraph-io/badger/pb"
+	"github.com/dgraph-io/badger/skl"
+	"github.com/dgraph-io/badger/table"
+	"github.com/dgraph-io/badger/y"
+	humanize "github.com/dustin/go-humanize"
+	"github.com/pkg/errors"
+	"golang.org/x/net/trace"
 )
 
 var (
-	badgerPrefix = []byte("!badger!")     // Prefix for internal keys used by badger.
-	head         = []byte("!badger!head") // For storing value offset for replay.
-	txnKey       = []byte("!badger!txn")  // For indicating end of entries in txn.
-	badgerMove   = []byte("!badger!move") // For key-value pairs which got moved during GC.
+	badgerPrefix      = []byte("!badger!")        // Prefix for internal keys used by badger.
+	head              = []byte("!badger!head")    // For storing value offset for replay.
+	txnKey            = []byte("!badger!txn")     // For indicating end of entries in txn.
+	badgerMove        = []byte("!badger!move")    // For key-value pairs which got moved during GC.
+	lfDiscardStatsKey = []byte("!badger!discard") // For storing lfDiscardStats
 )
 
 type closers struct {
@@ -51,8 +56,11 @@ type closers struct {
 	memtable   *y.Closer
 	writes     *y.Closer
 	valueGC    *y.Closer
+	pub        *y.Closer
 }
 
+type callback func(kv *pb.KVList)
+
 // DB provides the various functions required to interact with Badger.
 // DB is thread-safe.
 type DB struct {
@@ -73,10 +81,18 @@ type DB struct {
 	vhead     valuePointer // less than or equal to a pointer to the last vlog value put into mt
 	writeCh   chan *request
 	flushChan chan flushTask // For flushing memtables.
+	closeOnce sync.Once      // For closing DB only once.
+
+	// Number of log rotates since the last memtable flush. We will access this field via atomic
+	// functions. Since we are not going to use any 64bit atomic functions, there is no need for
+	// 64 bit alignment of this struct(see #311).
+	logRotates int32
 
 	blockWrites int32
 
 	orc *oracle
+
+	pub *publisher
 }
 
 const (
@@ -125,9 +141,10 @@ func (db *DB) replayFunction() func(Entry, valuePointer) error {
 		}
 
 		v := y.ValueStruct{
-			Value:    nv,
-			Meta:     meta,
-			UserMeta: e.UserMeta,
+			Value:     nv,
+			Meta:      meta,
+			UserMeta:  e.UserMeta,
+			ExpiresAt: e.ExpiresAt,
 		}
 
 		if e.meta&bitFinTxn > 0 {
@@ -182,6 +199,8 @@ func Open(opt Options) (db *DB, err error) {
 	if opt.ReadOnly {
 		// Can't truncate if the DB is read only.
 		opt.Truncate = false
+		// Do not perform compaction in read only mode.
+		opt.CompactL0OnClose = false
 	}
 
 	for _, path := range []string{opt.Dir, opt.ValueDir} {
@@ -256,6 +275,7 @@ func Open(opt Options) (db *DB, err error) {
 		dirLockGuard:  dirLockGuard,
 		valueDirGuard: valueDirLockGuard,
 		orc:           newOracle(opt),
+		pub:           newPublisher(),
 	}
 
 	// Calculate initial size.
@@ -300,7 +320,10 @@ func Open(opt Options) (db *DB, err error) {
 	// Let's advance nextTxnTs to one more than whatever we observed via
 	// replaying the logs.
 	db.orc.txnMark.Done(db.orc.nextTxnTs)
-	db.orc.nextTxnTs++
+	// In normal mode, we must update readMark so older versions of keys can be removed during
+	// compaction when run in offline mode via the flatten tool.
+	db.orc.readMark.Done(db.orc.nextTxnTs)
+	db.orc.incrementNextTs()
 
 	db.writeCh = make(chan *request, kvWriteChCapacity)
 	db.closers.writes = y.NewCloser(1)
@@ -309,16 +332,26 @@ func Open(opt Options) (db *DB, err error) {
 	db.closers.valueGC = y.NewCloser(1)
 	go db.vlog.waitOnGC(db.closers.valueGC)
 
+	db.closers.pub = y.NewCloser(1)
+	go db.pub.listenForUpdates(db.closers.pub)
+
 	valueDirLockGuard = nil
 	dirLockGuard = nil
 	manifestFile = nil
 	return db, nil
 }
 
-// Close closes a DB. It's crucial to call it to ensure all the pending updates
-// make their way to disk. Calling DB.Close() multiple times is not safe and would
-// cause panic.
-func (db *DB) Close() (err error) {
+// Close closes a DB. It's crucial to call it to ensure all the pending updates make their way to
+// disk. Calling DB.Close() multiple times would still only close the DB once.
+func (db *DB) Close() error {
+	var err error
+	db.closeOnce.Do(func() {
+		err = db.close()
+	})
+	return err
+}
+
+func (db *DB) close() (err error) {
 	db.elog.Printf("Closing database")
 	atomic.StoreInt32(&db.blockWrites, 1)
 
@@ -328,8 +361,10 @@ func (db *DB) Close() (err error) {
 	// Stop writes next.
 	db.closers.writes.SignalAndWait()
 
+	db.closers.pub.SignalAndWait()
+
 	// Now close the value log.
-	if vlogErr := db.vlog.Close(); err == nil {
+	if vlogErr := db.vlog.Close(); vlogErr != nil {
 		err = errors.Wrap(vlogErr, "DB.Close")
 	}
 
@@ -346,7 +381,7 @@ func (db *DB) Close() (err error) {
 				defer db.Unlock()
 				y.AssertTrue(db.mt != nil)
 				select {
-				case db.flushChan <- flushTask{db.mt, db.vhead}:
+				case db.flushChan <- flushTask{mt: db.mt, vptr: db.vhead}:
 					db.imm = append(db.imm, db.mt) // Flusher will attempt to remove this from s.imm.
 					db.mt = nil                    // Will segfault if we try writing!
 					db.elog.Printf("pushed to flush chan\n")
@@ -369,10 +404,15 @@ func (db *DB) Close() (err error) {
 	// Force Compact L0
 	// We don't need to care about cstatus since no parallel compaction is running.
 	if db.opt.CompactL0OnClose {
-		if err := db.lc.doCompact(compactionPriority{level: 0, score: 1.73}); err != nil {
-			db.opt.Warningf("While forcing compaction on level 0: %v", err)
-		} else {
+		err := db.lc.doCompact(compactionPriority{level: 0, score: 1.73})
+		switch err {
+		case errFillTables:
+			// This error only means that there might be enough tables to do a compaction. So, we
+			// should not report it to the end user to avoid confusing them.
+		case nil:
 			db.opt.Infof("Force compaction on level 0 done")
+		default:
+			db.opt.Warningf("While forcing compaction on level 0: %v", err)
 		}
 	}
 
@@ -416,6 +456,12 @@ const (
 	lockFile = "LOCK"
 )
 
+// Sync syncs database content to disk. This function provides
+// more control to user to sync data whenever required.
+func (db *DB) Sync() error {
+	return db.vlog.sync(math.MaxUint32)
+}
+
 // When you create or delete a file, you have to ensure the directory entry for the file is synced
 // in order to guarantee the file is visible (if the system crashes).  (See the man page for fsync,
 // or see https://github.com/coreos/etcd/issues/6368 for an example.)
@@ -585,6 +631,8 @@ func (db *DB) writeRequests(reqs []*request) error {
 		return err
 	}
 
+	db.elog.Printf("Sending updates to subscribers")
+	db.pub.sendUpdates(reqs)
 	db.elog.Printf("Writing to memtable")
 	var count int
 	for _, b := range reqs {
@@ -593,7 +641,7 @@ func (db *DB) writeRequests(reqs []*request) error {
 		}
 		count += len(b.Entries)
 		var i uint64
-		for err := db.ensureRoomForWrite(); err == errNoRoom; err = db.ensureRoomForWrite() {
+		for err = db.ensureRoomForWrite(); err == errNoRoom; err = db.ensureRoomForWrite() {
 			i++
 			if i%100 == 0 {
 				db.elog.Printf("Making room for writes")
@@ -637,6 +685,8 @@ func (db *DB) sendToWriteCh(entries []*Entry) (*request, error) {
 	req.Entries = entries
 	req.Wg = sync.WaitGroup{}
 	req.Wg.Add(1)
+	req.IncrRef()     // for db write
+	req.IncrRef()     // for publisher updates
 	db.writeCh <- req // Handled in doWrites.
 	y.NumPuts.Add(int64(len(entries)))
 
@@ -741,21 +791,36 @@ func (db *DB) ensureRoomForWrite() error {
 	var err error
 	db.Lock()
 	defer db.Unlock()
-	if db.mt.MemSize() < db.opt.MaxTableSize {
+
+	// Here we determine if we need to force flush memtable. Given we rotated log file, it would
+	// make sense to force flush a memtable, so the updated value head would have a chance to be
+	// pushed to L0. Otherwise, it would not go to L0, until the memtable has been fully filled,
+	// which can take a lot longer if the write load has fewer keys and larger values. This force
+	// flush, thus avoids the need to read through a lot of log files on a crash and restart.
+	// Above approach is quite simple with small drawback. We are calling ensureRoomForWrite before
+	// inserting every entry in Memtable. We will get latest db.head after all entries for a request
+	// are inserted in Memtable. If we have done >= db.logRotates rotations, then while inserting
+	// first entry in Memtable, below condition will be true and we will endup flushing old value of
+	// db.head. Hence we are limiting no of value log files to be read to db.logRotates only.
+	forceFlush := atomic.LoadInt32(&db.logRotates) >= db.opt.LogRotatesToFlush
+
+	if !forceFlush && db.mt.MemSize() < db.opt.MaxTableSize {
 		return nil
 	}
 
 	y.AssertTrue(db.mt != nil) // A nil mt indicates that DB is being closed.
 	select {
-	case db.flushChan <- flushTask{db.mt, db.vhead}:
-		db.elog.Printf("Flushing value log to disk if async mode.")
+	case db.flushChan <- flushTask{mt: db.mt, vptr: db.vhead}:
+		// After every memtable flush, let's reset the counter.
+		atomic.StoreInt32(&db.logRotates, 0)
+
 		// Ensure value log is synced to disk so this memtable's contents wouldn't be lost.
-		err = db.vlog.sync()
+		err = db.vlog.sync(db.vhead.Fid)
 		if err != nil {
 			return err
 		}
 
-		db.elog.Printf("Flushing memtable, mt.size=%d size of flushChan: %d\n",
+		db.opt.Debugf("Flushing memtable, mt.size=%d size of flushChan: %d\n",
 			db.mt.MemSize(), len(db.flushChan))
 		// We manage to push this task. Let's modify imm.
 		db.imm = append(db.imm, db.mt)
@@ -773,12 +838,15 @@ func arenaSize(opt Options) int64 {
 }
 
 // WriteLevel0Table flushes memtable.
-func writeLevel0Table(s *skl.Skiplist, f *os.File) error {
-	iter := s.NewIterator()
+func writeLevel0Table(ft flushTask, f io.Writer) error {
+	iter := ft.mt.NewIterator()
 	defer iter.Close()
 	b := table.NewTableBuilder()
 	defer b.Close()
 	for iter.SeekToFirst(); iter.Valid(); iter.Next() {
+		if len(ft.dropPrefix) > 0 && bytes.HasPrefix(iter.Key(), ft.dropPrefix) {
+			continue
+		}
 		if err := b.Add(iter.Key(), iter.Value()); err != nil {
 			return err
 		}
@@ -788,25 +856,34 @@ func writeLevel0Table(s *skl.Skiplist, f *os.File) error {
 }
 
 type flushTask struct {
-	mt   *skl.Skiplist
-	vptr valuePointer
+	mt         *skl.Skiplist
+	vptr       valuePointer
+	dropPrefix []byte
 }
 
 // handleFlushTask must be run serially.
 func (db *DB) handleFlushTask(ft flushTask) error {
-	if !ft.mt.Empty() {
-		// Store badger head even if vptr is zero, need it for readTs
-		db.opt.Infof("Storing value log head: %+v\n", ft.vptr)
-		db.elog.Printf("Storing offset: %+v\n", ft.vptr)
-		offset := make([]byte, vptrSize)
-		ft.vptr.Encode(offset)
-
-		// Pick the max commit ts, so in case of crash, our read ts would be higher than all the
-		// commits.
-		headTs := y.KeyWithTs(head, db.orc.nextTs())
-		ft.mt.Put(headTs, y.ValueStruct{Value: offset})
+	// There can be a scnerio, when empty memtable is flushed. For example, memtable is empty and
+	// after writing request to value log, rotation count exceeds db.LogRotatesToFlush.
+	if ft.mt.Empty() {
+		return nil
 	}
 
+	// Store badger head even if vptr is zero, need it for readTs
+	db.opt.Debugf("Storing value log head: %+v\n", ft.vptr)
+	db.elog.Printf("Storing offset: %+v\n", ft.vptr)
+	offset := make([]byte, vptrSize)
+	ft.vptr.Encode(offset)
+
+	// Pick the max commit ts, so in case of crash, our read ts would be higher than all the
+	// commits.
+	headTs := y.KeyWithTs(head, db.orc.nextTs())
+	ft.mt.Put(headTs, y.ValueStruct{Value: offset})
+
+	// Also store lfDiscardStats before flushing memtables
+	discardStatsKey := y.KeyWithTs(lfDiscardStatsKey, 1)
+	ft.mt.Put(discardStatsKey, y.ValueStruct{Value: db.vlog.encodedDiscardStats()})
+
 	fileID := db.lc.reserveFileID()
 	fd, err := y.CreateSyncedFile(table.NewFilename(fileID, db.opt.Dir), true)
 	if err != nil {
@@ -817,7 +894,7 @@ func (db *DB) handleFlushTask(ft flushTask) error {
 	dirSyncCh := make(chan error)
 	go func() { dirSyncCh <- syncDir(db.opt.Dir) }()
 
-	err = writeLevel0Table(ft.mt, fd)
+	err = writeLevel0Table(ft, fd)
 	dirSyncErr := <-dirSyncCh
 
 	if err != nil {
@@ -837,22 +914,7 @@ func (db *DB) handleFlushTask(ft flushTask) error {
 	// We own a ref on tbl.
 	err = db.lc.addLevel0Table(tbl) // This will incrRef (if we don't error, sure)
 	tbl.DecrRef()                   // Releases our ref.
-	if err != nil {
-		return err
-	}
-
-	// Update s.imm. Need a lock.
-	db.Lock()
-	defer db.Unlock()
-	// This is a single-threaded operation. ft.mt corresponds to the head of
-	// db.imm list. Once we flush it, we advance db.imm. The next ft.mt
-	// which would arrive here would match db.imm[0], because we acquire a
-	// lock over DB when pushing to flushChan.
-	// TODO: This logic is dirty AF. Any change and this could easily break.
-	y.AssertTrue(ft.mt == db.imm[0])
-	db.imm = db.imm[1:]
-	ft.mt.DecrRef() // Return memory.
-	return nil
+	return err
 }
 
 // flushMemtable must keep running until we send it an empty flushTask. If there
@@ -868,6 +930,18 @@ func (db *DB) flushMemtable(lc *y.Closer) error {
 		for {
 			err := db.handleFlushTask(ft)
 			if err == nil {
+				// Update s.imm. Need a lock.
+				db.Lock()
+				// This is a single-threaded operation. ft.mt corresponds to the head of
+				// db.imm list. Once we flush it, we advance db.imm. The next ft.mt
+				// which would arrive here would match db.imm[0], because we acquire a
+				// lock over DB when pushing to flushChan.
+				// TODO: This logic is dirty AF. Any change and this could easily break.
+				y.AssertTrue(ft.mt == db.imm[0])
+				db.imm = db.imm[1:]
+				ft.mt.DecrRef() // Return memory.
+				db.Unlock()
+
 				break
 			}
 			// Encountered error. Retry indefinitely.
@@ -994,7 +1068,7 @@ func (db *DB) RunValueLogGC(discardRatio float64) error {
 
 // Size returns the size of lsm and value log files in bytes. It can be used to decide how often to
 // call RunValueLogGC.
-func (db *DB) Size() (lsm int64, vlog int64) {
+func (db *DB) Size() (lsm, vlog int64) {
 	if y.LSMSize.Get(db.opt.Dir) == nil {
 		lsm, vlog = 0, 0
 		return
@@ -1104,16 +1178,18 @@ func (db *DB) GetSequence(key []byte, bandwidth uint64) (*Sequence, error) {
 	return seq, err
 }
 
-// Tables gets the TableInfo objects from the level controller.
-func (db *DB) Tables() []TableInfo {
-	return db.lc.getTableInfo()
+// Tables gets the TableInfo objects from the level controller. If withKeysCount
+// is true, TableInfo objects also contain counts of keys for the tables.
+func (db *DB) Tables(withKeysCount bool) []TableInfo {
+	return db.lc.getTableInfo(withKeysCount)
 }
 
 // KeySplits can be used to get rough key ranges to divide up iteration over
 // the DB.
 func (db *DB) KeySplits(prefix []byte) []string {
 	var splits []string
-	for _, ti := range db.Tables() {
+	// We just want table ranges here and not keys count.
+	for _, ti := range db.Tables(false) {
 		// We don't use ti.Left, because that has a tendency to store !badger
 		// keys.
 		if bytes.HasPrefix(ti.Right, prefix) {
@@ -1230,6 +1306,32 @@ func (db *DB) Flatten(workers int) error {
 	}
 }
 
+func (db *DB) prepareToDrop() func() {
+	if db.opt.ReadOnly {
+		panic("Attempting to drop data in read-only mode.")
+	}
+	// Stop accepting new writes.
+	atomic.StoreInt32(&db.blockWrites, 1)
+
+	// Make all pending writes finish. The following will also close writeCh.
+	db.closers.writes.SignalAndWait()
+	db.opt.Infof("Writes flushed. Stopping compactions now...")
+
+	// Stop all compactions.
+	db.stopCompactions()
+	return func() {
+		db.opt.Infof("Resuming writes")
+		db.startCompactions()
+
+		db.writeCh = make(chan *request, kvWriteChCapacity)
+		db.closers.writes = y.NewCloser(1)
+		go db.doWrites(db.closers.writes)
+
+		// Resume writes.
+		atomic.StoreInt32(&db.blockWrites, 0)
+	}
+}
+
 // DropAll would drop all the data stored in Badger. It does this in the following way.
 // - Stop accepting new writes.
 // - Pause memtable flushes and compactions.
@@ -1242,31 +1344,20 @@ func (db *DB) Flatten(workers int) error {
 // any reads while DropAll is going on, otherwise they may result in panics. Ideally, both reads and
 // writes are paused before running DropAll, and resumed after it is finished.
 func (db *DB) DropAll() error {
-	if db.opt.ReadOnly {
-		panic("Attempting to drop data in read-only mode.")
+	f, err := db.dropAll()
+	if err != nil {
+		return err
 	}
+	if f == nil {
+		panic("both error and returned function cannot be nil in DropAll")
+	}
+	f()
+	return nil
+}
+
+func (db *DB) dropAll() (func(), error) {
 	db.opt.Infof("DropAll called. Blocking writes...")
-	// Stop accepting new writes.
-	atomic.StoreInt32(&db.blockWrites, 1)
-
-	// Make all pending writes finish. The following will also close writeCh.
-	db.closers.writes.SignalAndWait()
-	db.opt.Infof("Writes flushed. Stopping compactions now...")
-
-	// Stop all compactions.
-	db.stopCompactions()
-	defer func() {
-		db.opt.Infof("Resuming writes")
-		db.startCompactions()
-
-		db.writeCh = make(chan *request, kvWriteChCapacity)
-		db.closers.writes = y.NewCloser(1)
-		go db.doWrites(db.closers.writes)
-
-		// Resume writes.
-		atomic.StoreInt32(&db.blockWrites, 0)
-	}()
-	db.opt.Infof("Compactions stopped. Dropping all SSTables...")
+	f := db.prepareToDrop()
 
 	// Block all foreign interactions with memory tables.
 	db.Lock()
@@ -1274,23 +1365,115 @@ func (db *DB) DropAll() error {
 
 	// Remove inmemory tables. Calling DecrRef for safety. Not sure if they're absolutely needed.
 	db.mt.DecrRef()
-	db.mt = skl.NewSkiplist(arenaSize(db.opt)) // Set it up for future writes.
 	for _, mt := range db.imm {
 		mt.DecrRef()
 	}
 	db.imm = db.imm[:0]
+	db.mt = skl.NewSkiplist(arenaSize(db.opt)) // Set it up for future writes.
 
-	num, err := db.lc.deleteLSMTree()
+	num, err := db.lc.dropTree()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	db.opt.Infof("Deleted %d SSTables. Now deleting value logs...\n", num)
 
 	num, err = db.vlog.dropAll()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	db.vhead = valuePointer{} // Zero it out.
+	db.lc.nextFileID = 1
 	db.opt.Infof("Deleted %d value log files. DropAll done.\n", num)
+	return f, nil
+}
+
+// DropPrefix would drop all the keys with the provided prefix. It does this in the following way:
+// - Stop accepting new writes.
+// - Stop memtable flushes and compactions.
+// - Flush out all memtables, skipping over keys with the given prefix, Kp.
+// - Write out the value log header to memtables when flushing, so we don't accidentally bring Kp
+//   back after a restart.
+// - Compact L0->L1, skipping over Kp.
+// - Compact rest of the levels, Li->Li, picking tables which have Kp.
+// - Resume memtable flushes, compactions and writes.
+func (db *DB) DropPrefix(prefix []byte) error {
+	db.opt.Infof("DropPrefix called on %s. Blocking writes...", hex.Dump(prefix))
+	f := db.prepareToDrop()
+	defer f()
+
+	// Block all foreign interactions with memory tables.
+	db.Lock()
+	defer db.Unlock()
+
+	db.imm = append(db.imm, db.mt)
+	for _, memtable := range db.imm {
+		if memtable.Empty() {
+			memtable.DecrRef()
+			continue
+		}
+		task := flushTask{
+			mt: memtable,
+			// Ensure that the head of value log gets persisted to disk.
+			vptr:       db.vhead,
+			dropPrefix: prefix,
+		}
+		db.opt.Debugf("Flushing memtable")
+		if err := db.handleFlushTask(task); err != nil {
+			db.opt.Errorf("While trying to flush memtable: %v", err)
+			return err
+		}
+		memtable.DecrRef()
+	}
+	db.imm = db.imm[:0]
+	db.mt = skl.NewSkiplist(arenaSize(db.opt))
+
+	// Drop prefixes from the levels.
+	if err := db.lc.dropPrefix(prefix); err != nil {
+		return err
+	}
+	db.opt.Infof("DropPrefix done")
 	return nil
 }
+
+// Subscribe can be used watch key changes for the given key prefix.
+func (db *DB) Subscribe(ctx context.Context, cb callback, prefix []byte, prefixes ...[]byte) error {
+	if cb == nil {
+		return ErrNilCallback
+	}
+	prefixes = append(prefixes, prefix)
+	c := y.NewCloser(1)
+	recvCh, id := db.pub.newSubscriber(c, prefixes...)
+	slurp := func(batch *pb.KVList) {
+		defer func() {
+			if len(batch.GetKv()) > 0 {
+				cb(batch)
+			}
+		}()
+		for {
+			select {
+			case kvs := <-recvCh:
+				batch.Kv = append(batch.Kv, kvs.Kv...)
+			default:
+				return
+			}
+		}
+	}
+	for {
+		select {
+		case <-c.HasBeenClosed():
+			slurp(new(pb.KVList))
+			// Drain if any pending updates.
+			c.Done()
+			// No need to delete here. Closer will be called only while
+			// closing DB. Subscriber will be deleted by cleanSubscribers.
+			return nil
+		case <-ctx.Done():
+			c.Done()
+			db.pub.deleteSubscriber(id)
+			// Delete the subscriber to avoid further updates.
+			return ctx.Err()
+		case batch := <-recvCh:
+			slurp(batch)
+		}
+	}
+}

vendor/github.com/dgraph-io/badger/db2_test.go

@@ -1,325 +0,0 @@
-/*
- * Copyright 2018 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"math/rand"
-	"os"
-	"path"
-	"regexp"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestTruncateVlogWithClose(t *testing.T) {
-	key := func(i int) []byte {
-		return []byte(fmt.Sprintf("%d%10d", i, i))
-	}
-	data := func(l int) []byte {
-		m := make([]byte, l)
-		_, err := rand.Read(m)
-		require.NoError(t, err)
-		return m
-	}
-
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	opt := getTestOptions(dir)
-	opt.SyncWrites = true
-	opt.Truncate = true
-	opt.ValueThreshold = 1 // Force all reads from value log.
-
-	db, err := Open(opt)
-	require.NoError(t, err)
-
-	err = db.Update(func(txn *Txn) error {
-		return txn.Set(key(0), data(4055))
-	})
-	require.NoError(t, err)
-
-	// Close the DB.
-	require.NoError(t, db.Close())
-	require.NoError(t, os.Truncate(path.Join(dir, "000000.vlog"), 4096))
-
-	// Reopen and write some new data.
-	db, err = Open(opt)
-	require.NoError(t, err)
-	for i := 0; i < 32; i++ {
-		err := db.Update(func(txn *Txn) error {
-			return txn.Set(key(i), data(10))
-		})
-		require.NoError(t, err)
-	}
-	// Read it back to ensure that we can read it now.
-	for i := 0; i < 32; i++ {
-		err := db.View(func(txn *Txn) error {
-			item, err := txn.Get(key(i))
-			require.NoError(t, err)
-			val := getItemValue(t, item)
-			require.Equal(t, 10, len(val))
-			return nil
-		})
-		require.NoError(t, err)
-	}
-	require.NoError(t, db.Close())
-
-	// Reopen and read the data again.
-	db, err = Open(opt)
-	require.NoError(t, err)
-	for i := 0; i < 32; i++ {
-		err := db.View(func(txn *Txn) error {
-			item, err := txn.Get(key(i))
-			require.NoError(t, err)
-			val := getItemValue(t, item)
-			require.Equal(t, 10, len(val))
-			return nil
-		})
-		require.NoError(t, err)
-	}
-	require.NoError(t, db.Close())
-}
-
-var manual = flag.Bool("manual", false, "Set when manually running some tests.")
-
-// The following 3 TruncateVlogNoClose tests should be run one after another.
-// None of these close the DB, simulating a crash. They should be run with a
-// script, which truncates the value log to 4096, lining up with the end of the
-// first entry in the txn. At <4096, it would cause the entry to be truncated
-// immediately, at >4096, same thing.
-func TestTruncateVlogNoClose(t *testing.T) {
-	if !*manual {
-		t.Skip("Skipping test meant to be run manually.")
-		return
-	}
-	fmt.Println("running")
-	dir := "p"
-	opts := getTestOptions(dir)
-	opts.SyncWrites = true
-	opts.Truncate = true
-
-	kv, err := Open(opts)
-	require.NoError(t, err)
-	key := func(i int) string {
-		return fmt.Sprintf("%d%10d", i, i)
-	}
-	data := fmt.Sprintf("%4055d", 1)
-	err = kv.Update(func(txn *Txn) error {
-		return txn.Set([]byte(key(0)), []byte(data))
-	})
-	require.NoError(t, err)
-}
-func TestTruncateVlogNoClose2(t *testing.T) {
-	if !*manual {
-		t.Skip("Skipping test meant to be run manually.")
-		return
-	}
-	dir := "p"
-	opts := getTestOptions(dir)
-	opts.SyncWrites = true
-	opts.Truncate = true
-
-	kv, err := Open(opts)
-	require.NoError(t, err)
-	key := func(i int) string {
-		return fmt.Sprintf("%d%10d", i, i)
-	}
-	data := fmt.Sprintf("%10d", 1)
-	for i := 32; i < 64; i++ {
-		err := kv.Update(func(txn *Txn) error {
-			return txn.Set([]byte(key(i)), []byte(data))
-		})
-		require.NoError(t, err)
-	}
-	for i := 32; i < 64; i++ {
-		require.NoError(t, kv.View(func(txn *Txn) error {
-			item, err := txn.Get([]byte(key(i)))
-			require.NoError(t, err)
-			val := getItemValue(t, item)
-			require.NotNil(t, val)
-			require.True(t, len(val) > 0)
-			return nil
-		}))
-	}
-}
-func TestTruncateVlogNoClose3(t *testing.T) {
-	if !*manual {
-		t.Skip("Skipping test meant to be run manually.")
-		return
-	}
-	fmt.Print("Running")
-	dir := "p"
-	opts := getTestOptions(dir)
-	opts.SyncWrites = true
-	opts.Truncate = true
-
-	kv, err := Open(opts)
-	require.NoError(t, err)
-	key := func(i int) string {
-		return fmt.Sprintf("%d%10d", i, i)
-	}
-	for i := 32; i < 64; i++ {
-		require.NoError(t, kv.View(func(txn *Txn) error {
-			item, err := txn.Get([]byte(key(i)))
-			require.NoError(t, err)
-			val := getItemValue(t, item)
-			require.NotNil(t, val)
-			require.True(t, len(val) > 0)
-			return nil
-		}))
-	}
-}
-
-func TestBigKeyValuePairs(t *testing.T) {
-	// This test takes too much memory. So, run separately.
-	if !*manual {
-		t.Skip("Skipping test meant to be run manually.")
-		return
-	}
-	opts := DefaultOptions
-	opts.MaxTableSize = 1 << 20
-	opts.ValueLogMaxEntries = 64
-	runBadgerTest(t, &opts, func(t *testing.T, db *DB) {
-		bigK := make([]byte, 65001)
-		bigV := make([]byte, db.opt.ValueLogFileSize+1)
-		small := make([]byte, 65000)
-
-		txn := db.NewTransaction(true)
-		require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, small))
-		require.Regexp(t, regexp.MustCompile("Value.*exceeded"), txn.Set(small, bigV))
-
-		require.NoError(t, txn.Set(small, small))
-		require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, bigV))
-
-		require.NoError(t, db.View(func(txn *Txn) error {
-			_, err := txn.Get(small)
-			require.Equal(t, ErrKeyNotFound, err)
-			return nil
-		}))
-
-		// Now run a longer test, which involves value log GC.
-		data := fmt.Sprintf("%100d", 1)
-		key := func(i int) string {
-			return fmt.Sprintf("%65000d", i)
-		}
-
-		saveByKey := func(key string, value []byte) error {
-			return db.Update(func(txn *Txn) error {
-				return txn.Set([]byte(key), value)
-			})
-		}
-
-		getByKey := func(key string) error {
-			return db.View(func(txn *Txn) error {
-				item, err := txn.Get([]byte(key))
-				if err != nil {
-					return err
-				}
-				return item.Value(func(val []byte) error {
-					if len(val) == 0 {
-						log.Fatalf("key not found %q", len(key))
-					}
-					return nil
-				})
-			})
-		}
-
-		for i := 0; i < 32; i++ {
-			if i < 30 {
-				require.NoError(t, saveByKey(key(i), []byte(data)))
-			} else {
-				require.NoError(t, saveByKey(key(i), []byte(fmt.Sprintf("%100d", i))))
-			}
-		}
-
-		for j := 0; j < 5; j++ {
-			for i := 0; i < 32; i++ {
-				if i < 30 {
-					require.NoError(t, saveByKey(key(i), []byte(data)))
-				} else {
-					require.NoError(t, saveByKey(key(i), []byte(fmt.Sprintf("%100d", i))))
-				}
-			}
-		}
-
-		for i := 0; i < 32; i++ {
-			require.NoError(t, getByKey(key(i)))
-		}
-
-		var loops int
-		var err error
-		for err == nil {
-			err = db.RunValueLogGC(0.5)
-			require.NotRegexp(t, regexp.MustCompile("truncate"), err)
-			loops++
-		}
-		t.Logf("Ran value log GC %d times. Last error: %v\n", loops, err)
-	})
-}
-
-// The following test checks for issue #585.
-func TestPushValueLogLimit(t *testing.T) {
-	// This test takes too much memory. So, run separately.
-	if !*manual {
-		t.Skip("Skipping test meant to be run manually.")
-		return
-	}
-	opt := DefaultOptions
-	opt.ValueLogMaxEntries = 64
-	opt.ValueLogFileSize = 2 << 30
-	runBadgerTest(t, &opt, func(t *testing.T, db *DB) {
-		data := []byte(fmt.Sprintf("%30d", 1))
-		key := func(i int) string {
-			return fmt.Sprintf("%100d", i)
-		}
-
-		for i := 0; i < 32; i++ {
-			if i == 4 {
-				v := make([]byte, 2<<30)
-				err := db.Update(func(txn *Txn) error {
-					return txn.Set([]byte(key(i)), v)
-				})
-				require.NoError(t, err)
-			} else {
-				err := db.Update(func(txn *Txn) error {
-					return txn.Set([]byte(key(i)), data)
-				})
-				require.NoError(t, err)
-			}
-		}
-
-		for i := 0; i < 32; i++ {
-			err := db.View(func(txn *Txn) error {
-				item, err := txn.Get([]byte(key(i)))
-				require.NoError(t, err, "Getting key: %s", key(i))
-				err = item.Value(func(v []byte) error {
-					_ = v
-					return nil
-				})
-				require.NoError(t, err, "Getting value: %s", key(i))
-				return nil
-			})
-			require.NoError(t, err)
-		}
-	})
-}

vendor/github.com/dgraph-io/badger/dir_unix.go

@@ -24,8 +24,8 @@ import (
 	"os"
 	"path/filepath"
 
-	"gx/ipfs/QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu/sys/unix"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 )
 
 // directoryLockGuard holds a lock on a directory and a pid file inside.  The pid file isn't part

vendor/github.com/dgraph-io/badger/dir_windows.go

@@ -24,7 +24,7 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/pkg/errors"
 )
 
 // FILE_ATTRIBUTE_TEMPORARY - A file that is being used for temporary storage.

vendor/github.com/dgraph-io/badger/errors.go

@@ -17,7 +17,7 @@
 package badger
 
 import (
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/pkg/errors"
 )
 
 var (
@@ -102,4 +102,7 @@ var (
 	// ErrBlockedWrites is returned if the user called DropAll. During the process of dropping all
 	// data from Badger, we stop accepting new writes, by returning this error.
 	ErrBlockedWrites = errors.New("Writes are blocked, possibly due to DropAll or Close")
+
+	// ErrNilCallback is returned when subscriber's callback is nil.
+	ErrNilCallback = errors.New("Callback cannot be nil")
 )

vendor/github.com/dgraph-io/badger/histogram.go

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2019 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package badger
+
+import (
+	"fmt"
+	"math"
+)
+
+// PrintHistogram builds and displays the key-value size histogram.
+// When keyPrefix is set, only the keys that have prefix "keyPrefix" are
+// considered for creating the histogram
+func (db *DB) PrintHistogram(keyPrefix []byte) {
+	if db == nil {
+		fmt.Println("\nCannot build histogram: DB is nil.")
+		return
+	}
+	histogram := db.buildHistogram(keyPrefix)
+	fmt.Printf("Histogram of key sizes (in bytes)\n")
+	histogram.keySizeHistogram.printHistogram()
+	fmt.Printf("Histogram of value sizes (in bytes)\n")
+	histogram.valueSizeHistogram.printHistogram()
+}
+
+// histogramData stores information about a histogram
+type histogramData struct {
+	bins        []int64
+	countPerBin []int64
+	totalCount  int64
+	min         int64
+	max         int64
+	sum         int64
+}
+
+// sizeHistogram contains keySize histogram and valueSize histogram
+type sizeHistogram struct {
+	keySizeHistogram, valueSizeHistogram histogramData
+}
+
+// newSizeHistogram returns a new instance of keyValueSizeHistogram with
+// properly initialized fields.
+func newSizeHistogram() *sizeHistogram {
+	// TODO(ibrahim): find appropriate bin size.
+	keyBins := createHistogramBins(1, 16)
+	valueBins := createHistogramBins(1, 30)
+	return &sizeHistogram{
+		keySizeHistogram: histogramData{
+			bins:        keyBins,
+			countPerBin: make([]int64, len(keyBins)+1),
+			max:         math.MinInt64,
+			min:         math.MaxInt64,
+			sum:         0,
+		},
+		valueSizeHistogram: histogramData{
+			bins:        valueBins,
+			countPerBin: make([]int64, len(valueBins)+1),
+			max:         math.MinInt64,
+			min:         math.MaxInt64,
+			sum:         0,
+		},
+	}
+}
+
+// createHistogramBins creates bins for an histogram. The bin sizes are powers
+// of two of the form [2^min_exponent, ..., 2^max_exponent].
+func createHistogramBins(minExponent, maxExponent uint32) []int64 {
+	var bins []int64
+	for i := minExponent; i <= maxExponent; i++ {
+		bins = append(bins, int64(1)<<i)
+	}
+	return bins
+}
+
+// Update the min and max fields if value is less than or greater than the
+// current min/max value.
+func (histogram *histogramData) Update(value int64) {
+	if value > histogram.max {
+		histogram.max = value
+	}
+	if value < histogram.min {
+		histogram.min = value
+	}
+
+	histogram.sum += value
+	histogram.totalCount++
+
+	for index := 0; index <= len(histogram.bins); index++ {
+		// Allocate value in the last buckets if we reached the end of the Bounds array.
+		if index == len(histogram.bins) {
+			histogram.countPerBin[index]++
+			break
+		}
+
+		// Check if the value should be added to the "index" bin
+		if value < int64(histogram.bins[index]) {
+			histogram.countPerBin[index]++
+			break
+		}
+	}
+}
+
+// buildHistogram builds the key-value size histogram.
+// When keyPrefix is set, only the keys that have prefix "keyPrefix" are
+// considered for creating the histogram
+func (db *DB) buildHistogram(keyPrefix []byte) *sizeHistogram {
+	txn := db.NewTransaction(false)
+	defer txn.Discard()
+
+	itr := txn.NewIterator(DefaultIteratorOptions)
+	defer itr.Close()
+
+	badgerHistogram := newSizeHistogram()
+
+	// Collect key and value sizes.
+	for itr.Seek(keyPrefix); itr.ValidForPrefix(keyPrefix); itr.Next() {
+		item := itr.Item()
+		badgerHistogram.keySizeHistogram.Update(item.KeySize())
+		badgerHistogram.valueSizeHistogram.Update(item.ValueSize())
+	}
+	return badgerHistogram
+}
+
+// printHistogram prints the histogram data in a human-readable format.
+func (histogram histogramData) printHistogram() {
+	fmt.Printf("Total count: %d\n", histogram.totalCount)
+	fmt.Printf("Min value: %d\n", histogram.min)
+	fmt.Printf("Max value: %d\n", histogram.max)
+	fmt.Printf("Mean: %.2f\n", float64(histogram.sum)/float64(histogram.totalCount))
+	fmt.Printf("%24s %9s\n", "Range", "Count")
+
+	numBins := len(histogram.bins)
+	for index, count := range histogram.countPerBin {
+		if count == 0 {
+			continue
+		}
+
+		// The last bin represents the bin that contains the range from
+		// the last bin up to infinity so it's processed differently than the
+		// other bins.
+		if index == len(histogram.countPerBin)-1 {
+			lowerBound := int(histogram.bins[numBins-1])
+			fmt.Printf("[%10d, %10s) %9d\n", lowerBound, "infinity", count)
+			continue
+		}
+
+		upperBound := int(histogram.bins[index])
+		lowerBound := 0
+		if index > 0 {
+			lowerBound = int(histogram.bins[index-1])
+		}
+
+		fmt.Printf("[%10d, %10d) %9d\n", lowerBound, upperBound, count)
+	}
+	fmt.Println()
+}

vendor/github.com/dgraph-io/badger/integration/testgc/.gitignore

@@ -1 +0,0 @@
-/testgc

vendor/github.com/dgraph-io/badger/integration/testgc/main.go

@@ -1,218 +0,0 @@
-package main
-
-import (
-	"encoding/binary"
-	"fmt"
-	"log"
-	"math/rand"
-	"net/http"
-	_ "net/http/pprof"
-	"os"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-)
-
-var maxValue int64 = 10000000
-var suffix = make([]byte, 128)
-
-type testSuite struct {
-	sync.Mutex
-	vals map[uint64]uint64
-
-	count uint64 // Not under mutex lock.
-}
-
-func encoded(i uint64) []byte {
-	out := make([]byte, 8)
-	binary.BigEndian.PutUint64(out, i)
-	return out
-}
-
-func (s *testSuite) write(db *badger.DB) error {
-	return db.Update(func(txn *badger.Txn) error {
-		for i := 0; i < 10; i++ {
-			// These keys would be overwritten.
-			keyi := uint64(rand.Int63n(maxValue))
-			key := encoded(keyi)
-			vali := atomic.AddUint64(&s.count, 1)
-			val := encoded(vali)
-			val = append(val, suffix...)
-			if err := txn.Set(key, val); err != nil {
-				return err
-			}
-		}
-		for i := 0; i < 20; i++ {
-			// These keys would be new and never overwritten.
-			keyi := atomic.AddUint64(&s.count, 1)
-			if keyi%1000000 == 0 {
-				log.Printf("Count: %d\n", keyi)
-			}
-			key := encoded(keyi)
-			val := append(key, suffix...)
-			if err := txn.Set(key, val); err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-}
-
-func (s *testSuite) read(db *badger.DB) error {
-	max := int64(atomic.LoadUint64(&s.count))
-	keyi := uint64(rand.Int63n(max))
-	key := encoded(keyi)
-
-	err := db.View(func(txn *badger.Txn) error {
-		item, err := txn.Get(key)
-		if err != nil {
-			return err
-		}
-		val, err := item.ValueCopy(nil)
-		if err != nil {
-			return err
-		}
-		y.AssertTruef(len(val) == len(suffix)+8, "Found val of len: %d\n", len(val))
-		vali := binary.BigEndian.Uint64(val[0:8])
-		s.Lock()
-		expected := s.vals[keyi]
-		if vali < expected {
-			log.Fatalf("Expected: %d. Found: %d. Key: %d\n", expected, vali, keyi)
-		} else if vali == expected {
-			// pass
-		} else {
-			s.vals[keyi] = vali
-		}
-		s.Unlock()
-		return nil
-	})
-	if err == badger.ErrKeyNotFound {
-		return nil
-	}
-	return err
-}
-
-func main() {
-	fmt.Println("Badger Integration test for value log GC.")
-
-	dir := "/mnt/drive/badgertest"
-	os.RemoveAll(dir)
-
-	opts := badger.DefaultOptions
-	opts.Dir = dir
-	opts.ValueDir = dir
-	opts.TableLoadingMode = options.MemoryMap
-	opts.ValueLogLoadingMode = options.FileIO
-	opts.SyncWrites = false
-
-	db, err := badger.Open(opts)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer db.Close()
-
-	go http.ListenAndServe("localhost:8080", nil)
-
-	closer := y.NewCloser(11)
-	go func() {
-		// Run value log GC.
-		defer closer.Done()
-		var count int
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-		for range ticker.C {
-		again:
-			select {
-			case <-closer.HasBeenClosed():
-				log.Printf("Num times value log GC was successful: %d\n", count)
-				return
-			default:
-			}
-			log.Printf("Starting a value log GC")
-			err := db.RunValueLogGC(0.1)
-			log.Printf("Result of value log GC: %v\n", err)
-			if err == nil {
-				count++
-				goto again
-			}
-		}
-	}()
-
-	s := testSuite{
-		count: uint64(maxValue),
-		vals:  make(map[uint64]uint64),
-	}
-	var numLoops uint64
-	ticker := time.NewTicker(5 * time.Second)
-	for i := 0; i < 10; i++ {
-		go func() {
-			defer closer.Done()
-			for {
-				if err := s.write(db); err != nil {
-					log.Fatal(err)
-				}
-				for j := 0; j < 10; j++ {
-					if err := s.read(db); err != nil {
-						log.Fatal(err)
-					}
-				}
-				nl := atomic.AddUint64(&numLoops, 1)
-				select {
-				case <-closer.HasBeenClosed():
-					return
-				case <-ticker.C:
-					log.Printf("Num loops: %d\n", nl)
-				default:
-				}
-			}
-		}()
-	}
-	time.Sleep(5 * time.Minute)
-	log.Println("Signaling...")
-	closer.SignalAndWait()
-	log.Println("Wait done. Now iterating over everything.")
-
-	err = db.View(func(txn *badger.Txn) error {
-		iopts := badger.DefaultIteratorOptions
-		itr := txn.NewIterator(iopts)
-		defer itr.Close()
-
-		var total, tested int
-		for itr.Rewind(); itr.Valid(); itr.Next() {
-			item := itr.Item()
-			key := item.Key()
-			keyi := binary.BigEndian.Uint64(key)
-			total++
-
-			val, err := item.ValueCopy(nil)
-			if err != nil {
-				return err
-			}
-			if len(val) < 8 {
-				log.Printf("Unexpected value: %x\n", val)
-				continue
-			}
-			vali := binary.BigEndian.Uint64(val[0:8])
-
-			expected, ok := s.vals[keyi] // Not all keys must be in vals map.
-			if ok {
-				tested++
-				if vali < expected {
-					// vali must be equal or greater than what's in the map.
-					log.Fatalf("Expected: %d. Got: %d. Key: %d\n", expected, vali, keyi)
-				}
-			}
-		}
-		log.Printf("Total iterated: %d. Tested values: %d\n", total, tested)
-		return nil
-	})
-	if err != nil {
-		log.Fatalf("Error while iterating: %v", err)
-	}
-	log.Println("Iteration done. Test successful.")
-	time.Sleep(time.Minute) // Time to do some poking around.
-}

vendor/github.com/dgraph-io/badger/iterator.go

@@ -24,10 +24,10 @@ import (
 	"sync/atomic"
 	"time"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table"
+	"github.com/dgraph-io/badger/options"
+	"github.com/dgraph-io/badger/table"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
+	"github.com/dgraph-io/badger/y"
 )
 
 type prefetchStatus uint8
@@ -140,7 +140,7 @@ func (item *Item) IsDeletedOrExpired() bool {
 	return isDeletedOrExpired(item.meta, item.expiresAt)
 }
 
-// DiscardEarlierVersions returns whether the iterator was created with the
+// DiscardEarlierVersions returns whether the item was created with the
 // option to discard earlier versions of a key when multiple are available.
 func (item *Item) DiscardEarlierVersions() bool {
 	return item.meta&bitDiscardEarlierVersions > 0
@@ -246,6 +246,12 @@ func (item *Item) EstimatedSize() int64 {
 	return int64(vp.Len) // includes key length.
 }
 
+// KeySize returns the size of the key.
+// Exact size of the key is key + 8 bytes of timestamp
+func (item *Item) KeySize() int64 {
+	return int64(len(item.key))
+}
+
 // ValueSize returns the exact size of the value.
 //
 // This can be called to quickly estimate the size of a value without fetching

vendor/github.com/dgraph-io/badger/iterator_test.go

@@ -1,244 +0,0 @@
-/*
- * Copyright 2018 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-)
-
-type tableMock struct {
-	left, right []byte
-}
-
-func (tm *tableMock) Smallest() []byte            { return tm.left }
-func (tm *tableMock) Biggest() []byte             { return tm.right }
-func (tm *tableMock) DoesNotHave(key []byte) bool { return false }
-
-func TestPickTables(t *testing.T) {
-	opt := DefaultIteratorOptions
-
-	within := func(prefix, left, right string) {
-		opt.Prefix = []byte(prefix)
-		tm := &tableMock{left: []byte(left), right: []byte(right)}
-		require.True(t, opt.pickTable(tm))
-	}
-	outside := func(prefix, left, right string) {
-		opt.Prefix = []byte(prefix)
-		tm := &tableMock{left: []byte(left), right: []byte(right)}
-		require.False(t, opt.pickTable(tm))
-	}
-	within("abc", "ab", "ad")
-	within("abc", "abc", "ad")
-	within("abc", "abb123", "ad")
-	within("abc", "abc123", "abd234")
-	within("abc", "abc123", "abc456")
-
-	outside("abd", "abe", "ad")
-	outside("abd", "ac", "ad")
-	outside("abd", "b", "e")
-	outside("abd", "a", "ab")
-	outside("abd", "ab", "abc")
-	outside("abd", "ab", "abc123")
-}
-
-func TestIteratePrefix(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		bkey := func(i int) []byte {
-			return []byte(fmt.Sprintf("%04d", i))
-		}
-		val := []byte("OK")
-		n := 10000
-
-		batch := db.NewWriteBatch()
-		for i := 0; i < n; i++ {
-			if (i % 1000) == 0 {
-				t.Logf("Put i=%d\n", i)
-			}
-			require.NoError(t, batch.Set(bkey(i), val, 0))
-		}
-		require.NoError(t, batch.Flush())
-
-		countKeys := func(prefix string) int {
-			t.Logf("Testing with prefix: %s", prefix)
-			var count int
-			opt := DefaultIteratorOptions
-			opt.Prefix = []byte(prefix)
-			err := db.View(func(txn *Txn) error {
-				itr := txn.NewIterator(opt)
-				defer itr.Close()
-				for itr.Rewind(); itr.Valid(); itr.Next() {
-					item := itr.Item()
-					err := item.Value(func(v []byte) error {
-						require.Equal(t, val, v)
-						return nil
-					})
-					require.NoError(t, err)
-					require.True(t, bytes.HasPrefix(item.Key(), opt.Prefix))
-					count++
-				}
-				return nil
-			})
-			require.NoError(t, err)
-			return count
-		}
-
-		countOneKey := func(key []byte) int {
-			var count int
-			err := db.View(func(txn *Txn) error {
-				itr := txn.NewKeyIterator(key, DefaultIteratorOptions)
-				defer itr.Close()
-				for itr.Rewind(); itr.Valid(); itr.Next() {
-					item := itr.Item()
-					err := item.Value(func(v []byte) error {
-						require.Equal(t, val, v)
-						return nil
-					})
-					require.NoError(t, err)
-					require.Equal(t, key, item.Key())
-					count++
-				}
-				return nil
-			})
-			require.NoError(t, err)
-			return count
-		}
-
-		for i := 0; i <= 9; i++ {
-			require.Equal(t, 1, countKeys(fmt.Sprintf("%d%d%d%d", i, i, i, i)))
-			require.Equal(t, 10, countKeys(fmt.Sprintf("%d%d%d", i, i, i)))
-			require.Equal(t, 100, countKeys(fmt.Sprintf("%d%d", i, i)))
-			require.Equal(t, 1000, countKeys(fmt.Sprintf("%d", i)))
-		}
-		require.Equal(t, 10000, countKeys(""))
-
-		t.Logf("Testing each key with key iterator")
-		for i := 0; i < n; i++ {
-			require.Equal(t, 1, countOneKey(bkey(i)))
-		}
-	})
-}
-
-// go test -v -run=XXX -bench=BenchmarkIterate -benchtime=3s
-// Benchmark with opt.Prefix set ===
-// goos: linux
-// goarch: amd64
-// pkg: github.com/dgraph-io/badger
-// BenchmarkIteratePrefixSingleKey/Key_lookups-4         	   10000	    365539 ns/op
-// --- BENCH: BenchmarkIteratePrefixSingleKey/Key_lookups-4
-// 	iterator_test.go:147: Inner b.N: 1
-// 	iterator_test.go:147: Inner b.N: 100
-// 	iterator_test.go:147: Inner b.N: 10000
-// --- BENCH: BenchmarkIteratePrefixSingleKey
-// 	iterator_test.go:143: LSM files: 79
-// 	iterator_test.go:145: Outer b.N: 1
-// PASS
-// ok  	github.com/dgraph-io/badger	41.586s
-//
-// Benchmark with NO opt.Prefix set ===
-// goos: linux
-// goarch: amd64
-// pkg: github.com/dgraph-io/badger
-// BenchmarkIteratePrefixSingleKey/Key_lookups-4         	   10000	    460924 ns/op
-// --- BENCH: BenchmarkIteratePrefixSingleKey/Key_lookups-4
-// 	iterator_test.go:147: Inner b.N: 1
-// 	iterator_test.go:147: Inner b.N: 100
-// 	iterator_test.go:147: Inner b.N: 10000
-// --- BENCH: BenchmarkIteratePrefixSingleKey
-// 	iterator_test.go:143: LSM files: 83
-// 	iterator_test.go:145: Outer b.N: 1
-// PASS
-// ok  	github.com/dgraph-io/badger	41.836s
-//
-// Only my laptop there's a 20% improvement in latency with ~80 files.
-func BenchmarkIteratePrefixSingleKey(b *testing.B) {
-	dir, err := ioutil.TempDir(".", "badger-test")
-	y.Check(err)
-	defer os.RemoveAll(dir)
-	opts := getTestOptions(dir)
-	opts.TableLoadingMode = options.LoadToRAM
-	db, err := Open(opts)
-	y.Check(err)
-	defer db.Close()
-
-	N := 100000 // Should generate around 80 SSTables.
-	val := []byte("OK")
-	bkey := func(i int) []byte {
-		return []byte(fmt.Sprintf("%06d", i))
-	}
-
-	batch := db.NewWriteBatch()
-	for i := 0; i < N; i++ {
-		y.Check(batch.Set(bkey(i), val, 0))
-	}
-	y.Check(batch.Flush())
-	var lsmFiles int
-	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if strings.HasSuffix(path, ".sst") {
-			lsmFiles++
-		}
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	y.Check(err)
-	b.Logf("LSM files: %d", lsmFiles)
-	b.Logf("Key splits: %v", db.KeySplits(nil))
-	b.Logf("Key splits with prefix: %v", db.KeySplits([]byte("09")))
-
-	b.Logf("Outer b.N: %d", b.N)
-	b.Run("Key lookups", func(b *testing.B) {
-		b.Logf("Inner b.N: %d", b.N)
-		for i := 0; i < b.N; i++ {
-			key := bkey(rand.Intn(N))
-			err := db.View(func(txn *Txn) error {
-				opt := DefaultIteratorOptions
-				// NOTE: Comment opt.Prefix out here to compare the performance
-				// difference between providing Prefix as an option, v/s not. I
-				// see a 20% improvement when there are ~80 SSTables.
-				opt.Prefix = key
-				opt.AllVersions = true
-
-				itr := txn.NewIterator(opt)
-				defer itr.Close()
-
-				var count int
-				for itr.Seek(key); itr.ValidForPrefix(key); itr.Next() {
-					count++
-				}
-				if count != 1 {
-					b.Fatalf("Count must be one key: %s. Found: %d", key, count)
-				}
-				return nil
-			})
-			if err != nil {
-				b.Fatalf("Error while View: %v", err)
-			}
-		}
-	})
-}

vendor/github.com/dgraph-io/badger/level_handler.go

@@ -21,9 +21,9 @@ import (
 	"sort"
 	"sync"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/table"
+	"github.com/dgraph-io/badger/y"
+	"github.com/pkg/errors"
 )
 
 type levelHandler struct {
@@ -102,48 +102,40 @@ func (s *levelHandler) deleteTables(toDel []*table.Table) error {
 
 // replaceTables will replace tables[left:right] with newTables. Note this EXCLUDES tables[right].
 // You must call decr() to delete the old tables _after_ writing the update to the manifest.
-func (s *levelHandler) replaceTables(newTables []*table.Table) error {
+func (s *levelHandler) replaceTables(toDel, toAdd []*table.Table) error {
 	// Need to re-search the range of tables in this level to be replaced as other goroutines might
 	// be changing it as well.  (They can't touch our tables, but if they add/remove other tables,
 	// the indices get shifted around.)
-	if len(newTables) == 0 {
-		return nil
-	}
-
 	s.Lock() // We s.Unlock() below.
 
+	toDelMap := make(map[uint64]struct{})
+	for _, t := range toDel {
+		toDelMap[t.ID()] = struct{}{}
+	}
+	var newTables []*table.Table
+	for _, t := range s.tables {
+		_, found := toDelMap[t.ID()]
+		if !found {
+			newTables = append(newTables, t)
+			continue
+		}
+		s.totalSize -= t.Size()
+	}
+
 	// Increase totalSize first.
-	for _, tbl := range newTables {
-		s.totalSize += tbl.Size()
-		tbl.IncrRef()
+	for _, t := range toAdd {
+		s.totalSize += t.Size()
+		t.IncrRef()
+		newTables = append(newTables, t)
 	}
 
-	kr := keyRange{
-		left:  newTables[0].Smallest(),
-		right: newTables[len(newTables)-1].Biggest(),
-	}
-	left, right := s.overlappingTables(levelHandlerRLocked{}, kr)
-
-	toDecr := make([]*table.Table, right-left)
-	// Update totalSize and reference counts.
-	for i := left; i < right; i++ {
-		tbl := s.tables[i]
-		s.totalSize -= tbl.Size()
-		toDecr[i-left] = tbl
-	}
-
-	// To be safe, just make a copy. TODO: Be more careful and avoid copying.
-	numDeleted := right - left
-	numAdded := len(newTables)
-	tables := make([]*table.Table, len(s.tables)-numDeleted+numAdded)
-	y.AssertTrue(left == copy(tables, s.tables[:left]))
-	t := tables[left:]
-	y.AssertTrue(numAdded == copy(t, newTables))
-	t = t[numAdded:]
-	y.AssertTrue(len(s.tables[right:]) == copy(t, s.tables[right:]))
-	s.tables = tables
+	// Assign tables.
+	s.tables = newTables
+	sort.Slice(s.tables, func(i, j int) bool {
+		return y.CompareKeys(s.tables[i].Smallest(), s.tables[j].Smallest()) < 0
+	})
 	s.Unlock() // s.Unlock before we DecrRef tables -- that can be slow.
-	return decrRefs(toDecr)
+	return decrRefs(toDel)
 }
 
 func decrRefs(tables []*table.Table) error {
@@ -294,6 +286,9 @@ type levelHandlerRLocked struct{}
 // This function should already have acquired a read lock, and this is so important the caller must
 // pass an empty parameter declaring such.
 func (s *levelHandler) overlappingTables(_ levelHandlerRLocked, kr keyRange) (int, int) {
+	if len(kr.left) == 0 || len(kr.right) == 0 {
+		return 0, 0
+	}
 	left := sort.Search(len(s.tables), func(i int) bool {
 		return y.CompareKeys(kr.left, s.tables[i].Biggest()) <= 0
 	})

vendor/github.com/dgraph-io/badger/levels.go

@@ -17,6 +17,7 @@
 package badger
 
 import (
+	"bytes"
 	"fmt"
 	"math"
 	"math/rand"
@@ -27,12 +28,12 @@ import (
 	"sync/atomic"
 	"time"
 
-	"gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace"
+	"golang.org/x/net/trace"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/pb"
+	"github.com/dgraph-io/badger/table"
+	"github.com/dgraph-io/badger/y"
+	"github.com/pkg/errors"
 )
 
 type levelsController struct {
@@ -215,10 +216,10 @@ func (s *levelsController) cleanupLevels() error {
 	return firstErr
 }
 
-// This function picks all tables from all levels, creates a manifest changeset,
+// dropTree picks all tables from all levels, creates a manifest changeset,
 // applies it, and then decrements the refs of these tables, which would result
 // in their deletion.
-func (s *levelsController) deleteLSMTree() (int, error) {
+func (s *levelsController) dropTree() (int, error) {
 	// First pick all tables, so we can create a manifest changelog.
 	var all []*table.Table
 	for _, l := range s.levels {
@@ -255,6 +256,72 @@ func (s *levelsController) deleteLSMTree() (int, error) {
 	return len(all), nil
 }
 
+// dropPrefix runs a L0->L1 compaction, and then runs same level compaction on the rest of the
+// levels. For L0->L1 compaction, it runs compactions normally, but skips over all the keys with the
+// provided prefix. For Li->Li compactions, it picks up the tables which would have the prefix. The
+// tables who only have keys with this prefix are quickly dropped. The ones which have other keys
+// are run through MergeIterator and compacted to create new tables. All the mechanisms of
+// compactions apply, i.e. level sizes and MANIFEST are updated as in the normal flow.
+func (s *levelsController) dropPrefix(prefix []byte) error {
+	opt := s.kv.opt
+	for _, l := range s.levels {
+		l.RLock()
+		if l.level == 0 {
+			size := len(l.tables)
+			l.RUnlock()
+
+			if size > 0 {
+				cp := compactionPriority{
+					level: 0,
+					score: 1.74,
+					// A unique number greater than 1.0 does two things. Helps identify this
+					// function in logs, and forces a compaction.
+					dropPrefix: prefix,
+				}
+				if err := s.doCompact(cp); err != nil {
+					opt.Warningf("While compacting level 0: %v", err)
+					return nil
+				}
+			}
+			continue
+		}
+
+		var tables []*table.Table
+		for _, table := range l.tables {
+			var absent bool
+			switch {
+			case bytes.HasPrefix(table.Smallest(), prefix):
+			case bytes.HasPrefix(table.Biggest(), prefix):
+			case bytes.Compare(prefix, table.Smallest()) > 0 &&
+				bytes.Compare(prefix, table.Biggest()) < 0:
+			default:
+				absent = true
+			}
+			if !absent {
+				tables = append(tables, table)
+			}
+		}
+		l.RUnlock()
+		if len(tables) == 0 {
+			continue
+		}
+
+		cd := compactDef{
+			elog:       trace.New(fmt.Sprintf("Badger.L%d", l.level), "Compact"),
+			thisLevel:  l,
+			nextLevel:  l,
+			top:        []*table.Table{},
+			bot:        tables,
+			dropPrefix: prefix,
+		}
+		if err := s.runCompactDef(l.level, cd); err != nil {
+			opt.Warningf("While running compact def: %+v. Error: %v", cd, err)
+			return err
+		}
+	}
+	return nil
+}
+
 func (s *levelsController) startCompact(lc *y.Closer) {
 	n := s.kv.opt.NumCompactors
 	lc.AddRunning(n - 1)
@@ -311,8 +378,9 @@ func (l *levelHandler) isCompactable(delSize int64) bool {
 }
 
 type compactionPriority struct {
-	level int
-	score float64
+	level      int
+	score      float64
+	dropPrefix []byte
 }
 
 // pickCompactLevel determines which level to compact.
@@ -350,7 +418,7 @@ func (s *levelsController) pickCompactLevels() (prios []compactionPriority) {
 
 // compactBuildTables merge topTables and botTables to form a list of new tables.
 func (s *levelsController) compactBuildTables(
-	l int, cd compactDef) ([]*table.Table, func() error, error) {
+	lev int, cd compactDef) ([]*table.Table, func() error, error) {
 	topTables := cd.top
 	botTables := cd.bot
 
@@ -358,7 +426,7 @@ func (s *levelsController) compactBuildTables(
 	{
 		kr := getKeyRange(cd.top)
 		for i, lh := range s.levels {
-			if i <= l { // Skip upper levels.
+			if i <= lev { // Skip upper levels.
 				continue
 			}
 			lh.RLock()
@@ -369,7 +437,6 @@ func (s *levelsController) compactBuildTables(
 				break
 			}
 		}
-		cd.elog.LazyPrintf("Key range overlaps with lower levels: %v", hasOverlap)
 	}
 
 	// Try to collect stats so that we can inform value log about GC. That would help us find which
@@ -385,15 +452,26 @@ func (s *levelsController) compactBuildTables(
 
 	// Create iterators across all the tables involved first.
 	var iters []y.Iterator
-	if l == 0 {
+	if lev == 0 {
 		iters = appendIteratorsReversed(iters, topTables, false)
-	} else {
+	} else if len(topTables) > 0 {
 		y.AssertTrue(len(topTables) == 1)
 		iters = []y.Iterator{topTables[0].NewIterator(false)}
 	}
 
 	// Next level has level>=1 and we can use ConcatIterator as key ranges do not overlap.
-	iters = append(iters, table.NewConcatIterator(botTables, false))
+	var valid []*table.Table
+	for _, table := range botTables {
+		if len(cd.dropPrefix) > 0 &&
+			bytes.HasPrefix(table.Smallest(), cd.dropPrefix) &&
+			bytes.HasPrefix(table.Biggest(), cd.dropPrefix) {
+			// All the keys in this table have the dropPrefix. So, this table does not need to be
+			// in the iterator and can be dropped immediately.
+			continue
+		}
+		valid = append(valid, table)
+	}
+	iters = append(iters, table.NewConcatIterator(valid, false))
 	it := y.NewMergeIterator(iters, false)
 	defer it.Close() // Important to close the iterator to do ref counting.
 
@@ -417,6 +495,13 @@ func (s *levelsController) compactBuildTables(
 		builder := table.NewTableBuilder()
 		var numKeys, numSkips uint64
 		for ; it.Valid(); it.Next() {
+			// See if we need to skip the prefix.
+			if len(cd.dropPrefix) > 0 && bytes.HasPrefix(it.Key(), cd.dropPrefix) {
+				numSkips++
+				updateStats(it.Value())
+				continue
+			}
+
 			// See if we need to skip this key.
 			if len(skipKey) > 0 {
 				if y.SameKey(it.Key(), skipKey) {
@@ -441,7 +526,8 @@ func (s *levelsController) compactBuildTables(
 
 			vs := it.Value()
 			version := y.ParseTs(it.Key())
-			if version <= discardTs {
+			// Do not discard entries inserted by merge operator. These entries will be discarded once they're merged
+			if version <= discardTs && vs.Meta&bitMergeEntry == 0 {
 				// Keep track of the number of versions encountered for this key. Only consider the
 				// versions which are below the minReadTs, otherwise, we might end up discarding the
 				// only valid version for a running transaction.
@@ -474,8 +560,8 @@ func (s *levelsController) compactBuildTables(
 		}
 		// It was true that it.Valid() at least once in the loop above, which means we
 		// called Add() at least once, and builder is not Empty().
-		cd.elog.LazyPrintf("Added %d keys. Skipped %d keys.", numKeys, numSkips)
-		cd.elog.LazyPrintf("LOG Compact. Iteration took: %v\n", time.Since(timeStart))
+		s.kv.opt.Debugf("LOG Compact. Added %d keys. Skipped %d keys. Iteration took: %v",
+			numKeys, numSkips, time.Since(timeStart))
 		if !builder.Empty() {
 			numBuilds++
 			fileID := s.reserveFileID()
@@ -533,8 +619,8 @@ func (s *levelsController) compactBuildTables(
 	sort.Slice(newTables, func(i, j int) bool {
 		return y.CompareKeys(newTables[i].Biggest(), newTables[j].Biggest()) < 0
 	})
-	s.kv.vlog.updateGCStats(discardStats)
-	cd.elog.LazyPrintf("Discard stats: %v", discardStats)
+	s.kv.vlog.updateDiscardStats(discardStats)
+	s.kv.opt.Debugf("Discard stats: %v", discardStats)
 	return newTables, func() error { return decrRefs(newTables) }, nil
 }
 
@@ -566,6 +652,8 @@ type compactDef struct {
 	nextRange keyRange
 
 	thisSize int64
+
+	dropPrefix []byte
 }
 
 func (cd *compactDef) lockLevels() {
@@ -689,7 +777,7 @@ func (s *levelsController) runCompactDef(l int, cd compactDef) (err error) {
 
 	// See comment earlier in this function about the ordering of these ops, and the order in which
 	// we access levels when reading.
-	if err := nextLevel.replaceTables(newTables); err != nil {
+	if err := nextLevel.replaceTables(cd.bot, newTables); err != nil {
 		return err
 	}
 	if err := thisLevel.deleteTables(cd.top); err != nil {
@@ -699,8 +787,9 @@ func (s *levelsController) runCompactDef(l int, cd compactDef) (err error) {
 	// Note: For level 0, while doCompact is running, it is possible that new tables are added.
 	// However, the tables are added only to the end, so it is ok to just delete the first table.
 
-	cd.elog.LazyPrintf("LOG Compact %d->%d, del %d tables, add %d tables, took %v\n",
-		l, l+1, len(cd.top)+len(cd.bot), len(newTables), time.Since(timeStart))
+	s.kv.opt.Infof("LOG Compact %d->%d, del %d tables, add %d tables, took %v\n",
+		thisLevel.level, nextLevel.level, len(cd.top)+len(cd.bot),
+		len(newTables), time.Since(timeStart))
 	return nil
 }
 
@@ -712,41 +801,40 @@ func (s *levelsController) doCompact(p compactionPriority) error {
 	y.AssertTrue(l+1 < s.kv.opt.MaxLevels) // Sanity check.
 
 	cd := compactDef{
-		elog:      trace.New(fmt.Sprintf("Badger.L%d", l), "Compact"),
-		thisLevel: s.levels[l],
-		nextLevel: s.levels[l+1],
+		elog:       trace.New(fmt.Sprintf("Badger.L%d", l), "Compact"),
+		thisLevel:  s.levels[l],
+		nextLevel:  s.levels[l+1],
+		dropPrefix: p.dropPrefix,
 	}
 	cd.elog.SetMaxEvents(100)
 	defer cd.elog.Finish()
 
-	cd.elog.LazyPrintf("Got compaction priority: %+v", p)
+	s.kv.opt.Infof("Got compaction priority: %+v", p)
 
 	// While picking tables to be compacted, both levels' tables are expected to
 	// remain unchanged.
 	if l == 0 {
 		if !s.fillTablesL0(&cd) {
-			cd.elog.LazyPrintf("fillTables failed for level: %d\n", l)
 			return errFillTables
 		}
 
 	} else {
 		if !s.fillTables(&cd) {
-			cd.elog.LazyPrintf("fillTables failed for level: %d\n", l)
 			return errFillTables
 		}
 	}
 	defer s.cstatus.delete(cd) // Remove the ranges from compaction status.
 
-	cd.elog.LazyPrintf("Running for level: %d\n", cd.thisLevel.level)
+	s.kv.opt.Infof("Running for level: %d\n", cd.thisLevel.level)
 	s.cstatus.toLog(cd.elog)
 	if err := s.runCompactDef(l, cd); err != nil {
 		// This compaction couldn't be done successfully.
-		cd.elog.LazyPrintf("\tLOG Compact FAILED with error: %+v: %+v", err, cd)
+		s.kv.opt.Warningf("LOG Compact FAILED with error: %+v: %+v", err, cd)
 		return err
 	}
 
 	s.cstatus.toLog(cd.elog)
-	cd.elog.LazyPrintf("Compaction for level: %d DONE", cd.thisLevel.level)
+	s.kv.opt.Infof("Compaction for level: %d DONE", cd.thisLevel.level)
 	return nil
 }
 
@@ -858,23 +946,35 @@ func (s *levelsController) appendIterators(
 
 // TableInfo represents the information about a table.
 type TableInfo struct {
-	ID    uint64
-	Level int
-	Left  []byte
-	Right []byte
+	ID       uint64
+	Level    int
+	Left     []byte
+	Right    []byte
+	KeyCount uint64 // Number of keys in the table
 }
 
-func (s *levelsController) getTableInfo() (result []TableInfo) {
+func (s *levelsController) getTableInfo(withKeysCount bool) (result []TableInfo) {
 	for _, l := range s.levels {
+		l.RLock()
 		for _, t := range l.tables {
+			var count uint64
+			if withKeysCount {
+				it := t.NewIterator(false)
+				for it.Rewind(); it.Valid(); it.Next() {
+					count++
+				}
+			}
+
 			info := TableInfo{
-				ID:    t.ID(),
-				Level: l.level,
-				Left:  t.Smallest(),
-				Right: t.Biggest(),
+				ID:       t.ID(),
+				Level:    l.level,
+				Left:     t.Smallest(),
+				Right:    t.Biggest(),
+				KeyCount: count,
 			}
 			result = append(result, info)
 		}
+		l.RUnlock()
 	}
 	sort.Slice(result, func(i, j int) bool {
 		if result[i].Level != result[j].Level {

vendor/github.com/dgraph-io/badger/logger.go

@@ -24,8 +24,9 @@ import (
 // Logger is implemented by any logging system that is used for standard logs.
 type Logger interface {
 	Errorf(string, ...interface{})
-	Infof(string, ...interface{})
 	Warningf(string, ...interface{})
+	Infof(string, ...interface{})
+	Debugf(string, ...interface{})
 }
 
 // Errorf logs an ERROR log message to the logger specified in opts or to the
@@ -53,6 +54,14 @@ func (opt *Options) Warningf(format string, v ...interface{}) {
 	opt.Logger.Warningf(format, v...)
 }
 
+// Debugf logs a DEBUG message to the logger specified in opts.
+func (opt *Options) Debugf(format string, v ...interface{}) {
+	if opt.Logger == nil {
+		return
+	}
+	opt.Logger.Debugf(format, v...)
+}
+
 type defaultLog struct {
 	*log.Logger
 }
@@ -63,10 +72,14 @@ func (l *defaultLog) Errorf(f string, v ...interface{}) {
 	l.Printf("ERROR: "+f, v...)
 }
 
+func (l *defaultLog) Warningf(f string, v ...interface{}) {
+	l.Printf("WARNING: "+f, v...)
+}
+
 func (l *defaultLog) Infof(f string, v ...interface{}) {
 	l.Printf("INFO: "+f, v...)
 }
 
-func (l *defaultLog) Warningf(f string, v ...interface{}) {
-	l.Printf("WARNING: "+f, v...)
+func (l *defaultLog) Debugf(f string, v ...interface{}) {
+	l.Printf("DEBUG: "+f, v...)
 }

vendor/github.com/dgraph-io/badger/logger_test.go

@@ -1,67 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-type mockLogger struct {
-	output string
-}
-
-func (l *mockLogger) Errorf(f string, v ...interface{}) {
-	l.output = fmt.Sprintf("ERROR: "+f, v...)
-}
-
-func (l *mockLogger) Infof(f string, v ...interface{}) {
-	l.output = fmt.Sprintf("INFO: "+f, v...)
-}
-
-func (l *mockLogger) Warningf(f string, v ...interface{}) {
-	l.output = fmt.Sprintf("WARNING: "+f, v...)
-}
-
-// Test that the DB-specific log is used instead of the global log.
-func TestDbLog(t *testing.T) {
-	l := &mockLogger{}
-	opt := Options{Logger: l}
-
-	opt.Errorf("test")
-	require.Equal(t, "ERROR: test", l.output)
-	opt.Infof("test")
-	require.Equal(t, "INFO: test", l.output)
-	opt.Warningf("test")
-	require.Equal(t, "WARNING: test", l.output)
-}
-
-// Test that the global logger is used when no logger is specified in Options.
-func TestNoDbLog(t *testing.T) {
-	l := &mockLogger{}
-	opt := Options{}
-	opt.Logger = l
-
-	opt.Errorf("test")
-	require.Equal(t, "ERROR: test", l.output)
-	opt.Infof("test")
-	require.Equal(t, "INFO: test", l.output)
-	opt.Warningf("test")
-	require.Equal(t, "WARNING: test", l.output)
-}

vendor/github.com/dgraph-io/badger/managed_db_test.go

@@ -1,353 +0,0 @@
-package badger
-
-import (
-	"io/ioutil"
-	"math"
-	"math/rand"
-	"os"
-	"runtime"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-)
-
-func val(large bool) []byte {
-	var buf []byte
-	if large {
-		buf = make([]byte, 8192)
-	} else {
-		buf = make([]byte, 16)
-	}
-	rand.Read(buf)
-	return buf
-}
-
-func numKeys(db *DB) int {
-	var count int
-	err := db.View(func(txn *Txn) error {
-		itr := txn.NewIterator(DefaultIteratorOptions)
-		defer itr.Close()
-
-		for itr.Rewind(); itr.Valid(); itr.Next() {
-			count++
-		}
-		return nil
-	})
-	y.Check(err)
-	return count
-}
-
-func numKeysManaged(db *DB, readTs uint64) int {
-	txn := db.NewTransactionAt(readTs, false)
-	defer txn.Discard()
-
-	itr := txn.NewIterator(DefaultIteratorOptions)
-	defer itr.Close()
-
-	var count int
-	for itr.Rewind(); itr.Valid(); itr.Next() {
-		count++
-	}
-	return count
-}
-
-func TestDropAllManaged(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opts := getTestOptions(dir)
-	opts.managedTxns = true
-	opts.ValueLogFileSize = 5 << 20
-	db, err := Open(opts)
-	require.NoError(t, err)
-
-	N := uint64(10000)
-	populate := func(db *DB, start uint64) {
-		var wg sync.WaitGroup
-		for i := start; i < start+N; i++ {
-			wg.Add(1)
-			txn := db.NewTransactionAt(math.MaxUint64, true)
-			require.NoError(t, txn.Set([]byte(key("key", int(i))), val(true)))
-			require.NoError(t, txn.CommitAt(uint64(i), func(err error) {
-				require.NoError(t, err)
-				wg.Done()
-			}))
-		}
-		wg.Wait()
-	}
-
-	populate(db, N)
-	require.Equal(t, int(N), numKeysManaged(db, math.MaxUint64))
-
-	require.NoError(t, db.DropAll())
-	require.NoError(t, db.DropAll()) // Just call it twice, for fun.
-	require.Equal(t, 0, numKeysManaged(db, math.MaxUint64))
-
-	// Check that we can still write to mdb, and using lower timestamps.
-	populate(db, 1)
-	require.Equal(t, int(N), numKeysManaged(db, math.MaxUint64))
-	db.Close()
-
-	// Ensure that value log is correctly replayed, that we are preserving badgerHead.
-	opts.managedTxns = true
-	db2, err := Open(opts)
-	require.NoError(t, err)
-	require.Equal(t, int(N), numKeysManaged(db2, math.MaxUint64))
-	db2.Close()
-}
-
-func TestDropAll(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opts := getTestOptions(dir)
-	opts.ValueLogFileSize = 5 << 20
-	db, err := Open(opts)
-	require.NoError(t, err)
-
-	N := uint64(10000)
-	populate := func(db *DB) {
-		writer := db.NewWriteBatch()
-		for i := uint64(0); i < N; i++ {
-			require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0))
-		}
-		require.NoError(t, writer.Flush())
-	}
-
-	populate(db)
-	require.Equal(t, int(N), numKeys(db))
-
-	require.NoError(t, db.DropAll())
-	require.Equal(t, 0, numKeys(db))
-
-	// Check that we can still write to mdb, and using lower timestamps.
-	populate(db)
-	require.Equal(t, int(N), numKeys(db))
-	db.Close()
-
-	// Ensure that value log is correctly replayed.
-	db2, err := Open(opts)
-	require.NoError(t, err)
-	require.Equal(t, int(N), numKeys(db2))
-	db2.Close()
-}
-
-func TestDropAllTwice(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opts := getTestOptions(dir)
-	opts.ValueLogFileSize = 5 << 20
-	db, err := Open(opts)
-	require.NoError(t, err)
-
-	N := uint64(10000)
-	populate := func(db *DB) {
-		writer := db.NewWriteBatch()
-		for i := uint64(0); i < N; i++ {
-			require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0))
-		}
-		require.NoError(t, writer.Flush())
-	}
-
-	populate(db)
-	require.Equal(t, int(N), numKeys(db))
-
-	require.NoError(t, db.DropAll())
-	require.Equal(t, 0, numKeys(db))
-
-	// Call DropAll again.
-	require.NoError(t, db.DropAll())
-}
-
-func TestDropAllWithPendingTxn(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opts := getTestOptions(dir)
-	opts.ValueLogFileSize = 5 << 20
-	db, err := Open(opts)
-	require.NoError(t, err)
-
-	N := uint64(10000)
-	populate := func(db *DB) {
-		writer := db.NewWriteBatch()
-		for i := uint64(0); i < N; i++ {
-			require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0))
-		}
-		require.NoError(t, writer.Flush())
-	}
-
-	populate(db)
-	require.Equal(t, int(N), numKeys(db))
-
-	txn := db.NewTransaction(true)
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		itr := txn.NewIterator(DefaultIteratorOptions)
-		defer itr.Close()
-
-		var keys []string
-		for {
-			var count int
-			for itr.Rewind(); itr.Valid(); itr.Next() {
-				count++
-				item := itr.Item()
-				keys = append(keys, string(item.KeyCopy(nil)))
-				_, err := item.ValueCopy(nil)
-				if err != nil {
-					t.Logf("Got error during value copy: %v", err)
-					return
-				}
-			}
-			t.Logf("Got number of keys: %d\n", count)
-			for _, key := range keys {
-				item, err := txn.Get([]byte(key))
-				if err != nil {
-					t.Logf("Got error during key lookup: %v", err)
-					return
-				}
-				if _, err := item.ValueCopy(nil); err != nil {
-					t.Logf("Got error during second value copy: %v", err)
-					return
-				}
-			}
-		}
-	}()
-	// Do not cancel txn.
-
-	go func() {
-		time.Sleep(2 * time.Second)
-		require.NoError(t, db.DropAll())
-	}()
-	wg.Wait()
-}
-
-func TestDropReadOnly(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opts := getTestOptions(dir)
-	opts.ValueLogFileSize = 5 << 20
-	db, err := Open(opts)
-	require.NoError(t, err)
-	N := uint64(1000)
-	populate := func(db *DB) {
-		writer := db.NewWriteBatch()
-		for i := uint64(0); i < N; i++ {
-			require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0))
-		}
-		require.NoError(t, writer.Flush())
-	}
-
-	populate(db)
-	require.Equal(t, int(N), numKeys(db))
-	require.NoError(t, db.Close())
-
-	opts.ReadOnly = true
-	db2, err := Open(opts)
-	// acquireDirectoryLock returns ErrWindowsNotSupported on Windows. It can be ignored safely.
-	if runtime.GOOS == "windows" {
-		require.Equal(t, err, ErrWindowsNotSupported)
-	} else {
-		require.NoError(t, err)
-	}
-	require.Panics(t, func() { db2.DropAll() })
-}
-
-func TestWriteAfterClose(t *testing.T) {
-	dir, err := ioutil.TempDir(".", "badger-test")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opts := getTestOptions(dir)
-	opts.ValueLogFileSize = 5 << 20
-	db, err := Open(opts)
-	require.NoError(t, err)
-	N := uint64(1000)
-	populate := func(db *DB) {
-		writer := db.NewWriteBatch()
-		for i := uint64(0); i < N; i++ {
-			require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0))
-		}
-		require.NoError(t, writer.Flush())
-	}
-
-	populate(db)
-	require.Equal(t, int(N), numKeys(db))
-	require.NoError(t, db.Close())
-	err = db.Update(func(txn *Txn) error {
-		return txn.Set([]byte("a"), []byte("b"))
-	})
-	require.Equal(t, ErrBlockedWrites, err)
-}
-
-func TestDropAllRace(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opts := getTestOptions(dir)
-	opts.managedTxns = true
-	db, err := Open(opts)
-	require.NoError(t, err)
-
-	N := 10000
-	// Start a goroutine to keep trying to write to DB while DropAll happens.
-	closer := y.NewCloser(1)
-	go func() {
-		defer closer.Done()
-		ticker := time.NewTicker(time.Millisecond)
-		defer ticker.Stop()
-
-		i := N + 1 // Writes would happen above N.
-		var errors int32
-		for {
-			select {
-			case <-ticker.C:
-				i++
-				txn := db.NewTransactionAt(math.MaxUint64, true)
-				require.NoError(t, txn.Set([]byte(key("key", i)), val(false)))
-				if err := txn.CommitAt(uint64(i), func(err error) {
-					if err != nil {
-						atomic.AddInt32(&errors, 1)
-					}
-				}); err != nil {
-					atomic.AddInt32(&errors, 1)
-				}
-			case <-closer.HasBeenClosed():
-				// The following causes a data race.
-				// t.Logf("i: %d. Number of (expected) write errors: %d.\n", i, errors)
-				return
-			}
-		}
-	}()
-
-	var wg sync.WaitGroup
-	for i := 1; i <= N; i++ {
-		wg.Add(1)
-		txn := db.NewTransactionAt(math.MaxUint64, true)
-		require.NoError(t, txn.Set([]byte(key("key", i)), val(false)))
-		require.NoError(t, txn.CommitAt(uint64(i), func(err error) {
-			require.NoError(t, err)
-			wg.Done()
-		}))
-	}
-	wg.Wait()
-
-	before := numKeysManaged(db, math.MaxUint64)
-	require.True(t, before > N)
-
-	require.NoError(t, db.DropAll())
-	closer.SignalAndWait()
-
-	after := numKeysManaged(db, math.MaxUint64)
-	t.Logf("Before: %d. After dropall: %d\n", before, after)
-	require.True(t, after < before)
-	db.Close()
-}

vendor/github.com/dgraph-io/badger/manifest.go

@@ -27,9 +27,9 @@ import (
 	"path/filepath"
 	"sync"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/pb"
+	"github.com/dgraph-io/badger/y"
+	"github.com/pkg/errors"
 )
 
 // Manifest represents the contents of the MANIFEST file in a Badger store.

vendor/github.com/dgraph-io/badger/manifest_test.go

@@ -1,244 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"sort"
-	"testing"
-
-	"gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace"
-
-	"github.com/stretchr/testify/require"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-)
-
-func TestManifestBasic(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	opt := getTestOptions(dir)
-	{
-		kv, err := Open(opt)
-		require.NoError(t, err)
-		n := 5000
-		for i := 0; i < n; i++ {
-			if (i % 10000) == 0 {
-				fmt.Printf("Putting i=%d\n", i)
-			}
-			k := []byte(fmt.Sprintf("%16x", rand.Int63()))
-			txnSet(t, kv, k, k, 0x00)
-		}
-		txnSet(t, kv, []byte("testkey"), []byte("testval"), 0x05)
-		kv.validate()
-		require.NoError(t, kv.Close())
-	}
-
-	kv, err := Open(opt)
-	require.NoError(t, err)
-
-	require.NoError(t, kv.View(func(txn *Txn) error {
-		item, err := txn.Get([]byte("testkey"))
-		require.NoError(t, err)
-		require.EqualValues(t, "testval", string(getItemValue(t, item)))
-		require.EqualValues(t, byte(0x05), item.UserMeta())
-		return nil
-	}))
-	require.NoError(t, kv.Close())
-}
-
-func helpTestManifestFileCorruption(t *testing.T, off int64, errorContent string) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	opt := getTestOptions(dir)
-	{
-		kv, err := Open(opt)
-		require.NoError(t, err)
-		require.NoError(t, kv.Close())
-	}
-	fp, err := os.OpenFile(filepath.Join(dir, ManifestFilename), os.O_RDWR, 0)
-	require.NoError(t, err)
-	// Mess with magic value or version to force error
-	_, err = fp.WriteAt([]byte{'X'}, off)
-	require.NoError(t, err)
-	require.NoError(t, fp.Close())
-	kv, err := Open(opt)
-	defer func() {
-		if kv != nil {
-			kv.Close()
-		}
-	}()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), errorContent)
-}
-
-func TestManifestMagic(t *testing.T) {
-	helpTestManifestFileCorruption(t, 3, "bad magic")
-}
-
-func TestManifestVersion(t *testing.T) {
-	helpTestManifestFileCorruption(t, 4, "unsupported version")
-}
-
-func key(prefix string, i int) string {
-	return prefix + fmt.Sprintf("%04d", i)
-}
-
-func buildTestTable(t *testing.T, prefix string, n int) *os.File {
-	y.AssertTrue(n <= 10000)
-	keyValues := make([][]string, n)
-	for i := 0; i < n; i++ {
-		k := key(prefix, i)
-		v := fmt.Sprintf("%d", i)
-		keyValues[i] = []string{k, v}
-	}
-	return buildTable(t, keyValues)
-}
-
-// TODO - Move these to somewhere where table package can also use it.
-// keyValues is n by 2 where n is number of pairs.
-func buildTable(t *testing.T, keyValues [][]string) *os.File {
-	b := table.NewTableBuilder()
-	defer b.Close()
-	// TODO: Add test for file garbage collection here. No files should be left after the tests here.
-
-	filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63())
-	f, err := y.OpenSyncedFile(filename, true)
-	if t != nil {
-		require.NoError(t, err)
-	} else {
-		y.Check(err)
-	}
-
-	sort.Slice(keyValues, func(i, j int) bool {
-		return keyValues[i][0] < keyValues[j][0]
-	})
-	for _, kv := range keyValues {
-		y.AssertTrue(len(kv) == 2)
-		err := b.Add(y.KeyWithTs([]byte(kv[0]), 10), y.ValueStruct{
-			Value:    []byte(kv[1]),
-			Meta:     'A',
-			UserMeta: 0,
-		})
-		if t != nil {
-			require.NoError(t, err)
-		} else {
-			y.Check(err)
-		}
-	}
-	f.Write(b.Finish())
-	f.Close()
-	f, _ = y.OpenSyncedFile(filename, true)
-	return f
-}
-
-func TestOverlappingKeyRangeError(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opt := DefaultOptions
-	opt.Dir = dir
-	opt.ValueDir = dir
-	kv, err := Open(opt)
-	require.NoError(t, err)
-
-	lh0 := newLevelHandler(kv, 0)
-	lh1 := newLevelHandler(kv, 1)
-	f := buildTestTable(t, "k", 2)
-	t1, err := table.OpenTable(f, options.MemoryMap, nil)
-	require.NoError(t, err)
-	defer t1.DecrRef()
-
-	done := lh0.tryAddLevel0Table(t1)
-	require.Equal(t, true, done)
-
-	cd := compactDef{
-		thisLevel: lh0,
-		nextLevel: lh1,
-		elog:      trace.New("Badger", "Compact"),
-	}
-
-	manifest := createManifest()
-	lc, err := newLevelsController(kv, &manifest)
-	require.NoError(t, err)
-	done = lc.fillTablesL0(&cd)
-	require.Equal(t, true, done)
-	lc.runCompactDef(0, cd)
-
-	f = buildTestTable(t, "l", 2)
-	t2, err := table.OpenTable(f, options.MemoryMap, nil)
-	require.NoError(t, err)
-	defer t2.DecrRef()
-	done = lh0.tryAddLevel0Table(t2)
-	require.Equal(t, true, done)
-
-	cd = compactDef{
-		thisLevel: lh0,
-		nextLevel: lh1,
-		elog:      trace.New("Badger", "Compact"),
-	}
-	lc.fillTablesL0(&cd)
-	lc.runCompactDef(0, cd)
-}
-
-func TestManifestRewrite(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	deletionsThreshold := 10
-	mf, m, err := helpOpenOrCreateManifestFile(dir, false, deletionsThreshold)
-	defer func() {
-		if mf != nil {
-			mf.close()
-		}
-	}()
-	require.NoError(t, err)
-	require.Equal(t, 0, m.Creations)
-	require.Equal(t, 0, m.Deletions)
-
-	err = mf.addChanges([]*pb.ManifestChange{
-		newCreateChange(0, 0, nil),
-	})
-	require.NoError(t, err)
-
-	for i := uint64(0); i < uint64(deletionsThreshold*3); i++ {
-		ch := []*pb.ManifestChange{
-			newCreateChange(i+1, 0, nil),
-			newDeleteChange(i),
-		}
-		err := mf.addChanges(ch)
-		require.NoError(t, err)
-	}
-	err = mf.close()
-	require.NoError(t, err)
-	mf = nil
-	mf, m, err = helpOpenOrCreateManifestFile(dir, false, deletionsThreshold)
-	require.NoError(t, err)
-	require.Equal(t, map[uint64]TableManifest{
-		uint64(deletionsThreshold * 3): {Level: 0, Checksum: []byte{}},
-	}, m.Tables)
-}

vendor/github.com/dgraph-io/badger/merge.go

@@ -20,8 +20,8 @@ import (
 	"sync"
 	"time"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/y"
+	"github.com/pkg/errors"
 )
 
 // MergeOperator represents a Badger merge operator.
@@ -63,11 +63,11 @@ var errNoMerge = errors.New("No need for merge")
 func (op *MergeOperator) iterateAndMerge(txn *Txn) (val []byte, err error) {
 	opt := DefaultIteratorOptions
 	opt.AllVersions = true
-	it := txn.NewIterator(opt)
+	it := txn.NewKeyIterator(op.key, opt)
 	defer it.Close()
 
 	var numVersions int
-	for it.Rewind(); it.ValidForPrefix(op.key); it.Next() {
+	for it.Rewind(); it.Valid(); it.Next() {
 		item := it.Item()
 		numVersions++
 		if numVersions == 1 {
@@ -107,8 +107,8 @@ func (op *MergeOperator) compact() error {
 		if err != nil {
 			return err
 		}
-
-		// Write value back to db
+		// Write value back to the DB. It is important that we do not set the bitMergeEntry bit
+		// here. When compaction happens, all the older merged entries will be removed.
 		return txn.SetWithDiscard(op.key, val, 0)
 	})
 
@@ -144,7 +144,7 @@ func (op *MergeOperator) runCompactions(dur time.Duration) {
 // routine into the values that were recorded by previous invocations to Add().
 func (op *MergeOperator) Add(val []byte) error {
 	return op.db.Update(func(txn *Txn) error {
-		return txn.Set(op.key, val)
+		return txn.setMergeEntry(op.key, val)
 	})
 }
 

vendor/github.com/dgraph-io/badger/options.go

@@ -17,7 +17,7 @@
 package badger
 
 import (
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
+	"github.com/dgraph-io/badger/options"
 )
 
 // NOTE: Keep the comments in the following to 75 chars width, so they
@@ -53,6 +53,18 @@ type Options struct {
 	// How many versions to keep per key.
 	NumVersionsToKeep int
 
+	// Open the DB as read-only. With this set, multiple processes can
+	// open the same Badger DB. Note: if the DB being opened had crashed
+	// before and has vlog data to be replayed, ReadOnly will cause Open
+	// to fail with an appropriate message.
+	ReadOnly bool
+
+	// Truncate value log to delete corrupt data, if any. Would not truncate if ReadOnly is set.
+	Truncate bool
+
+	// DB-specific logger which will override the global logger.
+	Logger Logger
+
 	// 3. Flags that user might want to review
 	// ----------------------------------------
 	// The following affect all levels of LSM tree.
@@ -89,6 +101,13 @@ type Options struct {
 	// efficient when the DB is opened later.
 	CompactL0OnClose bool
 
+	// After this many number of value log file rotates, there would be a force flushing of memtable
+	// to disk. This is useful in write loads with fewer keys and larger values. This work load
+	// would fill up the value logs quickly, while not filling up the Memtables. Thus, on a crash
+	// and restart, the value log head could cause the replay of a good number of value log files
+	// which can slow things on start.
+	LogRotatesToFlush int32
+
 	// Transaction start and commit timestamps are managed by end-user.
 	// This is only useful for databases built on top of Badger (like Dgraph).
 	// Not recommended for most users.
@@ -99,17 +118,6 @@ type Options struct {
 	maxBatchCount int64 // max entries in batch
 	maxBatchSize  int64 // max batch size in bytes
 
-	// Open the DB as read-only. With this set, multiple processes can
-	// open the same Badger DB. Note: if the DB being opened had crashed
-	// before and has vlog data to be replayed, ReadOnly will cause Open
-	// to fail with an appropriate message.
-	ReadOnly bool
-
-	// Truncate value log to delete corrupt data, if any. Would not truncate if ReadOnly is set.
-	Truncate bool
-
-	// DB-specific logger which will override the global logger.
-	Logger Logger
 }
 
 // DefaultOptions sets a list of recommended options for good performance.
@@ -117,7 +125,7 @@ type Options struct {
 var DefaultOptions = Options{
 	LevelOneSize:        256 << 20,
 	LevelSizeMultiplier: 10,
-	TableLoadingMode:    options.LoadToRAM,
+	TableLoadingMode:    options.MemoryMap,
 	ValueLogLoadingMode: options.MemoryMap,
 	// table.MemoryMap to mmap() the tables.
 	// table.Nothing to not preload the tables.
@@ -140,6 +148,7 @@ var DefaultOptions = Options{
 	ValueThreshold:     32,
 	Truncate:           false,
 	Logger:             defaultLogger,
+	LogRotatesToFlush:  2,
 }
 
 // LSMOnlyOptions follows from DefaultOptions, but sets a higher ValueThreshold

vendor/github.com/dgraph-io/badger/package.json

@@ -1,72 +0,0 @@
-{
-  "author": "dgraph-io",
-  "bugs": {
-    "url": "https://github.com/dgraph-io/badger"
-  },
-  "gx": {
-    "dvcsimport": "github.com/dgraph-io/badger"
-  },
-  "gxDependencies": [
-    {
-      "author": "whyrusleeping",
-      "hash": "QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy",
-      "name": "errors",
-      "version": "0.0.1"
-    },
-    {
-      "author": "magik6k",
-      "hash": "Qmbvv2urkn5Wtwws4yzjE85qRjB293EodchZofJsrTRuvN",
-      "name": "go-lz4",
-      "version": "1.0.0"
-    },
-    {
-      "author": "kubuxu",
-      "hash": "QmWaLViWQF8jgyoLLqqcSrnp6dJpHESiJfzor1vrfDyTZf",
-      "name": "bbloom",
-      "version": "0.1.2"
-    },
-    {
-      "author": "kubuxu",
-      "hash": "QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu",
-      "name": "sys",
-      "version": "0.2.0"
-    },
-    {
-      "author": "whyrusleeping",
-      "hash": "QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn",
-      "name": "go-net",
-      "version": "0.2.0"
-    },
-    {
-      "author": "magik6k",
-      "hash": "QmRFFHk2jw9tgjxv12bCuuTnSbVXxEvYQkuNCLMEv9eUwP",
-      "name": "go-farm",
-      "version": "1.0.0"
-    },
-    {
-      "author": "magik6k",
-      "hash": "QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy",
-      "name": "go-humanize",
-      "version": "1.0.1"
-    },
-    {
-      "author": "GoGo",
-      "hash": "QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB",
-      "name": "gogo-protobuf",
-      "version": "1.2.1"
-    },
-    {
-      "author": "magik6k",
-      "hash": "QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT",
-      "name": "cobra",
-      "version": "0.0.1"
-    }
-  ],
-  "gxVersion": "0.10.0",
-  "language": "go",
-  "license": "Apache 2.0",
-  "name": "badger",
-  "releaseCmd": "git commit -a -m \"gx publish $VERSION\"",
-  "version": "2.11.4"
-}
-

vendor/github.com/dgraph-io/badger/pb/gen.sh

@@ -4,4 +4,4 @@
 
 protos=${GOPATH-$HOME/go}/src/github.com/dgraph-io/badger/pb
 pushd $protos > /dev/null
-protoc --gogofaster_out=. -I=. pb.proto
+protoc --gofast_out=plugins=grpc:. -I=. pb.proto

vendor/github.com/dgraph-io/badger/pb/pb.pb.go

@@ -5,7 +5,7 @@ package pb
 
 import (
 	fmt "fmt"
-	proto "gx/ipfs/QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB/gogo-protobuf/proto"
+	proto "github.com/golang/protobuf/proto"
 	io "io"
 	math "math"
 )
@@ -19,7 +19,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
 type ManifestChange_Operation int32
 
@@ -53,6 +53,11 @@ type KV struct {
 	Version   uint64 `protobuf:"varint,4,opt,name=version,proto3" json:"version,omitempty"`
 	ExpiresAt uint64 `protobuf:"varint,5,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"`
 	Meta      []byte `protobuf:"bytes,6,opt,name=meta,proto3" json:"meta,omitempty"`
+	// Stream id is used to identify which stream the KV came from.
+	StreamId             uint32   `protobuf:"varint,10,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
 }
 
 func (m *KV) Reset()         { *m = KV{} }
@@ -130,8 +135,18 @@ func (m *KV) GetMeta() []byte {
 	return nil
 }
 
+func (m *KV) GetStreamId() uint32 {
+	if m != nil {
+		return m.StreamId
+	}
+	return 0
+}
+
 type KVList struct {
-	Kv []*KV `protobuf:"bytes,1,rep,name=kv,proto3" json:"kv,omitempty"`
+	Kv                   []*KV    `protobuf:"bytes,1,rep,name=kv,proto3" json:"kv,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
 }
 
 func (m *KVList) Reset()         { *m = KVList{} }
@@ -176,7 +191,10 @@ func (m *KVList) GetKv() []*KV {
 
 type ManifestChangeSet struct {
 	// A set of changes that are applied atomically.
-	Changes []*ManifestChange `protobuf:"bytes,1,rep,name=changes,proto3" json:"changes,omitempty"`
+	Changes              []*ManifestChange `protobuf:"bytes,1,rep,name=changes,proto3" json:"changes,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
 }
 
 func (m *ManifestChangeSet) Reset()         { *m = ManifestChangeSet{} }
@@ -220,10 +238,13 @@ func (m *ManifestChangeSet) GetChanges() []*ManifestChange {
 }
 
 type ManifestChange struct {
-	Id       uint64                   `protobuf:"varint,1,opt,name=Id,proto3" json:"Id,omitempty"`
-	Op       ManifestChange_Operation `protobuf:"varint,2,opt,name=Op,proto3,enum=pb.ManifestChange_Operation" json:"Op,omitempty"`
-	Level    uint32                   `protobuf:"varint,3,opt,name=Level,proto3" json:"Level,omitempty"`
-	Checksum []byte                   `protobuf:"bytes,4,opt,name=Checksum,proto3" json:"Checksum,omitempty"`
+	Id                   uint64                   `protobuf:"varint,1,opt,name=Id,proto3" json:"Id,omitempty"`
+	Op                   ManifestChange_Operation `protobuf:"varint,2,opt,name=Op,proto3,enum=pb.ManifestChange_Operation" json:"Op,omitempty"`
+	Level                uint32                   `protobuf:"varint,3,opt,name=Level,proto3" json:"Level,omitempty"`
+	Checksum             []byte                   `protobuf:"bytes,4,opt,name=Checksum,proto3" json:"Checksum,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}                 `json:"-"`
+	XXX_unrecognized     []byte                   `json:"-"`
+	XXX_sizecache        int32                    `json:"-"`
 }
 
 func (m *ManifestChange) Reset()         { *m = ManifestChange{} }
@@ -298,29 +319,30 @@ func init() {
 func init() { proto.RegisterFile("pb.proto", fileDescriptor_f80abaa17e25ccc8) }
 
 var fileDescriptor_f80abaa17e25ccc8 = []byte{
-	// 349 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x91, 0x4b, 0x4e, 0xc2, 0x40,
-	0x18, 0xc7, 0x3b, 0x43, 0x29, 0xf0, 0x89, 0x04, 0x27, 0xc6, 0x4c, 0x7c, 0x34, 0x4d, 0xdd, 0xb0,
-	0x20, 0x5d, 0xe0, 0x09, 0x10, 0xbb, 0x20, 0x40, 0x48, 0x46, 0xc2, 0x96, 0xb4, 0x30, 0x4a, 0x53,
-	0x68, 0x27, 0xed, 0xd0, 0xe8, 0x2d, 0x5c, 0x79, 0x03, 0xef, 0xe2, 0x92, 0xa5, 0x4b, 0x03, 0x17,
-	0x31, 0x1d, 0x1e, 0x09, 0x71, 0xf7, 0x7f, 0xcc, 0xf7, 0x5f, 0xfc, 0x06, 0xca, 0xc2, 0x77, 0x44,
-	0x12, 0xcb, 0x98, 0x60, 0xe1, 0xdb, 0x9f, 0x08, 0x70, 0x6f, 0x4c, 0xea, 0x50, 0x08, 0xf9, 0x3b,
-	0x45, 0x16, 0x6a, 0x54, 0x59, 0x2e, 0xc9, 0x25, 0x14, 0x33, 0x6f, 0xb1, 0xe2, 0x14, 0xab, 0x6c,
-	0x67, 0xc8, 0x0d, 0x54, 0x56, 0x29, 0x4f, 0x26, 0x4b, 0x2e, 0x3d, 0x5a, 0x50, 0x4d, 0x39, 0x0f,
-	0x06, 0x5c, 0x7a, 0x84, 0x42, 0x29, 0xe3, 0x49, 0x1a, 0xc4, 0x11, 0xd5, 0x2d, 0xd4, 0xd0, 0xd9,
-	0xc1, 0x92, 0x3b, 0x00, 0xfe, 0x26, 0x82, 0x84, 0xa7, 0x13, 0x4f, 0xd2, 0xa2, 0x2a, 0x2b, 0xfb,
-	0xa4, 0x2d, 0x09, 0x01, 0x5d, 0x0d, 0x1a, 0x6a, 0x50, 0x69, 0xdb, 0x02, 0xa3, 0x37, 0xee, 0x07,
-	0xa9, 0x24, 0x57, 0x80, 0xc3, 0x8c, 0x22, 0xab, 0xd0, 0x38, 0x6b, 0x19, 0x8e, 0xf0, 0x9d, 0xde,
-	0x98, 0xe1, 0x30, 0xb3, 0xdb, 0x70, 0x31, 0xf0, 0xa2, 0xe0, 0x85, 0xa7, 0xb2, 0x33, 0xf7, 0xa2,
-	0x57, 0xfe, 0xcc, 0x25, 0x69, 0x42, 0x69, 0xaa, 0x4c, 0xba, 0xbf, 0x20, 0xf9, 0xc5, 0xe9, 0x3b,
-	0x76, 0x78, 0x62, 0x7f, 0x21, 0xa8, 0x9d, 0x76, 0xa4, 0x06, 0xb8, 0x3b, 0x53, 0x20, 0x74, 0x86,
-	0xbb, 0x33, 0xd2, 0x04, 0x3c, 0x14, 0x0a, 0x42, 0xad, 0x75, 0xfb, 0x7f, 0xcb, 0x19, 0x0a, 0x9e,
-	0x78, 0x32, 0x88, 0x23, 0x86, 0x87, 0x22, 0xa7, 0xd6, 0xe7, 0x19, 0x5f, 0x28, 0x36, 0xe7, 0x6c,
-	0x67, 0xc8, 0x35, 0x94, 0x3b, 0x73, 0x3e, 0x0d, 0xd3, 0xd5, 0x52, 0x91, 0xa9, 0xb2, 0xa3, 0xb7,
-	0xef, 0xa1, 0x72, 0x9c, 0x20, 0x00, 0x46, 0x87, 0xb9, 0xed, 0x91, 0x5b, 0xd7, 0x72, 0xfd, 0xe4,
-	0xf6, 0xdd, 0x91, 0x5b, 0x47, 0x8f, 0xf4, 0x7b, 0x63, 0xa2, 0xf5, 0xc6, 0x44, 0xbf, 0x1b, 0x13,
-	0x7d, 0x6c, 0x4d, 0x6d, 0xbd, 0x35, 0xb5, 0x9f, 0xad, 0xa9, 0xf9, 0x86, 0xfa, 0xca, 0x87, 0xbf,
-	0x00, 0x00, 0x00, 0xff, 0xff, 0xc9, 0xf9, 0xca, 0x14, 0xd6, 0x01, 0x00, 0x00,
+	// 365 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x91, 0x4f, 0x8a, 0xdb, 0x30,
+	0x14, 0xc6, 0x47, 0x8a, 0xc7, 0xe3, 0xbc, 0xce, 0x04, 0x57, 0x94, 0x22, 0xfa, 0xc7, 0x18, 0x77,
+	0xe3, 0xc5, 0xe0, 0xc5, 0xf4, 0x04, 0x69, 0xea, 0x45, 0x48, 0x42, 0x40, 0x0d, 0xd9, 0x06, 0x39,
+	0x7e, 0x6d, 0x8c, 0x13, 0x5b, 0x58, 0x8a, 0x69, 0x6f, 0xd2, 0x0b, 0xf4, 0x04, 0xbd, 0x44, 0x97,
+	0x3d, 0x42, 0x49, 0x2f, 0x52, 0xac, 0xfc, 0x81, 0xd0, 0xdd, 0xfb, 0xbe, 0xef, 0xbd, 0x4f, 0xf0,
+	0x13, 0x78, 0x2a, 0x4b, 0x54, 0x53, 0x9b, 0x9a, 0x51, 0x95, 0x45, 0x3f, 0x09, 0xd0, 0xc9, 0x92,
+	0xf9, 0xd0, 0x2b, 0xf1, 0x1b, 0x27, 0x21, 0x89, 0xef, 0x45, 0x37, 0xb2, 0x17, 0x70, 0xdb, 0xca,
+	0xed, 0x1e, 0x39, 0xb5, 0xde, 0x51, 0xb0, 0xd7, 0xd0, 0xdf, 0x6b, 0x6c, 0x56, 0x3b, 0x34, 0x92,
+	0xf7, 0x6c, 0xe2, 0x75, 0xc6, 0x0c, 0x8d, 0x64, 0x1c, 0xee, 0x5a, 0x6c, 0x74, 0x51, 0x57, 0xdc,
+	0x09, 0x49, 0xec, 0x88, 0xb3, 0x64, 0x6f, 0x01, 0xf0, 0xab, 0x2a, 0x1a, 0xd4, 0x2b, 0x69, 0xf8,
+	0xad, 0x0d, 0xfb, 0x27, 0x67, 0x68, 0x18, 0x03, 0xc7, 0x16, 0xba, 0xb6, 0xd0, 0xce, 0xdd, 0x4b,
+	0xda, 0x34, 0x28, 0x77, 0xab, 0x22, 0xe7, 0x10, 0x92, 0xf8, 0x41, 0x78, 0x47, 0x63, 0x9c, 0x47,
+	0x21, 0xb8, 0x93, 0xe5, 0xb4, 0xd0, 0x86, 0xbd, 0x04, 0x5a, 0xb6, 0x9c, 0x84, 0xbd, 0xf8, 0xd9,
+	0x93, 0x9b, 0xa8, 0x2c, 0x99, 0x2c, 0x05, 0x2d, 0xdb, 0x68, 0x08, 0xcf, 0x67, 0xb2, 0x2a, 0x3e,
+	0xa3, 0x36, 0xa3, 0x8d, 0xac, 0xbe, 0xe0, 0x27, 0x34, 0xec, 0x11, 0xee, 0xd6, 0x56, 0xe8, 0xd3,
+	0x05, 0xeb, 0x2e, 0xae, 0xf7, 0xc4, 0x79, 0x25, 0xfa, 0x41, 0x60, 0x70, 0x9d, 0xb1, 0x01, 0xd0,
+	0x71, 0x6e, 0x29, 0x39, 0x82, 0x8e, 0x73, 0xf6, 0x08, 0x74, 0xae, 0x2c, 0xa1, 0xc1, 0xd3, 0x9b,
+	0xff, 0xbb, 0x92, 0xb9, 0xc2, 0x46, 0x9a, 0xa2, 0xae, 0x04, 0x9d, 0xab, 0x0e, 0xe9, 0x14, 0x5b,
+	0xdc, 0x5a, 0x70, 0x0f, 0xe2, 0x28, 0xd8, 0x2b, 0xf0, 0x46, 0x1b, 0x5c, 0x97, 0x7a, 0xbf, 0xb3,
+	0xd8, 0xee, 0xc5, 0x45, 0x47, 0xef, 0xa0, 0x7f, 0xa9, 0x60, 0x00, 0xee, 0x48, 0xa4, 0xc3, 0x45,
+	0xea, 0xdf, 0x74, 0xf3, 0xc7, 0x74, 0x9a, 0x2e, 0x52, 0x9f, 0x7c, 0xf0, 0x7f, 0x1d, 0x02, 0xf2,
+	0xfb, 0x10, 0x90, 0x3f, 0x87, 0x80, 0x7c, 0xff, 0x1b, 0xdc, 0x64, 0xae, 0xfd, 0xdf, 0xf7, 0xff,
+	0x02, 0x00, 0x00, 0xff, 0xff, 0xeb, 0x28, 0x5d, 0xcf, 0xeb, 0x01, 0x00, 0x00,
 }
 
 func (m *KV) Marshal() (dAtA []byte, err error) {
@@ -372,6 +394,14 @@ func (m *KV) MarshalTo(dAtA []byte) (int, error) {
 		i = encodeVarintPb(dAtA, i, uint64(len(m.Meta)))
 		i += copy(dAtA[i:], m.Meta)
 	}
+	if m.StreamId != 0 {
+		dAtA[i] = 0x50
+		i++
+		i = encodeVarintPb(dAtA, i, uint64(m.StreamId))
+	}
+	if m.XXX_unrecognized != nil {
+		i += copy(dAtA[i:], m.XXX_unrecognized)
+	}
 	return i, nil
 }
 
@@ -402,6 +432,9 @@ func (m *KVList) MarshalTo(dAtA []byte) (int, error) {
 			i += n
 		}
 	}
+	if m.XXX_unrecognized != nil {
+		i += copy(dAtA[i:], m.XXX_unrecognized)
+	}
 	return i, nil
 }
 
@@ -432,6 +465,9 @@ func (m *ManifestChangeSet) MarshalTo(dAtA []byte) (int, error) {
 			i += n
 		}
 	}
+	if m.XXX_unrecognized != nil {
+		i += copy(dAtA[i:], m.XXX_unrecognized)
+	}
 	return i, nil
 }
 
@@ -471,6 +507,9 @@ func (m *ManifestChange) MarshalTo(dAtA []byte) (int, error) {
 		i = encodeVarintPb(dAtA, i, uint64(len(m.Checksum)))
 		i += copy(dAtA[i:], m.Checksum)
 	}
+	if m.XXX_unrecognized != nil {
+		i += copy(dAtA[i:], m.XXX_unrecognized)
+	}
 	return i, nil
 }
 
@@ -511,6 +550,12 @@ func (m *KV) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovPb(uint64(l))
 	}
+	if m.StreamId != 0 {
+		n += 1 + sovPb(uint64(m.StreamId))
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
 	return n
 }
 
@@ -526,6 +571,9 @@ func (m *KVList) Size() (n int) {
 			n += 1 + l + sovPb(uint64(l))
 		}
 	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
 	return n
 }
 
@@ -541,6 +589,9 @@ func (m *ManifestChangeSet) Size() (n int) {
 			n += 1 + l + sovPb(uint64(l))
 		}
 	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
 	return n
 }
 
@@ -563,6 +614,9 @@ func (m *ManifestChange) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovPb(uint64(l))
 	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
 	return n
 }
 
@@ -782,6 +836,25 @@ func (m *KV) Unmarshal(dAtA []byte) error {
 				m.Meta = []byte{}
 			}
 			iNdEx = postIndex
+		case 10:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field StreamId", wireType)
+			}
+			m.StreamId = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPb
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.StreamId |= uint32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipPb(dAtA[iNdEx:])
@@ -797,6 +870,7 @@ func (m *KV) Unmarshal(dAtA []byte) error {
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
 			iNdEx += skippy
 		}
 	}
@@ -884,6 +958,7 @@ func (m *KVList) Unmarshal(dAtA []byte) error {
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
 			iNdEx += skippy
 		}
 	}
@@ -971,6 +1046,7 @@ func (m *ManifestChangeSet) Unmarshal(dAtA []byte) error {
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
 			iNdEx += skippy
 		}
 	}
@@ -1115,6 +1191,7 @@ func (m *ManifestChange) Unmarshal(dAtA []byte) error {
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
 			iNdEx += skippy
 		}
 	}

vendor/github.com/dgraph-io/badger/pb/pb.proto

@@ -26,6 +26,9 @@ message KV {
   uint64 version = 4;
   uint64 expires_at = 5;
   bytes meta = 6;
+
+  // Stream id is used to identify which stream the KV came from.
+  uint32 stream_id = 10;
 }
 
 message KVList {

vendor/github.com/dgraph-io/badger/publisher.go

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2019 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package badger
+
+import (
+	"bytes"
+	"sync"
+
+	"github.com/dgraph-io/badger/pb"
+	"github.com/dgraph-io/badger/y"
+)
+
+type subscriber struct {
+	prefixes  [][]byte
+	sendCh    chan<- *pb.KVList
+	subCloser *y.Closer
+}
+
+type publisher struct {
+	sync.Mutex
+	pubCh       chan requests
+	subscribers map[uint64]subscriber
+	nextID      uint64
+}
+
+func newPublisher() *publisher {
+	return &publisher{
+		pubCh:       make(chan requests, 1000),
+		subscribers: make(map[uint64]subscriber),
+		nextID:      0,
+	}
+}
+
+func (p *publisher) listenForUpdates(c *y.Closer) {
+	defer func() {
+		p.cleanSubscribers()
+		c.Done()
+	}()
+	slurp := func(batch []*request) {
+		for {
+			select {
+			case reqs := <-p.pubCh:
+				batch = append(batch, reqs...)
+			default:
+				p.publishUpdates(batch)
+				return
+			}
+		}
+	}
+	for {
+		select {
+		case <-c.HasBeenClosed():
+			return
+		case reqs := <-p.pubCh:
+			slurp(reqs)
+		}
+	}
+}
+
+func (p *publisher) publishUpdates(reqs requests) {
+	kvs := &pb.KVList{}
+	p.Lock()
+	defer func() {
+		p.Unlock()
+		// Release all the request.
+		reqs.DecrRef()
+	}()
+	for _, s := range p.subscribers {
+		for _, prefix := range s.prefixes {
+			for _, req := range reqs {
+				for _, e := range req.Entries {
+					// TODO: Use trie to find subscribers.
+					if bytes.HasPrefix(e.Key, prefix) {
+						k := y.SafeCopy(nil, e.Key)
+						kv := &pb.KV{
+							Key:       y.ParseKey(k),
+							Value:     y.SafeCopy(nil, e.Value),
+							Meta:      []byte{e.UserMeta},
+							ExpiresAt: e.ExpiresAt,
+							Version:   y.ParseTs(k),
+						}
+						kvs.Kv = append(kvs.Kv, kv)
+					}
+				}
+			}
+		}
+		if len(kvs.GetKv()) > 0 {
+			s.sendCh <- kvs
+		}
+	}
+}
+
+func (p *publisher) newSubscriber(c *y.Closer, prefixes ...[]byte) (<-chan *pb.KVList, uint64) {
+	p.Lock()
+	defer p.Unlock()
+	ch := make(chan *pb.KVList, 1000)
+	id := p.nextID
+	// Increment next ID.
+	p.nextID++
+	p.subscribers[id] = subscriber{
+		prefixes:  prefixes,
+		sendCh:    ch,
+		subCloser: c,
+	}
+	return ch, id
+}
+
+// cleanSubscribers stops all the subscribers. Ideally, It should be called while closing DB.
+func (p *publisher) cleanSubscribers() {
+	p.Lock()
+	defer p.Unlock()
+	for id, s := range p.subscribers {
+		delete(p.subscribers, id)
+		s.subCloser.SignalAndWait()
+	}
+}
+
+func (p *publisher) deleteSubscriber(id uint64) {
+	p.Lock()
+	defer p.Unlock()
+	if _, ok := p.subscribers[id]; !ok {
+		return
+	}
+	delete(p.subscribers, id)
+}
+
+func (p *publisher) sendUpdates(reqs []*request) {
+	// TODO: Prefix check before pushing into pubCh.
+	if p.noOfSubscribers() != 0 {
+		p.pubCh <- reqs
+	}
+}
+
+func (p *publisher) noOfSubscribers() int {
+	p.Lock()
+	defer p.Unlock()
+	return len(p.subscribers)
+}

vendor/github.com/dgraph-io/badger/skl/arena.go

@@ -20,7 +20,7 @@ import (
 	"sync/atomic"
 	"unsafe"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
+	"github.com/dgraph-io/badger/y"
 )
 
 const (

vendor/github.com/dgraph-io/badger/skl/skl.go

@@ -38,7 +38,7 @@ import (
 	"sync/atomic"
 	"unsafe"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
+	"github.com/dgraph-io/badger/y"
 )
 
 const (

vendor/github.com/dgraph-io/badger/skl/skl_test.go

@@ -1,475 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package skl
-
-import (
-	"encoding/binary"
-	"fmt"
-	"math/rand"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-)
-
-const arenaSize = 1 << 20
-
-func newValue(v int) []byte {
-	return []byte(fmt.Sprintf("%05d", v))
-}
-
-// length iterates over skiplist to give exact size.
-func length(s *Skiplist) int {
-	x := s.getNext(s.head, 0)
-	count := 0
-	for x != nil {
-		count++
-		x = s.getNext(x, 0)
-	}
-	return count
-}
-
-func TestEmpty(t *testing.T) {
-	key := []byte("aaa")
-	l := NewSkiplist(arenaSize)
-
-	v := l.Get(key)
-	require.True(t, v.Value == nil) // Cannot use require.Nil for unsafe.Pointer nil.
-
-	for _, less := range []bool{true, false} {
-		for _, allowEqual := range []bool{true, false} {
-			n, found := l.findNear(key, less, allowEqual)
-			require.Nil(t, n)
-			require.False(t, found)
-		}
-	}
-
-	it := l.NewIterator()
-	require.False(t, it.Valid())
-
-	it.SeekToFirst()
-	require.False(t, it.Valid())
-
-	it.SeekToLast()
-	require.False(t, it.Valid())
-
-	it.Seek(key)
-	require.False(t, it.Valid())
-
-	l.DecrRef()
-	require.True(t, l.valid()) // Check the reference counting.
-
-	it.Close()
-	require.False(t, l.valid()) // Check the reference counting.
-}
-
-// TestBasic tests single-threaded inserts and updates and gets.
-func TestBasic(t *testing.T) {
-	l := NewSkiplist(arenaSize)
-	val1 := newValue(42)
-	val2 := newValue(52)
-	val3 := newValue(62)
-	val4 := newValue(72)
-
-	// Try inserting values.
-	// Somehow require.Nil doesn't work when checking for unsafe.Pointer(nil).
-	l.Put(y.KeyWithTs([]byte("key1"), 0), y.ValueStruct{Value: val1, Meta: 55, UserMeta: 0})
-	l.Put(y.KeyWithTs([]byte("key2"), 2), y.ValueStruct{Value: val2, Meta: 56, UserMeta: 0})
-	l.Put(y.KeyWithTs([]byte("key3"), 0), y.ValueStruct{Value: val3, Meta: 57, UserMeta: 0})
-
-	v := l.Get(y.KeyWithTs([]byte("key"), 0))
-	require.True(t, v.Value == nil)
-
-	v = l.Get(y.KeyWithTs([]byte("key1"), 0))
-	require.True(t, v.Value != nil)
-	require.EqualValues(t, "00042", string(v.Value))
-	require.EqualValues(t, 55, v.Meta)
-
-	v = l.Get(y.KeyWithTs([]byte("key2"), 0))
-	require.True(t, v.Value == nil)
-
-	v = l.Get(y.KeyWithTs([]byte("key3"), 0))
-	require.True(t, v.Value != nil)
-	require.EqualValues(t, "00062", string(v.Value))
-	require.EqualValues(t, 57, v.Meta)
-
-	l.Put(y.KeyWithTs([]byte("key3"), 1), y.ValueStruct{Value: val4, Meta: 12, UserMeta: 0})
-	v = l.Get(y.KeyWithTs([]byte("key3"), 1))
-	require.True(t, v.Value != nil)
-	require.EqualValues(t, "00072", string(v.Value))
-	require.EqualValues(t, 12, v.Meta)
-}
-
-// TestConcurrentBasic tests concurrent writes followed by concurrent reads.
-func TestConcurrentBasic(t *testing.T) {
-	const n = 1000
-	l := NewSkiplist(arenaSize)
-	var wg sync.WaitGroup
-	key := func(i int) []byte {
-		return y.KeyWithTs([]byte(fmt.Sprintf("%05d", i)), 0)
-	}
-	for i := 0; i < n; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			l.Put(key(i),
-				y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0})
-		}(i)
-	}
-	wg.Wait()
-	// Check values. Concurrent reads.
-	for i := 0; i < n; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			v := l.Get(key(i))
-			require.True(t, v.Value != nil)
-			require.EqualValues(t, newValue(i), v.Value)
-		}(i)
-	}
-	wg.Wait()
-	require.EqualValues(t, n, length(l))
-}
-
-// TestOneKey will read while writing to one single key.
-func TestOneKey(t *testing.T) {
-	const n = 100
-	key := y.KeyWithTs([]byte("thekey"), 0)
-	l := NewSkiplist(arenaSize)
-	defer l.DecrRef()
-
-	var wg sync.WaitGroup
-	for i := 0; i < n; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			l.Put(key, y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0})
-		}(i)
-	}
-	// We expect that at least some write made it such that some read returns a value.
-	var sawValue int32
-	for i := 0; i < n; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			p := l.Get(key)
-			if p.Value == nil {
-				return
-			}
-			atomic.AddInt32(&sawValue, 1)
-			v, err := strconv.Atoi(string(p.Value))
-			require.NoError(t, err)
-			require.True(t, 0 <= v && v < n, fmt.Sprintf("invalid value %d", v))
-		}()
-	}
-	wg.Wait()
-	require.True(t, sawValue > 0)
-	require.EqualValues(t, 1, length(l))
-}
-
-func TestFindNear(t *testing.T) {
-	l := NewSkiplist(arenaSize)
-	defer l.DecrRef()
-	for i := 0; i < 1000; i++ {
-		key := fmt.Sprintf("%05d", i*10+5)
-		l.Put(y.KeyWithTs([]byte(key), 0), y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0})
-	}
-
-	n, eq := l.findNear(y.KeyWithTs([]byte("00001"), 0), false, false)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("00005"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("00001"), 0), false, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("00005"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("00001"), 0), true, false)
-	require.Nil(t, n)
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("00001"), 0), true, true)
-	require.Nil(t, n)
-	require.False(t, eq)
-
-	n, eq = l.findNear(y.KeyWithTs([]byte("00005"), 0), false, false)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("00015"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("00005"), 0), false, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("00005"), 0), string(n.key(l.arena)))
-	require.True(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("00005"), 0), true, false)
-	require.Nil(t, n)
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("00005"), 0), true, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("00005"), 0), string(n.key(l.arena)))
-	require.True(t, eq)
-
-	n, eq = l.findNear(y.KeyWithTs([]byte("05555"), 0), false, false)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("05565"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("05555"), 0), false, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("05555"), 0), string(n.key(l.arena)))
-	require.True(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("05555"), 0), true, false)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("05545"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("05555"), 0), true, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("05555"), 0), string(n.key(l.arena)))
-	require.True(t, eq)
-
-	n, eq = l.findNear(y.KeyWithTs([]byte("05558"), 0), false, false)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("05565"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("05558"), 0), false, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("05565"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("05558"), 0), true, false)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("05555"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("05558"), 0), true, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("05555"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-
-	n, eq = l.findNear(y.KeyWithTs([]byte("09995"), 0), false, false)
-	require.Nil(t, n)
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("09995"), 0), false, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("09995"), 0), string(n.key(l.arena)))
-	require.True(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("09995"), 0), true, false)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("09985"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("09995"), 0), true, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("09995"), 0), string(n.key(l.arena)))
-	require.True(t, eq)
-
-	n, eq = l.findNear(y.KeyWithTs([]byte("59995"), 0), false, false)
-	require.Nil(t, n)
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("59995"), 0), false, true)
-	require.Nil(t, n)
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("59995"), 0), true, false)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("09995"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-	n, eq = l.findNear(y.KeyWithTs([]byte("59995"), 0), true, true)
-	require.NotNil(t, n)
-	require.EqualValues(t, y.KeyWithTs([]byte("09995"), 0), string(n.key(l.arena)))
-	require.False(t, eq)
-}
-
-// TestIteratorNext tests a basic iteration over all nodes from the beginning.
-func TestIteratorNext(t *testing.T) {
-	const n = 100
-	l := NewSkiplist(arenaSize)
-	defer l.DecrRef()
-	it := l.NewIterator()
-	defer it.Close()
-	require.False(t, it.Valid())
-	it.SeekToFirst()
-	require.False(t, it.Valid())
-	for i := n - 1; i >= 0; i-- {
-		l.Put(y.KeyWithTs([]byte(fmt.Sprintf("%05d", i)), 0),
-			y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0})
-	}
-	it.SeekToFirst()
-	for i := 0; i < n; i++ {
-		require.True(t, it.Valid())
-		v := it.Value()
-		require.EqualValues(t, newValue(i), v.Value)
-		it.Next()
-	}
-	require.False(t, it.Valid())
-}
-
-// TestIteratorPrev tests a basic iteration over all nodes from the end.
-func TestIteratorPrev(t *testing.T) {
-	const n = 100
-	l := NewSkiplist(arenaSize)
-	defer l.DecrRef()
-	it := l.NewIterator()
-	defer it.Close()
-	require.False(t, it.Valid())
-	it.SeekToFirst()
-	require.False(t, it.Valid())
-	for i := 0; i < n; i++ {
-		l.Put(y.KeyWithTs([]byte(fmt.Sprintf("%05d", i)), 0),
-			y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0})
-	}
-	it.SeekToLast()
-	for i := n - 1; i >= 0; i-- {
-		require.True(t, it.Valid())
-		v := it.Value()
-		require.EqualValues(t, newValue(i), v.Value)
-		it.Prev()
-	}
-	require.False(t, it.Valid())
-}
-
-// TestIteratorSeek tests Seek and SeekForPrev.
-func TestIteratorSeek(t *testing.T) {
-	const n = 100
-	l := NewSkiplist(arenaSize)
-	defer l.DecrRef()
-
-	it := l.NewIterator()
-	defer it.Close()
-
-	require.False(t, it.Valid())
-	it.SeekToFirst()
-	require.False(t, it.Valid())
-	// 1000, 1010, 1020, ..., 1990.
-	for i := n - 1; i >= 0; i-- {
-		v := i*10 + 1000
-		l.Put(y.KeyWithTs([]byte(fmt.Sprintf("%05d", i*10+1000)), 0),
-			y.ValueStruct{Value: newValue(v), Meta: 0, UserMeta: 0})
-	}
-	it.SeekToFirst()
-	require.True(t, it.Valid())
-	v := it.Value()
-	require.EqualValues(t, "01000", v.Value)
-
-	it.Seek(y.KeyWithTs([]byte("01000"), 0))
-	require.True(t, it.Valid())
-	v = it.Value()
-	require.EqualValues(t, "01000", v.Value)
-
-	it.Seek(y.KeyWithTs([]byte("01005"), 0))
-	require.True(t, it.Valid())
-	v = it.Value()
-	require.EqualValues(t, "01010", v.Value)
-
-	it.Seek(y.KeyWithTs([]byte("01010"), 0))
-	require.True(t, it.Valid())
-	v = it.Value()
-	require.EqualValues(t, "01010", v.Value)
-
-	it.Seek(y.KeyWithTs([]byte("99999"), 0))
-	require.False(t, it.Valid())
-
-	// Try SeekForPrev.
-	it.SeekForPrev(y.KeyWithTs([]byte("00"), 0))
-	require.False(t, it.Valid())
-
-	it.SeekForPrev(y.KeyWithTs([]byte("01000"), 0))
-	require.True(t, it.Valid())
-	v = it.Value()
-	require.EqualValues(t, "01000", v.Value)
-
-	it.SeekForPrev(y.KeyWithTs([]byte("01005"), 0))
-	require.True(t, it.Valid())
-	v = it.Value()
-	require.EqualValues(t, "01000", v.Value)
-
-	it.SeekForPrev(y.KeyWithTs([]byte("01010"), 0))
-	require.True(t, it.Valid())
-	v = it.Value()
-	require.EqualValues(t, "01010", v.Value)
-
-	it.SeekForPrev(y.KeyWithTs([]byte("99999"), 0))
-	require.True(t, it.Valid())
-	v = it.Value()
-	require.EqualValues(t, "01990", v.Value)
-}
-
-func randomKey(rng *rand.Rand) []byte {
-	b := make([]byte, 8)
-	key := rng.Uint32()
-	key2 := rng.Uint32()
-	binary.LittleEndian.PutUint32(b, key)
-	binary.LittleEndian.PutUint32(b[4:], key2)
-	return y.KeyWithTs(b, 0)
-}
-
-// Standard test. Some fraction is read. Some fraction is write. Writes have
-// to go through mutex lock.
-func BenchmarkReadWrite(b *testing.B) {
-	value := newValue(123)
-	for i := 0; i <= 10; i++ {
-		readFrac := float32(i) / 10.0
-		b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) {
-			l := NewSkiplist(int64((b.N + 1) * MaxNodeSize))
-			defer l.DecrRef()
-			b.ResetTimer()
-			var count int
-			b.RunParallel(func(pb *testing.PB) {
-				rng := rand.New(rand.NewSource(time.Now().UnixNano()))
-				for pb.Next() {
-					if rng.Float32() < readFrac {
-						v := l.Get(randomKey(rng))
-						if v.Value != nil {
-							count++
-						}
-					} else {
-						l.Put(randomKey(rng), y.ValueStruct{Value: value, Meta: 0, UserMeta: 0})
-					}
-				}
-			})
-		})
-	}
-}
-
-// Standard test. Some fraction is read. Some fraction is write. Writes have
-// to go through mutex lock.
-func BenchmarkReadWriteMap(b *testing.B) {
-	value := newValue(123)
-	for i := 0; i <= 10; i++ {
-		readFrac := float32(i) / 10.0
-		b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) {
-			m := make(map[string][]byte)
-			var mutex sync.RWMutex
-			b.ResetTimer()
-			var count int
-			b.RunParallel(func(pb *testing.PB) {
-				rng := rand.New(rand.NewSource(time.Now().UnixNano()))
-				for pb.Next() {
-					if rand.Float32() < readFrac {
-						mutex.RLock()
-						_, ok := m[string(randomKey(rng))]
-						mutex.RUnlock()
-						if ok {
-							count++
-						}
-					} else {
-						mutex.Lock()
-						m[string(randomKey(rng))] = value
-						mutex.Unlock()
-					}
-				}
-			})
-		})
-	}
-}

vendor/github.com/dgraph-io/badger/stream.go

@@ -19,12 +19,14 @@ package badger
 import (
 	"bytes"
 	"context"
+	"math"
 	"sync"
+	"sync/atomic"
 	"time"
 
-	humanize "gx/ipfs/QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy/go-humanize"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
+	"github.com/dgraph-io/badger/pb"
+	"github.com/dgraph-io/badger/y"
+	humanize "github.com/dustin/go-humanize"
 )
 
 const pageSize = 4 << 20 // 4MB
@@ -66,10 +68,11 @@ type Stream struct {
 	// single goroutine, i.e. logic within Send method can expect single threaded execution.
 	Send func(*pb.KVList) error
 
-	readTs  uint64
-	db      *DB
-	rangeCh chan keyRange
-	kvChan  chan *pb.KVList
+	readTs       uint64
+	db           *DB
+	rangeCh      chan keyRange
+	kvChan       chan *pb.KVList
+	nextStreamId uint32
 }
 
 // ToList is a default implementation of KeyToList. It picks up all valid versions of the key,
@@ -113,6 +116,23 @@ func (st *Stream) ToList(key []byte, itr *Iterator) (*pb.KVList, error) {
 // end byte slices are owned by keyRange struct.
 func (st *Stream) produceRanges(ctx context.Context) {
 	splits := st.db.KeySplits(st.Prefix)
+
+	// We don't need to create more key ranges than NumGo goroutines. This way, we will have limited
+	// number of "streams" coming out, which then helps limit the memory used by SSWriter.
+	{
+		pickEvery := int(math.Floor(float64(len(splits)) / float64(st.NumGo)))
+		if pickEvery < 1 {
+			pickEvery = 1
+		}
+		filtered := splits[:0]
+		for i, split := range splits {
+			if (i+1)%pickEvery == 0 {
+				filtered = append(filtered, split)
+			}
+		}
+		splits = filtered
+	}
+
 	start := y.SafeCopy(nil, st.Prefix)
 	for _, key := range splits {
 		st.rangeCh <- keyRange{left: start, right: y.SafeCopy(nil, []byte(key))}
@@ -143,6 +163,9 @@ func (st *Stream) produceKVs(ctx context.Context) error {
 		itr := txn.NewIterator(iterOpts)
 		defer itr.Close()
 
+		// This unique stream id is used to identify all the keys from this iteration.
+		streamId := atomic.AddUint32(&st.nextStreamId, 1)
+
 		outList := new(pb.KVList)
 		var prevKey []byte
 		for itr.Seek(kr.left); itr.Valid(); {
@@ -174,13 +197,28 @@ func (st *Stream) produceKVs(ctx context.Context) error {
 			outList.Kv = append(outList.Kv, list.Kv...)
 			size += list.Size()
 			if size >= pageSize {
-				st.kvChan <- outList
+				for _, kv := range outList.Kv {
+					kv.StreamId = streamId
+				}
+				select {
+				case st.kvChan <- outList:
+				case <-ctx.Done():
+					return ctx.Err()
+				}
 				outList = new(pb.KVList)
 				size = 0
 			}
 		}
 		if len(outList.Kv) > 0 {
-			st.kvChan <- outList
+			for _, kv := range outList.Kv {
+				kv.StreamId = streamId
+			}
+			// TODO: Think of a way to indicate that a stream is over.
+			select {
+			case st.kvChan <- outList:
+			case <-ctx.Done():
+				return ctx.Err()
+			}
 		}
 		return nil
 	}
@@ -278,8 +316,8 @@ func (st *Stream) Orchestrate(ctx context.Context) error {
 
 	// kvChan should only have a small capacity to ensure that we don't buffer up too much data if
 	// sending is slow. Page size is set to 4MB, which is used to lazily cap the size of each
-	// KVList. To get around 64MB buffer, we can set the channel size to 16.
-	st.kvChan = make(chan *pb.KVList, 16)
+	// KVList. To get 128MB buffer, we can set the channel size to 32.
+	st.kvChan = make(chan *pb.KVList, 32)
 
 	if st.KeyToList == nil {
 		st.KeyToList = st.ToList

vendor/github.com/dgraph-io/badger/stream_test.go

@@ -1,169 +0,0 @@
-/*
- * Copyright 2018 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"math"
-	"os"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-	bpb "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-)
-
-func openManaged(dir string) (*DB, error) {
-	opt := DefaultOptions
-	opt.Dir = dir
-	opt.ValueDir = dir
-
-	return OpenManaged(opt)
-}
-
-func keyWithPrefix(prefix string, k int) []byte {
-	return []byte(fmt.Sprintf("%s-%d", prefix, k))
-}
-
-func keyToInt(k []byte) (string, int) {
-	splits := strings.Split(string(k), "-")
-	key, err := strconv.Atoi(splits[1])
-	y.Check(err)
-	return splits[0], key
-}
-
-func value(k int) []byte {
-	return []byte(fmt.Sprintf("%08d", k))
-}
-
-type collector struct {
-	kv []*bpb.KV
-}
-
-func (c *collector) Send(list *bpb.KVList) error {
-	c.kv = append(c.kv, list.Kv...)
-	return nil
-}
-
-var ctxb = context.Background()
-
-func TestStream(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	db, err := openManaged(dir)
-	require.NoError(t, err)
-
-	var count int
-	for _, prefix := range []string{"p0", "p1", "p2"} {
-		txn := db.NewTransactionAt(math.MaxUint64, true)
-		for i := 1; i <= 100; i++ {
-			require.NoError(t, txn.Set(keyWithPrefix(prefix, i), value(i)))
-			count++
-		}
-		require.NoError(t, txn.CommitAt(5, nil))
-	}
-
-	stream := db.NewStreamAt(math.MaxUint64)
-	stream.LogPrefix = "Testing"
-	c := &collector{}
-	stream.Send = func(list *bpb.KVList) error {
-		return c.Send(list)
-	}
-
-	// Test case 1. Retrieve everything.
-	err = stream.Orchestrate(ctxb)
-	require.NoError(t, err)
-	require.Equal(t, 300, len(c.kv), "Expected 300. Got: %d", len(c.kv))
-
-	m := make(map[string]int)
-	for _, kv := range c.kv {
-		prefix, ki := keyToInt(kv.Key)
-		expected := value(ki)
-		require.Equal(t, expected, kv.Value)
-		m[prefix]++
-	}
-	require.Equal(t, 3, len(m))
-	for pred, count := range m {
-		require.Equal(t, 100, count, "Count mismatch for pred: %s", pred)
-	}
-
-	// Test case 2. Retrieve only 1 predicate.
-	stream.Prefix = []byte("p1")
-	c.kv = c.kv[:0]
-	err = stream.Orchestrate(ctxb)
-	require.NoError(t, err)
-	require.Equal(t, 100, len(c.kv), "Expected 100. Got: %d", len(c.kv))
-
-	m = make(map[string]int)
-	for _, kv := range c.kv {
-		prefix, ki := keyToInt(kv.Key)
-		expected := value(ki)
-		require.Equal(t, expected, kv.Value)
-		m[prefix]++
-	}
-	require.Equal(t, 1, len(m))
-	for pred, count := range m {
-		require.Equal(t, 100, count, "Count mismatch for pred: %s", pred)
-	}
-
-	// Test case 3. Retrieve select keys within the predicate.
-	c.kv = c.kv[:0]
-	stream.ChooseKey = func(item *Item) bool {
-		_, k := keyToInt(item.Key())
-		return k%2 == 0
-	}
-	err = stream.Orchestrate(ctxb)
-	require.NoError(t, err)
-	require.Equal(t, 50, len(c.kv), "Expected 50. Got: %d", len(c.kv))
-
-	m = make(map[string]int)
-	for _, kv := range c.kv {
-		prefix, ki := keyToInt(kv.Key)
-		expected := value(ki)
-		require.Equal(t, expected, kv.Value)
-		m[prefix]++
-	}
-	require.Equal(t, 1, len(m))
-	for pred, count := range m {
-		require.Equal(t, 50, count, "Count mismatch for pred: %s", pred)
-	}
-
-	// Test case 4. Retrieve select keys from all predicates.
-	c.kv = c.kv[:0]
-	stream.Prefix = []byte{}
-	err = stream.Orchestrate(ctxb)
-	require.NoError(t, err)
-	require.Equal(t, 150, len(c.kv), "Expected 150. Got: %d", len(c.kv))
-
-	m = make(map[string]int)
-	for _, kv := range c.kv {
-		prefix, ki := keyToInt(kv.Key)
-		expected := value(ki)
-		require.Equal(t, expected, kv.Value)
-		m[prefix]++
-	}
-	require.Equal(t, 3, len(m))
-	for pred, count := range m {
-		require.Equal(t, 50, count, "Count mismatch for pred: %s", pred)
-	}
-}

vendor/github.com/dgraph-io/badger/stream_writer.go

@@ -0,0 +1,311 @@
+/*
+ * Copyright 2019 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package badger
+
+import (
+	"bytes"
+	"math"
+
+	"github.com/dgraph-io/badger/pb"
+	"github.com/dgraph-io/badger/table"
+	"github.com/dgraph-io/badger/y"
+	humanize "github.com/dustin/go-humanize"
+	"github.com/pkg/errors"
+)
+
+const headStreamId uint32 = math.MaxUint32
+
+// StreamWriter is used to write data coming from multiple streams. The streams must not have any
+// overlapping key ranges. Within each stream, the keys must be sorted. Badger Stream framework is
+// capable of generating such an output. So, this StreamWriter can be used at the other end to build
+// BadgerDB at a much faster pace by writing SSTables (and value logs) directly to LSM tree levels
+// without causing any compactions at all. This is way faster than using batched writer or using
+// transactions, but only applicable in situations where the keys are pre-sorted and the DB is being
+// bootstrapped. Existing data would get deleted when using this writer. So, this is only useful
+// when restoring from backup or replicating DB across servers.
+//
+// StreamWriter should not be called on in-use DB instances. It is designed only to bootstrap new
+// DBs.
+type StreamWriter struct {
+	db         *DB
+	done       func()
+	throttle   *y.Throttle
+	head       valuePointer
+	maxVersion uint64
+	writers    map[uint32]*sortedWriter
+}
+
+// NewStreamWriter creates a StreamWriter. Right after creating StreamWriter, Prepare must be
+// called. The memory usage of a StreamWriter is directly proportional to the number of streams
+// possible. So, efforts must be made to keep the number of streams low. Stream framework would
+// typically use 16 goroutines and hence create 16 streams.
+func (db *DB) NewStreamWriter() *StreamWriter {
+	return &StreamWriter{
+		db: db,
+		// throttle shouldn't make much difference. Memory consumption is based on the number of
+		// concurrent streams being processed.
+		throttle: y.NewThrottle(16),
+		writers:  make(map[uint32]*sortedWriter),
+	}
+}
+
+// Prepare should be called before writing any entry to StreamWriter. It deletes all data present in
+// existing DB, stops compactions and any writes being done by other means. Be very careful when
+// calling Prepare, because it could result in permanent data loss. Not calling Prepare would result
+// in a corrupt Badger instance.
+func (sw *StreamWriter) Prepare() error {
+	var err error
+	sw.done, err = sw.db.dropAll()
+	return err
+}
+
+// Write writes KVList to DB. Each KV within the list contains the stream id which StreamWriter
+// would use to demux the writes.
+func (sw *StreamWriter) Write(kvs *pb.KVList) error {
+	var entries []*Entry
+	for _, kv := range kvs.Kv {
+		var meta, userMeta byte
+		if len(kv.Meta) > 0 {
+			meta = kv.Meta[0]
+		}
+		if len(kv.UserMeta) > 0 {
+			userMeta = kv.UserMeta[0]
+		}
+		if sw.maxVersion < kv.Version {
+			sw.maxVersion = kv.Version
+		}
+		e := &Entry{
+			Key:       y.KeyWithTs(kv.Key, kv.Version),
+			Value:     kv.Value,
+			UserMeta:  userMeta,
+			ExpiresAt: kv.ExpiresAt,
+			meta:      meta,
+		}
+		// If the value can be colocated with the key in LSM tree, we can skip
+		// writing the value to value log.
+		e.skipVlog = sw.db.shouldWriteValueToLSM(*e)
+		entries = append(entries, e)
+	}
+	req := &request{
+		Entries: entries,
+	}
+	y.AssertTrue(len(kvs.Kv) == len(req.Entries))
+	if err := sw.db.vlog.write([]*request{req}); err != nil {
+		return err
+	}
+
+	for i, kv := range kvs.Kv {
+		e := req.Entries[i]
+		vptr := req.Ptrs[i]
+		if !vptr.IsZero() {
+			y.AssertTrue(sw.head.Less(vptr))
+			sw.head = vptr
+		}
+
+		writer, ok := sw.writers[kv.StreamId]
+		if !ok {
+			writer = sw.newWriter(kv.StreamId)
+			sw.writers[kv.StreamId] = writer
+		}
+
+		var vs y.ValueStruct
+		if e.skipVlog {
+			vs = y.ValueStruct{
+				Value:     e.Value,
+				Meta:      e.meta,
+				UserMeta:  e.UserMeta,
+				ExpiresAt: e.ExpiresAt,
+			}
+		} else {
+			vbuf := make([]byte, vptrSize)
+			vs = y.ValueStruct{
+				Value:     vptr.Encode(vbuf),
+				Meta:      e.meta | bitValuePointer,
+				UserMeta:  e.UserMeta,
+				ExpiresAt: e.ExpiresAt,
+			}
+		}
+		if err := writer.Add(e.Key, vs); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Flush is called once we are done writing all the entries. It syncs DB directories. It also
+// updates Oracle with maxVersion found in all entries (if DB is not managed).
+func (sw *StreamWriter) Flush() error {
+	defer sw.done()
+	for _, writer := range sw.writers {
+		if err := writer.Done(); err != nil {
+			return err
+		}
+	}
+
+	// Encode and write the value log head into a new table.
+	data := make([]byte, vptrSize)
+	sw.head.Encode(data)
+	headWriter := sw.newWriter(headStreamId)
+	if err := headWriter.Add(
+		y.KeyWithTs(head, sw.maxVersion),
+		y.ValueStruct{Value: data}); err != nil {
+		return err
+	}
+	if err := headWriter.Done(); err != nil {
+		return err
+	}
+
+	if !sw.db.opt.managedTxns {
+		sw.db.orc = newOracle(sw.db.opt)
+		sw.db.orc.nextTxnTs = sw.maxVersion
+		sw.db.orc.txnMark.Done(sw.maxVersion)
+		sw.db.orc.readMark.Done(sw.maxVersion)
+		sw.db.orc.incrementNextTs()
+	}
+
+	// Wait for all files to be written.
+	if err := sw.throttle.Finish(); err != nil {
+		return err
+	}
+
+	// Now sync the directories, so all the files are registered.
+	if sw.db.opt.ValueDir != sw.db.opt.Dir {
+		if err := syncDir(sw.db.opt.ValueDir); err != nil {
+			return err
+		}
+	}
+	return syncDir(sw.db.opt.Dir)
+}
+
+type sortedWriter struct {
+	db       *DB
+	throttle *y.Throttle
+
+	builder  *table.Builder
+	lastKey  []byte
+	streamId uint32
+}
+
+func (sw *StreamWriter) newWriter(streamId uint32) *sortedWriter {
+	return &sortedWriter{
+		db:       sw.db,
+		streamId: streamId,
+		throttle: sw.throttle,
+		builder:  table.NewTableBuilder(),
+	}
+}
+
+// ErrUnsortedKey is returned when any out of order key arrives at sortedWriter during call to Add.
+var ErrUnsortedKey = errors.New("Keys not in sorted order")
+
+// Add adds key and vs to sortedWriter.
+func (w *sortedWriter) Add(key []byte, vs y.ValueStruct) error {
+	if bytes.Compare(key, w.lastKey) <= 0 {
+		return ErrUnsortedKey
+	}
+	sameKey := y.SameKey(key, w.lastKey)
+	w.lastKey = y.SafeCopy(w.lastKey, key)
+
+	if err := w.builder.Add(key, vs); err != nil {
+		return err
+	}
+	// Same keys should go into the same SSTable.
+	if !sameKey && w.builder.ReachedCapacity(w.db.opt.MaxTableSize) {
+		return w.send()
+	}
+	return nil
+}
+
+func (w *sortedWriter) send() error {
+	if err := w.throttle.Do(); err != nil {
+		return err
+	}
+	go func(builder *table.Builder) {
+		data := builder.Finish()
+		err := w.createTable(data)
+		w.throttle.Done(err)
+	}(w.builder)
+	w.builder = table.NewTableBuilder()
+	return nil
+}
+
+// Done is called once we are done writing all keys and valueStructs
+// to sortedWriter. It completes writing current SST to disk.
+func (w *sortedWriter) Done() error {
+	if w.builder.Empty() {
+		return nil
+	}
+	return w.send()
+}
+
+func (w *sortedWriter) createTable(data []byte) error {
+	if len(data) == 0 {
+		return nil
+	}
+	fileID := w.db.lc.reserveFileID()
+	fd, err := y.CreateSyncedFile(table.NewFilename(fileID, w.db.opt.Dir), true)
+	if err != nil {
+		return err
+	}
+	if _, err := fd.Write(data); err != nil {
+		return err
+	}
+	tbl, err := table.OpenTable(fd, w.db.opt.TableLoadingMode, nil)
+	if err != nil {
+		return err
+	}
+	lc := w.db.lc
+
+	var lhandler *levelHandler
+	// We should start the levels from 1, because we need level 0 to set the !badger!head key. We
+	// cannot mix up this key with other keys from the DB, otherwise we would introduce a range
+	// overlap violation.
+	y.AssertTrue(len(lc.levels) > 1)
+	for _, l := range lc.levels[1:] {
+		ratio := float64(l.getTotalSize()) / float64(l.maxTotalSize)
+		if ratio < 1.0 {
+			lhandler = l
+			break
+		}
+	}
+	if lhandler == nil {
+		// If we're exceeding the size of the lowest level, shove it in the lowest level. Can't do
+		// better than that.
+		lhandler = lc.levels[len(lc.levels)-1]
+	}
+	if w.streamId == headStreamId {
+		// This is a special !badger!head key. We should store it at level 0, separate from all the
+		// other keys to avoid an overlap.
+		lhandler = lc.levels[0]
+	}
+	// Now that table can be opened successfully, let's add this to the MANIFEST.
+	change := &pb.ManifestChange{
+		Id:       tbl.ID(),
+		Op:       pb.ManifestChange_CREATE,
+		Level:    uint32(lhandler.level),
+		Checksum: tbl.Checksum,
+	}
+	if err := w.db.manifest.addChanges([]*pb.ManifestChange{change}); err != nil {
+		return err
+	}
+	if err := lhandler.replaceTables([]*table.Table{}, []*table.Table{tbl}); err != nil {
+		return err
+	}
+	w.db.opt.Infof("Table created: %d at level: %d for stream: %d. Size: %s\n",
+		fileID, lhandler.level, w.streamId, humanize.Bytes(uint64(tbl.Size())))
+	return nil
+}

vendor/github.com/dgraph-io/badger/structs.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"hash/crc32"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
+	"github.com/dgraph-io/badger/y"
 )
 
 type valuePointer struct {
@@ -85,7 +85,8 @@ type Entry struct {
 	meta      byte
 
 	// Fields maintained internally.
-	offset uint32
+	offset   uint32
+	skipVlog bool
 }
 
 func (e *Entry) estimateSize(threshold int) int {

vendor/github.com/dgraph-io/badger/table/builder.go

@@ -22,8 +22,8 @@ import (
 	"io"
 	"math"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmWaLViWQF8jgyoLLqqcSrnp6dJpHESiJfzor1vrfDyTZf/bbloom"
+	"github.com/AndreasBriese/bbloom"
+	"github.com/dgraph-io/badger/y"
 )
 
 var (
@@ -201,7 +201,7 @@ func (b *Builder) blockIndex() []byte {
 
 // Finish finishes the table by appending the index.
 func (b *Builder) Finish() []byte {
-	bf, _ := bbloom.New(float64(b.keyCount), 0.01)
+	bf := bbloom.New(float64(b.keyCount), 0.01)
 	var klen [2]byte
 	key := make([]byte, 1024)
 	for {
@@ -224,7 +224,7 @@ func (b *Builder) Finish() []byte {
 	b.buf.Write(index)
 
 	// Write bloom filter.
-	bdata, _ := bf.JSONMarshal()
+	bdata := bf.JSONMarshal()
 	n, err := b.buf.Write(bdata)
 	y.Check(err)
 	var buf [4]byte

vendor/github.com/dgraph-io/badger/table/iterator.go

@@ -22,8 +22,8 @@ import (
 	"math"
 	"sort"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/y"
+	"github.com/pkg/errors"
 )
 
 type blockIterator struct {

vendor/github.com/dgraph-io/badger/table/table.go

@@ -30,10 +30,10 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
-	"gx/ipfs/QmWaLViWQF8jgyoLLqqcSrnp6dJpHESiJfzor1vrfDyTZf/bbloom"
+	"github.com/AndreasBriese/bbloom"
+	"github.com/dgraph-io/badger/options"
+	"github.com/dgraph-io/badger/y"
+	"github.com/pkg/errors"
 )
 
 const fileSuffix = ".sst"
@@ -59,7 +59,7 @@ type Table struct {
 	tableSize int      // Initialized in OpenTable, using fd.Stat().
 
 	blockIndex []keyOffset
-	ref        int32 // For file garbage collection.  Atomic.
+	ref        int32 // For file garbage collection. Atomic.
 
 	loadingMode options.FileLoadingMode
 	mmap        []byte // Memory mapped.
@@ -197,7 +197,7 @@ func (t *Table) Close() error {
 	return t.fd.Close()
 }
 
-func (t *Table) read(off int, sz int) ([]byte, error) {
+func (t *Table) read(off, sz int) ([]byte, error) {
 	if len(t.mmap) > 0 {
 		if len(t.mmap[off:]) < sz {
 			return nil, y.ErrEOF
@@ -212,7 +212,7 @@ func (t *Table) read(off int, sz int) ([]byte, error) {
 	return res, err
 }
 
-func (t *Table) readNoFail(off int, sz int) []byte {
+func (t *Table) readNoFail(off, sz int) []byte {
 	res, err := t.read(off, sz)
 	y.Check(err)
 	return res
@@ -230,7 +230,7 @@ func (t *Table) readIndex() error {
 	bloomLen := int(binary.BigEndian.Uint32(buf))
 	readPos -= bloomLen
 	data := t.readNoFail(readPos, bloomLen)
-	t.bf = *bbloom.JSONUnmarshal(data)
+	t.bf = bbloom.JSONUnmarshal(data)
 
 	readPos -= 4
 	buf = t.readNoFail(readPos, 4)

vendor/github.com/dgraph-io/badger/table/table_test.go

@@ -1,729 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package table
-
-import (
-	"fmt"
-	"math/rand"
-	"os"
-	"sort"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-)
-
-func key(prefix string, i int) string {
-	return prefix + fmt.Sprintf("%04d", i)
-}
-
-func buildTestTable(t *testing.T, prefix string, n int) *os.File {
-	y.AssertTrue(n <= 10000)
-	keyValues := make([][]string, n)
-	for i := 0; i < n; i++ {
-		k := key(prefix, i)
-		v := fmt.Sprintf("%d", i)
-		keyValues[i] = []string{k, v}
-	}
-	return buildTable(t, keyValues)
-}
-
-// keyValues is n by 2 where n is number of pairs.
-func buildTable(t *testing.T, keyValues [][]string) *os.File {
-	b := NewTableBuilder()
-	defer b.Close()
-	// TODO: Add test for file garbage collection here. No files should be left after the tests here.
-
-	filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63())
-	f, err := y.OpenSyncedFile(filename, true)
-	if t != nil {
-		require.NoError(t, err)
-	} else {
-		y.Check(err)
-	}
-
-	sort.Slice(keyValues, func(i, j int) bool {
-		return keyValues[i][0] < keyValues[j][0]
-	})
-	for _, kv := range keyValues {
-		y.AssertTrue(len(kv) == 2)
-		err := b.Add(y.KeyWithTs([]byte(kv[0]), 0), y.ValueStruct{Value: []byte(kv[1]), Meta: 'A', UserMeta: 0})
-		if t != nil {
-			require.NoError(t, err)
-		} else {
-			y.Check(err)
-		}
-	}
-	f.Write(b.Finish())
-	f.Close()
-	f, _ = y.OpenSyncedFile(filename, true)
-	return f
-}
-
-func TestTableIterator(t *testing.T) {
-	for _, n := range []int{99, 100, 101} {
-		t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) {
-			f := buildTestTable(t, "key", n)
-			table, err := OpenTable(f, options.MemoryMap, nil)
-			require.NoError(t, err)
-			defer table.DecrRef()
-			it := table.NewIterator(false)
-			defer it.Close()
-			count := 0
-			for it.Rewind(); it.Valid(); it.Next() {
-				v := it.Value()
-				k := y.KeyWithTs([]byte(key("key", count)), 0)
-				require.EqualValues(t, k, it.Key())
-				require.EqualValues(t, fmt.Sprintf("%d", count), string(v.Value))
-				count++
-			}
-			require.Equal(t, count, n)
-		})
-	}
-}
-
-func TestSeekToFirst(t *testing.T) {
-	for _, n := range []int{99, 100, 101, 199, 200, 250, 9999, 10000} {
-		t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) {
-			f := buildTestTable(t, "key", n)
-			table, err := OpenTable(f, options.MemoryMap, nil)
-			require.NoError(t, err)
-			defer table.DecrRef()
-			it := table.NewIterator(false)
-			defer it.Close()
-			it.seekToFirst()
-			require.True(t, it.Valid())
-			v := it.Value()
-			require.EqualValues(t, "0", string(v.Value))
-			require.EqualValues(t, 'A', v.Meta)
-		})
-	}
-}
-
-func TestSeekToLast(t *testing.T) {
-	for _, n := range []int{99, 100, 101, 199, 200, 250, 9999, 10000} {
-		t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) {
-			f := buildTestTable(t, "key", n)
-			table, err := OpenTable(f, options.MemoryMap, nil)
-			require.NoError(t, err)
-			defer table.DecrRef()
-			it := table.NewIterator(false)
-			defer it.Close()
-			it.seekToLast()
-			require.True(t, it.Valid())
-			v := it.Value()
-			require.EqualValues(t, fmt.Sprintf("%d", n-1), string(v.Value))
-			require.EqualValues(t, 'A', v.Meta)
-			it.prev()
-			require.True(t, it.Valid())
-			v = it.Value()
-			require.EqualValues(t, fmt.Sprintf("%d", n-2), string(v.Value))
-			require.EqualValues(t, 'A', v.Meta)
-		})
-	}
-}
-
-func TestSeek(t *testing.T) {
-	f := buildTestTable(t, "k", 10000)
-	table, err := OpenTable(f, options.MemoryMap, nil)
-	require.NoError(t, err)
-	defer table.DecrRef()
-
-	it := table.NewIterator(false)
-	defer it.Close()
-
-	var data = []struct {
-		in    string
-		valid bool
-		out   string
-	}{
-		{"abc", true, "k0000"},
-		{"k0100", true, "k0100"},
-		{"k0100b", true, "k0101"}, // Test case where we jump to next block.
-		{"k1234", true, "k1234"},
-		{"k1234b", true, "k1235"},
-		{"k9999", true, "k9999"},
-		{"z", false, ""},
-	}
-
-	for _, tt := range data {
-		it.seek(y.KeyWithTs([]byte(tt.in), 0))
-		if !tt.valid {
-			require.False(t, it.Valid())
-			continue
-		}
-		require.True(t, it.Valid())
-		k := it.Key()
-		require.EqualValues(t, tt.out, string(y.ParseKey(k)))
-	}
-}
-
-func TestSeekForPrev(t *testing.T) {
-	f := buildTestTable(t, "k", 10000)
-	table, err := OpenTable(f, options.MemoryMap, nil)
-	require.NoError(t, err)
-	defer table.DecrRef()
-
-	it := table.NewIterator(false)
-	defer it.Close()
-
-	var data = []struct {
-		in    string
-		valid bool
-		out   string
-	}{
-		{"abc", false, ""},
-		{"k0100", true, "k0100"},
-		{"k0100b", true, "k0100"}, // Test case where we jump to next block.
-		{"k1234", true, "k1234"},
-		{"k1234b", true, "k1234"},
-		{"k9999", true, "k9999"},
-		{"z", true, "k9999"},
-	}
-
-	for _, tt := range data {
-		it.seekForPrev(y.KeyWithTs([]byte(tt.in), 0))
-		if !tt.valid {
-			require.False(t, it.Valid())
-			continue
-		}
-		require.True(t, it.Valid())
-		k := it.Key()
-		require.EqualValues(t, tt.out, string(y.ParseKey(k)))
-	}
-}
-
-func TestIterateFromStart(t *testing.T) {
-	// Vary the number of elements added.
-	for _, n := range []int{99, 100, 101, 199, 200, 250, 9999, 10000} {
-		t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) {
-			f := buildTestTable(t, "key", n)
-			table, err := OpenTable(f, options.MemoryMap, nil)
-			require.NoError(t, err)
-			defer table.DecrRef()
-			ti := table.NewIterator(false)
-			defer ti.Close()
-			ti.reset()
-			ti.seekToFirst()
-			require.True(t, ti.Valid())
-			// No need to do a Next.
-			// ti.Seek brings us to the first key >= "". Essentially a SeekToFirst.
-			var count int
-			for ; ti.Valid(); ti.next() {
-				v := ti.Value()
-				require.EqualValues(t, fmt.Sprintf("%d", count), string(v.Value))
-				require.EqualValues(t, 'A', v.Meta)
-				count++
-			}
-			require.EqualValues(t, n, count)
-		})
-	}
-}
-
-func TestIterateFromEnd(t *testing.T) {
-	// Vary the number of elements added.
-	for _, n := range []int{99, 100, 101, 199, 200, 250, 9999, 10000} {
-		t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) {
-			f := buildTestTable(t, "key", n)
-			table, err := OpenTable(f, options.FileIO, nil)
-			require.NoError(t, err)
-			defer table.DecrRef()
-			ti := table.NewIterator(false)
-			defer ti.Close()
-			ti.reset()
-			ti.seek(y.KeyWithTs([]byte("zzzzzz"), 0)) // Seek to end, an invalid element.
-			require.False(t, ti.Valid())
-			for i := n - 1; i >= 0; i-- {
-				ti.prev()
-				require.True(t, ti.Valid())
-				v := ti.Value()
-				require.EqualValues(t, fmt.Sprintf("%d", i), string(v.Value))
-				require.EqualValues(t, 'A', v.Meta)
-			}
-			ti.prev()
-			require.False(t, ti.Valid())
-		})
-	}
-}
-
-func TestTable(t *testing.T) {
-	f := buildTestTable(t, "key", 10000)
-	table, err := OpenTable(f, options.FileIO, nil)
-	require.NoError(t, err)
-	defer table.DecrRef()
-	ti := table.NewIterator(false)
-	defer ti.Close()
-	kid := 1010
-	seek := y.KeyWithTs([]byte(key("key", kid)), 0)
-	for ti.seek(seek); ti.Valid(); ti.next() {
-		k := ti.Key()
-		require.EqualValues(t, string(y.ParseKey(k)), key("key", kid))
-		kid++
-	}
-	if kid != 10000 {
-		t.Errorf("Expected kid: 10000. Got: %v", kid)
-	}
-
-	ti.seek(y.KeyWithTs([]byte(key("key", 99999)), 0))
-	require.False(t, ti.Valid())
-
-	ti.seek(y.KeyWithTs([]byte(key("key", -1)), 0))
-	require.True(t, ti.Valid())
-	k := ti.Key()
-	require.EqualValues(t, string(y.ParseKey(k)), key("key", 0))
-}
-
-func TestIterateBackAndForth(t *testing.T) {
-	f := buildTestTable(t, "key", 10000)
-	table, err := OpenTable(f, options.MemoryMap, nil)
-	require.NoError(t, err)
-	defer table.DecrRef()
-
-	seek := y.KeyWithTs([]byte(key("key", 1010)), 0)
-	it := table.NewIterator(false)
-	defer it.Close()
-	it.seek(seek)
-	require.True(t, it.Valid())
-	k := it.Key()
-	require.EqualValues(t, seek, k)
-
-	it.prev()
-	it.prev()
-	require.True(t, it.Valid())
-	k = it.Key()
-	require.EqualValues(t, key("key", 1008), string(y.ParseKey(k)))
-
-	it.next()
-	it.next()
-	require.True(t, it.Valid())
-	k = it.Key()
-	require.EqualValues(t, key("key", 1010), y.ParseKey(k))
-
-	it.seek(y.KeyWithTs([]byte(key("key", 2000)), 0))
-	require.True(t, it.Valid())
-	k = it.Key()
-	require.EqualValues(t, key("key", 2000), y.ParseKey(k))
-
-	it.prev()
-	require.True(t, it.Valid())
-	k = it.Key()
-	require.EqualValues(t, key("key", 1999), y.ParseKey(k))
-
-	it.seekToFirst()
-	k = it.Key()
-	require.EqualValues(t, key("key", 0), y.ParseKey(k))
-}
-
-func TestUniIterator(t *testing.T) {
-	f := buildTestTable(t, "key", 10000)
-	table, err := OpenTable(f, options.MemoryMap, nil)
-	require.NoError(t, err)
-	defer table.DecrRef()
-	{
-		it := table.NewIterator(false)
-		defer it.Close()
-		var count int
-		for it.Rewind(); it.Valid(); it.Next() {
-			v := it.Value()
-			require.EqualValues(t, fmt.Sprintf("%d", count), string(v.Value))
-			require.EqualValues(t, 'A', v.Meta)
-			count++
-		}
-		require.EqualValues(t, 10000, count)
-	}
-	{
-		it := table.NewIterator(true)
-		defer it.Close()
-		var count int
-		for it.Rewind(); it.Valid(); it.Next() {
-			v := it.Value()
-			require.EqualValues(t, fmt.Sprintf("%d", 10000-1-count), string(v.Value))
-			require.EqualValues(t, 'A', v.Meta)
-			count++
-		}
-		require.EqualValues(t, 10000, count)
-	}
-}
-
-// Try having only one table.
-func TestConcatIteratorOneTable(t *testing.T) {
-	f := buildTable(t, [][]string{
-		{"k1", "a1"},
-		{"k2", "a2"},
-	})
-
-	tbl, err := OpenTable(f, options.MemoryMap, nil)
-	require.NoError(t, err)
-	defer tbl.DecrRef()
-
-	it := NewConcatIterator([]*Table{tbl}, false)
-	defer it.Close()
-
-	it.Rewind()
-	require.True(t, it.Valid())
-	k := it.Key()
-	require.EqualValues(t, "k1", string(y.ParseKey(k)))
-	vs := it.Value()
-	require.EqualValues(t, "a1", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-}
-
-func TestConcatIterator(t *testing.T) {
-	f := buildTestTable(t, "keya", 10000)
-	f2 := buildTestTable(t, "keyb", 10000)
-	f3 := buildTestTable(t, "keyc", 10000)
-	tbl, err := OpenTable(f, options.MemoryMap, nil)
-	require.NoError(t, err)
-	defer tbl.DecrRef()
-	tbl2, err := OpenTable(f2, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer tbl2.DecrRef()
-	tbl3, err := OpenTable(f3, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer tbl3.DecrRef()
-
-	{
-		it := NewConcatIterator([]*Table{tbl, tbl2, tbl3}, false)
-		defer it.Close()
-		it.Rewind()
-		require.True(t, it.Valid())
-		var count int
-		for ; it.Valid(); it.Next() {
-			vs := it.Value()
-			require.EqualValues(t, fmt.Sprintf("%d", count%10000), string(vs.Value))
-			require.EqualValues(t, 'A', vs.Meta)
-			count++
-		}
-		require.EqualValues(t, 30000, count)
-
-		it.Seek(y.KeyWithTs([]byte("a"), 0))
-		require.EqualValues(t, "keya0000", string(y.ParseKey(it.Key())))
-		vs := it.Value()
-		require.EqualValues(t, "0", string(vs.Value))
-
-		it.Seek(y.KeyWithTs([]byte("keyb"), 0))
-		require.EqualValues(t, "keyb0000", string(y.ParseKey(it.Key())))
-		vs = it.Value()
-		require.EqualValues(t, "0", string(vs.Value))
-
-		it.Seek(y.KeyWithTs([]byte("keyb9999b"), 0))
-		require.EqualValues(t, "keyc0000", string(y.ParseKey(it.Key())))
-		vs = it.Value()
-		require.EqualValues(t, "0", string(vs.Value))
-
-		it.Seek(y.KeyWithTs([]byte("keyd"), 0))
-		require.False(t, it.Valid())
-	}
-	{
-		it := NewConcatIterator([]*Table{tbl, tbl2, tbl3}, true)
-		defer it.Close()
-		it.Rewind()
-		require.True(t, it.Valid())
-		var count int
-		for ; it.Valid(); it.Next() {
-			vs := it.Value()
-			require.EqualValues(t, fmt.Sprintf("%d", 10000-(count%10000)-1), string(vs.Value))
-			require.EqualValues(t, 'A', vs.Meta)
-			count++
-		}
-		require.EqualValues(t, 30000, count)
-
-		it.Seek(y.KeyWithTs([]byte("a"), 0))
-		require.False(t, it.Valid())
-
-		it.Seek(y.KeyWithTs([]byte("keyb"), 0))
-		require.EqualValues(t, "keya9999", string(y.ParseKey(it.Key())))
-		vs := it.Value()
-		require.EqualValues(t, "9999", string(vs.Value))
-
-		it.Seek(y.KeyWithTs([]byte("keyb9999b"), 0))
-		require.EqualValues(t, "keyb9999", string(y.ParseKey(it.Key())))
-		vs = it.Value()
-		require.EqualValues(t, "9999", string(vs.Value))
-
-		it.Seek(y.KeyWithTs([]byte("keyd"), 0))
-		require.EqualValues(t, "keyc9999", string(y.ParseKey(it.Key())))
-		vs = it.Value()
-		require.EqualValues(t, "9999", string(vs.Value))
-	}
-}
-
-func TestMergingIterator(t *testing.T) {
-	f1 := buildTable(t, [][]string{
-		{"k1", "a1"},
-		{"k2", "a2"},
-	})
-	f2 := buildTable(t, [][]string{
-		{"k1", "b1"},
-		{"k2", "b2"},
-	})
-	tbl1, err := OpenTable(f1, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer tbl1.DecrRef()
-	tbl2, err := OpenTable(f2, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer tbl2.DecrRef()
-	it1 := tbl1.NewIterator(false)
-	it2 := NewConcatIterator([]*Table{tbl2}, false)
-	it := y.NewMergeIterator([]y.Iterator{it1, it2}, false)
-	defer it.Close()
-
-	it.Rewind()
-	require.True(t, it.Valid())
-	k := it.Key()
-	require.EqualValues(t, "k1", string(y.ParseKey(k)))
-	vs := it.Value()
-	require.EqualValues(t, "a1", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-	it.Next()
-
-	require.True(t, it.Valid())
-	k = it.Key()
-	require.EqualValues(t, "k2", string(y.ParseKey(k)))
-	vs = it.Value()
-	require.EqualValues(t, "a2", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-	it.Next()
-
-	require.False(t, it.Valid())
-}
-
-func TestMergingIteratorReversed(t *testing.T) {
-	f1 := buildTable(t, [][]string{
-		{"k1", "a1"},
-		{"k2", "a2"},
-	})
-	f2 := buildTable(t, [][]string{
-		{"k1", "b1"},
-		{"k2", "b2"},
-	})
-	tbl1, err := OpenTable(f1, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer tbl1.DecrRef()
-	tbl2, err := OpenTable(f2, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer tbl2.DecrRef()
-	it1 := tbl1.NewIterator(true)
-	it2 := NewConcatIterator([]*Table{tbl2}, true)
-	it := y.NewMergeIterator([]y.Iterator{it1, it2}, true)
-	defer it.Close()
-
-	it.Rewind()
-	require.True(t, it.Valid())
-	k := it.Key()
-	require.EqualValues(t, "k2", string(y.ParseKey(k)))
-	vs := it.Value()
-	require.EqualValues(t, "a2", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-	it.Next()
-
-	require.True(t, it.Valid())
-	k = it.Key()
-	require.EqualValues(t, "k1", string(y.ParseKey(k)))
-	vs = it.Value()
-	require.EqualValues(t, "a1", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-	it.Next()
-
-	require.False(t, it.Valid())
-}
-
-// Take only the first iterator.
-func TestMergingIteratorTakeOne(t *testing.T) {
-	f1 := buildTable(t, [][]string{
-		{"k1", "a1"},
-		{"k2", "a2"},
-	})
-	f2 := buildTable(t, [][]string{})
-
-	t1, err := OpenTable(f1, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer t1.DecrRef()
-	t2, err := OpenTable(f2, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer t2.DecrRef()
-
-	it1 := NewConcatIterator([]*Table{t1}, false)
-	it2 := NewConcatIterator([]*Table{t2}, false)
-	it := y.NewMergeIterator([]y.Iterator{it1, it2}, false)
-	defer it.Close()
-
-	it.Rewind()
-	require.True(t, it.Valid())
-	k := it.Key()
-	require.EqualValues(t, "k1", string(y.ParseKey(k)))
-	vs := it.Value()
-	require.EqualValues(t, "a1", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-	it.Next()
-
-	require.True(t, it.Valid())
-	k = it.Key()
-	require.EqualValues(t, "k2", string(y.ParseKey(k)))
-	vs = it.Value()
-	require.EqualValues(t, "a2", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-	it.Next()
-
-	require.False(t, it.Valid())
-}
-
-// Take only the second iterator.
-func TestMergingIteratorTakeTwo(t *testing.T) {
-	f1 := buildTable(t, [][]string{})
-	f2 := buildTable(t, [][]string{
-		{"k1", "a1"},
-		{"k2", "a2"},
-	})
-
-	t1, err := OpenTable(f1, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer t1.DecrRef()
-	t2, err := OpenTable(f2, options.LoadToRAM, nil)
-	require.NoError(t, err)
-	defer t2.DecrRef()
-
-	it1 := NewConcatIterator([]*Table{t1}, false)
-	it2 := NewConcatIterator([]*Table{t2}, false)
-	it := y.NewMergeIterator([]y.Iterator{it1, it2}, false)
-	defer it.Close()
-
-	it.Rewind()
-	require.True(t, it.Valid())
-	k := it.Key()
-	require.EqualValues(t, "k1", string(y.ParseKey(k)))
-	vs := it.Value()
-	require.EqualValues(t, "a1", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-	it.Next()
-
-	require.True(t, it.Valid())
-	k = it.Key()
-	require.EqualValues(t, "k2", string(y.ParseKey(k)))
-	vs = it.Value()
-	require.EqualValues(t, "a2", string(vs.Value))
-	require.EqualValues(t, 'A', vs.Meta)
-	it.Next()
-
-	require.False(t, it.Valid())
-}
-
-func BenchmarkRead(b *testing.B) {
-	n := 5 << 20
-	builder := NewTableBuilder()
-	filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63())
-	f, err := y.OpenSyncedFile(filename, true)
-	y.Check(err)
-	for i := 0; i < n; i++ {
-		k := fmt.Sprintf("%016x", i)
-		v := fmt.Sprintf("%d", i)
-		y.Check(builder.Add([]byte(k), y.ValueStruct{Value: []byte(v), Meta: 123, UserMeta: 0}))
-	}
-
-	f.Write(builder.Finish())
-	tbl, err := OpenTable(f, options.MemoryMap, nil)
-	y.Check(err)
-	defer tbl.DecrRef()
-
-	//	y.Printf("Size of table: %d\n", tbl.Size())
-	b.ResetTimer()
-	// Iterate b.N times over the entire table.
-	for i := 0; i < b.N; i++ {
-		func() {
-			it := tbl.NewIterator(false)
-			defer it.Close()
-			for it.seekToFirst(); it.Valid(); it.next() {
-			}
-		}()
-	}
-}
-
-func BenchmarkReadAndBuild(b *testing.B) {
-	n := 5 << 20
-	builder := NewTableBuilder()
-	filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63())
-	f, err := y.OpenSyncedFile(filename, true)
-	y.Check(err)
-	for i := 0; i < n; i++ {
-		k := fmt.Sprintf("%016x", i)
-		v := fmt.Sprintf("%d", i)
-		y.Check(builder.Add([]byte(k), y.ValueStruct{Value: []byte(v), Meta: 123, UserMeta: 0}))
-	}
-
-	f.Write(builder.Finish())
-	tbl, err := OpenTable(f, options.MemoryMap, nil)
-	y.Check(err)
-	defer tbl.DecrRef()
-
-	//	y.Printf("Size of table: %d\n", tbl.Size())
-	b.ResetTimer()
-	// Iterate b.N times over the entire table.
-	for i := 0; i < b.N; i++ {
-		func() {
-			newBuilder := NewTableBuilder()
-			it := tbl.NewIterator(false)
-			defer it.Close()
-			for it.seekToFirst(); it.Valid(); it.next() {
-				vs := it.Value()
-				newBuilder.Add(it.Key(), vs)
-			}
-			newBuilder.Finish()
-		}()
-	}
-}
-
-func BenchmarkReadMerged(b *testing.B) {
-	n := 5 << 20
-	m := 5 // Number of tables.
-	y.AssertTrue((n % m) == 0)
-	tableSize := n / m
-	var tables []*Table
-	for i := 0; i < m; i++ {
-		filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63())
-		builder := NewTableBuilder()
-		f, err := y.OpenSyncedFile(filename, true)
-		y.Check(err)
-		for j := 0; j < tableSize; j++ {
-			id := j*m + i // Arrays are interleaved.
-			// id := i*tableSize+j (not interleaved)
-			k := fmt.Sprintf("%016x", id)
-			v := fmt.Sprintf("%d", id)
-			y.Check(builder.Add([]byte(k), y.ValueStruct{Value: []byte(v), Meta: 123, UserMeta: 0}))
-		}
-		f.Write(builder.Finish())
-		tbl, err := OpenTable(f, options.MemoryMap, nil)
-		y.Check(err)
-		tables = append(tables, tbl)
-		defer tbl.DecrRef()
-	}
-
-	b.ResetTimer()
-	// Iterate b.N times over the entire table.
-	for i := 0; i < b.N; i++ {
-		func() {
-			var iters []y.Iterator
-			for _, tbl := range tables {
-				iters = append(iters, tbl.NewIterator(false))
-			}
-			it := y.NewMergeIterator(iters, false)
-			defer it.Close()
-			for it.Rewind(); it.Valid(); it.Next() {
-			}
-		}()
-	}
-}

vendor/github.com/dgraph-io/badger/test.sh

@@ -17,8 +17,15 @@ go test -v --manual=true -run='TestTruncateVlogNoClose$' .
 truncate --size=4096 p/000000.vlog
 go test -v --manual=true -run='TestTruncateVlogNoClose2$' .
 go test -v --manual=true -run='TestTruncateVlogNoClose3$' .
-rm -R p
+rm -R p || true
 
 # Then the normal tests.
+echo
+echo "==> Starting tests with value log mmapped..."
+sleep 5
 go test -v --vlog_mmap=true -race ./...
+
+echo
+echo "==> Starting tests with value log not mmapped..."
+sleep 5
 go test -v --vlog_mmap=false -race ./...

vendor/github.com/dgraph-io/badger/txn.go

@@ -27,9 +27,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	farm "gx/ipfs/QmRFFHk2jw9tgjxv12bCuuTnSbVXxEvYQkuNCLMEv9eUwP/go-farm"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/y"
+	farm "github.com/dgryski/go-farm"
+	"github.com/pkg/errors"
 )
 
 type oracle struct {
@@ -127,6 +127,12 @@ func (o *oracle) nextTs() uint64 {
 	return o.nextTxnTs
 }
 
+func (o *oracle) incrementNextTs() {
+	o.Lock()
+	defer o.Unlock()
+	o.nextTxnTs++
+}
+
 // Any deleted or invalid versions at or below ts would be discarded during
 // compaction to reclaim disk space in LSM tree and thence value log.
 func (o *oracle) setDiscardTs(ts uint64) {
@@ -353,7 +359,7 @@ func (txn *Txn) SetWithDiscard(key, val []byte, meta byte) error {
 
 // SetWithTTL adds a key-value pair to the database, along with a time-to-live
 // (TTL) setting. A key stored with a TTL would automatically expire after the
-// time has elapsed , and be eligible for garbage collection.
+// time has elapsed, and be eligible for garbage collection.
 //
 // The current transaction keeps a reference to the key and val byte slice
 // arguments. Users must not modify key and val until the end of the
@@ -364,6 +370,12 @@ func (txn *Txn) SetWithTTL(key, val []byte, dur time.Duration) error {
 	return txn.SetEntry(e)
 }
 
+// setMergeEntry is similar to SetEntry but it sets the bitMergeEntry flag
+func (txn *Txn) setMergeEntry(key, val []byte) error {
+	e := &Entry{Key: key, Value: val, meta: bitMergeEntry}
+	return txn.SetEntry(e)
+}
+
 func exceedsSize(prefix string, max int64, key []byte) error {
 	return errors.Errorf("%s with size %d exceeded %d limit. %s:\n%s",
 		prefix, len(key), max, prefix, hex.Dump(key[:1<<10]))

vendor/github.com/dgraph-io/badger/txn_test.go

@@ -1,845 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"strconv"
-	"sync"
-	"testing"
-	"time"
-
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestTxnSimple(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-
-		txn := db.NewTransaction(true)
-
-		for i := 0; i < 10; i++ {
-			k := []byte(fmt.Sprintf("key=%d", i))
-			v := []byte(fmt.Sprintf("val=%d", i))
-			txn.Set(k, v)
-		}
-
-		item, err := txn.Get([]byte("key=8"))
-		require.NoError(t, err)
-
-		require.NoError(t, item.Value(func(val []byte) error {
-			require.Equal(t, []byte("val=8"), val)
-			return nil
-		}))
-
-		require.Panics(t, func() { txn.CommitAt(100, nil) })
-		require.NoError(t, txn.Commit())
-	})
-}
-
-func TestTxnReadAfterWrite(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		var wg sync.WaitGroup
-		N := 100
-		wg.Add(N)
-		for i := 0; i < N; i++ {
-			go func(i int) {
-				defer wg.Done()
-				key := []byte(fmt.Sprintf("key%d", i))
-				err := db.Update(func(tx *Txn) error {
-					return tx.Set(key, key)
-				})
-				require.NoError(t, err)
-				err = db.View(func(tx *Txn) error {
-					item, err := tx.Get(key)
-					require.NoError(t, err)
-					val, err := item.ValueCopy(nil)
-					require.NoError(t, err)
-					require.Equal(t, val, key)
-					return nil
-				})
-				require.NoError(t, err)
-			}(i)
-		}
-		wg.Wait()
-	})
-}
-
-func TestTxnCommitAsync(t *testing.T) {
-	key := func(i int) []byte {
-		return []byte(fmt.Sprintf("key=%d", i))
-	}
-
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		txn := db.NewTransaction(true)
-		for i := 0; i < 40; i++ {
-			err := txn.Set(key(i), []byte(strconv.Itoa(100)))
-			require.NoError(t, err)
-		}
-		require.NoError(t, txn.Commit())
-		txn.Discard()
-
-		closer := y.NewCloser(1)
-		go func() {
-			defer closer.Done()
-			for {
-				select {
-				case <-closer.HasBeenClosed():
-					return
-				default:
-				}
-				// Keep checking balance variant
-				txn := db.NewTransaction(false)
-				totalBalance := 0
-				for i := 0; i < 40; i++ {
-					item, err := txn.Get(key(i))
-					require.NoError(t, err)
-					val, err := item.ValueCopy(nil)
-					require.NoError(t, err)
-					bal, err := strconv.Atoi(string(val))
-					require.NoError(t, err)
-					totalBalance += bal
-				}
-				require.Equal(t, totalBalance, 4000)
-				txn.Discard()
-			}
-		}()
-
-		var wg sync.WaitGroup
-		wg.Add(100)
-		for i := 0; i < 100; i++ {
-			go func() {
-				txn := db.NewTransaction(true)
-				delta := rand.Intn(100)
-				for i := 0; i < 20; i++ {
-					err := txn.Set(key(i), []byte(strconv.Itoa(100-delta)))
-					require.NoError(t, err)
-				}
-				for i := 20; i < 40; i++ {
-					err := txn.Set(key(i), []byte(strconv.Itoa(100+delta)))
-					require.NoError(t, err)
-				}
-				// We are only doing writes, so there won't be any conflicts.
-				txn.CommitWith(func(err error) {})
-				txn.Discard()
-				wg.Done()
-			}()
-		}
-		wg.Wait()
-		closer.SignalAndWait()
-		time.Sleep(time.Millisecond * 10) // allow goroutine to complete.
-	})
-}
-
-func TestTxnVersions(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		k := []byte("key")
-		for i := 1; i < 10; i++ {
-			txn := db.NewTransaction(true)
-
-			txn.Set(k, []byte(fmt.Sprintf("valversion=%d", i)))
-			require.NoError(t, txn.Commit())
-			require.Equal(t, uint64(i), db.orc.readTs())
-		}
-
-		checkIterator := func(itr *Iterator, i int) {
-			defer itr.Close()
-			count := 0
-			for itr.Rewind(); itr.Valid(); itr.Next() {
-				item := itr.Item()
-				require.Equal(t, k, item.Key())
-
-				val, err := item.ValueCopy(nil)
-				require.NoError(t, err)
-				exp := fmt.Sprintf("valversion=%d", i)
-				require.Equal(t, exp, string(val), "i=%d", i)
-				count++
-			}
-			require.Equal(t, 1, count, "i=%d", i) // Should only loop once.
-		}
-
-		checkAllVersions := func(itr *Iterator, i int) {
-			var version uint64
-			if itr.opt.Reverse {
-				version = 1
-			} else {
-				version = uint64(i)
-			}
-
-			count := 0
-			for itr.Rewind(); itr.Valid(); itr.Next() {
-				item := itr.Item()
-				require.Equal(t, k, item.Key())
-				require.Equal(t, version, item.Version())
-
-				val, err := item.ValueCopy(nil)
-				require.NoError(t, err)
-				exp := fmt.Sprintf("valversion=%d", version)
-				require.Equal(t, exp, string(val), "v=%d", version)
-				count++
-
-				if itr.opt.Reverse {
-					version++
-				} else {
-					version--
-				}
-			}
-			require.Equal(t, i, count, "i=%d", i) // Should loop as many times as i.
-		}
-
-		for i := 1; i < 10; i++ {
-			txn := db.NewTransaction(true)
-			txn.readTs = uint64(i) // Read version at i.
-
-			item, err := txn.Get(k)
-			require.NoError(t, err)
-
-			val, err := item.ValueCopy(nil)
-			require.NoError(t, err)
-			require.Equal(t, []byte(fmt.Sprintf("valversion=%d", i)), val,
-				"Expected versions to match up at i=%d", i)
-
-			// Try retrieving the latest version forward and reverse.
-			itr := txn.NewIterator(DefaultIteratorOptions)
-			checkIterator(itr, i)
-
-			opt := DefaultIteratorOptions
-			opt.Reverse = true
-			itr = txn.NewIterator(opt)
-			checkIterator(itr, i)
-
-			// Now try retrieving all versions forward and reverse.
-			opt = DefaultIteratorOptions
-			opt.AllVersions = true
-			itr = txn.NewIterator(opt)
-			checkAllVersions(itr, i)
-			itr.Close()
-
-			opt = DefaultIteratorOptions
-			opt.AllVersions = true
-			opt.Reverse = true
-			itr = txn.NewIterator(opt)
-			checkAllVersions(itr, i)
-			itr.Close()
-
-			txn.Discard()
-		}
-		txn := db.NewTransaction(true)
-		defer txn.Discard()
-		item, err := txn.Get(k)
-		require.NoError(t, err)
-
-		val, err := item.ValueCopy(nil)
-		require.NoError(t, err)
-		require.Equal(t, []byte("valversion=9"), val)
-	})
-}
-
-func TestTxnWriteSkew(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		// Accounts
-		ax := []byte("x")
-		ay := []byte("y")
-
-		// Set balance to $100 in each account.
-		txn := db.NewTransaction(true)
-		defer txn.Discard()
-		val := []byte(strconv.Itoa(100))
-		txn.Set(ax, val)
-		txn.Set(ay, val)
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(1), db.orc.readTs())
-
-		getBal := func(txn *Txn, key []byte) (bal int) {
-			item, err := txn.Get(key)
-			require.NoError(t, err)
-
-			val, err := item.ValueCopy(nil)
-			require.NoError(t, err)
-			bal, err = strconv.Atoi(string(val))
-			require.NoError(t, err)
-			return bal
-		}
-
-		// Start two transactions, each would read both accounts and deduct from one account.
-		txn1 := db.NewTransaction(true)
-
-		sum := getBal(txn1, ax)
-		sum += getBal(txn1, ay)
-		require.Equal(t, 200, sum)
-		txn1.Set(ax, []byte("0")) // Deduct 100 from ax.
-
-		// Let's read this back.
-		sum = getBal(txn1, ax)
-		require.Equal(t, 0, sum)
-		sum += getBal(txn1, ay)
-		require.Equal(t, 100, sum)
-		// Don't commit yet.
-
-		txn2 := db.NewTransaction(true)
-
-		sum = getBal(txn2, ax)
-		sum += getBal(txn2, ay)
-		require.Equal(t, 200, sum)
-		txn2.Set(ay, []byte("0")) // Deduct 100 from ay.
-
-		// Let's read this back.
-		sum = getBal(txn2, ax)
-		require.Equal(t, 100, sum)
-		sum += getBal(txn2, ay)
-		require.Equal(t, 100, sum)
-
-		// Commit both now.
-		require.NoError(t, txn1.Commit())
-		require.Error(t, txn2.Commit()) // This should fail.
-
-		require.Equal(t, uint64(2), db.orc.readTs())
-	})
-}
-
-// a3, a2, b4 (del), b3, c2, c1
-// Read at ts=4 -> a3, c2
-// Read at ts=4(Uncommitted) -> a3, b4
-// Read at ts=3 -> a3, b3, c2
-// Read at ts=2 -> a2, c2
-// Read at ts=1 -> c1
-func TestTxnIterationEdgeCase(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		ka := []byte("a")
-		kb := []byte("b")
-		kc := []byte("c")
-
-		// c1
-		txn := db.NewTransaction(true)
-		txn.Set(kc, []byte("c1"))
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(1), db.orc.readTs())
-
-		// a2, c2
-		txn = db.NewTransaction(true)
-		txn.Set(ka, []byte("a2"))
-		txn.Set(kc, []byte("c2"))
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(2), db.orc.readTs())
-
-		// b3
-		txn = db.NewTransaction(true)
-		txn.Set(ka, []byte("a3"))
-		txn.Set(kb, []byte("b3"))
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(3), db.orc.readTs())
-
-		// b4, c4(del) (Uncommitted)
-		txn4 := db.NewTransaction(true)
-		require.NoError(t, txn4.Set(kb, []byte("b4")))
-		require.NoError(t, txn4.Delete(kc))
-		require.Equal(t, uint64(3), db.orc.readTs())
-
-		// b4 (del)
-		txn = db.NewTransaction(true)
-		txn.Delete(kb)
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(4), db.orc.readTs())
-
-		checkIterator := func(itr *Iterator, expected []string) {
-			defer itr.Close()
-			var i int
-			for itr.Rewind(); itr.Valid(); itr.Next() {
-				item := itr.Item()
-				val, err := item.ValueCopy(nil)
-				require.NoError(t, err)
-				require.Equal(t, expected[i], string(val), "readts=%d", itr.readTs)
-				i++
-			}
-			require.Equal(t, len(expected), i)
-		}
-
-		txn = db.NewTransaction(true)
-		defer txn.Discard()
-		itr := txn.NewIterator(DefaultIteratorOptions)
-		itr5 := txn4.NewIterator(DefaultIteratorOptions)
-		checkIterator(itr, []string{"a3", "c2"})
-		checkIterator(itr5, []string{"a3", "b4"})
-
-		rev := DefaultIteratorOptions
-		rev.Reverse = true
-		itr = txn.NewIterator(rev)
-		itr5 = txn4.NewIterator(rev)
-		checkIterator(itr, []string{"c2", "a3"})
-		checkIterator(itr5, []string{"b4", "a3"})
-
-		txn.readTs = 3
-		itr = txn.NewIterator(DefaultIteratorOptions)
-		checkIterator(itr, []string{"a3", "b3", "c2"})
-		itr = txn.NewIterator(rev)
-		checkIterator(itr, []string{"c2", "b3", "a3"})
-
-		txn.readTs = 2
-		itr = txn.NewIterator(DefaultIteratorOptions)
-		checkIterator(itr, []string{"a2", "c2"})
-		itr = txn.NewIterator(rev)
-		checkIterator(itr, []string{"c2", "a2"})
-
-		txn.readTs = 1
-		itr = txn.NewIterator(DefaultIteratorOptions)
-		checkIterator(itr, []string{"c1"})
-		itr = txn.NewIterator(rev)
-		checkIterator(itr, []string{"c1"})
-	})
-}
-
-// a2, a3, b4 (del), b3, c2, c1
-// Read at ts=4 -> a3, c2
-// Read at ts=3 -> a3, b3, c2
-// Read at ts=2 -> a2, c2
-// Read at ts=1 -> c1
-func TestTxnIterationEdgeCase2(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		ka := []byte("a")
-		kb := []byte("aa")
-		kc := []byte("aaa")
-
-		// c1
-		txn := db.NewTransaction(true)
-		txn.Set(kc, []byte("c1"))
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(1), db.orc.readTs())
-
-		// a2, c2
-		txn = db.NewTransaction(true)
-		txn.Set(ka, []byte("a2"))
-		txn.Set(kc, []byte("c2"))
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(2), db.orc.readTs())
-
-		// b3
-		txn = db.NewTransaction(true)
-		txn.Set(ka, []byte("a3"))
-		txn.Set(kb, []byte("b3"))
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(3), db.orc.readTs())
-
-		// b4 (del)
-		txn = db.NewTransaction(true)
-		txn.Delete(kb)
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(4), db.orc.readTs())
-
-		checkIterator := func(itr *Iterator, expected []string) {
-			defer itr.Close()
-			var i int
-			for itr.Rewind(); itr.Valid(); itr.Next() {
-				item := itr.Item()
-				val, err := item.ValueCopy(nil)
-				require.NoError(t, err)
-				require.Equal(t, expected[i], string(val), "readts=%d", itr.readTs)
-				i++
-			}
-			require.Equal(t, len(expected), i)
-		}
-		txn = db.NewTransaction(true)
-		defer txn.Discard()
-		rev := DefaultIteratorOptions
-		rev.Reverse = true
-
-		itr := txn.NewIterator(DefaultIteratorOptions)
-		checkIterator(itr, []string{"a3", "c2"})
-		itr = txn.NewIterator(rev)
-		checkIterator(itr, []string{"c2", "a3"})
-
-		txn.readTs = 5
-		itr = txn.NewIterator(DefaultIteratorOptions)
-		itr.Seek(ka)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), ka)
-		itr.Seek(kc)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kc)
-		itr.Close()
-
-		itr = txn.NewIterator(rev)
-		itr.Seek(ka)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), ka)
-		itr.Seek(kc)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kc)
-		itr.Close()
-
-		txn.readTs = 3
-		itr = txn.NewIterator(DefaultIteratorOptions)
-		checkIterator(itr, []string{"a3", "b3", "c2"})
-		itr = txn.NewIterator(rev)
-		checkIterator(itr, []string{"c2", "b3", "a3"})
-
-		txn.readTs = 2
-		itr = txn.NewIterator(DefaultIteratorOptions)
-		checkIterator(itr, []string{"a2", "c2"})
-		itr = txn.NewIterator(rev)
-		checkIterator(itr, []string{"c2", "a2"})
-
-		txn.readTs = 1
-		itr = txn.NewIterator(DefaultIteratorOptions)
-		checkIterator(itr, []string{"c1"})
-		itr = txn.NewIterator(rev)
-		checkIterator(itr, []string{"c1"})
-	})
-}
-
-func TestTxnIterationEdgeCase3(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		kb := []byte("abc")
-		kc := []byte("acd")
-		kd := []byte("ade")
-
-		// c1
-		txn := db.NewTransaction(true)
-		txn.Set(kc, []byte("c1"))
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(1), db.orc.readTs())
-
-		// b2
-		txn = db.NewTransaction(true)
-		txn.Set(kb, []byte("b2"))
-		require.NoError(t, txn.Commit())
-		require.Equal(t, uint64(2), db.orc.readTs())
-
-		txn2 := db.NewTransaction(true)
-		require.NoError(t, txn2.Set(kd, []byte("d2")))
-		require.NoError(t, txn2.Delete(kc))
-
-		txn = db.NewTransaction(true)
-		defer txn.Discard()
-		rev := DefaultIteratorOptions
-		rev.Reverse = true
-
-		itr := txn.NewIterator(DefaultIteratorOptions)
-		itr.Seek([]byte("ab"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Seek([]byte("ac"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kc)
-		itr.Seek(nil)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Seek([]byte("ac"))
-		itr.Rewind()
-		itr.Seek(nil)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Seek([]byte("ac"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kc)
-		itr.Close()
-
-		//  Keys: "abc", "ade"
-		// Read pending writes.
-		itr = txn2.NewIterator(DefaultIteratorOptions)
-		itr.Seek([]byte("ab"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Seek([]byte("ac"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kd)
-		itr.Seek(nil)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Seek([]byte("ac"))
-		itr.Rewind()
-		itr.Seek(nil)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Seek([]byte("ad"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kd)
-		itr.Close()
-
-		itr = txn.NewIterator(rev)
-		itr.Seek([]byte("ac"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Seek([]byte("ad"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kc)
-		itr.Seek(nil)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kc)
-		itr.Seek([]byte("ac"))
-		itr.Rewind()
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kc)
-		itr.Seek([]byte("ad"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kc)
-		itr.Close()
-
-		//  Keys: "abc", "ade"
-		itr = txn2.NewIterator(rev)
-		itr.Seek([]byte("ad"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Seek([]byte("ae"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kd)
-		itr.Seek(nil)
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kd)
-		itr.Seek([]byte("ab"))
-		itr.Rewind()
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kd)
-		itr.Seek([]byte("ac"))
-		require.True(t, itr.Valid())
-		require.Equal(t, itr.item.Key(), kb)
-		itr.Close()
-	})
-}
-
-func TestIteratorAllVersionsWithDeleted(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		// Write two keys
-		err := db.Update(func(txn *Txn) error {
-			txn.Set([]byte("answer1"), []byte("42"))
-			txn.Set([]byte("answer2"), []byte("43"))
-			return nil
-		})
-		require.NoError(t, err)
-
-		// Delete the specific key version from underlying db directly
-		err = db.View(func(txn *Txn) error {
-			item, err := txn.Get([]byte("answer1"))
-			require.NoError(t, err)
-			err = txn.db.batchSet([]*Entry{
-				{
-					Key:  y.KeyWithTs(item.key, item.version),
-					meta: bitDelete,
-				},
-			})
-			require.NoError(t, err)
-			return err
-		})
-		require.NoError(t, err)
-
-		opts := DefaultIteratorOptions
-		opts.AllVersions = true
-		opts.PrefetchValues = false
-
-		// Verify that deleted shows up when AllVersions is set.
-		err = db.View(func(txn *Txn) error {
-			it := txn.NewIterator(opts)
-			defer it.Close()
-			var count int
-			for it.Rewind(); it.Valid(); it.Next() {
-				count++
-				item := it.Item()
-				if count == 1 {
-					require.Equal(t, []byte("answer1"), item.Key())
-					require.True(t, item.meta&bitDelete > 0)
-				} else {
-					require.Equal(t, []byte("answer2"), item.Key())
-				}
-			}
-			require.Equal(t, 2, count)
-			return nil
-		})
-		require.NoError(t, err)
-	})
-}
-
-func TestIteratorAllVersionsWithDeleted2(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
-		// Set and delete alternatively
-		for i := 0; i < 4; i++ {
-			err := db.Update(func(txn *Txn) error {
-				if i%2 == 0 {
-					txn.Set([]byte("key"), []byte("value"))
-					return nil
-				}
-				txn.Delete([]byte("key"))
-				return nil
-			})
-			require.NoError(t, err)
-		}
-
-		opts := DefaultIteratorOptions
-		opts.AllVersions = true
-		opts.PrefetchValues = false
-
-		// Verify that deleted shows up when AllVersions is set.
-		err := db.View(func(txn *Txn) error {
-			it := txn.NewIterator(opts)
-			defer it.Close()
-			var count int
-			for it.Rewind(); it.Valid(); it.Next() {
-				item := it.Item()
-				require.Equal(t, []byte("key"), item.Key())
-				if count%2 != 0 {
-					val, err := item.ValueCopy(nil)
-					require.NoError(t, err)
-					require.Equal(t, val, []byte("value"))
-				} else {
-					require.True(t, item.meta&bitDelete > 0)
-				}
-				count++
-			}
-			require.Equal(t, 4, count)
-			return nil
-		})
-		require.NoError(t, err)
-	})
-}
-
-func TestManagedDB(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	opt := getTestOptions(dir)
-	opt.managedTxns = true
-	db, err := Open(opt)
-	require.NoError(t, err)
-	defer db.Close()
-
-	key := func(i int) []byte {
-		return []byte(fmt.Sprintf("key-%02d", i))
-	}
-
-	val := func(i int) []byte {
-		return []byte(fmt.Sprintf("val-%d", i))
-	}
-
-	require.Panics(t, func() {
-		db.Update(func(tx *Txn) error { return nil })
-	})
-
-	err = db.View(func(tx *Txn) error { return nil })
-	require.NoError(t, err)
-
-	// Write data at t=3.
-	txn := db.NewTransactionAt(3, true)
-	for i := 0; i <= 3; i++ {
-		require.NoError(t, txn.Set(key(i), val(i)))
-	}
-	require.Panics(t, func() { txn.Commit() })
-	require.NoError(t, txn.CommitAt(3, nil))
-
-	// Read data at t=2.
-	txn = db.NewTransactionAt(2, false)
-	for i := 0; i <= 3; i++ {
-		_, err := txn.Get(key(i))
-		require.Equal(t, ErrKeyNotFound, err)
-	}
-	txn.Discard()
-
-	// Read data at t=3.
-	txn = db.NewTransactionAt(3, false)
-	for i := 0; i <= 3; i++ {
-		item, err := txn.Get(key(i))
-		require.NoError(t, err)
-		require.Equal(t, uint64(3), item.Version())
-		v, err := item.ValueCopy(nil)
-		require.NoError(t, err)
-		require.Equal(t, val(i), v)
-	}
-	txn.Discard()
-
-	// Write data at t=7.
-	txn = db.NewTransactionAt(6, true)
-	for i := 0; i <= 7; i++ {
-		_, err := txn.Get(key(i))
-		if err == nil {
-			continue // Don't overwrite existing keys.
-		}
-		require.NoError(t, txn.Set(key(i), val(i)))
-	}
-	require.NoError(t, txn.CommitAt(7, nil))
-
-	// Read data at t=9.
-	txn = db.NewTransactionAt(9, false)
-	for i := 0; i <= 9; i++ {
-		item, err := txn.Get(key(i))
-		if i <= 7 {
-			require.NoError(t, err)
-		} else {
-			require.Equal(t, ErrKeyNotFound, err)
-		}
-
-		if i <= 3 {
-			require.Equal(t, uint64(3), item.Version())
-		} else if i <= 7 {
-			require.Equal(t, uint64(7), item.Version())
-		}
-		if i <= 7 {
-			v, err := item.ValueCopy(nil)
-			require.NoError(t, err)
-			require.Equal(t, val(i), v)
-		}
-	}
-	txn.Discard()
-}
-
-func TestArmV7Issue311Fix(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-
-	config := DefaultOptions
-	config.TableLoadingMode = options.MemoryMap
-	config.ValueLogFileSize = 16 << 20
-	config.LevelOneSize = 8 << 20
-	config.MaxTableSize = 2 << 20
-	config.Dir = dir
-	config.ValueDir = dir
-	config.SyncWrites = false
-
-	db, err := Open(config)
-	if err != nil {
-		t.Fatalf("cannot open db at location %s: %v", dir, err)
-	}
-
-	err = db.View(func(txn *Txn) error { return nil })
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db.Update(func(txn *Txn) error {
-		return txn.Set([]byte{0x11}, []byte{0x22})
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db.Update(func(txn *Txn) error {
-		return txn.Set([]byte{0x11}, []byte{0x22})
-	})
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err = db.Close(); err != nil {
-		t.Fatal(err)
-	}
-}

vendor/github.com/dgraph-io/badger/util.go

@@ -23,9 +23,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/table"
+	"github.com/dgraph-io/badger/y"
+	"github.com/pkg/errors"
 )
 
 // summary is produced when DB is closed. Currently it is used only for testing.

vendor/github.com/dgraph-io/badger/value.go

@@ -20,6 +20,7 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
+	"encoding/json"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -34,10 +35,10 @@ import (
 	"sync/atomic"
 	"time"
 
-	"gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/dgraph-io/badger/options"
+	"github.com/dgraph-io/badger/y"
+	"github.com/pkg/errors"
+	"golang.org/x/net/trace"
 )
 
 // Values have their first byte being byteData or byteDelete. This helps us distinguish between
@@ -46,7 +47,8 @@ const (
 	bitDelete                 byte = 1 << 0 // Set if the key has been deleted.
 	bitValuePointer           byte = 1 << 1 // Set if the value is NOT stored directly next to key.
 	bitDiscardEarlierVersions byte = 1 << 2 // Set if earlier versions can be discarded.
-
+	// Set if item shouldn't be discarded via compactions (used by merge operator)
+	bitMergeEntry byte = 1 << 3
 	// The MSB 2 bits are for transactions.
 	bitTxn    byte = 1 << 6 // Set if the entry is part of a txn.
 	bitFinTxn byte = 1 << 7 // Set if the entry is to indicate end of txn in value log.
@@ -555,6 +557,9 @@ func (vlog *valueLog) decrIteratorCount() error {
 }
 
 func (vlog *valueLog) deleteLogFile(lf *logFile) error {
+	if lf == nil {
+		return nil
+	}
 	path := vlog.fpath(lf.fid)
 	if err := lf.munmap(); err != nil {
 		_ = lf.fd.Close()
@@ -702,11 +707,15 @@ func errFile(err error, path string, msg string) error {
 }
 
 func (vlog *valueLog) replayLog(lf *logFile, offset uint32, replayFn logEntry) error {
-	// We should open the file in RW mode, so it can be truncated.
 	var err error
-	lf.fd, err = os.OpenFile(lf.path, os.O_RDWR, 0)
+	mode := os.O_RDONLY
+	if vlog.opt.Truncate {
+		// We should open the file in RW mode, so it can be truncated.
+		mode = os.O_RDWR
+	}
+	lf.fd, err = os.OpenFile(lf.path, mode, 0)
 	if err != nil {
-		return errFile(err, lf.path, "Open file in RW mode")
+		return errFile(err, lf.path, "Open file")
 	}
 	defer lf.fd.Close()
 
@@ -745,7 +754,10 @@ func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry) error {
 	vlog.db = db
 	vlog.elog = trace.NewEventLog("Badger", "Valuelog")
 	vlog.garbageCh = make(chan struct{}, 1) // Only allow one GC at a time.
-	vlog.lfDiscardStats = &lfDiscardStats{m: make(map[uint32]int64)}
+
+	if err := vlog.populateDiscardStats(); err != nil {
+		return err
+	}
 
 	if err := vlog.populateFilesMap(); err != nil {
 		return err
@@ -877,40 +889,68 @@ type request struct {
 	Ptrs []valuePointer
 	Wg   sync.WaitGroup
 	Err  error
+	ref  int32
+}
+
+func (req *request) IncrRef() {
+	atomic.AddInt32(&req.ref, 1)
+}
+
+func (req *request) DecrRef() {
+	nRef := atomic.AddInt32(&req.ref, -1)
+	if nRef > 0 {
+		return
+	}
+	req.Entries = nil
+	requestPool.Put(req)
 }
 
 func (req *request) Wait() error {
 	req.Wg.Wait()
-	req.Entries = nil
 	err := req.Err
-	requestPool.Put(req)
+	req.DecrRef() // DecrRef after writing to DB.
 	return err
 }
 
-// sync is thread-unsafe and should not be called concurrently with write.
-func (vlog *valueLog) sync() error {
+type requests []*request
+
+func (reqs requests) DecrRef() {
+	for _, req := range reqs {
+		req.DecrRef()
+	}
+}
+
+// sync function syncs content of latest value log file to disk. Syncing of value log directory is
+// not required here as it happens every time a value log file rotation happens(check createVlogFile
+// function). During rotation, previous value log file also gets synced to disk. It only syncs file
+// if fid >= vlog.maxFid. In some cases such as replay(while openning db), it might be called with
+// fid < vlog.maxFid. To sync irrespective of file id just call it with math.MaxUint32.
+func (vlog *valueLog) sync(fid uint32) error {
 	if vlog.opt.SyncWrites {
 		return nil
 	}
 
 	vlog.filesLock.RLock()
-	if len(vlog.filesMap) == 0 {
+	maxFid := atomic.LoadUint32(&vlog.maxFid)
+	// During replay it is possible to get sync call with fid less than maxFid.
+	// Because older file has already been synced, we can return from here.
+	if fid < maxFid || len(vlog.filesMap) == 0 {
 		vlog.filesLock.RUnlock()
 		return nil
 	}
-	maxFid := atomic.LoadUint32(&vlog.maxFid)
 	curlf := vlog.filesMap[maxFid]
+	// Sometimes it is possible that vlog.maxFid has been increased but file creation
+	// with same id is still in progress and this function is called. In those cases
+	// entry for the file might not be present in vlog.filesMap.
+	if curlf == nil {
+		vlog.filesLock.RUnlock()
+		return nil
+	}
 	curlf.lock.RLock()
 	vlog.filesLock.RUnlock()
 
-	dirSyncCh := make(chan error)
-	go func() { dirSyncCh <- syncDir(vlog.opt.ValueDir) }()
 	err := curlf.sync()
 	curlf.lock.RUnlock()
-	dirSyncErr := <-dirSyncCh
-	if err != nil {
-		err = dirSyncErr
-	}
 	return err
 }
 
@@ -955,6 +995,7 @@ func (vlog *valueLog) write(reqs []*request) error {
 				return err
 			}
 			curlf = newlf
+			atomic.AddInt32(&vlog.db.logRotates, 1)
 		}
 		return nil
 	}
@@ -962,8 +1003,13 @@ func (vlog *valueLog) write(reqs []*request) error {
 	for i := range reqs {
 		b := reqs[i]
 		b.Ptrs = b.Ptrs[:0]
+		var written int
 		for j := range b.Entries {
 			e := b.Entries[j]
+			if e.skipVlog {
+				b.Ptrs = append(b.Ptrs, valuePointer{})
+				continue
+			}
 			var p valuePointer
 
 			p.Fid = curlf.fid
@@ -975,8 +1021,9 @@ func (vlog *valueLog) write(reqs []*request) error {
 			}
 			p.Len = uint32(plen)
 			b.Ptrs = append(b.Ptrs, p)
+			written++
 		}
-		vlog.numEntriesWritten += uint32(len(b.Entries))
+		vlog.numEntriesWritten += uint32(written)
 		// We write to disk here so that all entries that are part of the same transaction are
 		// written to the same vlog file.
 		writeNow :=
@@ -1157,6 +1204,7 @@ func (vlog *valueLog) doRunGC(lf *logFile, discardRatio float64, tr trace.Trace)
 
 	// Set up the sampling window sizes.
 	sizeWindow := float64(fi.Size()) * 0.1                          // 10% of the file as window.
+	sizeWindowM := sizeWindow / (1 << 20)                           // in MBs.
 	countWindow := int(float64(vlog.opt.ValueLogMaxEntries) * 0.01) // 1% of num entries.
 	tr.LazyPrintf("Size window: %5.2f. Count window: %d.", sizeWindow, countWindow)
 
@@ -1185,7 +1233,7 @@ func (vlog *valueLog) doRunGC(lf *logFile, discardRatio float64, tr trace.Trace)
 			tr.LazyPrintf("Stopping sampling after %d entries.", countWindow)
 			return errStop
 		}
-		if r.total > sizeWindow {
+		if r.total > sizeWindowM {
 			tr.LazyPrintf("Stopping sampling after reaching window size.")
 			return errStop
 		}
@@ -1250,7 +1298,7 @@ func (vlog *valueLog) doRunGC(lf *logFile, discardRatio float64, tr trace.Trace)
 
 	// If we couldn't sample at least a 1000 KV pairs or at least 75% of the window size,
 	// and what we can discard is below the threshold, we should skip the rewrite.
-	if (r.count < countWindow && r.total < sizeWindow*0.75) || r.discard < discardRatio*r.total {
+	if (r.count < countWindow && r.total < sizeWindowM*0.75) || r.discard < discardRatio*r.total {
 		tr.LazyPrintf("Skipping GC on fid: %d", lf.fid)
 		return ErrNoRewrite
 	}
@@ -1305,10 +1353,44 @@ func (vlog *valueLog) runGC(discardRatio float64, head valuePointer) error {
 	}
 }
 
-func (vlog *valueLog) updateGCStats(stats map[uint32]int64) {
+func (vlog *valueLog) updateDiscardStats(stats map[uint32]int64) {
 	vlog.lfDiscardStats.Lock()
 	for fid, sz := range stats {
 		vlog.lfDiscardStats.m[fid] += sz
 	}
 	vlog.lfDiscardStats.Unlock()
 }
+
+// encodedDiscardStats returns []byte representation of lfDiscardStats
+// This will be called while storing stats in BadgerDB
+func (vlog *valueLog) encodedDiscardStats() []byte {
+	vlog.lfDiscardStats.Lock()
+	defer vlog.lfDiscardStats.Unlock()
+
+	encodedStats, _ := json.Marshal(vlog.lfDiscardStats.m)
+	return encodedStats
+}
+
+// populateDiscardStats populates vlog.lfDiscardStats
+// This function will be called while initializing valueLog
+func (vlog *valueLog) populateDiscardStats() error {
+	discardStatsKey := y.KeyWithTs(lfDiscardStatsKey, math.MaxUint64)
+	vs, err := vlog.db.get(discardStatsKey)
+	if err != nil {
+		return err
+	}
+
+	// check if value is Empty
+	if vs.Value == nil || len(vs.Value) == 0 {
+		vlog.lfDiscardStats = &lfDiscardStats{m: make(map[uint32]int64)}
+		return nil
+	}
+
+	var statsMap map[uint32]int64
+	if err := json.Unmarshal(vs.Value, &statsMap); err != nil {
+		return err
+	}
+	vlog.opt.Debugf("Value Log Discard stats: %v", statsMap)
+	vlog.lfDiscardStats = &lfDiscardStats{m: statsMap}
+	return nil
+}

vendor/github.com/dgraph-io/badger/value_test.go

@@ -1,880 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package badger
-
-import (
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"sync"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-	humanize "gx/ipfs/QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy/go-humanize"
-	"gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options"
-	"gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y"
-)
-
-func TestValueBasic(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	y.Check(err)
-	defer os.RemoveAll(dir)
-
-	kv, _ := Open(getTestOptions(dir))
-	defer kv.Close()
-	log := &kv.vlog
-
-	// Use value big enough that the value log writes them even if SyncWrites is false.
-	const val1 = "sampleval012345678901234567890123"
-	const val2 = "samplevalb012345678901234567890123"
-	require.True(t, len(val1) >= kv.opt.ValueThreshold)
-
-	e := &Entry{
-		Key:   []byte("samplekey"),
-		Value: []byte(val1),
-		meta:  bitValuePointer,
-	}
-	e2 := &Entry{
-		Key:   []byte("samplekeyb"),
-		Value: []byte(val2),
-		meta:  bitValuePointer,
-	}
-
-	b := new(request)
-	b.Entries = []*Entry{e, e2}
-
-	log.write([]*request{b})
-	require.Len(t, b.Ptrs, 2)
-	t.Logf("Pointer written: %+v %+v\n", b.Ptrs[0], b.Ptrs[1])
-
-	s := new(y.Slice)
-	buf1, cb1, err1 := log.readValueBytes(b.Ptrs[0], s)
-	buf2, cb2, err2 := log.readValueBytes(b.Ptrs[1], s)
-	require.NoError(t, err1)
-	require.NoError(t, err2)
-	defer runCallback(cb1)
-	defer runCallback(cb2)
-
-	readEntries := []Entry{valueBytesToEntry(buf1), valueBytesToEntry(buf2)}
-	require.EqualValues(t, []Entry{
-		{
-			Key:   []byte("samplekey"),
-			Value: []byte(val1),
-			meta:  bitValuePointer,
-		},
-		{
-			Key:   []byte("samplekeyb"),
-			Value: []byte(val2),
-			meta:  bitValuePointer,
-		},
-	}, readEntries)
-
-}
-
-func TestValueGCManaged(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	N := 10000
-	opt := getTestOptions(dir)
-	opt.ValueLogMaxEntries = uint32(N / 10)
-	opt.managedTxns = true
-	db, err := Open(opt)
-	require.NoError(t, err)
-	defer db.Close()
-
-	var ts uint64
-	newTs := func() uint64 {
-		ts++
-		return ts
-	}
-
-	sz := 64 << 10
-	var wg sync.WaitGroup
-	for i := 0; i < N; i++ {
-		v := make([]byte, sz)
-		rand.Read(v[:rand.Intn(sz)])
-
-		wg.Add(1)
-		txn := db.NewTransactionAt(newTs(), true)
-		require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v))
-		require.NoError(t, txn.CommitAt(newTs(), func(err error) {
-			wg.Done()
-			require.NoError(t, err)
-		}))
-	}
-
-	for i := 0; i < N; i++ {
-		wg.Add(1)
-		txn := db.NewTransactionAt(newTs(), true)
-		require.NoError(t, txn.Delete([]byte(fmt.Sprintf("key%d", i))))
-		require.NoError(t, txn.CommitAt(newTs(), func(err error) {
-			wg.Done()
-			require.NoError(t, err)
-		}))
-	}
-	wg.Wait()
-	files, err := ioutil.ReadDir(dir)
-	require.NoError(t, err)
-	for _, fi := range files {
-		t.Logf("File: %s. Size: %s\n", fi.Name(), humanize.Bytes(uint64(fi.Size())))
-	}
-
-	for i := 0; i < 100; i++ {
-		// Try at max 100 times to GC even a single value log file.
-		if err := db.RunValueLogGC(0.0001); err == nil {
-			return // Done
-		}
-	}
-	require.Fail(t, "Unable to GC even a single value log file.")
-}
-
-func TestValueGC(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opt := getTestOptions(dir)
-	opt.ValueLogFileSize = 1 << 20
-
-	kv, _ := Open(opt)
-	defer kv.Close()
-
-	sz := 32 << 10
-	txn := kv.NewTransaction(true)
-	for i := 0; i < 100; i++ {
-		v := make([]byte, sz)
-		rand.Read(v[:rand.Intn(sz)])
-		require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v))
-		if i%20 == 0 {
-			require.NoError(t, txn.Commit())
-			txn = kv.NewTransaction(true)
-		}
-	}
-	require.NoError(t, txn.Commit())
-
-	for i := 0; i < 45; i++ {
-		txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i)))
-	}
-
-	kv.vlog.filesLock.RLock()
-	lf := kv.vlog.filesMap[kv.vlog.sortedFids()[0]]
-	kv.vlog.filesLock.RUnlock()
-
-	//	lf.iterate(0, func(e Entry) bool {
-	//		e.print("lf")
-	//		return true
-	//	})
-
-	tr := trace.New("Test", "Test")
-	defer tr.Finish()
-	kv.vlog.rewrite(lf, tr)
-	for i := 45; i < 100; i++ {
-		key := []byte(fmt.Sprintf("key%d", i))
-
-		require.NoError(t, kv.View(func(txn *Txn) error {
-			item, err := txn.Get(key)
-			require.NoError(t, err)
-			val := getItemValue(t, item)
-			require.NotNil(t, val)
-			require.True(t, len(val) == sz, "Size found: %d", len(val))
-			return nil
-		}))
-	}
-}
-
-func TestValueGC2(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opt := getTestOptions(dir)
-	opt.ValueLogFileSize = 1 << 20
-
-	kv, _ := Open(opt)
-	defer kv.Close()
-
-	sz := 32 << 10
-	txn := kv.NewTransaction(true)
-	for i := 0; i < 100; i++ {
-		v := make([]byte, sz)
-		rand.Read(v[:rand.Intn(sz)])
-		require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v))
-		if i%20 == 0 {
-			require.NoError(t, txn.Commit())
-			txn = kv.NewTransaction(true)
-		}
-	}
-	require.NoError(t, txn.Commit())
-
-	for i := 0; i < 5; i++ {
-		txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i)))
-	}
-
-	for i := 5; i < 10; i++ {
-		v := []byte(fmt.Sprintf("value%d", i))
-		txnSet(t, kv, []byte(fmt.Sprintf("key%d", i)), v, 0)
-	}
-
-	kv.vlog.filesLock.RLock()
-	lf := kv.vlog.filesMap[kv.vlog.sortedFids()[0]]
-	kv.vlog.filesLock.RUnlock()
-
-	//	lf.iterate(0, func(e Entry) bool {
-	//		e.print("lf")
-	//		return true
-	//	})
-
-	tr := trace.New("Test", "Test")
-	defer tr.Finish()
-	kv.vlog.rewrite(lf, tr)
-	for i := 0; i < 5; i++ {
-		key := []byte(fmt.Sprintf("key%d", i))
-		require.NoError(t, kv.View(func(txn *Txn) error {
-			_, err := txn.Get(key)
-			require.Equal(t, ErrKeyNotFound, err)
-			return nil
-		}))
-	}
-	for i := 5; i < 10; i++ {
-		key := []byte(fmt.Sprintf("key%d", i))
-		require.NoError(t, kv.View(func(txn *Txn) error {
-			item, err := txn.Get(key)
-			require.NoError(t, err)
-			val := getItemValue(t, item)
-			require.NotNil(t, val)
-			require.Equal(t, string(val), fmt.Sprintf("value%d", i))
-			return nil
-		}))
-	}
-	for i := 10; i < 100; i++ {
-		key := []byte(fmt.Sprintf("key%d", i))
-		require.NoError(t, kv.View(func(txn *Txn) error {
-			item, err := txn.Get(key)
-			require.NoError(t, err)
-			val := getItemValue(t, item)
-			require.NotNil(t, val)
-			require.True(t, len(val) == sz, "Size found: %d", len(val))
-			return nil
-		}))
-	}
-}
-
-func TestValueGC3(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opt := getTestOptions(dir)
-	opt.ValueLogFileSize = 1 << 20
-
-	kv, err := Open(opt)
-	require.NoError(t, err)
-	defer kv.Close()
-
-	// We want to test whether an iterator can continue through a value log GC.
-
-	valueSize := 32 << 10
-
-	var value3 []byte
-	txn := kv.NewTransaction(true)
-	for i := 0; i < 100; i++ {
-		v := make([]byte, valueSize) // 32K * 100 will take >=3'276'800 B.
-		if i == 3 {
-			value3 = v
-		}
-		rand.Read(v[:])
-		// Keys key000, key001, key002, such that sorted order matches insertion order
-		require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%03d", i)), v))
-		if i%20 == 0 {
-			require.NoError(t, txn.Commit())
-			txn = kv.NewTransaction(true)
-		}
-	}
-	require.NoError(t, txn.Commit())
-
-	// Start an iterator to keys in the first value log file
-	itOpt := IteratorOptions{
-		PrefetchValues: false,
-		PrefetchSize:   0,
-		Reverse:        false,
-	}
-
-	txn = kv.NewTransaction(true)
-	it := txn.NewIterator(itOpt)
-	defer it.Close()
-	// Walk a few keys
-	it.Rewind()
-	require.True(t, it.Valid())
-	item := it.Item()
-	require.Equal(t, []byte("key000"), item.Key())
-	it.Next()
-	require.True(t, it.Valid())
-	item = it.Item()
-	require.Equal(t, []byte("key001"), item.Key())
-	it.Next()
-	require.True(t, it.Valid())
-	item = it.Item()
-	require.Equal(t, []byte("key002"), item.Key())
-
-	// Like other tests, we pull out a logFile to rewrite it directly
-
-	kv.vlog.filesLock.RLock()
-	logFile := kv.vlog.filesMap[kv.vlog.sortedFids()[0]]
-	kv.vlog.filesLock.RUnlock()
-
-	tr := trace.New("Test", "Test")
-	defer tr.Finish()
-	kv.vlog.rewrite(logFile, tr)
-	it.Next()
-	require.True(t, it.Valid())
-	item = it.Item()
-	require.Equal(t, []byte("key003"), item.Key())
-
-	v3, err := item.ValueCopy(nil)
-	require.NoError(t, err)
-	require.Equal(t, value3, v3)
-}
-
-func TestValueGC4(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opt := getTestOptions(dir)
-	opt.ValueLogFileSize = 1 << 20
-	opt.Truncate = true
-
-	kv, err := Open(opt)
-	require.NoError(t, err)
-	defer kv.Close()
-
-	sz := 128 << 10 // 5 entries per value log file.
-	txn := kv.NewTransaction(true)
-	for i := 0; i < 24; i++ {
-		v := make([]byte, sz)
-		rand.Read(v[:rand.Intn(sz)])
-		require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v))
-		if i%3 == 0 {
-			require.NoError(t, txn.Commit())
-			txn = kv.NewTransaction(true)
-		}
-	}
-	require.NoError(t, txn.Commit())
-
-	for i := 0; i < 8; i++ {
-		txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i)))
-	}
-
-	for i := 8; i < 16; i++ {
-		v := []byte(fmt.Sprintf("value%d", i))
-		txnSet(t, kv, []byte(fmt.Sprintf("key%d", i)), v, 0)
-	}
-
-	kv.vlog.filesLock.RLock()
-	lf0 := kv.vlog.filesMap[kv.vlog.sortedFids()[0]]
-	lf1 := kv.vlog.filesMap[kv.vlog.sortedFids()[1]]
-	kv.vlog.filesLock.RUnlock()
-
-	//	lf.iterate(0, func(e Entry) bool {
-	//		e.print("lf")
-	//		return true
-	//	})
-
-	tr := trace.New("Test", "Test")
-	defer tr.Finish()
-	kv.vlog.rewrite(lf0, tr)
-	kv.vlog.rewrite(lf1, tr)
-
-	err = kv.vlog.Close()
-	require.NoError(t, err)
-
-	err = kv.vlog.open(kv, valuePointer{Fid: 2}, kv.replayFunction())
-	require.NoError(t, err)
-
-	for i := 0; i < 8; i++ {
-		key := []byte(fmt.Sprintf("key%d", i))
-		require.NoError(t, kv.View(func(txn *Txn) error {
-			_, err := txn.Get(key)
-			require.Equal(t, ErrKeyNotFound, err)
-			return nil
-		}))
-	}
-	for i := 8; i < 16; i++ {
-		key := []byte(fmt.Sprintf("key%d", i))
-		require.NoError(t, kv.View(func(txn *Txn) error {
-			item, err := txn.Get(key)
-			require.NoError(t, err)
-			val := getItemValue(t, item)
-			require.NotNil(t, val)
-			require.Equal(t, string(val), fmt.Sprintf("value%d", i))
-			return nil
-		}))
-	}
-}
-
-func TestChecksums(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	// Set up SST with K1=V1
-	opts := getTestOptions(dir)
-	opts.Truncate = true
-	opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb
-	kv, err := Open(opts)
-	require.NoError(t, err)
-	require.NoError(t, kv.Close())
-
-	var (
-		k0 = []byte("k0")
-		k1 = []byte("k1")
-		k2 = []byte("k2")
-		k3 = []byte("k3")
-		v0 = []byte("value0-012345678901234567890123012345678901234567890123")
-		v1 = []byte("value1-012345678901234567890123012345678901234567890123")
-		v2 = []byte("value2-012345678901234567890123012345678901234567890123")
-		v3 = []byte("value3-012345678901234567890123012345678901234567890123")
-	)
-	// Make sure the value log would actually store the item
-	require.True(t, len(v0) >= kv.opt.ValueThreshold)
-
-	// Use a vlog with K0=V0 and a (corrupted) second transaction(k1,k2)
-	buf := createVlog(t, []*Entry{
-		{Key: k0, Value: v0},
-		{Key: k1, Value: v1},
-		{Key: k2, Value: v2},
-	})
-	buf[len(buf)-1]++ // Corrupt last byte
-	require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777))
-
-	// K1 should exist, but K2 shouldn't.
-	kv, err = Open(opts)
-	require.NoError(t, err)
-
-	require.NoError(t, kv.View(func(txn *Txn) error {
-		item, err := txn.Get(k0)
-		require.NoError(t, err)
-		require.Equal(t, getItemValue(t, item), v0)
-
-		_, err = txn.Get(k1)
-		require.Equal(t, ErrKeyNotFound, err)
-
-		_, err = txn.Get(k2)
-		require.Equal(t, ErrKeyNotFound, err)
-		return nil
-	}))
-
-	// Write K3 at the end of the vlog.
-	txnSet(t, kv, k3, v3, 0)
-	require.NoError(t, kv.Close())
-
-	// The vlog should contain K0 and K3 (K1 and k2 was lost when Badger started up
-	// last due to checksum failure).
-	kv, err = Open(opts)
-	require.NoError(t, err)
-
-	{
-		txn := kv.NewTransaction(false)
-
-		iter := txn.NewIterator(DefaultIteratorOptions)
-		iter.Seek(k0)
-		require.True(t, iter.Valid())
-		it := iter.Item()
-		require.Equal(t, it.Key(), k0)
-		require.Equal(t, getItemValue(t, it), v0)
-		iter.Next()
-		require.True(t, iter.Valid())
-		it = iter.Item()
-		require.Equal(t, it.Key(), k3)
-		require.Equal(t, getItemValue(t, it), v3)
-
-		iter.Close()
-		txn.Discard()
-	}
-
-	require.NoError(t, kv.Close())
-}
-
-func TestPartialAppendToValueLog(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	// Create skeleton files.
-	opts := getTestOptions(dir)
-	opts.Truncate = true
-	opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb
-	kv, err := Open(opts)
-	require.NoError(t, err)
-	require.NoError(t, kv.Close())
-
-	var (
-		k0 = []byte("k0")
-		k1 = []byte("k1")
-		k2 = []byte("k2")
-		k3 = []byte("k3")
-		v0 = []byte("value0-01234567890123456789012012345678901234567890123")
-		v1 = []byte("value1-01234567890123456789012012345678901234567890123")
-		v2 = []byte("value2-01234567890123456789012012345678901234567890123")
-		v3 = []byte("value3-01234567890123456789012012345678901234567890123")
-	)
-	// Values need to be long enough to actually get written to value log.
-	require.True(t, len(v3) >= kv.opt.ValueThreshold)
-
-	// Create truncated vlog to simulate a partial append.
-	// k0 - single transaction, k1 and k2 in another transaction
-	buf := createVlog(t, []*Entry{
-		{Key: k0, Value: v0},
-		{Key: k1, Value: v1},
-		{Key: k2, Value: v2},
-	})
-	buf = buf[:len(buf)-6]
-	require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777))
-
-	// Badger should now start up
-	kv, err = Open(opts)
-	require.NoError(t, err)
-
-	require.NoError(t, kv.View(func(txn *Txn) error {
-		item, err := txn.Get(k0)
-		require.NoError(t, err)
-		require.Equal(t, v0, getItemValue(t, item))
-
-		_, err = txn.Get(k1)
-		require.Equal(t, ErrKeyNotFound, err)
-		_, err = txn.Get(k2)
-		require.Equal(t, ErrKeyNotFound, err)
-		return nil
-	}))
-
-	// When K3 is set, it should be persisted after a restart.
-	txnSet(t, kv, k3, v3, 0)
-	require.NoError(t, kv.Close())
-	kv, err = Open(opts)
-	require.NoError(t, err)
-	checkKeys(t, kv, [][]byte{k3})
-
-	// Replay value log from beginning, badger head is past k2.
-	require.NoError(t, kv.vlog.Close())
-	require.NoError(t,
-		kv.vlog.open(kv, valuePointer{Fid: 0}, kv.replayFunction()))
-	require.NoError(t, kv.Close())
-}
-
-func TestReadOnlyOpenWithPartialAppendToValueLog(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	// Create skeleton files.
-	opts := getTestOptions(dir)
-	opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb
-	kv, err := Open(opts)
-	require.NoError(t, err)
-	require.NoError(t, kv.Close())
-
-	var (
-		k0 = []byte("k0")
-		k1 = []byte("k1")
-		k2 = []byte("k2")
-		v0 = []byte("value0-012345678901234567890123")
-		v1 = []byte("value1-012345678901234567890123")
-		v2 = []byte("value2-012345678901234567890123")
-	)
-
-	// Create truncated vlog to simulate a partial append.
-	// k0 - single transaction, k1 and k2 in another transaction
-	buf := createVlog(t, []*Entry{
-		{Key: k0, Value: v0},
-		{Key: k1, Value: v1},
-		{Key: k2, Value: v2},
-	})
-	buf = buf[:len(buf)-6]
-	require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777))
-
-	opts.ReadOnly = true
-	// Badger should fail a read-only open with values to replay
-	kv, err = Open(opts)
-	require.Error(t, err)
-	require.Regexp(t, "Database was not properly closed, cannot open read-only|Read-only mode is not supported on Windows", err.Error())
-}
-
-func TestValueLogTrigger(t *testing.T) {
-	t.Skip("Difficult to trigger compaction, so skipping. Re-enable after fixing #226")
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	opt := getTestOptions(dir)
-	opt.ValueLogFileSize = 1 << 20
-	kv, err := Open(opt)
-	require.NoError(t, err)
-
-	// Write a lot of data, so it creates some work for valug log GC.
-	sz := 32 << 10
-	txn := kv.NewTransaction(true)
-	for i := 0; i < 100; i++ {
-		v := make([]byte, sz)
-		rand.Read(v[:rand.Intn(sz)])
-		require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v))
-		if i%20 == 0 {
-			require.NoError(t, txn.Commit())
-			txn = kv.NewTransaction(true)
-		}
-	}
-	require.NoError(t, txn.Commit())
-
-	for i := 0; i < 45; i++ {
-		txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i)))
-	}
-
-	require.NoError(t, kv.RunValueLogGC(0.5))
-
-	require.NoError(t, kv.Close())
-
-	err = kv.RunValueLogGC(0.5)
-	require.Equal(t, ErrRejected, err, "Error should be returned after closing DB.")
-}
-
-func createVlog(t *testing.T, entries []*Entry) []byte {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	opts := getTestOptions(dir)
-	opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb
-	kv, err := Open(opts)
-	require.NoError(t, err)
-	txnSet(t, kv, entries[0].Key, entries[0].Value, entries[0].meta)
-	entries = entries[1:]
-	txn := kv.NewTransaction(true)
-	for _, entry := range entries {
-		require.NoError(t, txn.SetWithMeta(entry.Key, entry.Value, entry.meta))
-	}
-	require.NoError(t, txn.Commit())
-	require.NoError(t, kv.Close())
-
-	filename := vlogFilePath(dir, 0)
-	buf, err := ioutil.ReadFile(filename)
-	require.NoError(t, err)
-	return buf
-}
-
-func TestPenultimateLogCorruption(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-	opt := getTestOptions(dir)
-	opt.ValueLogLoadingMode = options.FileIO
-	// Each txn generates at least two entries. 3 txns will fit each file.
-	opt.ValueLogMaxEntries = 5
-
-	db0, err := Open(opt)
-	require.NoError(t, err)
-
-	h := testHelper{db: db0, t: t}
-	h.writeRange(0, 7)
-	h.readRange(0, 7)
-
-	for i := 2; i >= 0; i-- {
-		fpath := vlogFilePath(dir, uint32(i))
-		fi, err := os.Stat(fpath)
-		require.NoError(t, err)
-		require.True(t, fi.Size() > 0, "Empty file at log=%d", i)
-		if i == 0 {
-			err := os.Truncate(fpath, fi.Size()-1)
-			require.NoError(t, err)
-		}
-	}
-	// Simulate a crash by not closing db0, but releasing the locks.
-	if db0.dirLockGuard != nil {
-		require.NoError(t, db0.dirLockGuard.release())
-	}
-	if db0.valueDirGuard != nil {
-		require.NoError(t, db0.valueDirGuard.release())
-	}
-
-	opt.Truncate = true
-	db1, err := Open(opt)
-	require.NoError(t, err)
-	h.db = db1
-	h.readRange(0, 1) // Only 2 should be gone, because it is at the end of logfile 0.
-	h.readRange(3, 7)
-	err = db1.View(func(txn *Txn) error {
-		_, err := txn.Get(h.key(2)) // Verify that 2 is gone.
-		require.Equal(t, ErrKeyNotFound, err)
-		return nil
-	})
-	require.NoError(t, err)
-	require.NoError(t, db1.Close())
-}
-
-func checkKeys(t *testing.T, kv *DB, keys [][]byte) {
-	i := 0
-	txn := kv.NewTransaction(false)
-	iter := txn.NewIterator(IteratorOptions{})
-	for iter.Seek(keys[0]); iter.Valid(); iter.Next() {
-		require.Equal(t, iter.Item().Key(), keys[i])
-		i++
-	}
-	require.Equal(t, i, len(keys))
-}
-
-type testHelper struct {
-	db  *DB
-	t   *testing.T
-	val []byte
-}
-
-func (th *testHelper) key(i int) []byte {
-	return []byte(fmt.Sprintf("%010d", i))
-}
-func (th *testHelper) value() []byte {
-	if len(th.val) > 0 {
-		return th.val
-	}
-	th.val = make([]byte, 100)
-	y.Check2(rand.Read(th.val))
-	return th.val
-}
-
-// writeRange [from, to].
-func (th *testHelper) writeRange(from, to int) {
-	for i := from; i <= to; i++ {
-		err := th.db.Update(func(txn *Txn) error {
-			return txn.Set(th.key(i), th.value())
-		})
-		require.NoError(th.t, err)
-	}
-}
-
-func (th *testHelper) readRange(from, to int) {
-	for i := from; i <= to; i++ {
-		err := th.db.View(func(txn *Txn) error {
-			item, err := txn.Get(th.key(i))
-			if err != nil {
-				return err
-			}
-			return item.Value(func(val []byte) error {
-				require.Equal(th.t, val, th.value(), "key=%q", th.key(i))
-				return nil
-
-			})
-		})
-		require.NoError(th.t, err, "key=%q", th.key(i))
-	}
-}
-
-// Test Bug #578, which showed that if a value is moved during value log GC, an
-// older version can end up at a higher level in the LSM tree than a newer
-// version, causing the data to not be returned.
-func TestBug578(t *testing.T) {
-	dir, err := ioutil.TempDir("", "badger")
-	y.Check(err)
-	defer os.RemoveAll(dir)
-
-	opts := DefaultOptions
-	opts.Dir = dir
-	opts.ValueDir = dir
-	opts.ValueLogMaxEntries = 64
-	opts.MaxTableSize = 1 << 13
-
-	db, err := Open(opts)
-	require.NoError(t, err)
-
-	h := testHelper{db: db, t: t}
-
-	// Let's run this whole thing a few times.
-	for j := 0; j < 10; j++ {
-		t.Logf("Cycle: %d\n", j)
-		h.writeRange(0, 32)
-		h.writeRange(0, 10)
-		h.writeRange(50, 72)
-		h.writeRange(40, 72)
-		h.writeRange(40, 72)
-
-		// Run value log GC a few times.
-		for i := 0; i < 5; i++ {
-			db.RunValueLogGC(0.5)
-		}
-		h.readRange(0, 10)
-	}
-}
-
-func BenchmarkReadWrite(b *testing.B) {
-	rwRatio := []float32{
-		0.1, 0.2, 0.5, 1.0,
-	}
-	valueSize := []int{
-		64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384,
-	}
-
-	for _, vsz := range valueSize {
-		for _, rw := range rwRatio {
-			b.Run(fmt.Sprintf("%3.1f,%04d", rw, vsz), func(b *testing.B) {
-				dir, err := ioutil.TempDir("", "vlog-benchmark")
-				y.Check(err)
-				defer os.RemoveAll(dir)
-
-				db, err := Open(getTestOptions(dir))
-				y.Check(err)
-
-				vl := &db.vlog
-				b.ResetTimer()
-
-				for i := 0; i < b.N; i++ {
-					e := new(Entry)
-					e.Key = make([]byte, 16)
-					e.Value = make([]byte, vsz)
-					bl := new(request)
-					bl.Entries = []*Entry{e}
-
-					var ptrs []valuePointer
-
-					vl.write([]*request{bl})
-					ptrs = append(ptrs, bl.Ptrs...)
-
-					f := rand.Float32()
-					if f < rw {
-						vl.write([]*request{bl})
-
-					} else {
-						ln := len(ptrs)
-						if ln == 0 {
-							b.Fatalf("Zero length of ptrs")
-						}
-						idx := rand.Intn(ln)
-						s := new(y.Slice)
-						buf, cb, err := vl.readValueBytes(ptrs[idx], s)
-						if err != nil {
-							b.Fatalf("Benchmark Read: %v", err)
-						}
-
-						e := valueBytesToEntry(buf)
-						if len(e.Key) != 16 {
-							b.Fatalf("Key is invalid")
-						}
-						if len(e.Value) != vsz {
-							b.Fatalf("Value is invalid")
-						}
-						cb()
-					}
-				}
-			})
-		}
-	}
-}

vendor/github.com/dgraph-io/badger/y/error.go

@@ -32,7 +32,7 @@ import (
 	"fmt"
 	"log"
 
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/pkg/errors"
 )
 
 var debugMode = true

vendor/github.com/dgraph-io/badger/y/file_dsync.go

@@ -18,7 +18,7 @@
 
 package y
 
-import "gx/ipfs/QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu/sys/unix"
+import "golang.org/x/sys/unix"
 
 func init() {
 	datasyncFileFlag = unix.O_DSYNC

vendor/github.com/dgraph-io/badger/y/iterator.go

@@ -21,7 +21,7 @@ import (
 	"container/heap"
 	"encoding/binary"
 
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/pkg/errors"
 )
 
 // ValueStruct represents the value info that can be associated with a key, but also the internal

vendor/github.com/dgraph-io/badger/y/iterator_test.go

@@ -1,234 +0,0 @@
-/*
- * Copyright 2017 Dgraph Labs, Inc. and Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package y
-
-import (
-	"sort"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-type SimpleIterator struct {
-	keys     [][]byte
-	vals     [][]byte
-	idx      int
-	reversed bool
-}
-
-var (
-	closeCount int
-)
-
-func (s *SimpleIterator) Close() error { closeCount++; return nil }
-
-func (s *SimpleIterator) Next() {
-	if !s.reversed {
-		s.idx++
-	} else {
-		s.idx--
-	}
-}
-
-func (s *SimpleIterator) Rewind() {
-	if !s.reversed {
-		s.idx = 0
-	} else {
-		s.idx = len(s.keys) - 1
-	}
-}
-
-func (s *SimpleIterator) Seek(key []byte) {
-	key = KeyWithTs(key, 0)
-	if !s.reversed {
-		s.idx = sort.Search(len(s.keys), func(i int) bool {
-			return CompareKeys(s.keys[i], key) >= 0
-		})
-	} else {
-		n := len(s.keys)
-		s.idx = n - 1 - sort.Search(n, func(i int) bool {
-			return CompareKeys(s.keys[n-1-i], key) <= 0
-		})
-	}
-}
-
-func (s *SimpleIterator) Key() []byte { return s.keys[s.idx] }
-func (s *SimpleIterator) Value() ValueStruct {
-	return ValueStruct{
-		Value:    s.vals[s.idx],
-		UserMeta: 55,
-		Meta:     0,
-	}
-}
-func (s *SimpleIterator) Valid() bool {
-	return s.idx >= 0 && s.idx < len(s.keys)
-}
-
-func newSimpleIterator(keys []string, vals []string, reversed bool) *SimpleIterator {
-	k := make([][]byte, len(keys))
-	v := make([][]byte, len(vals))
-	AssertTrue(len(keys) == len(vals))
-	for i := 0; i < len(keys); i++ {
-		k[i] = KeyWithTs([]byte(keys[i]), 0)
-		v[i] = []byte(vals[i])
-	}
-	return &SimpleIterator{
-		keys:     k,
-		vals:     v,
-		idx:      -1,
-		reversed: reversed,
-	}
-}
-
-func getAll(it Iterator) ([]string, []string) {
-	var keys, vals []string
-	for ; it.Valid(); it.Next() {
-		k := it.Key()
-		keys = append(keys, string(ParseKey(k)))
-		v := it.Value()
-		vals = append(vals, string(v.Value))
-	}
-	return keys, vals
-}
-
-func closeAndCheck(t *testing.T, it Iterator, expected int) {
-	closeCount = 0
-	it.Close()
-	require.EqualValues(t, expected, closeCount)
-}
-
-func TestSimpleIterator(t *testing.T) {
-	keys := []string{"1", "2", "3"}
-	vals := []string{"v1", "v2", "v3"}
-	it := newSimpleIterator(keys, vals, false)
-	it.Rewind()
-	k, v := getAll(it)
-	require.EqualValues(t, keys, k)
-	require.EqualValues(t, vals, v)
-
-	closeAndCheck(t, it, 1)
-}
-
-func reversed(a []string) []string {
-	var out []string
-	for i := len(a) - 1; i >= 0; i-- {
-		out = append(out, a[i])
-	}
-	return out
-}
-
-func TestMergeSingle(t *testing.T) {
-	keys := []string{"1", "2", "3"}
-	vals := []string{"v1", "v2", "v3"}
-	it := newSimpleIterator(keys, vals, false)
-	mergeIt := NewMergeIterator([]Iterator{it}, false)
-	mergeIt.Rewind()
-	k, v := getAll(mergeIt)
-	require.EqualValues(t, keys, k)
-	require.EqualValues(t, vals, v)
-	closeAndCheck(t, mergeIt, 1)
-}
-
-func TestMergeSingleReversed(t *testing.T) {
-	keys := []string{"1", "2", "3"}
-	vals := []string{"v1", "v2", "v3"}
-	it := newSimpleIterator(keys, vals, true)
-	mergeIt := NewMergeIterator([]Iterator{it}, true)
-	mergeIt.Rewind()
-	k, v := getAll(mergeIt)
-	require.EqualValues(t, reversed(keys), k)
-	require.EqualValues(t, reversed(vals), v)
-	closeAndCheck(t, mergeIt, 1)
-}
-
-func TestMergeMore(t *testing.T) {
-	it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, false)
-	it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, false)
-	it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, false)
-	it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, false)
-
-	mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, false)
-	expectedKeys := []string{"1", "2", "3", "5", "7", "9"}
-	expectedVals := []string{"a1", "b2", "a3", "b5", "a7", "d9"}
-	mergeIt.Rewind()
-	k, v := getAll(mergeIt)
-	require.EqualValues(t, expectedKeys, k)
-	require.EqualValues(t, expectedVals, v)
-	closeAndCheck(t, mergeIt, 4)
-}
-
-// Ensure MergeIterator satisfies the Iterator interface
-func TestMergeIteratorNested(t *testing.T) {
-	keys := []string{"1", "2", "3"}
-	vals := []string{"v1", "v2", "v3"}
-	it := newSimpleIterator(keys, vals, false)
-	mergeIt := NewMergeIterator([]Iterator{it}, false)
-	mergeIt2 := NewMergeIterator([]Iterator{mergeIt}, false)
-	mergeIt2.Rewind()
-	k, v := getAll(mergeIt2)
-	require.EqualValues(t, keys, k)
-	require.EqualValues(t, vals, v)
-	closeAndCheck(t, mergeIt2, 1)
-}
-
-func TestMergeIteratorSeek(t *testing.T) {
-	it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, false)
-	it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, false)
-	it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, false)
-	it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, false)
-	mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, false)
-	mergeIt.Seek([]byte("4"))
-	k, v := getAll(mergeIt)
-	require.EqualValues(t, []string{"5", "7", "9"}, k)
-	require.EqualValues(t, []string{"b5", "a7", "d9"}, v)
-	closeAndCheck(t, mergeIt, 4)
-}
-
-func TestMergeIteratorSeekReversed(t *testing.T) {
-	it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, true)
-	it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, true)
-	it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, true)
-	it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, true)
-	mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, true)
-	mergeIt.Seek([]byte("5"))
-	k, v := getAll(mergeIt)
-	require.EqualValues(t, []string{"5", "3", "2", "1"}, k)
-	require.EqualValues(t, []string{"b5", "a3", "b2", "a1"}, v)
-	closeAndCheck(t, mergeIt, 4)
-}
-
-func TestMergeIteratorSeekInvalid(t *testing.T) {
-	it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, false)
-	it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, false)
-	it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, false)
-	it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, false)
-	mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, false)
-	mergeIt.Seek([]byte("f"))
-	require.False(t, mergeIt.Valid())
-	closeAndCheck(t, mergeIt, 4)
-}
-
-func TestMergeIteratorSeekInvalidReversed(t *testing.T) {
-	it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, true)
-	it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, true)
-	it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, true)
-	it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, true)
-	mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, true)
-	mergeIt.Seek([]byte("0"))
-	require.False(t, mergeIt.Valid())
-	closeAndCheck(t, mergeIt, 4)
-}

vendor/github.com/dgraph-io/badger/y/mmap_unix.go

@@ -23,7 +23,7 @@ import (
 	"syscall"
 	"unsafe"
 
-	"gx/ipfs/QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu/sys/unix"
+	"golang.org/x/sys/unix"
 )
 
 // Mmap uses the mmap system call to memory-map a file. If writable is true,

vendor/github.com/dgraph-io/badger/y/watermark.go

@@ -21,15 +21,15 @@ import (
 	"context"
 	"sync/atomic"
 
-	"gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace"
+	"golang.org/x/net/trace"
 )
 
 type uint64Heap []uint64
 
-func (u uint64Heap) Len() int               { return len(u) }
-func (u uint64Heap) Less(i int, j int) bool { return u[i] < u[j] }
-func (u uint64Heap) Swap(i int, j int)      { u[i], u[j] = u[j], u[i] }
-func (u *uint64Heap) Push(x interface{})    { *u = append(*u, x.(uint64)) }
+func (u uint64Heap) Len() int            { return len(u) }
+func (u uint64Heap) Less(i, j int) bool  { return u[i] < u[j] }
+func (u uint64Heap) Swap(i, j int)       { u[i], u[j] = u[j], u[i] }
+func (u *uint64Heap) Push(x interface{}) { *u = append(*u, x.(uint64)) }
 func (u *uint64Heap) Pop() interface{} {
 	old := *u
 	n := len(old)

vendor/github.com/dgraph-io/badger/y/y.go

@@ -26,7 +26,7 @@ import (
 	"sync"
 	"time"
 
-	"gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors"
+	"github.com/pkg/errors"
 )
 
 // ErrEOF indicates an end of file when trying to read from a memory mapped file
@@ -48,6 +48,9 @@ var (
 
 	// CastagnoliCrcTable is a CRC32 polynomial table
 	CastagnoliCrcTable = crc32.MakeTable(crc32.Castagnoli)
+
+	// Dummy channel for nil closers.
+	dummyCloserChan = make(chan struct{})
 )
 
 // OpenExistingFile opens an existing file, errors if it doesn't exist.
@@ -91,7 +94,7 @@ func OpenTruncFile(filename string, sync bool) (*os.File, error) {
 }
 
 // SafeCopy does append(a[:0], src...).
-func SafeCopy(a []byte, src []byte) []byte {
+func SafeCopy(a, src []byte) []byte {
 	return append(a[:0], src...)
 }
 
@@ -122,7 +125,7 @@ func ParseTs(key []byte) uint64 {
 // is same.
 // a<timestamp> would be sorted higher than aa<timestamp> if we use bytes.compare
 // All keys should have timestamp.
-func CompareKeys(key1 []byte, key2 []byte) int {
+func CompareKeys(key1, key2 []byte) int {
 	AssertTrue(len(key1) > 8 && len(key2) > 8)
 	if cmp := bytes.Compare(key1[:len(key1)-8], key2[:len(key2)-8]); cmp != 0 {
 		return cmp
@@ -203,11 +206,17 @@ func (lc *Closer) Signal() {
 
 // HasBeenClosed gets signaled when Signal() is called.
 func (lc *Closer) HasBeenClosed() <-chan struct{} {
+	if lc == nil {
+		return dummyCloserChan
+	}
 	return lc.closed
 }
 
 // Done calls Done() on the WaitGroup.
 func (lc *Closer) Done() {
+	if lc == nil {
+		return
+	}
 	lc.waiting.Done()
 }
 
@@ -227,9 +236,11 @@ func (lc *Closer) SignalAndWait() {
 // provides a mechanism to check for errors encountered by workers and wait for
 // them to finish.
 type Throttle struct {
-	wg    sync.WaitGroup
-	ch    chan struct{}
-	errCh chan error
+	once      sync.Once
+	wg        sync.WaitGroup
+	ch        chan struct{}
+	errCh     chan error
+	finishErr error
 }
 
 // NewThrottle creates a new throttle with a max number of workers.
@@ -271,16 +282,21 @@ func (t *Throttle) Done(err error) {
 	t.wg.Done()
 }
 
-// Finish waits until all workers have finished working. It would return any
-// error passed by Done.
+// Finish waits until all workers have finished working. It would return any error passed by Done.
+// If Finish is called multiple time, it will wait for workers to finish only once(first time).
+// From next calls, it will return same error as found on first call.
 func (t *Throttle) Finish() error {
-	t.wg.Wait()
-	close(t.ch)
-	close(t.errCh)
-	for err := range t.errCh {
-		if err != nil {
-			return err
+	t.once.Do(func() {
+		t.wg.Wait()
+		close(t.ch)
+		close(t.errCh)
+		for err := range t.errCh {
+			if err != nil {
+				t.finishErr = err
+				return
+			}
 		}
-	}
-	return nil
+	})
+
+	return t.finishErr
 }