diff --git a/Gopkg.lock b/Gopkg.lock index 8bbf00c8..036fef7d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -129,19 +129,18 @@ version = "v1.7.1" [[projects]] - digest = "1:5f5090f05382959db941fa45acbeb7f4c5241aa8ac0f8f4393dec696e5953f53" + digest = "1:98436a9785f711ffee66f06265296dd36afd6e3dc532410c98a9051b5fd4c2f1" name = "github.com/dgraph-io/badger" packages = [ ".", "options", - "protos", + "pb", "skl", "table", "y", ] pruneopts = "UT" - revision = "99233d725dbdd26d156c61b2f42ae1671b794656" - version = "gx/v1.5.4" + revision = "0ce1d2e26af1ba8b8a72ea864145a3e1e3b382cd" [[projects]] branch = "master" @@ -2289,6 +2288,7 @@ "github.com/ethereum/go-ethereum/accounts/abi/bind", "github.com/ethereum/go-ethereum/common", "github.com/ethereum/go-ethereum/common/hexutil", + "github.com/ethereum/go-ethereum/core", "github.com/ethereum/go-ethereum/core/rawdb", "github.com/ethereum/go-ethereum/core/state", "github.com/ethereum/go-ethereum/core/types", diff --git a/Gopkg.toml b/Gopkg.toml index bbb400a0..db32d945 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -63,8 +63,8 @@ version = "0.4.20" [[override]] - name = "github.com/ipfs/go-ds-badger" - version = "0.0.3" + name = "github.com/dgraph-io/badger" + revision = "0ce1d2e26af1ba8b8a72ea864145a3e1e3b382cd" [prune] go-tests = true diff --git a/cmd/streamSubscribe.go b/cmd/streamSubscribe.go index d2ac134b..3b66c8de 100644 --- a/cmd/streamSubscribe.go +++ b/cmd/streamSubscribe.go @@ -17,7 +17,9 @@ package cmd import ( "bytes" + "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" @@ -48,6 +50,7 @@ func init() { } func streamSubscribe() { + log.SetLevel(log.DebugLevel) // Prep the subscription config/filters to be sent to the server subscriptionConfig() @@ -61,27 +64,26 @@ func streamSubscribe() { // Subscribe to the seed node service with the given config/filter parameters sub, err := str.Stream(payloadChan, subConfig) if err != nil { - println(err.Error()) log.Fatal(err) } - + log.Info("awaiting payloads") // Receive response payloads and print out the results for { select { case payload := <-payloadChan: - if payload.Err != nil { - log.Error(payload.Err) + if payload.ErrMsg != "" { + log.Error(payload.ErrMsg) + continue } for _, headerRlp := range payload.HeadersRlp { var header types.Header err = rlp.Decode(bytes.NewBuffer(headerRlp), &header) if err != nil { - println(err.Error()) log.Error(err) + continue } - println("Header") - println(header.Hash().Hex()) - println(header.Number.Int64()) + fmt.Printf("Header number %d, hash %s\n", header.Number.Int64(), header.Hash().Hex()) + fmt.Printf("header: %v\n", header) } for _, trxRlp := range payload.TransactionsRlp { var trx types.Transaction @@ -89,11 +91,11 @@ func streamSubscribe() { stream := rlp.NewStream(buff, 0) err := trx.DecodeRLP(stream) if err != nil { - println(err.Error()) log.Error(err) + continue } - println("Trx") - println(trx.Hash().Hex()) + fmt.Printf("Transaction with hash %s\n", trx.Hash().Hex()) + fmt.Printf("trx: %v", trx) } for _, rctRlp := range payload.ReceiptsRlp { var rct types.Receipt @@ -101,52 +103,59 @@ func streamSubscribe() { stream := rlp.NewStream(buff, 0) err = rct.DecodeRLP(stream) if err != nil { - println(err.Error()) log.Error(err) + continue } - println("Rct") + fmt.Printf("Receipt with block hash %s, trx hash %s\n", rct.BlockHash.Hex(), rct.TxHash.Hex()) + fmt.Printf("rct: %v", rct) for _, l := range rct.Logs { - println("log") - println(l.BlockHash.Hex()) - println(l.TxHash.Hex()) - println(l.Address.Hex()) + if len(l.Topics) < 1 { + log.Error(fmt.Sprintf("log only has %d topics", len(l.Topics))) + continue + } + fmt.Printf("Log for block hash %s, trx hash %s, address %s, and with topic0 %s\n", + l.BlockHash.Hex(), l.TxHash.Hex(), l.Address.Hex(), l.Topics[0].Hex()) + fmt.Printf("log: %v\n", l) } } + // This assumes leafs only for key, stateRlp := range payload.StateNodesRlp { var acct state.Account err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct) if err != nil { - println(err.Error()) log.Error(err) + continue } - println("State") - print("key: ") - println(key.Hex()) - print("root: ") - println(acct.Root.Hex()) - print("balance: ") - println(acct.Balance.Int64()) + fmt.Printf("Account for key %s, and root %s, with balance %d\n", + key.Hex(), acct.Root.Hex(), acct.Balance.Int64()) + fmt.Printf("state account: %v\n", acct) } for stateKey, mappedRlp := range payload.StorageNodesRlp { - println("Storage") - print("state key: ") - println(stateKey.Hex()) + fmt.Printf("Storage for state key %s ", stateKey.Hex()) for storageKey, storageRlp := range mappedRlp { - println("Storage") - print("key: ") - println(storageKey.Hex()) + fmt.Printf("with storage key %s\n", storageKey.Hex()) var i []interface{} err := rlp.DecodeBytes(storageRlp, i) if err != nil { - println(err.Error()) log.Error(err) + continue + } + // if a leaf node + if len(i) == 2 { + keyBytes, ok := i[0].([]byte) + if !ok { + continue + } + valueBytes, ok := i[0].([]byte) + if !ok { + continue + } + fmt.Printf("Storage leaf key: %s, and value hash: %s\n", + common.BytesToHash(keyBytes).Hex(), common.BytesToHash(valueBytes).Hex()) } - print("bytes: ") - println(storageRlp) } } case err = <-sub.Err(): - println(err.Error()) log.Fatal(err) } } diff --git a/cmd/syncPublishScreenAndServe.go b/cmd/syncPublishScreenAndServe.go index b80f31f2..6512e8fa 100644 --- a/cmd/syncPublishScreenAndServe.go +++ b/cmd/syncPublishScreenAndServe.go @@ -47,6 +47,7 @@ func init() { } func syncPublishScreenAndServe() { + log.SetLevel(log.DebugLevel) blockChain, ethClient, rpcClient := getBlockChainAndClients() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) diff --git a/pkg/ipfs/api.go b/pkg/ipfs/api.go index aecb0763..f5169e5c 100644 --- a/pkg/ipfs/api.go +++ b/pkg/ipfs/api.go @@ -64,8 +64,13 @@ func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters config.S for { select { case packet := <-payloadChannel: - if err := notifier.Notify(rpcSub.ID, packet); err != nil { - log.Error("Failed to send state diff packet", "err", err) + if notifyErr := notifier.Notify(rpcSub.ID, packet); notifyErr != nil { + log.Error("Failed to send state diff packet", "err", notifyErr) + unSubErr := api.snp.Unsubscribe(rpcSub.ID) + if unSubErr != nil { + log.Error("Failed to unsubscribe from the state diff service", unSubErr) + } + return } case <-rpcSub.Err(): err := api.snp.Unsubscribe(rpcSub.ID) diff --git a/pkg/ipfs/converter_test.go b/pkg/ipfs/converter_test.go index 88b8c403..545dea67 100644 --- a/pkg/ipfs/converter_test.go +++ b/pkg/ipfs/converter_test.go @@ -20,19 +20,18 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" ) var _ = Describe("Converter", func() { Describe("Convert", func() { It("Converts StatediffPayloads into IPLDPayloads", func() { mockConverter := mocks.PayloadConverter{} - mockConverter.ReturnIPLDPayload = &helpers.MockIPLDPayload - ipldPayload, err := mockConverter.Convert(helpers.MockStatediffPayload) + mockConverter.ReturnIPLDPayload = &mocks.MockIPLDPayload + ipldPayload, err := mockConverter.Convert(mocks.MockStatediffPayload) Expect(err).ToNot(HaveOccurred()) - Expect(ipldPayload).To(Equal(&helpers.MockIPLDPayload)) - Expect(mockConverter.PassedStatediffPayload).To(Equal(helpers.MockStatediffPayload)) + Expect(ipldPayload).To(Equal(&mocks.MockIPLDPayload)) + Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStatediffPayload)) }) }) }) diff --git a/pkg/ipfs/fetcher.go b/pkg/ipfs/fetcher.go index 4319efca..1d50d1be 100644 --- a/pkg/ipfs/fetcher.go +++ b/pkg/ipfs/fetcher.go @@ -20,7 +20,6 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" @@ -50,6 +49,7 @@ func NewIPLDFetcher(ipfsPath string) (*EthIPLDFetcher, error) { // FetchCIDs is the exported method for fetching and returning all the cids passed in a CidWrapper func (f *EthIPLDFetcher) FetchCIDs(cids CidWrapper) (*IpldWrapper, error) { + log.Debug("fetching iplds") blocks := &IpldWrapper{ Headers: make([]blocks.Block, 0), Transactions: make([]blocks.Block, 0), @@ -85,6 +85,7 @@ func (f *EthIPLDFetcher) FetchCIDs(cids CidWrapper) (*IpldWrapper, error) { // fetchHeaders fetches headers // It uses the f.fetchBatch method func (f *EthIPLDFetcher) fetchHeaders(cids CidWrapper, blocks *IpldWrapper) error { + log.Debug("fetching header iplds") headerCids := make([]cid.Cid, 0, len(cids.Headers)) for _, c := range cids.Headers { dc, err := cid.Decode(c) @@ -103,6 +104,7 @@ func (f *EthIPLDFetcher) fetchHeaders(cids CidWrapper, blocks *IpldWrapper) erro // fetchTrxs fetches transactions // It uses the f.fetchBatch method func (f *EthIPLDFetcher) fetchTrxs(cids CidWrapper, blocks *IpldWrapper) error { + log.Debug("fetching transaction iplds") trxCids := make([]cid.Cid, 0, len(cids.Transactions)) for _, c := range cids.Transactions { dc, err := cid.Decode(c) @@ -121,6 +123,7 @@ func (f *EthIPLDFetcher) fetchTrxs(cids CidWrapper, blocks *IpldWrapper) error { // fetchRcts fetches receipts // It uses the f.fetchBatch method func (f *EthIPLDFetcher) fetchRcts(cids CidWrapper, blocks *IpldWrapper) error { + log.Debug("fetching receipt iplds") rctCids := make([]cid.Cid, 0, len(cids.Receipts)) for _, c := range cids.Receipts { dc, err := cid.Decode(c) @@ -140,6 +143,7 @@ func (f *EthIPLDFetcher) fetchRcts(cids CidWrapper, blocks *IpldWrapper) error { // It uses the single f.fetch method instead of the batch fetch, because it // needs to maintain the data's relation to state keys func (f *EthIPLDFetcher) fetchState(cids CidWrapper, blocks *IpldWrapper) error { + log.Debug("fetching state iplds") for _, stateNode := range cids.StateNodes { if stateNode.CID == "" || stateNode.Key == "" { continue @@ -161,6 +165,7 @@ func (f *EthIPLDFetcher) fetchState(cids CidWrapper, blocks *IpldWrapper) error // It uses the single f.fetch method instead of the batch fetch, because it // needs to maintain the data's relation to state and storage keys func (f *EthIPLDFetcher) fetchStorage(cids CidWrapper, blocks *IpldWrapper) error { + log.Debug("fetching storage iplds") for _, storageNode := range cids.StorageNodes { if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" { continue diff --git a/pkg/ipfs/helpers/mocks/api.go b/pkg/ipfs/mocks/api.go similarity index 100% rename from pkg/ipfs/helpers/mocks/api.go rename to pkg/ipfs/mocks/api.go diff --git a/pkg/ipfs/helpers/mocks/converter.go b/pkg/ipfs/mocks/converter.go similarity index 100% rename from pkg/ipfs/helpers/mocks/converter.go rename to pkg/ipfs/mocks/converter.go diff --git a/pkg/ipfs/helpers/mocks/publisher.go b/pkg/ipfs/mocks/publisher.go similarity index 100% rename from pkg/ipfs/helpers/mocks/publisher.go rename to pkg/ipfs/mocks/publisher.go diff --git a/pkg/ipfs/helpers/mocks/repository.go b/pkg/ipfs/mocks/repository.go similarity index 100% rename from pkg/ipfs/helpers/mocks/repository.go rename to pkg/ipfs/mocks/repository.go diff --git a/pkg/ipfs/helpers/mocks/screener.go b/pkg/ipfs/mocks/screener.go similarity index 100% rename from pkg/ipfs/helpers/mocks/screener.go rename to pkg/ipfs/mocks/screener.go diff --git a/pkg/ipfs/helpers/mocks/streamer.go b/pkg/ipfs/mocks/streamer.go similarity index 100% rename from pkg/ipfs/helpers/mocks/streamer.go rename to pkg/ipfs/mocks/streamer.go diff --git a/pkg/ipfs/helpers/test_data.go b/pkg/ipfs/mocks/test_data.go similarity index 99% rename from pkg/ipfs/helpers/test_data.go rename to pkg/ipfs/mocks/test_data.go index 33daefd0..564098e7 100644 --- a/pkg/ipfs/helpers/test_data.go +++ b/pkg/ipfs/mocks/test_data.go @@ -14,20 +14,20 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package helpers +package mocks import ( "errors" "math/big" "math/rand" + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" - - "github.com/vulcanize/vulcanizedb/pkg/ipfs" ) // Test variables diff --git a/pkg/ipfs/publisher_test.go b/pkg/ipfs/publisher_test.go index 2e14a814..e3279d06 100644 --- a/pkg/ipfs/publisher_test.go +++ b/pkg/ipfs/publisher_test.go @@ -20,19 +20,18 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" ) var _ = Describe("Publisher", func() { Describe("Publish", func() { It("Publishes IPLDPayload to IPFS", func() { mockPublisher := mocks.IPLDPublisher{} - mockPublisher.ReturnCIDPayload = &helpers.MockCIDPayload - cidPayload, err := mockPublisher.Publish(&helpers.MockIPLDPayload) + mockPublisher.ReturnCIDPayload = &mocks.MockCIDPayload + cidPayload, err := mockPublisher.Publish(&mocks.MockIPLDPayload) Expect(err).ToNot(HaveOccurred()) - Expect(cidPayload).To(Equal(&helpers.MockCIDPayload)) - Expect(mockPublisher.PassedIPLDPayload).To(Equal(&helpers.MockIPLDPayload)) + Expect(cidPayload).To(Equal(&mocks.MockCIDPayload)) + Expect(mockPublisher.PassedIPLDPayload).To(Equal(&mocks.MockIPLDPayload)) }) }) }) diff --git a/pkg/ipfs/repository_test.go b/pkg/ipfs/repository_test.go index 0f127224..dcffd0fe 100644 --- a/pkg/ipfs/repository_test.go +++ b/pkg/ipfs/repository_test.go @@ -20,17 +20,16 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" ) var _ = Describe("Repository", func() { Describe("Index", func() { It("Indexes CIDs against their metadata", func() { mockRepo := mocks.CIDRepository{} - err := mockRepo.Index(&helpers.MockCIDPayload) + err := mockRepo.Index(&mocks.MockCIDPayload) Expect(err).ToNot(HaveOccurred()) - Expect(mockRepo.PassedCIDPayload).To(Equal(&helpers.MockCIDPayload)) + Expect(mockRepo.PassedCIDPayload).To(Equal(&mocks.MockCIDPayload)) }) }) }) diff --git a/pkg/ipfs/resolver.go b/pkg/ipfs/resolver.go index 1cf2baf2..90fe92fd 100644 --- a/pkg/ipfs/resolver.go +++ b/pkg/ipfs/resolver.go @@ -18,7 +18,6 @@ package ipfs import ( "github.com/ethereum/go-ethereum/common" - "github.com/ipfs/go-block-format" ) diff --git a/pkg/ipfs/retreiver.go b/pkg/ipfs/retreiver.go index cb113b0f..6bf8e583 100644 --- a/pkg/ipfs/retreiver.go +++ b/pkg/ipfs/retreiver.go @@ -19,6 +19,8 @@ package ipfs import ( "github.com/jmoiron/sqlx" "github.com/lib/pq" + log "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -49,6 +51,7 @@ func (ecr *EthCIDRetriever) GetLastBlockNumber() (int64, error) { // RetrieveCIDs is used to retrieve all of the CIDs which conform to the passed StreamFilters func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription) ([]CidWrapper, error) { + log.Debug("retrieving cids") var endingBlock int64 var err error if streamFilters.EndingBlock <= 0 || streamFilters.EndingBlock <= streamFilters.StartingBlock { @@ -63,70 +66,68 @@ func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription) ([]C return nil, err } for i := streamFilters.StartingBlock; i <= endingBlock; i++ { - cw := &CidWrapper{ - BlockNumber: i, - Headers: make([]string, 0), - Transactions: make([]string, 0), - Receipts: make([]string, 0), - StateNodes: make([]StateNodeCID, 0), - StorageNodes: make([]StorageNodeCID, 0), - } + cw := CidWrapper{} if !streamFilters.HeaderFilter.Off { - err = ecr.retrieveHeaderCIDs(tx, streamFilters, cw, i) + cw.Headers, err = ecr.retrieveHeaderCIDs(tx, streamFilters, i) if err != nil { tx.Rollback() + log.Error("header cid retrieval error") return nil, err } } var trxIds []int64 if !streamFilters.TrxFilter.Off { - trxIds, err = ecr.retrieveTrxCIDs(tx, streamFilters, cw, i) + cw.Transactions, trxIds, err = ecr.retrieveTrxCIDs(tx, streamFilters, i) if err != nil { tx.Rollback() + log.Error("transaction cid retrieval error") return nil, err } } if !streamFilters.ReceiptFilter.Off { - err = ecr.retrieveRctCIDs(tx, streamFilters, cw, i, trxIds) + cw.Receipts, err = ecr.retrieveRctCIDs(tx, streamFilters, i, trxIds) if err != nil { tx.Rollback() + log.Error("receipt cid retrieval error") return nil, err } } if !streamFilters.StateFilter.Off { - err = ecr.retrieveStateCIDs(tx, streamFilters, cw, i) + cw.StateNodes, err = ecr.retrieveStateCIDs(tx, streamFilters, i) if err != nil { tx.Rollback() + log.Error("state cid retrieval error") return nil, err } } if !streamFilters.StorageFilter.Off { - err = ecr.retrieveStorageCIDs(tx, streamFilters, cw, i) + cw.StorageNodes, err = ecr.retrieveStorageCIDs(tx, streamFilters, i) if err != nil { tx.Rollback() + log.Error("storage cid retrieval error") return nil, err } } - cids = append(cids, *cw) + cids = append(cids, cw) } - return cids, err + return cids, tx.Commit() } -func (ecr *EthCIDRetriever) retrieveHeaderCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error { - var pgStr string - if streamFilters.HeaderFilter.FinalOnly { - pgStr = `SELECT cid FROM header_cids - WHERE block_number = $1 - AND final IS TRUE` - } else { - pgStr = `SELECT cid FROM header_cids +func (ecr *EthCIDRetriever) retrieveHeaderCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]string, error) { + log.Debug("retrieving header cids") + headers := make([]string, 0) + pgStr := `SELECT cid FROM header_cids WHERE block_number = $1` + if streamFilters.HeaderFilter.FinalOnly { + pgStr += ` AND final IS TRUE` } - return tx.Select(cids.Headers, pgStr, blockNumber) + err := tx.Select(&headers, pgStr, blockNumber) + return headers, err } -func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) ([]int64, error) { +func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]string, []int64, error) { + log.Debug("retrieving transaction cids") args := make([]interface{}, 0, 3) type result struct { ID int64 `db:"id"` @@ -144,19 +145,21 @@ func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters config.Su pgStr += ` AND transaction_cids.src = ANY($3::VARCHAR(66)[])` args = append(args, pq.Array(streamFilters.TrxFilter.Src)) } - err := tx.Select(results, pgStr, args...) + err := tx.Select(&results, pgStr, args...) if err != nil { - return nil, err + return nil, nil, err } - ids := make([]int64, 0) + ids := make([]int64, 0, len(results)) + cids := make([]string, 0, len(results)) for _, res := range results { - cids.Transactions = append(cids.Transactions, res.Cid) + cids = append(cids, res.Cid) ids = append(ids, res.ID) } - return ids, nil + return cids, ids, nil } -func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64, trxIds []int64) error { +func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64, trxIds []int64) ([]string, error) { + log.Debug("retrieving receipt cids") args := make([]interface{}, 0, 2) pgStr := `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids WHERE receipt_cids.tx_id = transaction_cids.id @@ -173,10 +176,13 @@ func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Su } else { pgStr += `)` } - return tx.Select(cids.Receipts, pgStr, args...) + receiptCids := make([]string, 0) + err := tx.Select(&receiptCids, pgStr, args...) + return receiptCids, err } -func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error { +func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]StateNodeCID, error) { + log.Debug("retrieving state cids") args := make([]interface{}, 0, 2) pgStr := `SELECT state_cids.cid, state_cids.state_key FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id) WHERE header_cids.block_number = $1` @@ -190,10 +196,16 @@ func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config. pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])` args = append(args, pq.Array(keys)) } - return tx.Select(cids.StateNodes, pgStr, args...) + if !streamFilters.StorageFilter.IntermediateNodes { + pgStr += ` AND state_cids.leaf = TRUE` + } + stateNodeCIDs := make([]StateNodeCID, 0) + err := tx.Select(&stateNodeCIDs, pgStr, args...) + return stateNodeCIDs, err } -func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error { +func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]StorageNodeCID, error) { + log.Debug("retrieving storage cids") args := make([]interface{}, 0, 3) pgStr := `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key FROM storage_cids, state_cids, header_cids WHERE storage_cids.state_id = state_cids.id @@ -213,7 +225,10 @@ func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters confi pgStr += ` AND storage_cids.storage_key = ANY($3::VARCHAR(66)[])` args = append(args, pq.Array(streamFilters.StorageFilter.StorageKeys)) } - return tx.Select(cids.StorageNodes, pgStr, args...) + if !streamFilters.StorageFilter.IntermediateNodes { + pgStr += ` AND storage_cids.leaf = TRUE` + } + storageNodeCIDs := make([]StorageNodeCID, 0) + err := tx.Select(&storageNodeCIDs, pgStr, args...) + return storageNodeCIDs, err } - -// ADD IF LEAF ONLY!! diff --git a/pkg/ipfs/screener.go b/pkg/ipfs/screener.go index 5468870c..8305a84d 100644 --- a/pkg/ipfs/screener.go +++ b/pkg/ipfs/screener.go @@ -19,12 +19,11 @@ package ipfs import ( "bytes" - "github.com/vulcanize/vulcanizedb/pkg/config" - - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" + + "github.com/vulcanize/vulcanizedb/pkg/config" ) // ResponseScreener is the inteface used to screen eth data and package appropriate data into a response payload diff --git a/pkg/ipfs/service.go b/pkg/ipfs/service.go index a89907b8..54e9da67 100644 --- a/pkg/ipfs/service.go +++ b/pkg/ipfs/service.go @@ -121,7 +121,8 @@ func (sap *Service) APIs() []rpc.API { } // SyncAndPublish is the backend processing loop which streams data from geth, converts it to iplds, publishes them to ipfs, and indexes their cids -// It then forwards the data to the Serve() loop which filters and sends relevant data to client subscriptions +// This continues on no matter if or how many subscribers there are, it then forwards the data to the ScreenAndServe() loop +// which filters and sends relevant data to client subscriptions, if there are any func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<- IPLDPayload, forwardQuitchan chan<- bool) error { sub, err := sap.Streamer.Stream(sap.PayloadChan) if err != nil { @@ -173,7 +174,8 @@ func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<- return nil } -// ScreenAndServe is the processing loop used to screen data streamed from the state diffing eth node and send the appropriate data to a requesting client subscription +// ScreenAndServe is the loop used to screen data streamed from the state diffing eth node +// and send the appropriate portions of it to a requesting client subscription, according to their subscription configuration func (sap *Service) ScreenAndServe(wg *sync.WaitGroup, receivePayloadChan <-chan IPLDPayload, receiveQuitchan <-chan bool) { wg.Add(1) go func() { @@ -206,32 +208,32 @@ func (sap *Service) processResponse(payload IPLDPayload) error { // Subscribe is used by the API to subscribe to the service loop func (sap *Service) Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters *config.Subscription) { - log.Info("Subscribing to the statediff service") + log.Info("Subscribing to the seed node service") sap.Lock() sap.Subscriptions[id] = Subscription{ PayloadChan: sub, QuitChan: quitChan, StreamFilters: streamFilters, } - sap.Unlock() // If the subscription requests a backfill, use the Postgres index to lookup and retrieve historical data // Otherwise we only filter new data as it is streamed in from the state diffing geth node - if streamFilters.BackFill { + if streamFilters.BackFill || streamFilters.BackFillOnly { + log.Debug("back-filling data for id", id) // Retrieve cached CIDs relevant to this subscriber - cids, err := sap.Retriever.RetrieveCIDs(*streamFilters) + cidWrappers, err := sap.Retriever.RetrieveCIDs(*streamFilters) if err != nil { log.Error(err) sap.serve(id, ResponsePayload{ - Err: err, + ErrMsg: "CID retrieval error: " + err.Error(), }) return } - for _, cid := range cids { - blocksWrapper, err := sap.Fetcher.FetchCIDs(cid) + for _, cidWrapper := range cidWrappers { + blocksWrapper, err := sap.Fetcher.FetchCIDs(cidWrapper) if err != nil { log.Error(err) sap.serve(id, ResponsePayload{ - Err: err, + ErrMsg: "IPLD fetching error: " + err.Error(), }) return } @@ -239,18 +241,22 @@ func (sap *Service) Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan ch if err != nil { log.Error(err) sap.serve(id, ResponsePayload{ - Err: err, + ErrMsg: "IPLD resolving error: " + err.Error(), }) return } sap.serve(id, *backFillIplds) } + if streamFilters.BackFillOnly { + delete(sap.Subscriptions, id) + } } + sap.Unlock() } // Unsubscribe is used to unsubscribe to the StateDiffingService loop func (sap *Service) Unsubscribe(id rpc.ID) error { - log.Info("Unsubscribing from the statediff service") + log.Info("Unsubscribing from the seed node service") sap.Lock() _, ok := sap.Subscriptions[id] if !ok { @@ -263,7 +269,7 @@ func (sap *Service) Unsubscribe(id rpc.ID) error { // Start is used to begin the service func (sap *Service) Start(*p2p.Server) error { - log.Info("Starting statediff service") + log.Info("Starting seed node service") wg := new(sync.WaitGroup) payloadChan := make(chan IPLDPayload, payloadChanBufferSize) quitChan := make(chan bool, 1) @@ -276,7 +282,7 @@ func (sap *Service) Start(*p2p.Server) error { // Stop is used to close down the service func (sap *Service) Stop() error { - log.Info("Stopping statediff service") + log.Info("Stopping seed node service") close(sap.QuitChan) return nil } diff --git a/pkg/ipfs/service_test.go b/pkg/ipfs/service_test.go index d5f95071..fbe0963f 100644 --- a/pkg/ipfs/service_test.go +++ b/pkg/ipfs/service_test.go @@ -26,8 +26,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/ipfs" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" ) var _ = Describe("Service", func() { @@ -41,18 +40,18 @@ var _ = Describe("Service", func() { ReturnErr: nil, } mockPublisher := &mocks.IPLDPublisher{ - ReturnCIDPayload: &helpers.MockCIDPayload, + ReturnCIDPayload: &mocks.MockCIDPayload, ReturnErr: nil, } mockStreamer := &mocks.StateDiffStreamer{ ReturnSub: &rpc.ClientSubscription{}, StreamPayloads: []statediff.Payload{ - helpers.MockStatediffPayload, + mocks.MockStatediffPayload, }, ReturnErr: nil, } mockConverter := &mocks.PayloadConverter{ - ReturnIPLDPayload: &helpers.MockIPLDPayload, + ReturnIPLDPayload: &mocks.MockIPLDPayload, ReturnErr: nil, } processor := &ipfs.Service{ @@ -68,9 +67,9 @@ var _ = Describe("Service", func() { time.Sleep(2 * time.Second) quitChan <- true wg.Wait() - Expect(mockConverter.PassedStatediffPayload).To(Equal(helpers.MockStatediffPayload)) - Expect(mockCidRepo.PassedCIDPayload).To(Equal(&helpers.MockCIDPayload)) - Expect(mockPublisher.PassedIPLDPayload).To(Equal(&helpers.MockIPLDPayload)) + Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStatediffPayload)) + Expect(mockCidRepo.PassedCIDPayload).To(Equal(&mocks.MockCIDPayload)) + Expect(mockPublisher.PassedIPLDPayload).To(Equal(&mocks.MockIPLDPayload)) Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan)) }) }) diff --git a/pkg/ipfs/streamer_test.go b/pkg/ipfs/streamer_test.go index acbc506e..70387142 100644 --- a/pkg/ipfs/streamer_test.go +++ b/pkg/ipfs/streamer_test.go @@ -22,8 +22,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" ) var _ = Describe("Streamer", func() { @@ -32,7 +31,7 @@ var _ = Describe("Streamer", func() { mockStreamer := mocks.StateDiffStreamer{} mockStreamer.ReturnSub = &rpc.ClientSubscription{} mockStreamer.StreamPayloads = []statediff.Payload{ - helpers.MockStatediffPayload, + mocks.MockStatediffPayload, } payloadChan := make(chan statediff.Payload, 1) sub, err := mockStreamer.Stream(payloadChan) @@ -40,7 +39,7 @@ var _ = Describe("Streamer", func() { Expect(sub).To(Equal(&rpc.ClientSubscription{})) Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan)) streamedPayload := <-payloadChan - Expect(streamedPayload).To(Equal(helpers.MockStatediffPayload)) + Expect(streamedPayload).To(Equal(mocks.MockStatediffPayload)) }) }) }) diff --git a/pkg/ipfs/types.go b/pkg/ipfs/types.go index bdf23113..0a33cf51 100644 --- a/pkg/ipfs/types.go +++ b/pkg/ipfs/types.go @@ -43,7 +43,7 @@ type ResponsePayload struct { ReceiptsRlp [][]byte `json:"receiptsRlp"` StateNodesRlp map[common.Hash][]byte `json:"stateNodesRlp"` StorageNodesRlp map[common.Hash]map[common.Hash][]byte `json:"storageNodesRlp"` - Err error `json:"error"` + ErrMsg string `json:"errMsg"` encoded []byte err error diff --git a/vendor/github.com/dgraph-io/badger/.gitignore b/vendor/github.com/dgraph-io/badger/.gitignore new file mode 100644 index 00000000..11b9bcb1 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/.gitignore @@ -0,0 +1 @@ +p/ diff --git a/vendor/github.com/dgraph-io/badger/.golangci.yml b/vendor/github.com/dgraph-io/badger/.golangci.yml new file mode 100644 index 00000000..22f245f2 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/.golangci.yml @@ -0,0 +1,20 @@ +run: + tests: false + +linters-settings: + lll: + line-length: 100 + +linters: + disable-all: true + enable: + - errcheck + - ineffassign + - gas + - gofmt + - golint + - gosimple + - govet + - lll + - varcheck + - unused diff --git a/vendor/github.com/dgraph-io/badger/.travis.yml b/vendor/github.com/dgraph-io/badger/.travis.yml new file mode 100644 index 00000000..43bf4cdc --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/.travis.yml @@ -0,0 +1,25 @@ +language: go + +go: + - "1.9" + - "1.10" + - "1.11" + +matrix: + include: + - os: osx +notifications: + email: false + slack: + secure: X7uBLWYbuUhf8QFE16CoS5z7WvFR8EN9j6cEectMW6mKZ3vwXGwVXRIPsgUq/606DsQdCCx34MR8MRWYGlu6TBolbSe9y0EP0i46yipPz22YtuT7umcVUbGEyx8MZKgG0v1u/zA0O4aCsOBpGAA3gxz8h3JlEHDt+hv6U8xRsSllVLzLSNb5lwxDtcfEDxVVqP47GMEgjLPM28Pyt5qwjk7o5a4YSVzkfdxBXxd3gWzFUWzJ5E3cTacli50dK4GVfiLcQY2aQYoYO7AAvDnvP+TPfjDkBlUEE4MUz5CDIN51Xb+WW33sX7g+r3Bj7V5IRcF973RiYkpEh+3eoiPnyWyxhDZBYilty3b+Hysp6d4Ov/3I3ll7Bcny5+cYjakjkMH3l9w3gs6Y82GlpSLSJshKWS8vPRsxFe0Pstj6QSJXTd9EBaFr+l1ScXjJv/Sya9j8N9FfTuOTESWuaL1auX4Y7zEEVHlA8SCNOO8K0eTfxGZnC/YcIHsR8rePEAcFxfOYQppkyLF/XvAtnb/LMUuu0g4y2qNdme6Oelvyar1tFEMRtbl4mRCdu/krXBFtkrsfUaVY6WTPdvXAGotsFJ0wuA53zGVhlcd3+xAlSlR3c1QX95HIMeivJKb5L4nTjP+xnrmQNtnVk+tG4LSH2ltuwcZSSczModtcBmRefrk= + +env: + global: + - secure: CRkV2+/jlO0gXzzS50XGxfMS117FNwiVjxNY/LeWq06RKD+dDCPxTJl3JCNe3l0cYEPAglV2uMMYukDiTqJ7e+HI4nh4N4mv6lwx39N8dAvJe1x5ITS2T4qk4kTjuQb1Q1vw/ZOxoQqmvNKj2uRmBdJ/HHmysbRJ1OzCWML3OXdUwJf0AYlJzTjpMfkOKr7sTtE4rwyyQtd4tKH1fGdurgI9ZuFd9qvYxK2qcJhsQ6CNqMXt+7FkVkN1rIPmofjjBTNryzUr4COFXuWH95aDAif19DeBW4lbNgo1+FpDsrgmqtuhl6NAuptI8q/imow2KXBYJ8JPXsxW8DVFj0IIp0RCd3GjaEnwBEbxAyiIHLfW7AudyTS/dJOvZffPqXnuJ8xj3OPIdNe4xY0hWl8Ju2HhKfLOAHq7VadHZWd3IHLil70EiL4/JLD1rNbMImUZisFaA8pyrcIvYYebjOnk4TscwKFLedClRSX1XsMjWWd0oykQtrdkHM2IxknnBpaLu7mFnfE07f6dkG0nlpyu4SCLey7hr5FdcEmljA0nIxTSYDg6035fQkBEAbe7hlESOekkVNT9IZPwG+lmt3vU4ofi6NqNbJecOuSB+h36IiZ9s4YQtxYNnLgW14zjuFGGyT5smc3IjBT7qngDjKIgyrSVoRkY/8udy9qbUgvBeW8= + +before_script: +- go get github.com/mattn/goveralls +script: +- bash contrib/cover.sh $HOME/build coverage.out || travis_terminate 1 +- goveralls -service=travis-ci -coverprofile=coverage.out || true +- goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/vendor/github.com/dgraph-io/badger/README.md b/vendor/github.com/dgraph-io/badger/README.md index 4133210f..a25d8e1f 100644 --- a/vendor/github.com/dgraph-io/badger/README.md +++ b/vendor/github.com/dgraph-io/badger/README.md @@ -58,7 +58,7 @@ To start using Badger, install Go 1.8 or above and run `go get`: $ go get github.com/dgraph-io/badger/... ``` -This will retrieve the library and install the `badger_info` command line +This will retrieve the library and install the `badger` command line utility into your `$GOBIN` path. @@ -144,7 +144,7 @@ txn := db.NewTransaction(true) for k,v := range updates { if err := txn.Set([]byte(k),[]byte(v)); err == ErrTxnTooBig { _ = txn.Commit() - txn = db.NewTransaction(..) + txn = db.NewTransaction(true) _ = txn.Set([]byte(k),[]byte(v)) } } @@ -673,6 +673,9 @@ Below is a list of known projects that use Badger: * [Babble](https://github.com/mosaicnetworks/babble) - BFT Consensus platform for distributed applications. * [Tormenta](https://github.com/jpincas/tormenta) - Embedded object-persistence layer / simple JSON database for Go projects. * [BadgerHold](https://github.com/timshannon/badgerhold) - An embeddable NoSQL store for querying Go types built on Badger +* [Goblero](https://github.com/didil/goblero) - Pure Go embedded persistent job queue backed by BadgerDB +* [Surfline](https://www.surfline.com) - Serving global wave and weather forecast data with Badger. +* [Cete](https://github.com/mosuka/cete) - Simple and highly available distributed key-value store built on Badger. Makes it easy bringing up a cluster of Badger with Raft consensus algorithm by hashicorp/raft. If you are using Badger in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/dgraph-io/badger/backup.go b/vendor/github.com/dgraph-io/badger/backup.go index 170d4c37..0bc3b328 100644 --- a/vendor/github.com/dgraph-io/badger/backup.go +++ b/vendor/github.com/dgraph-io/badger/backup.go @@ -22,10 +22,9 @@ import ( "context" "encoding/binary" "io" - "sync" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" + "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/y" ) // Backup is a wrapper function over Stream.Backup to generate full and incremental backups of the @@ -106,11 +105,8 @@ func (stream *Stream) Backup(w io.Writer, since uint64) (uint64, error) { if maxVersion < kv.Version { maxVersion = kv.Version } - if err := writeTo(kv, w); err != nil { - return err - } } - return nil + return writeTo(list, w) } if err := stream.Orchestrate(context.Background()); err != nil { @@ -119,11 +115,11 @@ func (stream *Stream) Backup(w io.Writer, since uint64) (uint64, error) { return maxVersion, nil } -func writeTo(entry *pb.KV, w io.Writer) error { - if err := binary.Write(w, binary.LittleEndian, uint64(entry.Size())); err != nil { +func writeTo(list *pb.KVList, w io.Writer) error { + if err := binary.Write(w, binary.LittleEndian, uint64(list.Size())); err != nil { return err } - buf, err := entry.Marshal() + buf, err := list.Marshal() if err != nil { return err } @@ -131,38 +127,73 @@ func writeTo(entry *pb.KV, w io.Writer) error { return err } +type loader struct { + db *DB + throttle *y.Throttle + entries []*Entry +} + +func (db *DB) newLoader(maxPendingWrites int) *loader { + return &loader{ + db: db, + throttle: y.NewThrottle(maxPendingWrites), + } +} + +func (l *loader) set(kv *pb.KV) error { + var userMeta, meta byte + if len(kv.UserMeta) > 0 { + userMeta = kv.UserMeta[0] + } + if len(kv.Meta) > 0 { + meta = kv.Meta[0] + } + + l.entries = append(l.entries, &Entry{ + Key: y.KeyWithTs(kv.Key, kv.Version), + Value: kv.Value, + UserMeta: userMeta, + ExpiresAt: kv.ExpiresAt, + meta: meta, + }) + if len(l.entries) >= 1000 { + return l.send() + } + return nil +} + +func (l *loader) send() error { + if err := l.throttle.Do(); err != nil { + return err + } + l.db.batchSetAsync(l.entries, func(err error) { + l.throttle.Done(err) + }) + + l.entries = make([]*Entry, 0, 1000) + return nil +} + +func (l *loader) finish() error { + if len(l.entries) > 0 { + if err := l.send(); err != nil { + return err + } + } + return l.throttle.Finish() +} + // Load reads a protobuf-encoded list of all entries from a reader and writes // them to the database. This can be used to restore the database from a backup // made by calling DB.Backup(). // // DB.Load() should be called on a database that is not running any other // concurrent transactions while it is running. -func (db *DB) Load(r io.Reader) error { +func (db *DB) Load(r io.Reader, maxPendingWrites int) error { br := bufio.NewReaderSize(r, 16<<10) unmarshalBuf := make([]byte, 1<<10) - var entries []*Entry - var wg sync.WaitGroup - errChan := make(chan error, 1) - - // func to check for pending error before sending off a batch for writing - batchSetAsyncIfNoErr := func(entries []*Entry) error { - select { - case err := <-errChan: - return err - default: - wg.Add(1) - return db.batchSetAsync(entries, func(err error) { - defer wg.Done() - if err != nil { - select { - case errChan <- err: - default: - } - } - }) - } - } + ldr := db.newLoader(maxPendingWrites) for { var sz uint64 err := binary.Read(br, binary.LittleEndian, &sz) @@ -176,51 +207,31 @@ func (db *DB) Load(r io.Reader) error { unmarshalBuf = make([]byte, sz) } - e := &pb.KV{} if _, err = io.ReadFull(br, unmarshalBuf[:sz]); err != nil { return err } - if err = e.Unmarshal(unmarshalBuf[:sz]); err != nil { + + list := &pb.KVList{} + if err := list.Unmarshal(unmarshalBuf[:sz]); err != nil { return err } - var userMeta byte - if len(e.UserMeta) > 0 { - userMeta = e.UserMeta[0] - } - entries = append(entries, &Entry{ - Key: y.KeyWithTs(e.Key, e.Version), - Value: e.Value, - UserMeta: userMeta, - ExpiresAt: e.ExpiresAt, - meta: e.Meta[0], - }) - // Update nextTxnTs, memtable stores this timestamp in badger head - // when flushed. - if e.Version >= db.orc.nextTxnTs { - db.orc.nextTxnTs = e.Version + 1 - } - if len(entries) == 1000 { - if err := batchSetAsyncIfNoErr(entries); err != nil { + for _, kv := range list.Kv { + if err := ldr.set(kv); err != nil { return err } - entries = make([]*Entry, 0, 1000) + + // Update nextTxnTs, memtable stores this + // timestamp in badger head when flushed. + if kv.Version >= db.orc.nextTxnTs { + db.orc.nextTxnTs = kv.Version + 1 + } } } - if len(entries) > 0 { - if err := batchSetAsyncIfNoErr(entries); err != nil { - return err - } - } - wg.Wait() - - select { - case err := <-errChan: + if err := ldr.finish(); err != nil { return err - default: - // Mark all versions done up until nextTxnTs. - db.orc.txnMark.Done(db.orc.nextTxnTs - 1) - return nil } + db.orc.txnMark.Done(db.orc.nextTxnTs - 1) + return nil } diff --git a/vendor/github.com/dgraph-io/badger/backup_test.go b/vendor/github.com/dgraph-io/badger/backup_test.go deleted file mode 100644 index 3e8a1150..00000000 --- a/vendor/github.com/dgraph-io/badger/backup_test.go +++ /dev/null @@ -1,519 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "bytes" - "fmt" - "io/ioutil" - "math/rand" - "os" - "path/filepath" - "reflect" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/require" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb" -) - -func TestBackupRestore1(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - db, err := Open(getTestOptions(dir)) - require.NoError(t, err) - - // Write some stuff - entries := []struct { - key []byte - val []byte - userMeta byte - version uint64 - }{ - {key: []byte("answer1"), val: []byte("42"), version: 1}, - {key: []byte("answer2"), val: []byte("43"), userMeta: 1, version: 2}, - } - - err = db.Update(func(txn *Txn) error { - e := entries[0] - err := txn.SetWithMeta(e.key, e.val, e.userMeta) - if err != nil { - return err - } - return nil - }) - require.NoError(t, err) - - err = db.Update(func(txn *Txn) error { - e := entries[1] - err := txn.SetWithMeta(e.key, e.val, e.userMeta) - if err != nil { - return err - } - return nil - }) - require.NoError(t, err) - - // Use different directory. - dir, err = ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - bak, err := ioutil.TempFile(dir, "badgerbak") - require.NoError(t, err) - ts, err := db.Backup(bak, 0) - t.Logf("New ts: %d\n", ts) - require.NoError(t, err) - require.NoError(t, bak.Close()) - require.NoError(t, db.Close()) - - db, err = Open(getTestOptions(dir)) - require.NoError(t, err) - defer db.Close() - bak, err = os.Open(bak.Name()) - require.NoError(t, err) - defer bak.Close() - - require.NoError(t, db.Load(bak)) - - err = db.View(func(txn *Txn) error { - opts := DefaultIteratorOptions - opts.AllVersions = true - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - val, err := item.ValueCopy(nil) - if err != nil { - return err - } - require.Equal(t, entries[count].key, item.Key()) - require.Equal(t, entries[count].val, val) - require.Equal(t, entries[count].version, item.Version()) - require.Equal(t, entries[count].userMeta, item.UserMeta()) - count++ - } - require.Equal(t, count, 2) - return nil - }) - require.NoError(t, err) -} - -func TestBackupRestore2(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "badger-test") - if err != nil { - t.Fatal(err) - } - defer func() { - os.RemoveAll(tmpdir) - }() - - s1Path := filepath.Join(tmpdir, "test1") - s2Path := filepath.Join(tmpdir, "test2") - s3Path := filepath.Join(tmpdir, "test3") - - opts := DefaultOptions - opts.Dir = s1Path - opts.ValueDir = s1Path - db1, err := Open(opts) - if err != nil { - t.Fatal(err) - } - key1 := []byte("key1") - key2 := []byte("key2") - rawValue := []byte("NotLongValue") - N := byte(251) - err = db1.Update(func(tx *Txn) error { - if err := tx.Set(key1, rawValue); err != nil { - return err - } - return tx.Set(key2, rawValue) - }) - if err != nil { - t.Fatal(err) - } - for i := byte(1); i < N; i++ { - err = db1.Update(func(tx *Txn) error { - if err := tx.Set(append(key1, i), rawValue); err != nil { - return err - } - return tx.Set(append(key2, i), rawValue) - }) - if err != nil { - t.Fatal(err) - } - } - var backup bytes.Buffer - _, err = db1.Backup(&backup, 0) - if err != nil { - t.Fatal(err) - } - fmt.Println("backup1 length:", backup.Len()) - - opts = DefaultOptions - opts.Dir = s2Path - opts.ValueDir = s2Path - db2, err := Open(opts) - if err != nil { - t.Fatal(err) - } - err = db2.Load(&backup) - if err != nil { - t.Fatal(err) - } - - for i := byte(1); i < N; i++ { - err = db2.View(func(tx *Txn) error { - k := append(key1, i) - item, err := tx.Get(k) - if err != nil { - if err == ErrKeyNotFound { - return fmt.Errorf("Key %q has been not found, but was set\n", k) - } - return err - } - v, err := item.ValueCopy(nil) - if err != nil { - return err - } - if !reflect.DeepEqual(v, rawValue) { - return fmt.Errorf("Values not match, got %v, expected %v", v, rawValue) - } - return nil - }) - if err != nil { - t.Fatal(err) - } - } - - for i := byte(1); i < N; i++ { - err = db2.Update(func(tx *Txn) error { - if err := tx.Set(append(key1, i), rawValue); err != nil { - return err - } - return tx.Set(append(key2, i), rawValue) - }) - if err != nil { - t.Fatal(err) - } - } - - backup.Reset() - _, err = db2.Backup(&backup, 0) - if err != nil { - t.Fatal(err) - } - fmt.Println("backup2 length:", backup.Len()) - opts = DefaultOptions - opts.Dir = s3Path - opts.ValueDir = s3Path - db3, err := Open(opts) - if err != nil { - t.Fatal(err) - } - - err = db3.Load(&backup) - if err != nil { - t.Fatal(err) - } - - for i := byte(1); i < N; i++ { - err = db3.View(func(tx *Txn) error { - k := append(key1, i) - item, err := tx.Get(k) - if err != nil { - if err == ErrKeyNotFound { - return fmt.Errorf("Key %q has been not found, but was set\n", k) - } - return err - } - v, err := item.ValueCopy(nil) - if err != nil { - return err - } - if !reflect.DeepEqual(v, rawValue) { - return fmt.Errorf("Values not match, got %v, expected %v", v, rawValue) - } - return nil - }) - if err != nil { - t.Fatal(err) - } - } - -} - -var randSrc = rand.NewSource(time.Now().UnixNano()) - -func createEntries(n int) []*pb.KV { - entries := make([]*pb.KV, n) - for i := 0; i < n; i++ { - entries[i] = &pb.KV{ - Key: []byte(fmt.Sprint("key", i)), - Value: []byte{1}, - UserMeta: []byte{0}, - Meta: []byte{0}, - } - } - return entries -} - -func populateEntries(db *DB, entries []*pb.KV) error { - return db.Update(func(txn *Txn) error { - var err error - for i, e := range entries { - if err = txn.Set(e.Key, e.Value); err != nil { - return err - } - entries[i].Version = 1 - } - return nil - }) -} - -func TestBackup(t *testing.T) { - var bb bytes.Buffer - - tmpdir, err := ioutil.TempDir("", "badger-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - opts := DefaultOptions - opts.Dir = filepath.Join(tmpdir, "backup0") - opts.ValueDir = opts.Dir - db1, err := Open(opts) - if err != nil { - t.Fatal(err) - } - - N := 1000 - entries := createEntries(N) - require.NoError(t, populateEntries(db1, entries)) - - _, err = db1.Backup(&bb, 0) - require.NoError(t, err) - - err = db1.View(func(txn *Txn) error { - opts := DefaultIteratorOptions - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - idx, err := strconv.Atoi(string(item.Key())[3:]) - if err != nil { - return err - } - if idx > N || !bytes.Equal(entries[idx].Key, item.Key()) { - return fmt.Errorf("%s: %s", string(item.Key()), ErrKeyNotFound) - } - count++ - } - if N != count { - return fmt.Errorf("wrong number of items: %d expected, %d actual", N, count) - } - return nil - }) - require.NoError(t, err) -} - -func TestBackupRestore3(t *testing.T) { - var bb bytes.Buffer - - tmpdir, err := ioutil.TempDir("", "badger-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - opts := DefaultOptions - N := 1000 - entries := createEntries(N) - - // backup - { - opts.Dir = filepath.Join(tmpdir, "backup1") - opts.ValueDir = opts.Dir - db1, err := Open(opts) - if err != nil { - t.Fatal(err) - } - - require.NoError(t, populateEntries(db1, entries)) - - _, err = db1.Backup(&bb, 0) - require.NoError(t, err) - require.NoError(t, db1.Close()) - } - require.True(t, len(entries) == N) - require.True(t, bb.Len() > 0) - - // restore - opts.Dir = filepath.Join(tmpdir, "restore1") - opts.ValueDir = opts.Dir - db2, err := Open(opts) - if err != nil { - t.Fatal(err) - } - require.NoError(t, db2.Load(&bb)) - - // verify - err = db2.View(func(txn *Txn) error { - opts := DefaultIteratorOptions - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - idx, err := strconv.Atoi(string(item.Key())[3:]) - if err != nil { - return err - } - if idx > N || !bytes.Equal(entries[idx].Key, item.Key()) { - return fmt.Errorf("%s: %s", string(item.Key()), ErrKeyNotFound) - } - count++ - } - if N != count { - return fmt.Errorf("wrong number of items: %d expected, %d actual", N, count) - } - return nil - }) - require.NoError(t, err) -} - -func TestBackupLoadIncremental(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "badger-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - opts := DefaultOptions - N := 100 - entries := createEntries(N) - updates := make(map[int]byte) - var bb bytes.Buffer - - // backup - { - opts.Dir = filepath.Join(tmpdir, "backup2") - opts.ValueDir = opts.Dir - db1, err := Open(opts) - if err != nil { - t.Fatal(err) - } - - require.NoError(t, populateEntries(db1, entries)) - since, err := db1.Backup(&bb, 0) - require.NoError(t, err) - - ints := rand.New(randSrc).Perm(N) - - // pick 10 items to mark as deleted. - err = db1.Update(func(txn *Txn) error { - for _, i := range ints[:10] { - if err := txn.Delete(entries[i].Key); err != nil { - return err - } - updates[i] = bitDelete - } - return nil - }) - require.NoError(t, err) - since, err = db1.Backup(&bb, since) - require.NoError(t, err) - - // pick 5 items to mark as expired. - err = db1.Update(func(txn *Txn) error { - for _, i := range (ints)[10:15] { - if err := txn.SetWithTTL( - entries[i].Key, entries[i].Value, -time.Hour); err != nil { - return err - } - updates[i] = bitDelete // expired - } - return nil - }) - require.NoError(t, err) - since, err = db1.Backup(&bb, since) - require.NoError(t, err) - - // pick 5 items to mark as discard. - err = db1.Update(func(txn *Txn) error { - for _, i := range ints[15:20] { - if err := txn.SetWithDiscard(entries[i].Key, entries[i].Value, 0); err != nil { - return err - } - updates[i] = bitDiscardEarlierVersions - } - return nil - }) - require.NoError(t, err) - _, err = db1.Backup(&bb, since) - require.NoError(t, err) - require.NoError(t, db1.Close()) - } - require.True(t, len(entries) == N) - require.True(t, bb.Len() > 0) - - // restore - opts.Dir = filepath.Join(tmpdir, "restore2") - opts.ValueDir = opts.Dir - db2, err := Open(opts) - if err != nil { - t.Fatal(err) - } - require.NoError(t, db2.Load(&bb)) - - // verify - actual := make(map[int]byte) - err = db2.View(func(txn *Txn) error { - opts := DefaultIteratorOptions - opts.AllVersions = true - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - idx, err := strconv.Atoi(string(item.Key())[3:]) - if err != nil { - return err - } - if item.IsDeletedOrExpired() { - _, ok := updates[idx] - if !ok { - return fmt.Errorf("%s: not expected to be updated but it is", - string(item.Key())) - } - actual[idx] = item.meta - count++ - continue - } - } - if len(updates) != count { - return fmt.Errorf("mismatched updated items: %d expected, %d actual", - len(updates), count) - } - return nil - }) - require.NoError(t, err, "%v %v", updates, actual) -} diff --git a/vendor/github.com/dgraph-io/badger/badger/.gitignore b/vendor/github.com/dgraph-io/badger/badger/.gitignore deleted file mode 100644 index a8e6bd9e..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/badger diff --git a/vendor/github.com/dgraph-io/badger/badger/cmd/backup.go b/vendor/github.com/dgraph-io/badger/badger/cmd/backup.go deleted file mode 100644 index 1b47bd4f..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/cmd/backup.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "os" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger" - "gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra" -) - -var backupFile string -var truncate bool - -// backupCmd represents the backup command -var backupCmd = &cobra.Command{ - Use: "backup", - Short: "Backup Badger database.", - Long: `Backup Badger database to a file in a version-agnostic manner. - -Iterates over each key-value pair, encodes it along with its metadata and -version in protocol buffers and writes them to a file. This file can later be -used by the restore command to create an identical copy of the -database.`, - RunE: doBackup, -} - -func init() { - RootCmd.AddCommand(backupCmd) - backupCmd.Flags().StringVarP(&backupFile, "backup-file", "f", - "badger.bak", "File to backup to") - backupCmd.Flags().BoolVarP(&truncate, "truncate", "t", - false, "Allow value log truncation if required.") -} - -func doBackup(cmd *cobra.Command, args []string) error { - // Open DB - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.Truncate = truncate - db, err := badger.Open(opts) - if err != nil { - return err - } - defer db.Close() - - // Create File - f, err := os.Create(backupFile) - if err != nil { - return err - } - defer f.Close() - - // Run Backup - _, err = db.Backup(f, 0) - return err -} diff --git a/vendor/github.com/dgraph-io/badger/badger/cmd/bank.go b/vendor/github.com/dgraph-io/badger/badger/cmd/bank.go deleted file mode 100644 index 90039eac..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/cmd/bank.go +++ /dev/null @@ -1,451 +0,0 @@ -/* - * Copyright 2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "bytes" - "errors" - "fmt" - "log" - "math" - "math/rand" - "strconv" - "sync" - "sync/atomic" - "time" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra" -) - -var testCmd = &cobra.Command{ - Use: "bank", - Short: "Run bank test on Badger.", - Long: ` -This command runs bank test on Badger, inspired by Jepsen. It creates many -accounts and moves money among them transactionally. It also reads the sum total -of all the accounts, to ensure that the total never changes. -`, -} - -var bankTest = &cobra.Command{ - Use: "test", - Short: "Execute bank test on Badger.", - RunE: runTest, -} - -var bankDisect = &cobra.Command{ - Use: "disect", - Short: "Disect the bank output.", - Long: ` -Disect the bank output BadgerDB to find the first transaction which causes -failure of the total invariant. -`, - RunE: runDisect, -} - -var numGoroutines, numAccounts, numPrevious int -var duration string -var stopAll int32 -var mmap bool - -const keyPrefix = "account:" - -const initialBal uint64 = 100 - -func init() { - RootCmd.AddCommand(testCmd) - testCmd.AddCommand(bankTest) - testCmd.AddCommand(bankDisect) - - testCmd.Flags().IntVarP( - &numAccounts, "accounts", "a", 10000, "Number of accounts in the bank.") - bankTest.Flags().IntVarP( - &numGoroutines, "conc", "c", 16, "Number of concurrent transactions to run.") - bankTest.Flags().StringVarP(&duration, "duration", "d", "3m", "How long to run the test.") - bankTest.Flags().BoolVarP(&mmap, "mmap", "m", false, "If true, mmap LSM tree. Default is RAM.") - bankDisect.Flags().IntVarP(&numPrevious, "previous", "p", 12, - "Starting from the violation txn, how many previous versions to retrieve.") -} - -func key(account int) []byte { - return []byte(fmt.Sprintf("%s%s", keyPrefix, strconv.Itoa(account))) -} - -func toAccount(key []byte) int { - i, err := strconv.Atoi(string(key[len(keyPrefix):])) - y.Check(err) - return i -} - -func toUint64(val []byte) uint64 { - u, err := strconv.ParseUint(string(val), 10, 64) - y.Check(err) - return uint64(u) -} - -func toSlice(bal uint64) []byte { - return []byte(strconv.FormatUint(bal, 10)) -} - -func getBalance(txn *badger.Txn, account int) (uint64, error) { - item, err := txn.Get(key(account)) - if err != nil { - return 0, err - } - - var bal uint64 - err = item.Value(func(v []byte) error { - bal = toUint64(v) - return nil - }) - return bal, err -} - -func putBalance(txn *badger.Txn, account int, bal uint64) error { - return txn.Set(key(account), toSlice(bal)) -} - -func min(a, b uint64) uint64 { - if a < b { - return a - } - return b -} - -var errAbandoned = errors.New("Transaction abandonded due to insufficient balance") - -func moveMoney(db *badger.DB, from, to int) error { - return db.Update(func(txn *badger.Txn) error { - balf, err := getBalance(txn, from) - if err != nil { - return err - } - balt, err := getBalance(txn, to) - if err != nil { - return err - } - - floor := min(balf, balt) - if floor < 5 { - return errAbandoned - } - // Move the money. - balf -= 5 - balt += 5 - - if err = putBalance(txn, from, balf); err != nil { - return err - } - return putBalance(txn, to, balt) - }) -} - -type account struct { - Id int - Bal uint64 -} - -func diff(a, b []account) string { - var buf bytes.Buffer - y.AssertTruef(len(a) == len(b), "len(a)=%d. len(b)=%d\n", len(a), len(b)) - for i := range a { - ai := a[i] - bi := b[i] - if ai.Id != bi.Id || ai.Bal != bi.Bal { - buf.WriteString(fmt.Sprintf("Index: %d. Account [%+v] -> [%+v]\n", i, ai, bi)) - } - } - return buf.String() -} - -var errFailure = errors.New("Found an balance mismatch. Test failed.") - -// seekTotal retrives the total of all accounts by seeking for each account key. -func seekTotal(txn *badger.Txn) ([]account, error) { - expected := uint64(numAccounts) * uint64(initialBal) - var accounts []account - - var total uint64 - for i := 0; i < numAccounts; i++ { - item, err := txn.Get(key(i)) - if err != nil { - log.Printf("Error for account: %d. err=%v. key=%q\n", i, err, key(i)) - return accounts, err - } - val, err := item.ValueCopy(nil) - if err != nil { - return accounts, err - } - acc := account{ - Id: i, - Bal: toUint64(val), - } - accounts = append(accounts, acc) - total += acc.Bal - } - if total != expected { - log.Printf("Balance did NOT match up. Expected: %d. Received: %d", - expected, total) - atomic.AddInt32(&stopAll, 1) - return accounts, errFailure - } - return accounts, nil -} - -// Range is [lowTs, highTs). -func findFirstInvalidTxn(db *badger.DB, lowTs, highTs uint64) uint64 { - checkAt := func(ts uint64) error { - txn := db.NewTransactionAt(ts, false) - _, err := seekTotal(txn) - txn.Discard() - return err - } - - if highTs-lowTs < 1 { - log.Printf("Checking at lowTs: %d\n", lowTs) - err := checkAt(lowTs) - if err == errFailure { - fmt.Printf("Violation at ts: %d\n", lowTs) - return lowTs - } else if err != nil { - log.Printf("Error at lowTs: %d. Err=%v\n", lowTs, err) - return 0 - } - fmt.Printf("No violation found at ts: %d\n", lowTs) - return 0 - } - - midTs := (lowTs + highTs) / 2 - log.Println() - log.Printf("Checking. low=%d. high=%d. mid=%d\n", lowTs, highTs, midTs) - err := checkAt(midTs) - if err == badger.ErrKeyNotFound || err == nil { - // If no failure, move to higher ts. - return findFirstInvalidTxn(db, midTs+1, highTs) - } - // Found an error. - return findFirstInvalidTxn(db, lowTs, midTs) -} - -func compareTwo(db *badger.DB, before, after uint64) { - fmt.Printf("Comparing @ts=%d with @ts=%d\n", before, after) - txn := db.NewTransactionAt(before, false) - prev, err := seekTotal(txn) - if err == errFailure { - // pass - } else { - y.Check(err) - } - txn.Discard() - - txn = db.NewTransactionAt(after, false) - now, err := seekTotal(txn) - if err == errFailure { - // pass - } else { - y.Check(err) - } - txn.Discard() - - fmt.Println(diff(prev, now)) -} - -func runDisect(cmd *cobra.Command, args []string) error { - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.ReadOnly = true - - // The total did not match up. So, let's disect the DB to find the - // transction which caused the total mismatch. - db, err := badger.OpenManaged(opts) - if err != nil { - return err - } - fmt.Println("opened db") - - var min, max uint64 = math.MaxUint64, 0 - { - txn := db.NewTransactionAt(uint64(math.MaxUint32), false) - iopt := badger.DefaultIteratorOptions - iopt.AllVersions = true - itr := txn.NewIterator(iopt) - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - if min > item.Version() { - min = item.Version() - } - if max < item.Version() { - max = item.Version() - } - } - itr.Close() - txn.Discard() - } - - log.Printf("min=%d. max=%d\n", min, max) - ts := findFirstInvalidTxn(db, min, max) - fmt.Println() - if ts == 0 { - fmt.Println("Nothing found. Exiting.") - return nil - } - - for i := 0; i < numPrevious; i++ { - compareTwo(db, ts-1-uint64(i), ts-uint64(i)) - } - return nil -} - -func runTest(cmd *cobra.Command, args []string) error { - rand.Seed(time.Now().UnixNano()) - - // Open DB - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.MaxTableSize = 4 << 20 // Force more compactions. - opts.NumLevelZeroTables = 2 - opts.NumMemtables = 2 - // Do not GC any versions, because we need them for the disect. - opts.NumVersionsToKeep = int(math.MaxInt32) - opts.ValueThreshold = 1 // Make all values go to value log. - if mmap { - opts.TableLoadingMode = options.MemoryMap - } - log.Printf("Opening DB with options: %+v\n", opts) - - db, err := badger.Open(opts) - if err != nil { - return err - } - defer db.Close() - - wb := db.NewWriteBatch() - for i := 0; i < numAccounts; i++ { - y.Check(wb.Set(key(i), toSlice(initialBal), 0)) - } - log.Println("Waiting for writes to be done...") - y.Check(wb.Flush()) - - log.Println("Bank initialization OK. Commencing test.") - log.Printf("Running with %d accounts, and %d goroutines.\n", numAccounts, numGoroutines) - log.Printf("Using keyPrefix: %s\n", keyPrefix) - - dur, err := time.ParseDuration(duration) - y.Check(err) - - // startTs := time.Now() - endTs := time.Now().Add(dur) - var total, errors, reads uint64 - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - - for range ticker.C { - if atomic.LoadInt32(&stopAll) > 0 { - // Do not proceed. - return - } - // log.Printf("[%6s] Total: %d. Errors: %d Reads: %d.\n", - // time.Since(startTs).Round(time.Second).String(), - // atomic.LoadUint64(&total), - // atomic.LoadUint64(&errors), - // atomic.LoadUint64(&reads)) - if time.Now().After(endTs) { - return - } - } - }() - - // RW goroutines. - for i := 0; i < numGoroutines; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - ticker := time.NewTicker(10 * time.Microsecond) - defer ticker.Stop() - - for range ticker.C { - if atomic.LoadInt32(&stopAll) > 0 { - // Do not proceed. - return - } - if time.Now().After(endTs) { - return - } - from := rand.Intn(numAccounts) - to := rand.Intn(numAccounts) - if from == to { - continue - } - err := moveMoney(db, from, to) - atomic.AddUint64(&total, 1) - if err == nil { - log.Printf("Moved $5. %d -> %d\n", from, to) - } else { - atomic.AddUint64(&errors, 1) - } - } - }() - } - - // RO goroutine. - wg.Add(1) - go func() { - defer wg.Done() - - ticker := time.NewTicker(10 * time.Microsecond) - defer ticker.Stop() - - for range ticker.C { - if atomic.LoadInt32(&stopAll) > 0 { - // Do not proceed. - return - } - if time.Now().After(endTs) { - return - } - - y.Check(db.View(func(txn *badger.Txn) error { - _, err := seekTotal(txn) - if err != nil { - log.Printf("Error while calculating total: %v", err) - } else { - atomic.AddUint64(&reads, 1) - } - return nil - })) - } - }() - wg.Wait() - - if atomic.LoadInt32(&stopAll) == 0 { - log.Println("Test OK") - return nil - } - log.Println("Test FAILED") - return fmt.Errorf("Test FAILED") -} diff --git a/vendor/github.com/dgraph-io/badger/badger/cmd/fill.go b/vendor/github.com/dgraph-io/badger/badger/cmd/fill.go deleted file mode 100644 index 6b103bb2..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/cmd/fill.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "crypto/rand" - "time" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra" -) - -var fillCmd = &cobra.Command{ - Use: "fill", - Short: "Fill Badger with random data.", - Long: ` -This command would fill Badger with random data. Useful for testing and performance analysis. -`, - RunE: fill, -} - -var keySz, valSz int -var numKeys float64 -var force bool - -const mil float64 = 1e6 - -func init() { - RootCmd.AddCommand(fillCmd) - fillCmd.Flags().IntVarP(&keySz, "key-size", "k", 32, "Size of key") - fillCmd.Flags().IntVarP(&valSz, "val-size", "v", 128, "Size of value") - fillCmd.Flags().Float64VarP(&numKeys, "keys-mil", "m", 10.0, - "Number of keys to add in millions") - fillCmd.Flags().BoolVarP(&force, "force-compact", "f", true, "Force compact level 0 on close.") -} - -func fill(cmd *cobra.Command, args []string) error { - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.Truncate = truncate - opts.SyncWrites = false - opts.CompactL0OnClose = force - opts.TableLoadingMode = options.FileIO - opts.ValueLogLoadingMode = options.FileIO - - db, err := badger.Open(opts) - if err != nil { - return err - } - defer func() { - start := time.Now() - err := db.Close() - opts.Infof("DB.Close. Error: %v. Time taken: %s", err, time.Since(start)) - }() - - start := time.Now() - batch := db.NewWriteBatch() - num := int64(numKeys * mil) - for i := int64(1); i <= num; i++ { - k := make([]byte, keySz) - v := make([]byte, valSz) - y.Check2(rand.Read(k)) - y.Check2(rand.Read(v)) - if err := batch.Set(k, v, 0); err != nil { - return err - } - if i%1e5 == 0 { - opts.Infof("Written keys: %d\n", i) - } - } - if err := batch.Flush(); err != nil { - return err - } - opts.Infof("%d keys written. Time taken: %s\n", num, time.Since(start)) - return nil -} diff --git a/vendor/github.com/dgraph-io/badger/badger/cmd/flatten.go b/vendor/github.com/dgraph-io/badger/badger/cmd/flatten.go deleted file mode 100644 index 4b67c982..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/cmd/flatten.go +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger" - "gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra" -) - -var flattenCmd = &cobra.Command{ - Use: "flatten", - Short: "Flatten the LSM tree.", - Long: ` -This command would compact all the LSM tables into one level. -`, - RunE: flatten, -} - -var numWorkers int - -func init() { - RootCmd.AddCommand(flattenCmd) - flattenCmd.Flags().IntVarP(&numWorkers, "num-workers", "w", 1, - "Number of concurrent compactors to run. More compactors would use more"+ - " server resources to potentially achieve faster compactions.") -} - -func flatten(cmd *cobra.Command, args []string) error { - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.Truncate = truncate - opts.NumCompactors = 0 - - db, err := badger.Open(opts) - if err != nil { - return err - } - defer db.Close() - - return db.Flatten(numWorkers) -} diff --git a/vendor/github.com/dgraph-io/badger/badger/cmd/info.go b/vendor/github.com/dgraph-io/badger/badger/cmd/info.go deleted file mode 100644 index ca8c28b5..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/cmd/info.go +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strings" - "time" - - humanize "gx/ipfs/QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy/go-humanize" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra" -) - -var infoCmd = &cobra.Command{ - Use: "info", - Short: "Health info about Badger database.", - Long: ` -This command prints information about the badger key-value store. It reads MANIFEST and prints its -info. It also prints info about missing/extra files, and general information about the value log -files (which are not referenced by the manifest). Use this tool to report any issues about Badger -to the Dgraph team. -`, - Run: func(cmd *cobra.Command, args []string) { - err := printInfo(sstDir, vlogDir) - if err != nil { - fmt.Println("Error:", err.Error()) - os.Exit(1) - } - if !showTables { - return - } - err = tableInfo(sstDir, vlogDir) - if err != nil { - fmt.Println("Error:", err.Error()) - os.Exit(1) - } - }, -} - -var showTables bool - -func init() { - RootCmd.AddCommand(infoCmd) - infoCmd.Flags().BoolVarP(&showTables, "show-tables", "s", false, - "If set to true, show tables as well.") -} - -func hbytes(sz int64) string { - return humanize.Bytes(uint64(sz)) -} - -func dur(src, dst time.Time) string { - return humanize.RelTime(dst, src, "earlier", "later") -} - -func tableInfo(dir, valueDir string) error { - // Open DB - opts := badger.DefaultOptions - opts.TableLoadingMode = options.MemoryMap - opts.Dir = sstDir - opts.ValueDir = vlogDir - opts.ReadOnly = true - - db, err := badger.Open(opts) - if err != nil { - return err - } - defer db.Close() - - tables := db.Tables() - for _, t := range tables { - lk, lv := y.ParseKey(t.Left), y.ParseTs(t.Left) - rk, rv := y.ParseKey(t.Right), y.ParseTs(t.Right) - fmt.Printf("SSTable [L%d, %03d] [%20X, v%-10d -> %20X, v%-10d]\n", - t.Level, t.ID, lk, lv, rk, rv) - } - return nil -} - -func printInfo(dir, valueDir string) error { - if dir == "" { - return fmt.Errorf("--dir not supplied") - } - if valueDir == "" { - valueDir = dir - } - fp, err := os.Open(filepath.Join(dir, badger.ManifestFilename)) - if err != nil { - return err - } - defer func() { - if fp != nil { - fp.Close() - } - }() - manifest, truncOffset, err := badger.ReplayManifestFile(fp) - if err != nil { - return err - } - fp.Close() - fp = nil - - fileinfos, err := ioutil.ReadDir(dir) - if err != nil { - return err - } - fileinfoByName := make(map[string]os.FileInfo) - fileinfoMarked := make(map[string]bool) - for _, info := range fileinfos { - fileinfoByName[info.Name()] = info - fileinfoMarked[info.Name()] = false - } - - fmt.Println() - var baseTime time.Time - // fmt.Print("\n[Manifest]\n") - manifestTruncated := false - manifestInfo, ok := fileinfoByName[badger.ManifestFilename] - if ok { - fileinfoMarked[badger.ManifestFilename] = true - truncatedString := "" - if truncOffset != manifestInfo.Size() { - truncatedString = fmt.Sprintf(" [TRUNCATED to %d]", truncOffset) - manifestTruncated = true - } - - baseTime = manifestInfo.ModTime() - fmt.Printf("[%25s] %-12s %6s MA%s\n", manifestInfo.ModTime().Format(time.RFC3339), - manifestInfo.Name(), hbytes(manifestInfo.Size()), truncatedString) - } else { - fmt.Printf("%s [MISSING]\n", manifestInfo.Name()) - } - - numMissing := 0 - numEmpty := 0 - - levelSizes := make([]int64, len(manifest.Levels)) - for level, lm := range manifest.Levels { - // fmt.Printf("\n[Level %d]\n", level) - // We create a sorted list of table ID's so that output is in consistent order. - tableIDs := make([]uint64, 0, len(lm.Tables)) - for id := range lm.Tables { - tableIDs = append(tableIDs, id) - } - sort.Slice(tableIDs, func(i, j int) bool { - return tableIDs[i] < tableIDs[j] - }) - for _, tableID := range tableIDs { - tableFile := table.IDToFilename(tableID) - tm, ok1 := manifest.Tables[tableID] - file, ok2 := fileinfoByName[tableFile] - if ok1 && ok2 { - fileinfoMarked[tableFile] = true - emptyString := "" - fileSize := file.Size() - if fileSize == 0 { - emptyString = " [EMPTY]" - numEmpty++ - } - levelSizes[level] += fileSize - // (Put level on every line to make easier to process with sed/perl.) - fmt.Printf("[%25s] %-12s %6s L%d %x%s\n", dur(baseTime, file.ModTime()), - tableFile, hbytes(fileSize), level, tm.Checksum, emptyString) - } else { - fmt.Printf("%s [MISSING]\n", tableFile) - numMissing++ - } - } - } - - valueDirFileinfos := fileinfos - if valueDir != dir { - valueDirFileinfos, err = ioutil.ReadDir(valueDir) - if err != nil { - return err - } - } - - // If valueDir is different from dir, holds extra files in the value dir. - valueDirExtras := []os.FileInfo{} - - valueLogSize := int64(0) - // fmt.Print("\n[Value Log]\n") - for _, file := range valueDirFileinfos { - if !strings.HasSuffix(file.Name(), ".vlog") { - if valueDir != dir { - valueDirExtras = append(valueDirExtras, file) - } - continue - } - - fileSize := file.Size() - emptyString := "" - if fileSize == 0 { - emptyString = " [EMPTY]" - numEmpty++ - } - valueLogSize += fileSize - fmt.Printf("[%25s] %-12s %6s VL%s\n", dur(baseTime, file.ModTime()), file.Name(), - hbytes(fileSize), emptyString) - - fileinfoMarked[file.Name()] = true - } - - numExtra := 0 - for _, file := range fileinfos { - if fileinfoMarked[file.Name()] { - continue - } - if numExtra == 0 { - fmt.Print("\n[EXTRA]\n") - } - fmt.Printf("[%s] %-12s %6s\n", file.ModTime().Format(time.RFC3339), - file.Name(), hbytes(file.Size())) - numExtra++ - } - - numValueDirExtra := 0 - for _, file := range valueDirExtras { - if numValueDirExtra == 0 { - fmt.Print("\n[ValueDir EXTRA]\n") - } - fmt.Printf("[%s] %-12s %6s\n", file.ModTime().Format(time.RFC3339), - file.Name(), hbytes(file.Size())) - numValueDirExtra++ - } - - fmt.Print("\n[Summary]\n") - totalIndexSize := int64(0) - for i, sz := range levelSizes { - fmt.Printf("Level %d size: %12s\n", i, hbytes(sz)) - totalIndexSize += sz - } - - fmt.Printf("Total index size: %8s\n", hbytes(totalIndexSize)) - fmt.Printf("Value log size: %10s\n", hbytes(valueLogSize)) - fmt.Println() - totalExtra := numExtra + numValueDirExtra - if totalExtra == 0 && numMissing == 0 && numEmpty == 0 && !manifestTruncated { - fmt.Println("Abnormalities: None.") - } else { - fmt.Println("Abnormalities:") - } - fmt.Printf("%d extra %s.\n", totalExtra, pluralFiles(totalExtra)) - fmt.Printf("%d missing %s.\n", numMissing, pluralFiles(numMissing)) - fmt.Printf("%d empty %s.\n", numEmpty, pluralFiles(numEmpty)) - fmt.Printf("%d truncated %s.\n", boolToNum(manifestTruncated), - pluralManifest(manifestTruncated)) - - return nil -} - -func boolToNum(x bool) int { - if x { - return 1 - } - return 0 -} - -func pluralManifest(manifestTruncated bool) string { - if manifestTruncated { - return "manifest" - } - return "manifests" -} - -func pluralFiles(count int) string { - if count == 1 { - return "file" - } - return "files" -} diff --git a/vendor/github.com/dgraph-io/badger/badger/cmd/restore.go b/vendor/github.com/dgraph-io/badger/badger/cmd/restore.go deleted file mode 100644 index fc54e1ce..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/cmd/restore.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "errors" - "os" - "path" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger" - "gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra" -) - -var restoreFile string - -// restoreCmd represents the restore command -var restoreCmd = &cobra.Command{ - Use: "restore", - Short: "Restore Badger database.", - Long: `Restore Badger database from a file. - -It reads a file generated using the backup command (or by calling the -DB.Backup() API method) and writes each key-value pair found in the file to -the Badger database. - -Restore creates a new database, and currently does not work on an already -existing database.`, - RunE: doRestore, -} - -func init() { - RootCmd.AddCommand(restoreCmd) - restoreCmd.Flags().StringVarP(&restoreFile, "backup-file", "f", - "badger.bak", "File to restore from") -} - -func doRestore(cmd *cobra.Command, args []string) error { - // Check if the DB already exists - manifestFile := path.Join(sstDir, badger.ManifestFilename) - if _, err := os.Stat(manifestFile); err == nil { // No error. File already exists. - return errors.New("Cannot restore to an already existing database") - } else if os.IsNotExist(err) { - // pass - } else { // Return an error if anything other than the error above - return err - } - - // Open DB - opts := badger.DefaultOptions - opts.Dir = sstDir - opts.ValueDir = vlogDir - db, err := badger.Open(opts) - if err != nil { - return err - } - defer db.Close() - - // Open File - f, err := os.Open(restoreFile) - if err != nil { - return err - } - defer f.Close() - - // Run restore - return db.Load(f) -} diff --git a/vendor/github.com/dgraph-io/badger/badger/cmd/root.go b/vendor/github.com/dgraph-io/badger/badger/cmd/root.go deleted file mode 100644 index 1506ce53..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/cmd/root.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "errors" - "fmt" - "os" - "strings" - - "gx/ipfs/QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT/cobra" -) - -var sstDir, vlogDir string - -// RootCmd represents the base command when called without any subcommands -var RootCmd = &cobra.Command{ - Use: "badger", - Short: "Tools to manage Badger database.", - PersistentPreRunE: validateRootCmdArgs, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := RootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func init() { - RootCmd.PersistentFlags().StringVar(&sstDir, "dir", "", - "Directory where the LSM tree files are located. (required)") - - RootCmd.PersistentFlags().StringVar(&vlogDir, "vlog-dir", "", - "Directory where the value log files are located, if different from --dir") -} - -func validateRootCmdArgs(cmd *cobra.Command, args []string) error { - if strings.HasPrefix(cmd.Use, "help ") { // No need to validate if it is help - return nil - } - if sstDir == "" { - return errors.New("--dir not specified") - } - if vlogDir == "" { - vlogDir = sstDir - } - return nil -} diff --git a/vendor/github.com/dgraph-io/badger/badger/main.go b/vendor/github.com/dgraph-io/badger/badger/main.go deleted file mode 100644 index e5864d8f..00000000 --- a/vendor/github.com/dgraph-io/badger/badger/main.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "fmt" - "net/http" - _ "net/http/pprof" - "runtime" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/badger/cmd" -) - -func main() { - go func() { - for i := 8080; i < 9080; i++ { - fmt.Printf("Listening for /debug HTTP requests at port: %d\n", i) - if err := http.ListenAndServe(fmt.Sprintf("localhost:%d", i), nil); err != nil { - fmt.Println("Port busy. Trying another one...") - continue - - } - } - }() - runtime.SetBlockProfileRate(100) - runtime.GOMAXPROCS(128) - cmd.Execute() -} diff --git a/vendor/github.com/dgraph-io/badger/batch.go b/vendor/github.com/dgraph-io/badger/batch.go index 2c26d4b0..bfbc239b 100644 --- a/vendor/github.com/dgraph-io/badger/batch.go +++ b/vendor/github.com/dgraph-io/badger/batch.go @@ -19,15 +19,17 @@ package badger import ( "sync" "time" + + "github.com/dgraph-io/badger/y" ) // WriteBatch holds the necessary info to perform batched writes. type WriteBatch struct { sync.Mutex - txn *Txn - db *DB - wg sync.WaitGroup - err error + txn *Txn + db *DB + throttle *y.Throttle + err error } // NewWriteBatch creates a new WriteBatch. This provides a way to conveniently do a lot of writes, @@ -36,7 +38,18 @@ type WriteBatch struct { // creating and committing transactions. Due to the nature of SSI guaratees provided by Badger, // blind writes can never encounter transaction conflicts (ErrConflict). func (db *DB) NewWriteBatch() *WriteBatch { - return &WriteBatch{db: db, txn: db.newTransaction(true, true)} + return &WriteBatch{ + db: db, + txn: db.newTransaction(true, true), + throttle: y.NewThrottle(16), + } +} + +// SetMaxPendingTxns sets a limit on maximum number of pending transactions while writing batches. +// This function should be called before using WriteBatch. Default value of MaxPendingTxns is +// 16 to minimise memory usage. +func (wb *WriteBatch) SetMaxPendingTxns(max int) { + wb.throttle = y.NewThrottle(max) } // Cancel function must be called if there's a chance that Flush might not get @@ -47,13 +60,15 @@ func (db *DB) NewWriteBatch() *WriteBatch { // // Note that any committed writes would still go through despite calling Cancel. func (wb *WriteBatch) Cancel() { - wb.wg.Wait() + if err := wb.throttle.Finish(); err != nil { + wb.db.opt.Errorf("WatchBatch.Cancel error while finishing: %v", err) + } wb.txn.Discard() } func (wb *WriteBatch) callback(err error) { // sync.WaitGroup is thread-safe, so it doesn't need to be run inside wb.Lock. - defer wb.wg.Done() + defer wb.throttle.Done(err) if err == nil { return } @@ -123,9 +138,9 @@ func (wb *WriteBatch) commit() error { if wb.err != nil { return wb.err } - // Get a new txn before we commit this one. So, the new txn doesn't need - // to wait for this one to commit. - wb.wg.Add(1) + if err := wb.throttle.Do(); err != nil { + return err + } wb.txn.CommitWith(wb.callback) wb.txn = wb.db.newTransaction(true, true) wb.txn.readTs = 0 // We're not reading anything. @@ -140,8 +155,10 @@ func (wb *WriteBatch) Flush() error { wb.txn.Discard() wb.Unlock() - wb.wg.Wait() - // Safe to access error without any synchronization here. + if err := wb.throttle.Finish(); err != nil { + return err + } + return wb.err } diff --git a/vendor/github.com/dgraph-io/badger/batch_test.go b/vendor/github.com/dgraph-io/badger/batch_test.go deleted file mode 100644 index 041fe92a..00000000 --- a/vendor/github.com/dgraph-io/badger/batch_test.go +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestWriteBatch(t *testing.T) { - key := func(i int) []byte { - return []byte(fmt.Sprintf("%10d", i)) - } - val := func(i int) []byte { - return []byte(fmt.Sprintf("%128d", i)) - } - - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - wb := db.NewWriteBatch() - defer wb.Cancel() - - N, M := 50000, 1000 - start := time.Now() - - for i := 0; i < N; i++ { - require.NoError(t, wb.Set(key(i), val(i), 0)) - } - for i := 0; i < M; i++ { - require.NoError(t, wb.Delete(key(i))) - } - require.NoError(t, wb.Flush()) - t.Logf("Time taken for %d writes (w/ test options): %s\n", N+M, time.Since(start)) - - err := db.View(func(txn *Txn) error { - itr := txn.NewIterator(DefaultIteratorOptions) - defer itr.Close() - - i := M - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - require.Equal(t, string(key(i)), string(item.Key())) - valcopy, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, val(i), valcopy) - i++ - } - require.Equal(t, N, i) - return nil - }) - require.NoError(t, err) - }) -} diff --git a/vendor/github.com/dgraph-io/badger/compaction.go b/vendor/github.com/dgraph-io/badger/compaction.go index 7fd1b174..931d5666 100644 --- a/vendor/github.com/dgraph-io/badger/compaction.go +++ b/vendor/github.com/dgraph-io/badger/compaction.go @@ -23,10 +23,10 @@ import ( "math" "sync" - "gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace" + "golang.org/x/net/trace" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" + "github.com/dgraph-io/badger/table" + "github.com/dgraph-io/badger/y" ) type keyRange struct { @@ -65,7 +65,9 @@ func (r keyRange) overlapsWith(dst keyRange) bool { } func getKeyRange(tables []*table.Table) keyRange { - y.AssertTrue(len(tables) > 0) + if len(tables) == 0 { + return keyRange{} + } smallest := tables[0].Smallest() biggest := tables[0].Biggest() for i := 1; i < len(tables); i++ { @@ -129,7 +131,7 @@ func (cs *compactStatus) toLog(tr trace.Trace) { tr.LazyPrintf("Compaction status:") for i, l := range cs.levels { - if len(l.debug()) == 0 { + if l.debug() == "" { continue } tr.LazyPrintf("[%d] %s", i, l.debug()) diff --git a/vendor/github.com/dgraph-io/badger/contrib/cover.sh b/vendor/github.com/dgraph-io/badger/contrib/cover.sh deleted file mode 100644 index 5e2c179a..00000000 --- a/vendor/github.com/dgraph-io/badger/contrib/cover.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -SRC="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." -TMP=$(mktemp /tmp/badger-coverage-XXXXX.txt) - -BUILD=$1 -OUT=$2 - -set -e - -pushd $SRC &> /dev/null - -# create coverage output -echo 'mode: atomic' > $OUT -for PKG in $(go list ./...|grep -v -E 'vendor'); do - go test -covermode=atomic -coverprofile=$TMP $PKG - tail -n +2 $TMP >> $OUT -done - -# Another round of tests after turning off mmap -go test -v -vlog_mmap=false github.com/dgraph-io/badger - -popd &> /dev/null diff --git a/vendor/github.com/dgraph-io/badger/db.go b/vendor/github.com/dgraph-io/badger/db.go index 8c374bdc..24e7b1a4 100644 --- a/vendor/github.com/dgraph-io/badger/db.go +++ b/vendor/github.com/dgraph-io/badger/db.go @@ -18,8 +18,11 @@ package badger import ( "bytes" + "context" "encoding/binary" + "encoding/hex" "expvar" + "io" "math" "os" "path/filepath" @@ -29,20 +32,22 @@ import ( "sync/atomic" "time" - humanize "gx/ipfs/QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy/go-humanize" - "gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/skl" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/skl" + "github.com/dgraph-io/badger/table" + "github.com/dgraph-io/badger/y" + humanize "github.com/dustin/go-humanize" + "github.com/pkg/errors" + "golang.org/x/net/trace" ) var ( - badgerPrefix = []byte("!badger!") // Prefix for internal keys used by badger. - head = []byte("!badger!head") // For storing value offset for replay. - txnKey = []byte("!badger!txn") // For indicating end of entries in txn. - badgerMove = []byte("!badger!move") // For key-value pairs which got moved during GC. + badgerPrefix = []byte("!badger!") // Prefix for internal keys used by badger. + head = []byte("!badger!head") // For storing value offset for replay. + txnKey = []byte("!badger!txn") // For indicating end of entries in txn. + badgerMove = []byte("!badger!move") // For key-value pairs which got moved during GC. + lfDiscardStatsKey = []byte("!badger!discard") // For storing lfDiscardStats ) type closers struct { @@ -51,8 +56,11 @@ type closers struct { memtable *y.Closer writes *y.Closer valueGC *y.Closer + pub *y.Closer } +type callback func(kv *pb.KVList) + // DB provides the various functions required to interact with Badger. // DB is thread-safe. type DB struct { @@ -73,10 +81,18 @@ type DB struct { vhead valuePointer // less than or equal to a pointer to the last vlog value put into mt writeCh chan *request flushChan chan flushTask // For flushing memtables. + closeOnce sync.Once // For closing DB only once. + + // Number of log rotates since the last memtable flush. We will access this field via atomic + // functions. Since we are not going to use any 64bit atomic functions, there is no need for + // 64 bit alignment of this struct(see #311). + logRotates int32 blockWrites int32 orc *oracle + + pub *publisher } const ( @@ -125,9 +141,10 @@ func (db *DB) replayFunction() func(Entry, valuePointer) error { } v := y.ValueStruct{ - Value: nv, - Meta: meta, - UserMeta: e.UserMeta, + Value: nv, + Meta: meta, + UserMeta: e.UserMeta, + ExpiresAt: e.ExpiresAt, } if e.meta&bitFinTxn > 0 { @@ -182,6 +199,8 @@ func Open(opt Options) (db *DB, err error) { if opt.ReadOnly { // Can't truncate if the DB is read only. opt.Truncate = false + // Do not perform compaction in read only mode. + opt.CompactL0OnClose = false } for _, path := range []string{opt.Dir, opt.ValueDir} { @@ -256,6 +275,7 @@ func Open(opt Options) (db *DB, err error) { dirLockGuard: dirLockGuard, valueDirGuard: valueDirLockGuard, orc: newOracle(opt), + pub: newPublisher(), } // Calculate initial size. @@ -300,7 +320,10 @@ func Open(opt Options) (db *DB, err error) { // Let's advance nextTxnTs to one more than whatever we observed via // replaying the logs. db.orc.txnMark.Done(db.orc.nextTxnTs) - db.orc.nextTxnTs++ + // In normal mode, we must update readMark so older versions of keys can be removed during + // compaction when run in offline mode via the flatten tool. + db.orc.readMark.Done(db.orc.nextTxnTs) + db.orc.incrementNextTs() db.writeCh = make(chan *request, kvWriteChCapacity) db.closers.writes = y.NewCloser(1) @@ -309,16 +332,26 @@ func Open(opt Options) (db *DB, err error) { db.closers.valueGC = y.NewCloser(1) go db.vlog.waitOnGC(db.closers.valueGC) + db.closers.pub = y.NewCloser(1) + go db.pub.listenForUpdates(db.closers.pub) + valueDirLockGuard = nil dirLockGuard = nil manifestFile = nil return db, nil } -// Close closes a DB. It's crucial to call it to ensure all the pending updates -// make their way to disk. Calling DB.Close() multiple times is not safe and would -// cause panic. -func (db *DB) Close() (err error) { +// Close closes a DB. It's crucial to call it to ensure all the pending updates make their way to +// disk. Calling DB.Close() multiple times would still only close the DB once. +func (db *DB) Close() error { + var err error + db.closeOnce.Do(func() { + err = db.close() + }) + return err +} + +func (db *DB) close() (err error) { db.elog.Printf("Closing database") atomic.StoreInt32(&db.blockWrites, 1) @@ -328,8 +361,10 @@ func (db *DB) Close() (err error) { // Stop writes next. db.closers.writes.SignalAndWait() + db.closers.pub.SignalAndWait() + // Now close the value log. - if vlogErr := db.vlog.Close(); err == nil { + if vlogErr := db.vlog.Close(); vlogErr != nil { err = errors.Wrap(vlogErr, "DB.Close") } @@ -346,7 +381,7 @@ func (db *DB) Close() (err error) { defer db.Unlock() y.AssertTrue(db.mt != nil) select { - case db.flushChan <- flushTask{db.mt, db.vhead}: + case db.flushChan <- flushTask{mt: db.mt, vptr: db.vhead}: db.imm = append(db.imm, db.mt) // Flusher will attempt to remove this from s.imm. db.mt = nil // Will segfault if we try writing! db.elog.Printf("pushed to flush chan\n") @@ -369,10 +404,15 @@ func (db *DB) Close() (err error) { // Force Compact L0 // We don't need to care about cstatus since no parallel compaction is running. if db.opt.CompactL0OnClose { - if err := db.lc.doCompact(compactionPriority{level: 0, score: 1.73}); err != nil { - db.opt.Warningf("While forcing compaction on level 0: %v", err) - } else { + err := db.lc.doCompact(compactionPriority{level: 0, score: 1.73}) + switch err { + case errFillTables: + // This error only means that there might be enough tables to do a compaction. So, we + // should not report it to the end user to avoid confusing them. + case nil: db.opt.Infof("Force compaction on level 0 done") + default: + db.opt.Warningf("While forcing compaction on level 0: %v", err) } } @@ -416,6 +456,12 @@ const ( lockFile = "LOCK" ) +// Sync syncs database content to disk. This function provides +// more control to user to sync data whenever required. +func (db *DB) Sync() error { + return db.vlog.sync(math.MaxUint32) +} + // When you create or delete a file, you have to ensure the directory entry for the file is synced // in order to guarantee the file is visible (if the system crashes). (See the man page for fsync, // or see https://github.com/coreos/etcd/issues/6368 for an example.) @@ -585,6 +631,8 @@ func (db *DB) writeRequests(reqs []*request) error { return err } + db.elog.Printf("Sending updates to subscribers") + db.pub.sendUpdates(reqs) db.elog.Printf("Writing to memtable") var count int for _, b := range reqs { @@ -593,7 +641,7 @@ func (db *DB) writeRequests(reqs []*request) error { } count += len(b.Entries) var i uint64 - for err := db.ensureRoomForWrite(); err == errNoRoom; err = db.ensureRoomForWrite() { + for err = db.ensureRoomForWrite(); err == errNoRoom; err = db.ensureRoomForWrite() { i++ if i%100 == 0 { db.elog.Printf("Making room for writes") @@ -637,6 +685,8 @@ func (db *DB) sendToWriteCh(entries []*Entry) (*request, error) { req.Entries = entries req.Wg = sync.WaitGroup{} req.Wg.Add(1) + req.IncrRef() // for db write + req.IncrRef() // for publisher updates db.writeCh <- req // Handled in doWrites. y.NumPuts.Add(int64(len(entries))) @@ -741,21 +791,36 @@ func (db *DB) ensureRoomForWrite() error { var err error db.Lock() defer db.Unlock() - if db.mt.MemSize() < db.opt.MaxTableSize { + + // Here we determine if we need to force flush memtable. Given we rotated log file, it would + // make sense to force flush a memtable, so the updated value head would have a chance to be + // pushed to L0. Otherwise, it would not go to L0, until the memtable has been fully filled, + // which can take a lot longer if the write load has fewer keys and larger values. This force + // flush, thus avoids the need to read through a lot of log files on a crash and restart. + // Above approach is quite simple with small drawback. We are calling ensureRoomForWrite before + // inserting every entry in Memtable. We will get latest db.head after all entries for a request + // are inserted in Memtable. If we have done >= db.logRotates rotations, then while inserting + // first entry in Memtable, below condition will be true and we will endup flushing old value of + // db.head. Hence we are limiting no of value log files to be read to db.logRotates only. + forceFlush := atomic.LoadInt32(&db.logRotates) >= db.opt.LogRotatesToFlush + + if !forceFlush && db.mt.MemSize() < db.opt.MaxTableSize { return nil } y.AssertTrue(db.mt != nil) // A nil mt indicates that DB is being closed. select { - case db.flushChan <- flushTask{db.mt, db.vhead}: - db.elog.Printf("Flushing value log to disk if async mode.") + case db.flushChan <- flushTask{mt: db.mt, vptr: db.vhead}: + // After every memtable flush, let's reset the counter. + atomic.StoreInt32(&db.logRotates, 0) + // Ensure value log is synced to disk so this memtable's contents wouldn't be lost. - err = db.vlog.sync() + err = db.vlog.sync(db.vhead.Fid) if err != nil { return err } - db.elog.Printf("Flushing memtable, mt.size=%d size of flushChan: %d\n", + db.opt.Debugf("Flushing memtable, mt.size=%d size of flushChan: %d\n", db.mt.MemSize(), len(db.flushChan)) // We manage to push this task. Let's modify imm. db.imm = append(db.imm, db.mt) @@ -773,12 +838,15 @@ func arenaSize(opt Options) int64 { } // WriteLevel0Table flushes memtable. -func writeLevel0Table(s *skl.Skiplist, f *os.File) error { - iter := s.NewIterator() +func writeLevel0Table(ft flushTask, f io.Writer) error { + iter := ft.mt.NewIterator() defer iter.Close() b := table.NewTableBuilder() defer b.Close() for iter.SeekToFirst(); iter.Valid(); iter.Next() { + if len(ft.dropPrefix) > 0 && bytes.HasPrefix(iter.Key(), ft.dropPrefix) { + continue + } if err := b.Add(iter.Key(), iter.Value()); err != nil { return err } @@ -788,25 +856,34 @@ func writeLevel0Table(s *skl.Skiplist, f *os.File) error { } type flushTask struct { - mt *skl.Skiplist - vptr valuePointer + mt *skl.Skiplist + vptr valuePointer + dropPrefix []byte } // handleFlushTask must be run serially. func (db *DB) handleFlushTask(ft flushTask) error { - if !ft.mt.Empty() { - // Store badger head even if vptr is zero, need it for readTs - db.opt.Infof("Storing value log head: %+v\n", ft.vptr) - db.elog.Printf("Storing offset: %+v\n", ft.vptr) - offset := make([]byte, vptrSize) - ft.vptr.Encode(offset) - - // Pick the max commit ts, so in case of crash, our read ts would be higher than all the - // commits. - headTs := y.KeyWithTs(head, db.orc.nextTs()) - ft.mt.Put(headTs, y.ValueStruct{Value: offset}) + // There can be a scnerio, when empty memtable is flushed. For example, memtable is empty and + // after writing request to value log, rotation count exceeds db.LogRotatesToFlush. + if ft.mt.Empty() { + return nil } + // Store badger head even if vptr is zero, need it for readTs + db.opt.Debugf("Storing value log head: %+v\n", ft.vptr) + db.elog.Printf("Storing offset: %+v\n", ft.vptr) + offset := make([]byte, vptrSize) + ft.vptr.Encode(offset) + + // Pick the max commit ts, so in case of crash, our read ts would be higher than all the + // commits. + headTs := y.KeyWithTs(head, db.orc.nextTs()) + ft.mt.Put(headTs, y.ValueStruct{Value: offset}) + + // Also store lfDiscardStats before flushing memtables + discardStatsKey := y.KeyWithTs(lfDiscardStatsKey, 1) + ft.mt.Put(discardStatsKey, y.ValueStruct{Value: db.vlog.encodedDiscardStats()}) + fileID := db.lc.reserveFileID() fd, err := y.CreateSyncedFile(table.NewFilename(fileID, db.opt.Dir), true) if err != nil { @@ -817,7 +894,7 @@ func (db *DB) handleFlushTask(ft flushTask) error { dirSyncCh := make(chan error) go func() { dirSyncCh <- syncDir(db.opt.Dir) }() - err = writeLevel0Table(ft.mt, fd) + err = writeLevel0Table(ft, fd) dirSyncErr := <-dirSyncCh if err != nil { @@ -837,22 +914,7 @@ func (db *DB) handleFlushTask(ft flushTask) error { // We own a ref on tbl. err = db.lc.addLevel0Table(tbl) // This will incrRef (if we don't error, sure) tbl.DecrRef() // Releases our ref. - if err != nil { - return err - } - - // Update s.imm. Need a lock. - db.Lock() - defer db.Unlock() - // This is a single-threaded operation. ft.mt corresponds to the head of - // db.imm list. Once we flush it, we advance db.imm. The next ft.mt - // which would arrive here would match db.imm[0], because we acquire a - // lock over DB when pushing to flushChan. - // TODO: This logic is dirty AF. Any change and this could easily break. - y.AssertTrue(ft.mt == db.imm[0]) - db.imm = db.imm[1:] - ft.mt.DecrRef() // Return memory. - return nil + return err } // flushMemtable must keep running until we send it an empty flushTask. If there @@ -868,6 +930,18 @@ func (db *DB) flushMemtable(lc *y.Closer) error { for { err := db.handleFlushTask(ft) if err == nil { + // Update s.imm. Need a lock. + db.Lock() + // This is a single-threaded operation. ft.mt corresponds to the head of + // db.imm list. Once we flush it, we advance db.imm. The next ft.mt + // which would arrive here would match db.imm[0], because we acquire a + // lock over DB when pushing to flushChan. + // TODO: This logic is dirty AF. Any change and this could easily break. + y.AssertTrue(ft.mt == db.imm[0]) + db.imm = db.imm[1:] + ft.mt.DecrRef() // Return memory. + db.Unlock() + break } // Encountered error. Retry indefinitely. @@ -994,7 +1068,7 @@ func (db *DB) RunValueLogGC(discardRatio float64) error { // Size returns the size of lsm and value log files in bytes. It can be used to decide how often to // call RunValueLogGC. -func (db *DB) Size() (lsm int64, vlog int64) { +func (db *DB) Size() (lsm, vlog int64) { if y.LSMSize.Get(db.opt.Dir) == nil { lsm, vlog = 0, 0 return @@ -1104,16 +1178,18 @@ func (db *DB) GetSequence(key []byte, bandwidth uint64) (*Sequence, error) { return seq, err } -// Tables gets the TableInfo objects from the level controller. -func (db *DB) Tables() []TableInfo { - return db.lc.getTableInfo() +// Tables gets the TableInfo objects from the level controller. If withKeysCount +// is true, TableInfo objects also contain counts of keys for the tables. +func (db *DB) Tables(withKeysCount bool) []TableInfo { + return db.lc.getTableInfo(withKeysCount) } // KeySplits can be used to get rough key ranges to divide up iteration over // the DB. func (db *DB) KeySplits(prefix []byte) []string { var splits []string - for _, ti := range db.Tables() { + // We just want table ranges here and not keys count. + for _, ti := range db.Tables(false) { // We don't use ti.Left, because that has a tendency to store !badger // keys. if bytes.HasPrefix(ti.Right, prefix) { @@ -1230,6 +1306,32 @@ func (db *DB) Flatten(workers int) error { } } +func (db *DB) prepareToDrop() func() { + if db.opt.ReadOnly { + panic("Attempting to drop data in read-only mode.") + } + // Stop accepting new writes. + atomic.StoreInt32(&db.blockWrites, 1) + + // Make all pending writes finish. The following will also close writeCh. + db.closers.writes.SignalAndWait() + db.opt.Infof("Writes flushed. Stopping compactions now...") + + // Stop all compactions. + db.stopCompactions() + return func() { + db.opt.Infof("Resuming writes") + db.startCompactions() + + db.writeCh = make(chan *request, kvWriteChCapacity) + db.closers.writes = y.NewCloser(1) + go db.doWrites(db.closers.writes) + + // Resume writes. + atomic.StoreInt32(&db.blockWrites, 0) + } +} + // DropAll would drop all the data stored in Badger. It does this in the following way. // - Stop accepting new writes. // - Pause memtable flushes and compactions. @@ -1242,31 +1344,20 @@ func (db *DB) Flatten(workers int) error { // any reads while DropAll is going on, otherwise they may result in panics. Ideally, both reads and // writes are paused before running DropAll, and resumed after it is finished. func (db *DB) DropAll() error { - if db.opt.ReadOnly { - panic("Attempting to drop data in read-only mode.") + f, err := db.dropAll() + if err != nil { + return err } + if f == nil { + panic("both error and returned function cannot be nil in DropAll") + } + f() + return nil +} + +func (db *DB) dropAll() (func(), error) { db.opt.Infof("DropAll called. Blocking writes...") - // Stop accepting new writes. - atomic.StoreInt32(&db.blockWrites, 1) - - // Make all pending writes finish. The following will also close writeCh. - db.closers.writes.SignalAndWait() - db.opt.Infof("Writes flushed. Stopping compactions now...") - - // Stop all compactions. - db.stopCompactions() - defer func() { - db.opt.Infof("Resuming writes") - db.startCompactions() - - db.writeCh = make(chan *request, kvWriteChCapacity) - db.closers.writes = y.NewCloser(1) - go db.doWrites(db.closers.writes) - - // Resume writes. - atomic.StoreInt32(&db.blockWrites, 0) - }() - db.opt.Infof("Compactions stopped. Dropping all SSTables...") + f := db.prepareToDrop() // Block all foreign interactions with memory tables. db.Lock() @@ -1274,23 +1365,115 @@ func (db *DB) DropAll() error { // Remove inmemory tables. Calling DecrRef for safety. Not sure if they're absolutely needed. db.mt.DecrRef() - db.mt = skl.NewSkiplist(arenaSize(db.opt)) // Set it up for future writes. for _, mt := range db.imm { mt.DecrRef() } db.imm = db.imm[:0] + db.mt = skl.NewSkiplist(arenaSize(db.opt)) // Set it up for future writes. - num, err := db.lc.deleteLSMTree() + num, err := db.lc.dropTree() if err != nil { - return err + return nil, err } db.opt.Infof("Deleted %d SSTables. Now deleting value logs...\n", num) num, err = db.vlog.dropAll() if err != nil { - return err + return nil, err } db.vhead = valuePointer{} // Zero it out. + db.lc.nextFileID = 1 db.opt.Infof("Deleted %d value log files. DropAll done.\n", num) + return f, nil +} + +// DropPrefix would drop all the keys with the provided prefix. It does this in the following way: +// - Stop accepting new writes. +// - Stop memtable flushes and compactions. +// - Flush out all memtables, skipping over keys with the given prefix, Kp. +// - Write out the value log header to memtables when flushing, so we don't accidentally bring Kp +// back after a restart. +// - Compact L0->L1, skipping over Kp. +// - Compact rest of the levels, Li->Li, picking tables which have Kp. +// - Resume memtable flushes, compactions and writes. +func (db *DB) DropPrefix(prefix []byte) error { + db.opt.Infof("DropPrefix called on %s. Blocking writes...", hex.Dump(prefix)) + f := db.prepareToDrop() + defer f() + + // Block all foreign interactions with memory tables. + db.Lock() + defer db.Unlock() + + db.imm = append(db.imm, db.mt) + for _, memtable := range db.imm { + if memtable.Empty() { + memtable.DecrRef() + continue + } + task := flushTask{ + mt: memtable, + // Ensure that the head of value log gets persisted to disk. + vptr: db.vhead, + dropPrefix: prefix, + } + db.opt.Debugf("Flushing memtable") + if err := db.handleFlushTask(task); err != nil { + db.opt.Errorf("While trying to flush memtable: %v", err) + return err + } + memtable.DecrRef() + } + db.imm = db.imm[:0] + db.mt = skl.NewSkiplist(arenaSize(db.opt)) + + // Drop prefixes from the levels. + if err := db.lc.dropPrefix(prefix); err != nil { + return err + } + db.opt.Infof("DropPrefix done") return nil } + +// Subscribe can be used watch key changes for the given key prefix. +func (db *DB) Subscribe(ctx context.Context, cb callback, prefix []byte, prefixes ...[]byte) error { + if cb == nil { + return ErrNilCallback + } + prefixes = append(prefixes, prefix) + c := y.NewCloser(1) + recvCh, id := db.pub.newSubscriber(c, prefixes...) + slurp := func(batch *pb.KVList) { + defer func() { + if len(batch.GetKv()) > 0 { + cb(batch) + } + }() + for { + select { + case kvs := <-recvCh: + batch.Kv = append(batch.Kv, kvs.Kv...) + default: + return + } + } + } + for { + select { + case <-c.HasBeenClosed(): + slurp(new(pb.KVList)) + // Drain if any pending updates. + c.Done() + // No need to delete here. Closer will be called only while + // closing DB. Subscriber will be deleted by cleanSubscribers. + return nil + case <-ctx.Done(): + c.Done() + db.pub.deleteSubscriber(id) + // Delete the subscriber to avoid further updates. + return ctx.Err() + case batch := <-recvCh: + slurp(batch) + } + } +} diff --git a/vendor/github.com/dgraph-io/badger/db2_test.go b/vendor/github.com/dgraph-io/badger/db2_test.go deleted file mode 100644 index 03a7b265..00000000 --- a/vendor/github.com/dgraph-io/badger/db2_test.go +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright 2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "flag" - "fmt" - "io/ioutil" - "log" - "math/rand" - "os" - "path" - "regexp" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestTruncateVlogWithClose(t *testing.T) { - key := func(i int) []byte { - return []byte(fmt.Sprintf("%d%10d", i, i)) - } - data := func(l int) []byte { - m := make([]byte, l) - _, err := rand.Read(m) - require.NoError(t, err) - return m - } - - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opt := getTestOptions(dir) - opt.SyncWrites = true - opt.Truncate = true - opt.ValueThreshold = 1 // Force all reads from value log. - - db, err := Open(opt) - require.NoError(t, err) - - err = db.Update(func(txn *Txn) error { - return txn.Set(key(0), data(4055)) - }) - require.NoError(t, err) - - // Close the DB. - require.NoError(t, db.Close()) - require.NoError(t, os.Truncate(path.Join(dir, "000000.vlog"), 4096)) - - // Reopen and write some new data. - db, err = Open(opt) - require.NoError(t, err) - for i := 0; i < 32; i++ { - err := db.Update(func(txn *Txn) error { - return txn.Set(key(i), data(10)) - }) - require.NoError(t, err) - } - // Read it back to ensure that we can read it now. - for i := 0; i < 32; i++ { - err := db.View(func(txn *Txn) error { - item, err := txn.Get(key(i)) - require.NoError(t, err) - val := getItemValue(t, item) - require.Equal(t, 10, len(val)) - return nil - }) - require.NoError(t, err) - } - require.NoError(t, db.Close()) - - // Reopen and read the data again. - db, err = Open(opt) - require.NoError(t, err) - for i := 0; i < 32; i++ { - err := db.View(func(txn *Txn) error { - item, err := txn.Get(key(i)) - require.NoError(t, err) - val := getItemValue(t, item) - require.Equal(t, 10, len(val)) - return nil - }) - require.NoError(t, err) - } - require.NoError(t, db.Close()) -} - -var manual = flag.Bool("manual", false, "Set when manually running some tests.") - -// The following 3 TruncateVlogNoClose tests should be run one after another. -// None of these close the DB, simulating a crash. They should be run with a -// script, which truncates the value log to 4096, lining up with the end of the -// first entry in the txn. At <4096, it would cause the entry to be truncated -// immediately, at >4096, same thing. -func TestTruncateVlogNoClose(t *testing.T) { - if !*manual { - t.Skip("Skipping test meant to be run manually.") - return - } - fmt.Println("running") - dir := "p" - opts := getTestOptions(dir) - opts.SyncWrites = true - opts.Truncate = true - - kv, err := Open(opts) - require.NoError(t, err) - key := func(i int) string { - return fmt.Sprintf("%d%10d", i, i) - } - data := fmt.Sprintf("%4055d", 1) - err = kv.Update(func(txn *Txn) error { - return txn.Set([]byte(key(0)), []byte(data)) - }) - require.NoError(t, err) -} -func TestTruncateVlogNoClose2(t *testing.T) { - if !*manual { - t.Skip("Skipping test meant to be run manually.") - return - } - dir := "p" - opts := getTestOptions(dir) - opts.SyncWrites = true - opts.Truncate = true - - kv, err := Open(opts) - require.NoError(t, err) - key := func(i int) string { - return fmt.Sprintf("%d%10d", i, i) - } - data := fmt.Sprintf("%10d", 1) - for i := 32; i < 64; i++ { - err := kv.Update(func(txn *Txn) error { - return txn.Set([]byte(key(i)), []byte(data)) - }) - require.NoError(t, err) - } - for i := 32; i < 64; i++ { - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get([]byte(key(i))) - require.NoError(t, err) - val := getItemValue(t, item) - require.NotNil(t, val) - require.True(t, len(val) > 0) - return nil - })) - } -} -func TestTruncateVlogNoClose3(t *testing.T) { - if !*manual { - t.Skip("Skipping test meant to be run manually.") - return - } - fmt.Print("Running") - dir := "p" - opts := getTestOptions(dir) - opts.SyncWrites = true - opts.Truncate = true - - kv, err := Open(opts) - require.NoError(t, err) - key := func(i int) string { - return fmt.Sprintf("%d%10d", i, i) - } - for i := 32; i < 64; i++ { - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get([]byte(key(i))) - require.NoError(t, err) - val := getItemValue(t, item) - require.NotNil(t, val) - require.True(t, len(val) > 0) - return nil - })) - } -} - -func TestBigKeyValuePairs(t *testing.T) { - // This test takes too much memory. So, run separately. - if !*manual { - t.Skip("Skipping test meant to be run manually.") - return - } - opts := DefaultOptions - opts.MaxTableSize = 1 << 20 - opts.ValueLogMaxEntries = 64 - runBadgerTest(t, &opts, func(t *testing.T, db *DB) { - bigK := make([]byte, 65001) - bigV := make([]byte, db.opt.ValueLogFileSize+1) - small := make([]byte, 65000) - - txn := db.NewTransaction(true) - require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, small)) - require.Regexp(t, regexp.MustCompile("Value.*exceeded"), txn.Set(small, bigV)) - - require.NoError(t, txn.Set(small, small)) - require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, bigV)) - - require.NoError(t, db.View(func(txn *Txn) error { - _, err := txn.Get(small) - require.Equal(t, ErrKeyNotFound, err) - return nil - })) - - // Now run a longer test, which involves value log GC. - data := fmt.Sprintf("%100d", 1) - key := func(i int) string { - return fmt.Sprintf("%65000d", i) - } - - saveByKey := func(key string, value []byte) error { - return db.Update(func(txn *Txn) error { - return txn.Set([]byte(key), value) - }) - } - - getByKey := func(key string) error { - return db.View(func(txn *Txn) error { - item, err := txn.Get([]byte(key)) - if err != nil { - return err - } - return item.Value(func(val []byte) error { - if len(val) == 0 { - log.Fatalf("key not found %q", len(key)) - } - return nil - }) - }) - } - - for i := 0; i < 32; i++ { - if i < 30 { - require.NoError(t, saveByKey(key(i), []byte(data))) - } else { - require.NoError(t, saveByKey(key(i), []byte(fmt.Sprintf("%100d", i)))) - } - } - - for j := 0; j < 5; j++ { - for i := 0; i < 32; i++ { - if i < 30 { - require.NoError(t, saveByKey(key(i), []byte(data))) - } else { - require.NoError(t, saveByKey(key(i), []byte(fmt.Sprintf("%100d", i)))) - } - } - } - - for i := 0; i < 32; i++ { - require.NoError(t, getByKey(key(i))) - } - - var loops int - var err error - for err == nil { - err = db.RunValueLogGC(0.5) - require.NotRegexp(t, regexp.MustCompile("truncate"), err) - loops++ - } - t.Logf("Ran value log GC %d times. Last error: %v\n", loops, err) - }) -} - -// The following test checks for issue #585. -func TestPushValueLogLimit(t *testing.T) { - // This test takes too much memory. So, run separately. - if !*manual { - t.Skip("Skipping test meant to be run manually.") - return - } - opt := DefaultOptions - opt.ValueLogMaxEntries = 64 - opt.ValueLogFileSize = 2 << 30 - runBadgerTest(t, &opt, func(t *testing.T, db *DB) { - data := []byte(fmt.Sprintf("%30d", 1)) - key := func(i int) string { - return fmt.Sprintf("%100d", i) - } - - for i := 0; i < 32; i++ { - if i == 4 { - v := make([]byte, 2<<30) - err := db.Update(func(txn *Txn) error { - return txn.Set([]byte(key(i)), v) - }) - require.NoError(t, err) - } else { - err := db.Update(func(txn *Txn) error { - return txn.Set([]byte(key(i)), data) - }) - require.NoError(t, err) - } - } - - for i := 0; i < 32; i++ { - err := db.View(func(txn *Txn) error { - item, err := txn.Get([]byte(key(i))) - require.NoError(t, err, "Getting key: %s", key(i)) - err = item.Value(func(v []byte) error { - _ = v - return nil - }) - require.NoError(t, err, "Getting value: %s", key(i)) - return nil - }) - require.NoError(t, err) - } - }) -} diff --git a/vendor/github.com/dgraph-io/badger/db_test.go b/vendor/github.com/dgraph-io/badger/db_test.go deleted file mode 100644 index 4d7c05bf..00000000 --- a/vendor/github.com/dgraph-io/badger/db_test.go +++ /dev/null @@ -1,1708 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "bytes" - "encoding/binary" - "flag" - "fmt" - "io/ioutil" - "log" - "math" - "math/rand" - "net/http" - "os" - "path/filepath" - "runtime" - "sort" - "sync" - "testing" - "time" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - - "github.com/stretchr/testify/require" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -var mmap = flag.Bool("vlog_mmap", true, "Specify if value log must be memory-mapped") - -func getTestOptions(dir string) Options { - opt := DefaultOptions - opt.MaxTableSize = 1 << 15 // Force more compaction. - opt.LevelOneSize = 4 << 15 // Force more compaction. - opt.Dir = dir - opt.ValueDir = dir - opt.SyncWrites = false - if !*mmap { - opt.ValueLogLoadingMode = options.FileIO - } - return opt -} - -func getItemValue(t *testing.T, item *Item) (val []byte) { - t.Helper() - var v []byte - size := item.ValueSize() - err := item.Value(func(val []byte) error { - if val == nil { - v = nil - } else { - v = append([]byte{}, val...) - } - return nil - }) - if err != nil { - t.Error(err) - } - if int64(len(v)) != size { - t.Errorf("incorrect size: expected %d, got %d", len(v), size) - } - if v == nil { - return nil - } - another, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, v, another) - return v -} - -func txnSet(t *testing.T, kv *DB, key []byte, val []byte, meta byte) { - txn := kv.NewTransaction(true) - require.NoError(t, txn.SetWithMeta(key, val, meta)) - require.NoError(t, txn.Commit()) -} - -func txnDelete(t *testing.T, kv *DB, key []byte) { - txn := kv.NewTransaction(true) - require.NoError(t, txn.Delete(key)) - require.NoError(t, txn.Commit()) -} - -// Opens a badger db and runs a a test on it. -func runBadgerTest(t *testing.T, opts *Options, test func(t *testing.T, db *DB)) { - dir, err := ioutil.TempDir(".", "badger-test") - require.NoError(t, err) - defer os.RemoveAll(dir) - if opts == nil { - opts = new(Options) - *opts = getTestOptions(dir) - } else { - opts.Dir = dir - opts.ValueDir = dir - } - db, err := Open(*opts) - require.NoError(t, err) - defer db.Close() - test(t, db) -} - -func TestWrite(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - for i := 0; i < 100; i++ { - txnSet(t, db, []byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("val%d", i)), 0x00) - } - }) -} - -func TestUpdateAndView(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - err := db.Update(func(txn *Txn) error { - for i := 0; i < 10; i++ { - err := txn.Set([]byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("val%d", i))) - if err != nil { - return err - } - } - return nil - }) - require.NoError(t, err) - - err = db.View(func(txn *Txn) error { - for i := 0; i < 10; i++ { - item, err := txn.Get([]byte(fmt.Sprintf("key%d", i))) - if err != nil { - return err - } - - expected := []byte(fmt.Sprintf("val%d", i)) - if err := item.Value(func(val []byte) error { - require.Equal(t, expected, val, - "Invalid value for key %q. expected: %q, actual: %q", - item.Key(), expected, val) - return nil - }); err != nil { - return err - } - } - return nil - }) - require.NoError(t, err) - }) -} - -func TestConcurrentWrite(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // Not a benchmark. Just a simple test for concurrent writes. - n := 20 - m := 500 - var wg sync.WaitGroup - for i := 0; i < n; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - for j := 0; j < m; j++ { - txnSet(t, db, []byte(fmt.Sprintf("k%05d_%08d", i, j)), - []byte(fmt.Sprintf("v%05d_%08d", i, j)), byte(j%127)) - } - }(i) - } - wg.Wait() - - t.Log("Starting iteration") - - opt := IteratorOptions{} - opt.Reverse = false - opt.PrefetchSize = 10 - opt.PrefetchValues = true - - txn := db.NewTransaction(true) - it := txn.NewIterator(opt) - defer it.Close() - var i, j int - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - k := item.Key() - if k == nil { - break // end of iteration. - } - - require.EqualValues(t, fmt.Sprintf("k%05d_%08d", i, j), string(k)) - v := getItemValue(t, item) - require.EqualValues(t, fmt.Sprintf("v%05d_%08d", i, j), string(v)) - require.Equal(t, item.UserMeta(), byte(j%127)) - j++ - if j == m { - i++ - j = 0 - } - } - require.EqualValues(t, n, i) - require.EqualValues(t, 0, j) - }) -} - -func TestGet(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - txnSet(t, db, []byte("key1"), []byte("val1"), 0x08) - - txn := db.NewTransaction(false) - item, err := txn.Get([]byte("key1")) - require.NoError(t, err) - require.EqualValues(t, "val1", getItemValue(t, item)) - require.Equal(t, byte(0x08), item.UserMeta()) - txn.Discard() - - txnSet(t, db, []byte("key1"), []byte("val2"), 0x09) - - txn = db.NewTransaction(false) - item, err = txn.Get([]byte("key1")) - require.NoError(t, err) - require.EqualValues(t, "val2", getItemValue(t, item)) - require.Equal(t, byte(0x09), item.UserMeta()) - txn.Discard() - - txnDelete(t, db, []byte("key1")) - - txn = db.NewTransaction(false) - _, err = txn.Get([]byte("key1")) - require.Equal(t, ErrKeyNotFound, err) - txn.Discard() - - txnSet(t, db, []byte("key1"), []byte("val3"), 0x01) - - txn = db.NewTransaction(false) - item, err = txn.Get([]byte("key1")) - require.NoError(t, err) - require.EqualValues(t, "val3", getItemValue(t, item)) - require.Equal(t, byte(0x01), item.UserMeta()) - - longVal := make([]byte, 1000) - txnSet(t, db, []byte("key1"), longVal, 0x00) - - txn = db.NewTransaction(false) - item, err = txn.Get([]byte("key1")) - require.NoError(t, err) - require.EqualValues(t, longVal, getItemValue(t, item)) - txn.Discard() - }) -} - -func TestGetAfterDelete(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // populate with one entry - key := []byte("key") - txnSet(t, db, key, []byte("val1"), 0x00) - require.NoError(t, db.Update(func(txn *Txn) error { - err := txn.Delete(key) - require.NoError(t, err) - - _, err = txn.Get(key) - require.Equal(t, ErrKeyNotFound, err) - return nil - })) - }) -} - -func TestTxnTooBig(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - data := func(i int) []byte { - return []byte(fmt.Sprintf("%b", i)) - } - // n := 500000 - n := 1000 - txn := db.NewTransaction(true) - for i := 0; i < n; { - if err := txn.Set(data(i), data(i)); err != nil { - require.NoError(t, txn.Commit()) - txn = db.NewTransaction(true) - } else { - i++ - } - } - require.NoError(t, txn.Commit()) - - txn = db.NewTransaction(true) - for i := 0; i < n; { - if err := txn.Delete(data(i)); err != nil { - require.NoError(t, txn.Commit()) - txn = db.NewTransaction(true) - } else { - i++ - } - } - require.NoError(t, txn.Commit()) - }) -} - -func TestForceCompactL0(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opts := getTestOptions(dir) - opts.ValueLogFileSize = 15 << 20 - opts.managedTxns = true - db, err := Open(opts) - require.NoError(t, err) - - data := func(i int) []byte { - return []byte(fmt.Sprintf("%b", i)) - } - n := 80 - m := 45 // Increasing would cause ErrTxnTooBig - sz := 32 << 10 - v := make([]byte, sz) - for i := 0; i < n; i += 2 { - version := uint64(i) - txn := db.NewTransactionAt(version, true) - for j := 0; j < m; j++ { - require.NoError(t, txn.Set(data(j), v)) - } - require.NoError(t, txn.CommitAt(version+1, nil)) - } - db.Close() - - opts.managedTxns = true - db, err = Open(opts) - require.NoError(t, err) - require.Equal(t, len(db.lc.levels[0].tables), 0) - require.NoError(t, db.Close()) -} - -// Put a lot of data to move some data to disk. -// WARNING: This test might take a while but it should pass! -func TestGetMore(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - - data := func(i int) []byte { - return []byte(fmt.Sprintf("%b", i)) - } - // n := 500000 - n := 10000 - m := 45 // Increasing would cause ErrTxnTooBig - for i := 0; i < n; i += m { - txn := db.NewTransaction(true) - for j := i; j < i+m && j < n; j++ { - require.NoError(t, txn.Set(data(j), data(j))) - } - require.NoError(t, txn.Commit()) - } - require.NoError(t, db.validate()) - - for i := 0; i < n; i++ { - txn := db.NewTransaction(false) - item, err := txn.Get(data(i)) - if err != nil { - t.Error(err) - } - require.EqualValues(t, string(data(i)), string(getItemValue(t, item))) - txn.Discard() - } - - // Overwrite - for i := 0; i < n; i += m { - txn := db.NewTransaction(true) - for j := i; j < i+m && j < n; j++ { - require.NoError(t, txn.Set(data(j), - // Use a long value that will certainly exceed value threshold. - []byte(fmt.Sprintf("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%9d", j)))) - } - require.NoError(t, txn.Commit()) - } - require.NoError(t, db.validate()) - - for i := 0; i < n; i++ { - expectedValue := fmt.Sprintf("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%9d", i) - k := data(i) - txn := db.NewTransaction(false) - item, err := txn.Get(k) - if err != nil { - t.Error(err) - } - got := string(getItemValue(t, item)) - if expectedValue != got { - - vs, err := db.get(y.KeyWithTs(k, math.MaxUint64)) - require.NoError(t, err) - fmt.Printf("wanted=%q Item: %s\n", k, item) - fmt.Printf("on re-run, got version: %+v\n", vs) - - txn := db.NewTransaction(false) - itr := txn.NewIterator(DefaultIteratorOptions) - for itr.Seek(k); itr.Valid(); itr.Next() { - item := itr.Item() - fmt.Printf("item=%s\n", item) - if !bytes.Equal(item.Key(), k) { - break - } - } - itr.Close() - txn.Discard() - } - require.EqualValues(t, expectedValue, string(getItemValue(t, item)), "wanted=%q Item: %s\n", k, item) - txn.Discard() - } - - // "Delete" key. - for i := 0; i < n; i += m { - if (i % 10000) == 0 { - fmt.Printf("Deleting i=%d\n", i) - } - txn := db.NewTransaction(true) - for j := i; j < i+m && j < n; j++ { - require.NoError(t, txn.Delete(data(j))) - } - require.NoError(t, txn.Commit()) - } - db.validate() - for i := 0; i < n; i++ { - if (i % 10000) == 0 { - // Display some progress. Right now, it's not very fast with no caching. - fmt.Printf("Testing i=%d\n", i) - } - k := data(i) - txn := db.NewTransaction(false) - _, err := txn.Get([]byte(k)) - require.Equal(t, ErrKeyNotFound, err, "should not have found k: %q", k) - txn.Discard() - } - }) -} - -// Put a lot of data to move some data to disk. -// WARNING: This test might take a while but it should pass! -func TestExistsMore(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // n := 500000 - n := 10000 - m := 45 - for i := 0; i < n; i += m { - if (i % 1000) == 0 { - t.Logf("Putting i=%d\n", i) - } - txn := db.NewTransaction(true) - for j := i; j < i+m && j < n; j++ { - require.NoError(t, txn.Set([]byte(fmt.Sprintf("%09d", j)), - []byte(fmt.Sprintf("%09d", j)))) - } - require.NoError(t, txn.Commit()) - } - db.validate() - - for i := 0; i < n; i++ { - if (i % 1000) == 0 { - fmt.Printf("Testing i=%d\n", i) - } - k := fmt.Sprintf("%09d", i) - require.NoError(t, db.View(func(txn *Txn) error { - _, err := txn.Get([]byte(k)) - require.NoError(t, err) - return nil - })) - } - require.NoError(t, db.View(func(txn *Txn) error { - _, err := txn.Get([]byte("non-exists")) - require.Error(t, err) - return nil - })) - - // "Delete" key. - for i := 0; i < n; i += m { - if (i % 1000) == 0 { - fmt.Printf("Deleting i=%d\n", i) - } - txn := db.NewTransaction(true) - for j := i; j < i+m && j < n; j++ { - require.NoError(t, txn.Delete([]byte(fmt.Sprintf("%09d", j)))) - } - require.NoError(t, txn.Commit()) - } - db.validate() - for i := 0; i < n; i++ { - if (i % 10000) == 0 { - // Display some progress. Right now, it's not very fast with no caching. - fmt.Printf("Testing i=%d\n", i) - } - k := fmt.Sprintf("%09d", i) - - require.NoError(t, db.View(func(txn *Txn) error { - _, err := txn.Get([]byte(k)) - require.Error(t, err) - return nil - })) - } - fmt.Println("Done and closing") - }) -} - -func TestIterate2Basic(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - - bkey := func(i int) []byte { - return []byte(fmt.Sprintf("%09d", i)) - } - bval := func(i int) []byte { - return []byte(fmt.Sprintf("%025d", i)) - } - - // n := 500000 - n := 10000 - for i := 0; i < n; i++ { - if (i % 1000) == 0 { - t.Logf("Put i=%d\n", i) - } - txnSet(t, db, bkey(i), bval(i), byte(i%127)) - } - - opt := IteratorOptions{} - opt.PrefetchValues = true - opt.PrefetchSize = 10 - - txn := db.NewTransaction(false) - it := txn.NewIterator(opt) - { - var count int - rewind := true - t.Log("Starting first basic iteration") - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - key := item.Key() - if rewind && count == 5000 { - // Rewind would skip /head/ key, and it.Next() would skip 0. - count = 1 - it.Rewind() - t.Log("Rewinding from 5000 to zero.") - rewind = false - continue - } - require.EqualValues(t, bkey(count), string(key)) - val := getItemValue(t, item) - require.EqualValues(t, bval(count), string(val)) - require.Equal(t, byte(count%127), item.UserMeta()) - count++ - } - require.EqualValues(t, n, count) - } - - { - t.Log("Starting second basic iteration") - idx := 5030 - for it.Seek(bkey(idx)); it.Valid(); it.Next() { - item := it.Item() - require.EqualValues(t, bkey(idx), string(item.Key())) - require.EqualValues(t, bval(idx), string(getItemValue(t, item))) - idx++ - } - } - it.Close() - }) -} - -func TestLoad(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - n := 10000 - { - kv, err := Open(getTestOptions(dir)) - require.NoError(t, err) - for i := 0; i < n; i++ { - if (i % 10000) == 0 { - fmt.Printf("Putting i=%d\n", i) - } - k := []byte(fmt.Sprintf("%09d", i)) - txnSet(t, kv, k, k, 0x00) - } - kv.Close() - } - - kv, err := Open(getTestOptions(dir)) - require.NoError(t, err) - require.Equal(t, uint64(10001), kv.orc.readTs()) - - for i := 0; i < n; i++ { - if (i % 10000) == 0 { - fmt.Printf("Testing i=%d\n", i) - } - k := fmt.Sprintf("%09d", i) - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get([]byte(k)) - require.NoError(t, err) - require.EqualValues(t, k, string(getItemValue(t, item))) - return nil - })) - } - kv.Close() - summary := kv.lc.getSummary() - - // Check that files are garbage collected. - idMap := getIDMap(dir) - for fileID := range idMap { - // Check that name is in summary.filenames. - require.True(t, summary.fileIDs[fileID], "%d", fileID) - } - require.EqualValues(t, len(idMap), len(summary.fileIDs)) - - var fileIDs []uint64 - for k := range summary.fileIDs { // Map to array. - fileIDs = append(fileIDs, k) - } - sort.Slice(fileIDs, func(i, j int) bool { return fileIDs[i] < fileIDs[j] }) - fmt.Printf("FileIDs: %v\n", fileIDs) -} - -func TestIterateDeleted(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - txnSet(t, db, []byte("Key1"), []byte("Value1"), 0x00) - txnSet(t, db, []byte("Key2"), []byte("Value2"), 0x00) - - iterOpt := DefaultIteratorOptions - iterOpt.PrefetchValues = false - txn := db.NewTransaction(false) - idxIt := txn.NewIterator(iterOpt) - defer idxIt.Close() - - count := 0 - txn2 := db.NewTransaction(true) - prefix := []byte("Key") - for idxIt.Seek(prefix); idxIt.ValidForPrefix(prefix); idxIt.Next() { - key := idxIt.Item().Key() - count++ - newKey := make([]byte, len(key)) - copy(newKey, key) - require.NoError(t, txn2.Delete(newKey)) - } - require.Equal(t, 2, count) - require.NoError(t, txn2.Commit()) - - for _, prefetch := range [...]bool{true, false} { - t.Run(fmt.Sprintf("Prefetch=%t", prefetch), func(t *testing.T) { - txn := db.NewTransaction(false) - iterOpt = DefaultIteratorOptions - iterOpt.PrefetchValues = prefetch - idxIt = txn.NewIterator(iterOpt) - - var estSize int64 - var idxKeys []string - for idxIt.Seek(prefix); idxIt.Valid(); idxIt.Next() { - item := idxIt.Item() - key := item.Key() - estSize += item.EstimatedSize() - if !bytes.HasPrefix(key, prefix) { - break - } - idxKeys = append(idxKeys, string(key)) - t.Logf("%+v\n", idxIt.Item()) - } - require.Equal(t, 0, len(idxKeys)) - require.Equal(t, int64(0), estSize) - }) - } - }) -} - -func TestIterateParallel(t *testing.T) { - key := func(account int) []byte { - var b [4]byte - binary.BigEndian.PutUint32(b[:], uint32(account)) - return append([]byte("account-"), b[:]...) - } - - N := 100000 - iterate := func(txn *Txn, wg *sync.WaitGroup) { - defer wg.Done() - itr := txn.NewIterator(DefaultIteratorOptions) - defer itr.Close() - - var count int - for itr.Rewind(); itr.Valid(); itr.Next() { - count++ - item := itr.Item() - require.Equal(t, "account-", string(item.Key()[0:8])) - err := item.Value(func(val []byte) error { - require.Equal(t, "1000", string(val)) - return nil - }) - require.NoError(t, err) - } - require.Equal(t, N, count) - itr.Close() // Double close. - } - - opt := DefaultOptions - runBadgerTest(t, &opt, func(t *testing.T, db *DB) { - var wg sync.WaitGroup - var txns []*Txn - for i := 0; i < N; i++ { - wg.Add(1) - txn := db.NewTransaction(true) - require.NoError(t, txn.Set(key(i), []byte("1000"))) - txns = append(txns, txn) - } - for _, txn := range txns { - txn.CommitWith(func(err error) { - y.Check(err) - wg.Done() - }) - } - - wg.Wait() - - // Check that a RW txn can't run multiple iterators. - txn := db.NewTransaction(true) - itr := txn.NewIterator(DefaultIteratorOptions) - require.Panics(t, func() { - txn.NewIterator(DefaultIteratorOptions) - }) - require.Panics(t, txn.Discard) - itr.Close() - txn.Discard() - - // Run multiple iterators for a RO txn. - txn = db.NewTransaction(false) - defer txn.Discard() - wg.Add(3) - go iterate(txn, &wg) - go iterate(txn, &wg) - go iterate(txn, &wg) - wg.Wait() - }) -} - -func TestDeleteWithoutSyncWrite(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - kv, err := Open(opt) - if err != nil { - t.Error(err) - t.Fail() - } - - key := []byte("k1") - // Set a value with size > value threshold so that its written to value log. - txnSet(t, kv, key, []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789FOOBARZOGZOG"), 0x00) - txnDelete(t, kv, key) - kv.Close() - - // Reopen KV - kv, err = Open(opt) - if err != nil { - t.Error(err) - t.Fail() - } - defer kv.Close() - - require.NoError(t, kv.View(func(txn *Txn) error { - _, err := txn.Get(key) - require.Equal(t, ErrKeyNotFound, err) - return nil - })) -} - -func TestPidFile(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // Reopen database - _, err := Open(getTestOptions(db.opt.Dir)) - require.Error(t, err) - require.Contains(t, err.Error(), "Another process is using this Badger database") - }) -} - -func TestInvalidKey(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - err := db.Update(func(txn *Txn) error { - err := txn.Set([]byte("!badger!head"), nil) - require.Equal(t, ErrInvalidKey, err) - - err = txn.Set([]byte("!badger!"), nil) - require.Equal(t, ErrInvalidKey, err) - - err = txn.Set([]byte("!badger"), []byte("BadgerDB")) - require.NoError(t, err) - return err - }) - require.NoError(t, err) - - require.NoError(t, db.View(func(txn *Txn) error { - item, err := txn.Get([]byte("!badger")) - if err != nil { - return err - } - require.NoError(t, item.Value(func(val []byte) error { - require.Equal(t, []byte("BadgerDB"), val) - return nil - })) - return nil - })) - }) -} - -func TestIteratorPrefetchSize(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - - bkey := func(i int) []byte { - return []byte(fmt.Sprintf("%09d", i)) - } - bval := func(i int) []byte { - return []byte(fmt.Sprintf("%025d", i)) - } - - n := 100 - for i := 0; i < n; i++ { - // if (i % 10) == 0 { - // t.Logf("Put i=%d\n", i) - // } - txnSet(t, db, bkey(i), bval(i), byte(i%127)) - } - - getIteratorCount := func(prefetchSize int) int { - opt := IteratorOptions{} - opt.PrefetchValues = true - opt.PrefetchSize = prefetchSize - - var count int - txn := db.NewTransaction(false) - it := txn.NewIterator(opt) - { - t.Log("Starting first basic iteration") - for it.Rewind(); it.Valid(); it.Next() { - count++ - } - require.EqualValues(t, n, count) - } - return count - } - - var sizes = []int{-10, 0, 1, 10} - for _, size := range sizes { - c := getIteratorCount(size) - require.Equal(t, 100, c) - } - }) -} - -func TestSetIfAbsentAsync(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - kv, _ := Open(getTestOptions(dir)) - - bkey := func(i int) []byte { - return []byte(fmt.Sprintf("%09d", i)) - } - - f := func(err error) {} - - n := 1000 - for i := 0; i < n; i++ { - // if (i % 10) == 0 { - // t.Logf("Put i=%d\n", i) - // } - txn := kv.NewTransaction(true) - _, err = txn.Get(bkey(i)) - require.Equal(t, ErrKeyNotFound, err) - require.NoError(t, txn.SetWithMeta(bkey(i), nil, byte(i%127))) - txn.CommitWith(f) - } - - require.NoError(t, kv.Close()) - kv, err = Open(getTestOptions(dir)) - require.NoError(t, err) - - opt := DefaultIteratorOptions - txn := kv.NewTransaction(false) - var count int - it := txn.NewIterator(opt) - { - t.Log("Starting first basic iteration") - for it.Rewind(); it.Valid(); it.Next() { - count++ - } - require.EqualValues(t, n, count) - } - require.Equal(t, n, count) - require.NoError(t, kv.Close()) -} - -func TestGetSetRace(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - - data := make([]byte, 4096) - _, err := rand.Read(data) - require.NoError(t, err) - - var ( - numOp = 100 - wg sync.WaitGroup - keyCh = make(chan string) - ) - - // writer - wg.Add(1) - go func() { - defer func() { - wg.Done() - close(keyCh) - }() - - for i := 0; i < numOp; i++ { - key := fmt.Sprintf("%d", i) - txnSet(t, db, []byte(key), data, 0x00) - keyCh <- key - } - }() - - // reader - wg.Add(1) - go func() { - defer wg.Done() - - for key := range keyCh { - require.NoError(t, db.View(func(txn *Txn) error { - item, err := txn.Get([]byte(key)) - require.NoError(t, err) - err = item.Value(nil) - require.NoError(t, err) - return nil - })) - } - }() - - wg.Wait() - }) -} - -func TestDiscardVersionsBelow(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // Write 4 versions of the same key - for i := 0; i < 4; i++ { - err := db.Update(func(txn *Txn) error { - return txn.Set([]byte("answer"), []byte(fmt.Sprintf("%d", i))) - }) - require.NoError(t, err) - } - - opts := DefaultIteratorOptions - opts.AllVersions = true - opts.PrefetchValues = false - - // Verify that there are 4 versions, and record 3rd version (2nd from top in iteration) - db.View(func(txn *Txn) error { - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - count++ - item := it.Item() - require.Equal(t, []byte("answer"), item.Key()) - if item.DiscardEarlierVersions() { - break - } - } - require.Equal(t, 4, count) - return nil - }) - - // Set new version and discard older ones. - err := db.Update(func(txn *Txn) error { - return txn.SetWithDiscard([]byte("answer"), []byte("5"), 0) - }) - require.NoError(t, err) - - // Verify that there are only 2 versions left, and versions - // below ts have been deleted. - db.View(func(txn *Txn) error { - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - count++ - item := it.Item() - require.Equal(t, []byte("answer"), item.Key()) - if item.DiscardEarlierVersions() { - break - } - } - require.Equal(t, 1, count) - return nil - }) - }) -} - -func TestExpiry(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // Write two keys, one with a TTL - err := db.Update(func(txn *Txn) error { - return txn.Set([]byte("answer1"), []byte("42")) - }) - require.NoError(t, err) - - err = db.Update(func(txn *Txn) error { - return txn.SetWithTTL([]byte("answer2"), []byte("43"), 1*time.Second) - }) - require.NoError(t, err) - - time.Sleep(2 * time.Second) - - // Verify that only unexpired key is found during iteration - err = db.View(func(txn *Txn) error { - _, err := txn.Get([]byte("answer1")) - require.NoError(t, err) - - _, err = txn.Get([]byte("answer2")) - require.Equal(t, ErrKeyNotFound, err) - return nil - }) - require.NoError(t, err) - - // Verify that only one key is found during iteration - opts := DefaultIteratorOptions - opts.PrefetchValues = false - err = db.View(func(txn *Txn) error { - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - count++ - item := it.Item() - require.Equal(t, []byte("answer1"), item.Key()) - } - require.Equal(t, 1, count) - return nil - }) - require.NoError(t, err) - }) -} - -func randBytes(n int) []byte { - recv := make([]byte, n) - in, err := rand.Read(recv) - if err != nil { - log.Fatal(err) - } - return recv[:in] -} - -var benchmarkData = []struct { - key, value []byte -}{ - {randBytes(100), nil}, - {randBytes(1000), []byte("foo")}, - {[]byte("foo"), randBytes(1000)}, - {[]byte(""), randBytes(1000)}, - {nil, randBytes(1000000)}, - {randBytes(100000), nil}, - {randBytes(1000000), nil}, -} - -func TestLargeKeys(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opts := new(Options) - *opts = DefaultOptions - opts.ValueLogFileSize = 1024 * 1024 * 1024 - opts.Dir = dir - opts.ValueDir = dir - - db, err := Open(*opts) - if err != nil { - t.Fatal(err) - } - for i := 0; i < 1000; i++ { - tx := db.NewTransaction(true) - for _, kv := range benchmarkData { - k := make([]byte, len(kv.key)) - copy(k, kv.key) - - v := make([]byte, len(kv.value)) - copy(v, kv.value) - if err := tx.Set(k, v); err != nil { - // Skip over this record. - } - } - if err := tx.Commit(); err != nil { - t.Fatalf("#%d: batchSet err: %v", i, err) - } - } -} - -func TestCreateDirs(t *testing.T) { - dir, err := ioutil.TempDir("", "parent") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opts := DefaultOptions - dir = filepath.Join(dir, "badger") - opts.Dir = dir - opts.ValueDir = dir - db, err := Open(opts) - require.NoError(t, err) - db.Close() - _, err = os.Stat(dir) - require.NoError(t, err) -} - -func TestGetSetDeadlock(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - fmt.Println(dir) - require.NoError(t, err) - defer os.RemoveAll(dir) - - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - opt.ValueLogFileSize = 1 << 20 - db, err := Open(opt) - require.NoError(t, err) - defer db.Close() - - val := make([]byte, 1<<19) - key := []byte("key1") - require.NoError(t, db.Update(func(txn *Txn) error { - rand.Read(val) - require.NoError(t, txn.Set(key, val)) - return nil - })) - - timeout, done := time.After(10*time.Second), make(chan bool) - - go func() { - db.Update(func(txn *Txn) error { - item, err := txn.Get(key) - require.NoError(t, err) - err = item.Value(nil) // This take a RLock on file - require.NoError(t, err) - - rand.Read(val) - require.NoError(t, txn.Set(key, val)) - require.NoError(t, txn.Set([]byte("key2"), val)) - return nil - }) - done <- true - }() - - select { - case <-timeout: - t.Fatal("db.Update did not finish within 10s, assuming deadlock.") - case <-done: - t.Log("db.Update finished.") - } -} - -func TestWriteDeadlock(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - fmt.Println(dir) - require.NoError(t, err) - defer os.RemoveAll(dir) - - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - opt.ValueLogFileSize = 10 << 20 - db, err := Open(opt) - require.NoError(t, err) - - print := func(count *int) { - *count++ - if *count%100 == 0 { - fmt.Printf("%05d\r", *count) - } - } - - var count int - val := make([]byte, 10000) - require.NoError(t, db.Update(func(txn *Txn) error { - for i := 0; i < 1500; i++ { - key := fmt.Sprintf("%d", i) - rand.Read(val) - require.NoError(t, txn.Set([]byte(key), val)) - print(&count) - } - return nil - })) - - count = 0 - fmt.Println("\nWrites done. Iteration and updates starting...") - err = db.Update(func(txn *Txn) error { - opt := DefaultIteratorOptions - opt.PrefetchValues = false - it := txn.NewIterator(opt) - defer it.Close() - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - - // Using Value() would cause deadlock. - // item.Value() - out, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, len(val), len(out)) - - key := y.Copy(item.Key()) - rand.Read(val) - require.NoError(t, txn.Set(key, val)) - print(&count) - } - return nil - }) - require.NoError(t, err) -} - -func TestSequence(t *testing.T) { - key0 := []byte("seq0") - key1 := []byte("seq1") - - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - seq0, err := db.GetSequence(key0, 10) - require.NoError(t, err) - seq1, err := db.GetSequence(key1, 100) - require.NoError(t, err) - - for i := uint64(0); i < uint64(105); i++ { - num, err := seq0.Next() - require.NoError(t, err) - require.Equal(t, i, num) - - num, err = seq1.Next() - require.NoError(t, err) - require.Equal(t, i, num) - } - err = db.View(func(txn *Txn) error { - item, err := txn.Get(key0) - if err != nil { - return err - } - var num0 uint64 - if err := item.Value(func(val []byte) error { - num0 = binary.BigEndian.Uint64(val) - return nil - }); err != nil { - return err - } - require.Equal(t, uint64(110), num0) - - item, err = txn.Get(key1) - if err != nil { - return err - } - var num1 uint64 - if err := item.Value(func(val []byte) error { - num1 = binary.BigEndian.Uint64(val) - return nil - }); err != nil { - return err - } - require.Equal(t, uint64(200), num1) - return nil - }) - require.NoError(t, err) - }) -} - -func TestSequence_Release(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // get sequence, use once and release - key := []byte("key") - seq, err := db.GetSequence(key, 1000) - require.NoError(t, err) - num, err := seq.Next() - require.NoError(t, err) - require.Equal(t, uint64(0), num) - require.NoError(t, seq.Release()) - - // we used up 0 and 1 should be stored now - err = db.View(func(txn *Txn) error { - item, err := txn.Get(key) - if err != nil { - return err - } - val, err := item.ValueCopy(nil) - if err != nil { - return err - } - require.Equal(t, num+1, binary.BigEndian.Uint64(val)) - return nil - }) - require.NoError(t, err) - - // using it again will lease 1+1000 - num, err = seq.Next() - require.NoError(t, err) - require.Equal(t, uint64(1), num) - err = db.View(func(txn *Txn) error { - item, err := txn.Get(key) - if err != nil { - return err - } - val, err := item.ValueCopy(nil) - if err != nil { - return err - } - require.Equal(t, uint64(1001), binary.BigEndian.Uint64(val)) - return nil - }) - require.NoError(t, err) - }) -} - -func uint64ToBytes(i uint64) []byte { - var buf [8]byte - binary.BigEndian.PutUint64(buf[:], i) - return buf[:] -} - -func bytesToUint64(b []byte) uint64 { - return binary.BigEndian.Uint64(b) -} - -// Merge function to add two uint64 numbers -func add(existing, new []byte) []byte { - return uint64ToBytes( - bytesToUint64(existing) + - bytesToUint64(new)) -} - -func TestMergeOperatorGetBeforeAdd(t *testing.T) { - key := []byte("merge") - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - m := db.GetMergeOperator(key, add, 200*time.Millisecond) - defer m.Stop() - - _, err := m.Get() - require.Equal(t, ErrKeyNotFound, err) - }) -} - -func TestMergeOperatorBeforeAdd(t *testing.T) { - key := []byte("merge") - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - m := db.GetMergeOperator(key, add, 200*time.Millisecond) - defer m.Stop() - time.Sleep(time.Second) - }) -} - -func TestMergeOperatorAddAndGet(t *testing.T) { - key := []byte("merge") - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - m := db.GetMergeOperator(key, add, 200*time.Millisecond) - defer m.Stop() - - err := m.Add(uint64ToBytes(1)) - require.NoError(t, err) - m.Add(uint64ToBytes(2)) - require.NoError(t, err) - m.Add(uint64ToBytes(3)) - require.NoError(t, err) - - res, err := m.Get() - require.NoError(t, err) - require.Equal(t, uint64(6), bytesToUint64(res)) - }) -} - -func TestMergeOperatorCompactBeforeGet(t *testing.T) { - key := []byte("merge") - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - m := db.GetMergeOperator(key, add, 200*time.Millisecond) - defer m.Stop() - - err := m.Add(uint64ToBytes(1)) - require.NoError(t, err) - m.Add(uint64ToBytes(2)) - require.NoError(t, err) - m.Add(uint64ToBytes(3)) - require.NoError(t, err) - - time.Sleep(250 * time.Millisecond) // wait for merge to happen - - res, err := m.Get() - require.NoError(t, err) - require.Equal(t, uint64(6), bytesToUint64(res)) - }) -} - -func TestMergeOperatorGetAfterStop(t *testing.T) { - key := []byte("merge") - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - m := db.GetMergeOperator(key, add, 1*time.Second) - - err := m.Add(uint64ToBytes(1)) - require.NoError(t, err) - m.Add(uint64ToBytes(2)) - require.NoError(t, err) - m.Add(uint64ToBytes(3)) - require.NoError(t, err) - - m.Stop() - res, err := m.Get() - require.NoError(t, err) - require.Equal(t, uint64(6), bytesToUint64(res)) - }) -} - -func TestReadOnly(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - - // Create the DB - db, err := Open(opts) - require.NoError(t, err) - for i := 0; i < 10000; i++ { - txnSet(t, db, []byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)), 0x00) - } - - // Attempt a read-only open while it's open read-write. - opts.ReadOnly = true - _, err = Open(opts) - require.Error(t, err) - if err == ErrWindowsNotSupported { - return - } - require.Contains(t, err.Error(), "Another process is using this Badger database") - db.Close() - - // Open one read-only - opts.ReadOnly = true - kv1, err := Open(opts) - require.NoError(t, err) - defer kv1.Close() - - // Open another read-only - kv2, err := Open(opts) - require.NoError(t, err) - defer kv2.Close() - - // Attempt a read-write open while it's open for read-only - opts.ReadOnly = false - _, err = Open(opts) - require.Error(t, err) - require.Contains(t, err.Error(), "Another process is using this Badger database") - - // Get a thing from the DB - txn1 := kv1.NewTransaction(true) - v1, err := txn1.Get([]byte("key1")) - require.NoError(t, err) - b1, err := v1.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, b1, []byte("value1")) - err = txn1.Commit() - require.NoError(t, err) - - // Get a thing from the DB via the other connection - txn2 := kv2.NewTransaction(true) - v2, err := txn2.Get([]byte("key2000")) - require.NoError(t, err) - b2, err := v2.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, b2, []byte("value2000")) - err = txn2.Commit() - require.NoError(t, err) - - // Attempt to set a value on a read-only connection - txn := kv1.NewTransaction(true) - err = txn.SetWithMeta([]byte("key"), []byte("value"), 0x00) - require.Error(t, err) - require.Contains(t, err.Error(), "No sets or deletes are allowed in a read-only transaction") - err = txn.Commit() - require.NoError(t, err) -} - -func TestLSMOnly(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opts := LSMOnlyOptions - opts.Dir = dir - opts.ValueDir = dir - - dopts := DefaultOptions - require.NotEqual(t, dopts.ValueThreshold, opts.ValueThreshold) - - dopts.ValueThreshold = 1 << 16 - _, err = Open(dopts) - require.Equal(t, ErrValueThreshold, err) - - opts.ValueLogMaxEntries = 100 - db, err := Open(opts) - require.NoError(t, err) - if err != nil { - t.Fatal(err) - } - - value := make([]byte, 128) - _, err = rand.Read(value) - for i := 0; i < 500; i++ { - require.NoError(t, err) - txnSet(t, db, []byte(fmt.Sprintf("key%d", i)), value, 0x00) - } - require.NoError(t, db.Close()) // Close to force compactions, so Value log GC would run. - - db, err = Open(opts) - require.NoError(t, err) - if err != nil { - t.Fatal(err) - } - defer db.Close() - require.NoError(t, db.RunValueLogGC(0.2)) -} - -// This test function is doing some intricate sorcery. -func TestMinReadTs(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - for i := 0; i < 10; i++ { - require.NoError(t, db.Update(func(txn *Txn) error { - return txn.Set([]byte("x"), []byte("y")) - })) - } - time.Sleep(time.Millisecond) - - readTxn0 := db.NewTransaction(false) - require.Equal(t, uint64(10), readTxn0.readTs) - - min := db.orc.readMark.DoneUntil() - require.Equal(t, uint64(9), min) - - readTxn := db.NewTransaction(false) - for i := 0; i < 10; i++ { - require.NoError(t, db.Update(func(txn *Txn) error { - return txn.Set([]byte("x"), []byte("y")) - })) - } - require.Equal(t, uint64(20), db.orc.readTs()) - - time.Sleep(time.Millisecond) - require.Equal(t, min, db.orc.readMark.DoneUntil()) - - readTxn0.Discard() - readTxn.Discard() - time.Sleep(time.Millisecond) - require.Equal(t, uint64(19), db.orc.readMark.DoneUntil()) - db.orc.readMark.Done(uint64(20)) // Because we called readTs. - - for i := 0; i < 10; i++ { - db.View(func(txn *Txn) error { - return nil - }) - } - time.Sleep(time.Millisecond) - require.Equal(t, uint64(20), db.orc.readMark.DoneUntil()) - }) -} - -func TestGoroutineLeak(t *testing.T) { - before := runtime.NumGoroutine() - t.Logf("Num go: %d", before) - for i := 0; i < 12; i++ { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - err := db.Update(func(txn *Txn) error { - return txn.Set([]byte("key"), []byte("value")) - }) - require.NoError(t, err) - }) - } - require.Equal(t, before, runtime.NumGoroutine()) -} - -func ExampleOpen() { - dir, err := ioutil.TempDir("", "badger") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(dir) - opts := DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - db, err := Open(opts) - if err != nil { - log.Fatal(err) - } - defer db.Close() - - err = db.View(func(txn *Txn) error { - _, err := txn.Get([]byte("key")) - // We expect ErrKeyNotFound - fmt.Println(err) - return nil - }) - - if err != nil { - log.Fatal(err) - } - - txn := db.NewTransaction(true) // Read-write txn - err = txn.Set([]byte("key"), []byte("value")) - if err != nil { - log.Fatal(err) - } - err = txn.Commit() - if err != nil { - log.Fatal(err) - } - - err = db.View(func(txn *Txn) error { - item, err := txn.Get([]byte("key")) - if err != nil { - return err - } - val, err := item.ValueCopy(nil) - if err != nil { - return err - } - fmt.Printf("%s\n", string(val)) - return nil - }) - - if err != nil { - log.Fatal(err) - } - - // Output: - // Key not found - // value -} - -func ExampleTxn_NewIterator() { - dir, err := ioutil.TempDir("", "badger") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(dir) - - opts := DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - - db, err := Open(opts) - if err != nil { - log.Fatal(err) - } - defer db.Close() - - bkey := func(i int) []byte { - return []byte(fmt.Sprintf("%09d", i)) - } - bval := func(i int) []byte { - return []byte(fmt.Sprintf("%025d", i)) - } - - txn := db.NewTransaction(true) - - // Fill in 1000 items - n := 1000 - for i := 0; i < n; i++ { - err := txn.Set(bkey(i), bval(i)) - if err != nil { - log.Fatal(err) - } - } - - err = txn.Commit() - if err != nil { - log.Fatal(err) - } - - opt := DefaultIteratorOptions - opt.PrefetchSize = 10 - - // Iterate over 1000 items - var count int - err = db.View(func(txn *Txn) error { - it := txn.NewIterator(opt) - defer it.Close() - for it.Rewind(); it.Valid(); it.Next() { - count++ - } - return nil - }) - if err != nil { - log.Fatal(err) - } - fmt.Printf("Counted %d elements", count) - // Output: - // Counted 1000 elements -} - -func TestMain(m *testing.M) { - // call flag.Parse() here if TestMain uses flags - go func() { - if err := http.ListenAndServe("localhost:8080", nil); err != nil { - log.Fatalf("Unable to open http port at 8080") - } - }() - os.Exit(m.Run()) -} diff --git a/vendor/github.com/dgraph-io/badger/dir_unix.go b/vendor/github.com/dgraph-io/badger/dir_unix.go index 4f809b66..a5e0fa33 100644 --- a/vendor/github.com/dgraph-io/badger/dir_unix.go +++ b/vendor/github.com/dgraph-io/badger/dir_unix.go @@ -24,8 +24,8 @@ import ( "os" "path/filepath" - "gx/ipfs/QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu/sys/unix" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/pkg/errors" + "golang.org/x/sys/unix" ) // directoryLockGuard holds a lock on a directory and a pid file inside. The pid file isn't part diff --git a/vendor/github.com/dgraph-io/badger/dir_windows.go b/vendor/github.com/dgraph-io/badger/dir_windows.go index 248bd27e..28ccb7aa 100644 --- a/vendor/github.com/dgraph-io/badger/dir_windows.go +++ b/vendor/github.com/dgraph-io/badger/dir_windows.go @@ -24,7 +24,7 @@ import ( "path/filepath" "syscall" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/pkg/errors" ) // FILE_ATTRIBUTE_TEMPORARY - A file that is being used for temporary storage. diff --git a/vendor/github.com/dgraph-io/badger/errors.go b/vendor/github.com/dgraph-io/badger/errors.go index 00836109..cad66cb1 100644 --- a/vendor/github.com/dgraph-io/badger/errors.go +++ b/vendor/github.com/dgraph-io/badger/errors.go @@ -17,7 +17,7 @@ package badger import ( - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/pkg/errors" ) var ( @@ -102,4 +102,7 @@ var ( // ErrBlockedWrites is returned if the user called DropAll. During the process of dropping all // data from Badger, we stop accepting new writes, by returning this error. ErrBlockedWrites = errors.New("Writes are blocked, possibly due to DropAll or Close") + + // ErrNilCallback is returned when subscriber's callback is nil. + ErrNilCallback = errors.New("Callback cannot be nil") ) diff --git a/vendor/github.com/dgraph-io/badger/histogram.go b/vendor/github.com/dgraph-io/badger/histogram.go new file mode 100644 index 00000000..d8c94bb7 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/histogram.go @@ -0,0 +1,169 @@ +/* + * Copyright 2019 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package badger + +import ( + "fmt" + "math" +) + +// PrintHistogram builds and displays the key-value size histogram. +// When keyPrefix is set, only the keys that have prefix "keyPrefix" are +// considered for creating the histogram +func (db *DB) PrintHistogram(keyPrefix []byte) { + if db == nil { + fmt.Println("\nCannot build histogram: DB is nil.") + return + } + histogram := db.buildHistogram(keyPrefix) + fmt.Printf("Histogram of key sizes (in bytes)\n") + histogram.keySizeHistogram.printHistogram() + fmt.Printf("Histogram of value sizes (in bytes)\n") + histogram.valueSizeHistogram.printHistogram() +} + +// histogramData stores information about a histogram +type histogramData struct { + bins []int64 + countPerBin []int64 + totalCount int64 + min int64 + max int64 + sum int64 +} + +// sizeHistogram contains keySize histogram and valueSize histogram +type sizeHistogram struct { + keySizeHistogram, valueSizeHistogram histogramData +} + +// newSizeHistogram returns a new instance of keyValueSizeHistogram with +// properly initialized fields. +func newSizeHistogram() *sizeHistogram { + // TODO(ibrahim): find appropriate bin size. + keyBins := createHistogramBins(1, 16) + valueBins := createHistogramBins(1, 30) + return &sizeHistogram{ + keySizeHistogram: histogramData{ + bins: keyBins, + countPerBin: make([]int64, len(keyBins)+1), + max: math.MinInt64, + min: math.MaxInt64, + sum: 0, + }, + valueSizeHistogram: histogramData{ + bins: valueBins, + countPerBin: make([]int64, len(valueBins)+1), + max: math.MinInt64, + min: math.MaxInt64, + sum: 0, + }, + } +} + +// createHistogramBins creates bins for an histogram. The bin sizes are powers +// of two of the form [2^min_exponent, ..., 2^max_exponent]. +func createHistogramBins(minExponent, maxExponent uint32) []int64 { + var bins []int64 + for i := minExponent; i <= maxExponent; i++ { + bins = append(bins, int64(1)< histogram.max { + histogram.max = value + } + if value < histogram.min { + histogram.min = value + } + + histogram.sum += value + histogram.totalCount++ + + for index := 0; index <= len(histogram.bins); index++ { + // Allocate value in the last buckets if we reached the end of the Bounds array. + if index == len(histogram.bins) { + histogram.countPerBin[index]++ + break + } + + // Check if the value should be added to the "index" bin + if value < int64(histogram.bins[index]) { + histogram.countPerBin[index]++ + break + } + } +} + +// buildHistogram builds the key-value size histogram. +// When keyPrefix is set, only the keys that have prefix "keyPrefix" are +// considered for creating the histogram +func (db *DB) buildHistogram(keyPrefix []byte) *sizeHistogram { + txn := db.NewTransaction(false) + defer txn.Discard() + + itr := txn.NewIterator(DefaultIteratorOptions) + defer itr.Close() + + badgerHistogram := newSizeHistogram() + + // Collect key and value sizes. + for itr.Seek(keyPrefix); itr.ValidForPrefix(keyPrefix); itr.Next() { + item := itr.Item() + badgerHistogram.keySizeHistogram.Update(item.KeySize()) + badgerHistogram.valueSizeHistogram.Update(item.ValueSize()) + } + return badgerHistogram +} + +// printHistogram prints the histogram data in a human-readable format. +func (histogram histogramData) printHistogram() { + fmt.Printf("Total count: %d\n", histogram.totalCount) + fmt.Printf("Min value: %d\n", histogram.min) + fmt.Printf("Max value: %d\n", histogram.max) + fmt.Printf("Mean: %.2f\n", float64(histogram.sum)/float64(histogram.totalCount)) + fmt.Printf("%24s %9s\n", "Range", "Count") + + numBins := len(histogram.bins) + for index, count := range histogram.countPerBin { + if count == 0 { + continue + } + + // The last bin represents the bin that contains the range from + // the last bin up to infinity so it's processed differently than the + // other bins. + if index == len(histogram.countPerBin)-1 { + lowerBound := int(histogram.bins[numBins-1]) + fmt.Printf("[%10d, %10s) %9d\n", lowerBound, "infinity", count) + continue + } + + upperBound := int(histogram.bins[index]) + lowerBound := 0 + if index > 0 { + lowerBound = int(histogram.bins[index-1]) + } + + fmt.Printf("[%10d, %10d) %9d\n", lowerBound, upperBound, count) + } + fmt.Println() +} diff --git a/vendor/github.com/dgraph-io/badger/images/benchmarks-rocksdb.png b/vendor/github.com/dgraph-io/badger/images/benchmarks-rocksdb.png deleted file mode 100644 index 27081e81..00000000 Binary files a/vendor/github.com/dgraph-io/badger/images/benchmarks-rocksdb.png and /dev/null differ diff --git a/vendor/github.com/dgraph-io/badger/images/diggy-shadow.png b/vendor/github.com/dgraph-io/badger/images/diggy-shadow.png deleted file mode 100644 index 19ba3f2c..00000000 Binary files a/vendor/github.com/dgraph-io/badger/images/diggy-shadow.png and /dev/null differ diff --git a/vendor/github.com/dgraph-io/badger/integration/testgc/.gitignore b/vendor/github.com/dgraph-io/badger/integration/testgc/.gitignore deleted file mode 100644 index f6600666..00000000 --- a/vendor/github.com/dgraph-io/badger/integration/testgc/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/testgc diff --git a/vendor/github.com/dgraph-io/badger/integration/testgc/main.go b/vendor/github.com/dgraph-io/badger/integration/testgc/main.go deleted file mode 100644 index a1852bb1..00000000 --- a/vendor/github.com/dgraph-io/badger/integration/testgc/main.go +++ /dev/null @@ -1,218 +0,0 @@ -package main - -import ( - "encoding/binary" - "fmt" - "log" - "math/rand" - "net/http" - _ "net/http/pprof" - "os" - "sync" - "sync/atomic" - "time" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -var maxValue int64 = 10000000 -var suffix = make([]byte, 128) - -type testSuite struct { - sync.Mutex - vals map[uint64]uint64 - - count uint64 // Not under mutex lock. -} - -func encoded(i uint64) []byte { - out := make([]byte, 8) - binary.BigEndian.PutUint64(out, i) - return out -} - -func (s *testSuite) write(db *badger.DB) error { - return db.Update(func(txn *badger.Txn) error { - for i := 0; i < 10; i++ { - // These keys would be overwritten. - keyi := uint64(rand.Int63n(maxValue)) - key := encoded(keyi) - vali := atomic.AddUint64(&s.count, 1) - val := encoded(vali) - val = append(val, suffix...) - if err := txn.Set(key, val); err != nil { - return err - } - } - for i := 0; i < 20; i++ { - // These keys would be new and never overwritten. - keyi := atomic.AddUint64(&s.count, 1) - if keyi%1000000 == 0 { - log.Printf("Count: %d\n", keyi) - } - key := encoded(keyi) - val := append(key, suffix...) - if err := txn.Set(key, val); err != nil { - return err - } - } - return nil - }) -} - -func (s *testSuite) read(db *badger.DB) error { - max := int64(atomic.LoadUint64(&s.count)) - keyi := uint64(rand.Int63n(max)) - key := encoded(keyi) - - err := db.View(func(txn *badger.Txn) error { - item, err := txn.Get(key) - if err != nil { - return err - } - val, err := item.ValueCopy(nil) - if err != nil { - return err - } - y.AssertTruef(len(val) == len(suffix)+8, "Found val of len: %d\n", len(val)) - vali := binary.BigEndian.Uint64(val[0:8]) - s.Lock() - expected := s.vals[keyi] - if vali < expected { - log.Fatalf("Expected: %d. Found: %d. Key: %d\n", expected, vali, keyi) - } else if vali == expected { - // pass - } else { - s.vals[keyi] = vali - } - s.Unlock() - return nil - }) - if err == badger.ErrKeyNotFound { - return nil - } - return err -} - -func main() { - fmt.Println("Badger Integration test for value log GC.") - - dir := "/mnt/drive/badgertest" - os.RemoveAll(dir) - - opts := badger.DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - opts.TableLoadingMode = options.MemoryMap - opts.ValueLogLoadingMode = options.FileIO - opts.SyncWrites = false - - db, err := badger.Open(opts) - if err != nil { - log.Fatal(err) - } - defer db.Close() - - go http.ListenAndServe("localhost:8080", nil) - - closer := y.NewCloser(11) - go func() { - // Run value log GC. - defer closer.Done() - var count int - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - for range ticker.C { - again: - select { - case <-closer.HasBeenClosed(): - log.Printf("Num times value log GC was successful: %d\n", count) - return - default: - } - log.Printf("Starting a value log GC") - err := db.RunValueLogGC(0.1) - log.Printf("Result of value log GC: %v\n", err) - if err == nil { - count++ - goto again - } - } - }() - - s := testSuite{ - count: uint64(maxValue), - vals: make(map[uint64]uint64), - } - var numLoops uint64 - ticker := time.NewTicker(5 * time.Second) - for i := 0; i < 10; i++ { - go func() { - defer closer.Done() - for { - if err := s.write(db); err != nil { - log.Fatal(err) - } - for j := 0; j < 10; j++ { - if err := s.read(db); err != nil { - log.Fatal(err) - } - } - nl := atomic.AddUint64(&numLoops, 1) - select { - case <-closer.HasBeenClosed(): - return - case <-ticker.C: - log.Printf("Num loops: %d\n", nl) - default: - } - } - }() - } - time.Sleep(5 * time.Minute) - log.Println("Signaling...") - closer.SignalAndWait() - log.Println("Wait done. Now iterating over everything.") - - err = db.View(func(txn *badger.Txn) error { - iopts := badger.DefaultIteratorOptions - itr := txn.NewIterator(iopts) - defer itr.Close() - - var total, tested int - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - key := item.Key() - keyi := binary.BigEndian.Uint64(key) - total++ - - val, err := item.ValueCopy(nil) - if err != nil { - return err - } - if len(val) < 8 { - log.Printf("Unexpected value: %x\n", val) - continue - } - vali := binary.BigEndian.Uint64(val[0:8]) - - expected, ok := s.vals[keyi] // Not all keys must be in vals map. - if ok { - tested++ - if vali < expected { - // vali must be equal or greater than what's in the map. - log.Fatalf("Expected: %d. Got: %d. Key: %d\n", expected, vali, keyi) - } - } - } - log.Printf("Total iterated: %d. Tested values: %d\n", total, tested) - return nil - }) - if err != nil { - log.Fatalf("Error while iterating: %v", err) - } - log.Println("Iteration done. Test successful.") - time.Sleep(time.Minute) // Time to do some poking around. -} diff --git a/vendor/github.com/dgraph-io/badger/iterator.go b/vendor/github.com/dgraph-io/badger/iterator.go index 1584a94f..c071324d 100644 --- a/vendor/github.com/dgraph-io/badger/iterator.go +++ b/vendor/github.com/dgraph-io/badger/iterator.go @@ -24,10 +24,10 @@ import ( "sync/atomic" "time" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table" + "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/table" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" + "github.com/dgraph-io/badger/y" ) type prefetchStatus uint8 @@ -140,7 +140,7 @@ func (item *Item) IsDeletedOrExpired() bool { return isDeletedOrExpired(item.meta, item.expiresAt) } -// DiscardEarlierVersions returns whether the iterator was created with the +// DiscardEarlierVersions returns whether the item was created with the // option to discard earlier versions of a key when multiple are available. func (item *Item) DiscardEarlierVersions() bool { return item.meta&bitDiscardEarlierVersions > 0 @@ -246,6 +246,12 @@ func (item *Item) EstimatedSize() int64 { return int64(vp.Len) // includes key length. } +// KeySize returns the size of the key. +// Exact size of the key is key + 8 bytes of timestamp +func (item *Item) KeySize() int64 { + return int64(len(item.key)) +} + // ValueSize returns the exact size of the value. // // This can be called to quickly estimate the size of a value without fetching diff --git a/vendor/github.com/dgraph-io/badger/iterator_test.go b/vendor/github.com/dgraph-io/badger/iterator_test.go deleted file mode 100644 index 9404b075..00000000 --- a/vendor/github.com/dgraph-io/badger/iterator_test.go +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright 2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "bytes" - "fmt" - "io/ioutil" - "math/rand" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -type tableMock struct { - left, right []byte -} - -func (tm *tableMock) Smallest() []byte { return tm.left } -func (tm *tableMock) Biggest() []byte { return tm.right } -func (tm *tableMock) DoesNotHave(key []byte) bool { return false } - -func TestPickTables(t *testing.T) { - opt := DefaultIteratorOptions - - within := func(prefix, left, right string) { - opt.Prefix = []byte(prefix) - tm := &tableMock{left: []byte(left), right: []byte(right)} - require.True(t, opt.pickTable(tm)) - } - outside := func(prefix, left, right string) { - opt.Prefix = []byte(prefix) - tm := &tableMock{left: []byte(left), right: []byte(right)} - require.False(t, opt.pickTable(tm)) - } - within("abc", "ab", "ad") - within("abc", "abc", "ad") - within("abc", "abb123", "ad") - within("abc", "abc123", "abd234") - within("abc", "abc123", "abc456") - - outside("abd", "abe", "ad") - outside("abd", "ac", "ad") - outside("abd", "b", "e") - outside("abd", "a", "ab") - outside("abd", "ab", "abc") - outside("abd", "ab", "abc123") -} - -func TestIteratePrefix(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - bkey := func(i int) []byte { - return []byte(fmt.Sprintf("%04d", i)) - } - val := []byte("OK") - n := 10000 - - batch := db.NewWriteBatch() - for i := 0; i < n; i++ { - if (i % 1000) == 0 { - t.Logf("Put i=%d\n", i) - } - require.NoError(t, batch.Set(bkey(i), val, 0)) - } - require.NoError(t, batch.Flush()) - - countKeys := func(prefix string) int { - t.Logf("Testing with prefix: %s", prefix) - var count int - opt := DefaultIteratorOptions - opt.Prefix = []byte(prefix) - err := db.View(func(txn *Txn) error { - itr := txn.NewIterator(opt) - defer itr.Close() - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - err := item.Value(func(v []byte) error { - require.Equal(t, val, v) - return nil - }) - require.NoError(t, err) - require.True(t, bytes.HasPrefix(item.Key(), opt.Prefix)) - count++ - } - return nil - }) - require.NoError(t, err) - return count - } - - countOneKey := func(key []byte) int { - var count int - err := db.View(func(txn *Txn) error { - itr := txn.NewKeyIterator(key, DefaultIteratorOptions) - defer itr.Close() - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - err := item.Value(func(v []byte) error { - require.Equal(t, val, v) - return nil - }) - require.NoError(t, err) - require.Equal(t, key, item.Key()) - count++ - } - return nil - }) - require.NoError(t, err) - return count - } - - for i := 0; i <= 9; i++ { - require.Equal(t, 1, countKeys(fmt.Sprintf("%d%d%d%d", i, i, i, i))) - require.Equal(t, 10, countKeys(fmt.Sprintf("%d%d%d", i, i, i))) - require.Equal(t, 100, countKeys(fmt.Sprintf("%d%d", i, i))) - require.Equal(t, 1000, countKeys(fmt.Sprintf("%d", i))) - } - require.Equal(t, 10000, countKeys("")) - - t.Logf("Testing each key with key iterator") - for i := 0; i < n; i++ { - require.Equal(t, 1, countOneKey(bkey(i))) - } - }) -} - -// go test -v -run=XXX -bench=BenchmarkIterate -benchtime=3s -// Benchmark with opt.Prefix set === -// goos: linux -// goarch: amd64 -// pkg: github.com/dgraph-io/badger -// BenchmarkIteratePrefixSingleKey/Key_lookups-4 10000 365539 ns/op -// --- BENCH: BenchmarkIteratePrefixSingleKey/Key_lookups-4 -// iterator_test.go:147: Inner b.N: 1 -// iterator_test.go:147: Inner b.N: 100 -// iterator_test.go:147: Inner b.N: 10000 -// --- BENCH: BenchmarkIteratePrefixSingleKey -// iterator_test.go:143: LSM files: 79 -// iterator_test.go:145: Outer b.N: 1 -// PASS -// ok github.com/dgraph-io/badger 41.586s -// -// Benchmark with NO opt.Prefix set === -// goos: linux -// goarch: amd64 -// pkg: github.com/dgraph-io/badger -// BenchmarkIteratePrefixSingleKey/Key_lookups-4 10000 460924 ns/op -// --- BENCH: BenchmarkIteratePrefixSingleKey/Key_lookups-4 -// iterator_test.go:147: Inner b.N: 1 -// iterator_test.go:147: Inner b.N: 100 -// iterator_test.go:147: Inner b.N: 10000 -// --- BENCH: BenchmarkIteratePrefixSingleKey -// iterator_test.go:143: LSM files: 83 -// iterator_test.go:145: Outer b.N: 1 -// PASS -// ok github.com/dgraph-io/badger 41.836s -// -// Only my laptop there's a 20% improvement in latency with ~80 files. -func BenchmarkIteratePrefixSingleKey(b *testing.B) { - dir, err := ioutil.TempDir(".", "badger-test") - y.Check(err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - opts.TableLoadingMode = options.LoadToRAM - db, err := Open(opts) - y.Check(err) - defer db.Close() - - N := 100000 // Should generate around 80 SSTables. - val := []byte("OK") - bkey := func(i int) []byte { - return []byte(fmt.Sprintf("%06d", i)) - } - - batch := db.NewWriteBatch() - for i := 0; i < N; i++ { - y.Check(batch.Set(bkey(i), val, 0)) - } - y.Check(batch.Flush()) - var lsmFiles int - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if strings.HasSuffix(path, ".sst") { - lsmFiles++ - } - if err != nil { - return err - } - return nil - }) - y.Check(err) - b.Logf("LSM files: %d", lsmFiles) - b.Logf("Key splits: %v", db.KeySplits(nil)) - b.Logf("Key splits with prefix: %v", db.KeySplits([]byte("09"))) - - b.Logf("Outer b.N: %d", b.N) - b.Run("Key lookups", func(b *testing.B) { - b.Logf("Inner b.N: %d", b.N) - for i := 0; i < b.N; i++ { - key := bkey(rand.Intn(N)) - err := db.View(func(txn *Txn) error { - opt := DefaultIteratorOptions - // NOTE: Comment opt.Prefix out here to compare the performance - // difference between providing Prefix as an option, v/s not. I - // see a 20% improvement when there are ~80 SSTables. - opt.Prefix = key - opt.AllVersions = true - - itr := txn.NewIterator(opt) - defer itr.Close() - - var count int - for itr.Seek(key); itr.ValidForPrefix(key); itr.Next() { - count++ - } - if count != 1 { - b.Fatalf("Count must be one key: %s. Found: %d", key, count) - } - return nil - }) - if err != nil { - b.Fatalf("Error while View: %v", err) - } - } - }) -} diff --git a/vendor/github.com/dgraph-io/badger/level_handler.go b/vendor/github.com/dgraph-io/badger/level_handler.go index a33124d8..147967fb 100644 --- a/vendor/github.com/dgraph-io/badger/level_handler.go +++ b/vendor/github.com/dgraph-io/badger/level_handler.go @@ -21,9 +21,9 @@ import ( "sort" "sync" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/table" + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" ) type levelHandler struct { @@ -102,48 +102,40 @@ func (s *levelHandler) deleteTables(toDel []*table.Table) error { // replaceTables will replace tables[left:right] with newTables. Note this EXCLUDES tables[right]. // You must call decr() to delete the old tables _after_ writing the update to the manifest. -func (s *levelHandler) replaceTables(newTables []*table.Table) error { +func (s *levelHandler) replaceTables(toDel, toAdd []*table.Table) error { // Need to re-search the range of tables in this level to be replaced as other goroutines might // be changing it as well. (They can't touch our tables, but if they add/remove other tables, // the indices get shifted around.) - if len(newTables) == 0 { - return nil - } - s.Lock() // We s.Unlock() below. + toDelMap := make(map[uint64]struct{}) + for _, t := range toDel { + toDelMap[t.ID()] = struct{}{} + } + var newTables []*table.Table + for _, t := range s.tables { + _, found := toDelMap[t.ID()] + if !found { + newTables = append(newTables, t) + continue + } + s.totalSize -= t.Size() + } + // Increase totalSize first. - for _, tbl := range newTables { - s.totalSize += tbl.Size() - tbl.IncrRef() + for _, t := range toAdd { + s.totalSize += t.Size() + t.IncrRef() + newTables = append(newTables, t) } - kr := keyRange{ - left: newTables[0].Smallest(), - right: newTables[len(newTables)-1].Biggest(), - } - left, right := s.overlappingTables(levelHandlerRLocked{}, kr) - - toDecr := make([]*table.Table, right-left) - // Update totalSize and reference counts. - for i := left; i < right; i++ { - tbl := s.tables[i] - s.totalSize -= tbl.Size() - toDecr[i-left] = tbl - } - - // To be safe, just make a copy. TODO: Be more careful and avoid copying. - numDeleted := right - left - numAdded := len(newTables) - tables := make([]*table.Table, len(s.tables)-numDeleted+numAdded) - y.AssertTrue(left == copy(tables, s.tables[:left])) - t := tables[left:] - y.AssertTrue(numAdded == copy(t, newTables)) - t = t[numAdded:] - y.AssertTrue(len(s.tables[right:]) == copy(t, s.tables[right:])) - s.tables = tables + // Assign tables. + s.tables = newTables + sort.Slice(s.tables, func(i, j int) bool { + return y.CompareKeys(s.tables[i].Smallest(), s.tables[j].Smallest()) < 0 + }) s.Unlock() // s.Unlock before we DecrRef tables -- that can be slow. - return decrRefs(toDecr) + return decrRefs(toDel) } func decrRefs(tables []*table.Table) error { @@ -294,6 +286,9 @@ type levelHandlerRLocked struct{} // This function should already have acquired a read lock, and this is so important the caller must // pass an empty parameter declaring such. func (s *levelHandler) overlappingTables(_ levelHandlerRLocked, kr keyRange) (int, int) { + if len(kr.left) == 0 || len(kr.right) == 0 { + return 0, 0 + } left := sort.Search(len(s.tables), func(i int) bool { return y.CompareKeys(kr.left, s.tables[i].Biggest()) <= 0 }) diff --git a/vendor/github.com/dgraph-io/badger/levels.go b/vendor/github.com/dgraph-io/badger/levels.go index 8b8ad3c7..df90164c 100644 --- a/vendor/github.com/dgraph-io/badger/levels.go +++ b/vendor/github.com/dgraph-io/badger/levels.go @@ -17,6 +17,7 @@ package badger import ( + "bytes" "fmt" "math" "math/rand" @@ -27,12 +28,12 @@ import ( "sync/atomic" "time" - "gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace" + "golang.org/x/net/trace" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/table" + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" ) type levelsController struct { @@ -215,10 +216,10 @@ func (s *levelsController) cleanupLevels() error { return firstErr } -// This function picks all tables from all levels, creates a manifest changeset, +// dropTree picks all tables from all levels, creates a manifest changeset, // applies it, and then decrements the refs of these tables, which would result // in their deletion. -func (s *levelsController) deleteLSMTree() (int, error) { +func (s *levelsController) dropTree() (int, error) { // First pick all tables, so we can create a manifest changelog. var all []*table.Table for _, l := range s.levels { @@ -255,6 +256,72 @@ func (s *levelsController) deleteLSMTree() (int, error) { return len(all), nil } +// dropPrefix runs a L0->L1 compaction, and then runs same level compaction on the rest of the +// levels. For L0->L1 compaction, it runs compactions normally, but skips over all the keys with the +// provided prefix. For Li->Li compactions, it picks up the tables which would have the prefix. The +// tables who only have keys with this prefix are quickly dropped. The ones which have other keys +// are run through MergeIterator and compacted to create new tables. All the mechanisms of +// compactions apply, i.e. level sizes and MANIFEST are updated as in the normal flow. +func (s *levelsController) dropPrefix(prefix []byte) error { + opt := s.kv.opt + for _, l := range s.levels { + l.RLock() + if l.level == 0 { + size := len(l.tables) + l.RUnlock() + + if size > 0 { + cp := compactionPriority{ + level: 0, + score: 1.74, + // A unique number greater than 1.0 does two things. Helps identify this + // function in logs, and forces a compaction. + dropPrefix: prefix, + } + if err := s.doCompact(cp); err != nil { + opt.Warningf("While compacting level 0: %v", err) + return nil + } + } + continue + } + + var tables []*table.Table + for _, table := range l.tables { + var absent bool + switch { + case bytes.HasPrefix(table.Smallest(), prefix): + case bytes.HasPrefix(table.Biggest(), prefix): + case bytes.Compare(prefix, table.Smallest()) > 0 && + bytes.Compare(prefix, table.Biggest()) < 0: + default: + absent = true + } + if !absent { + tables = append(tables, table) + } + } + l.RUnlock() + if len(tables) == 0 { + continue + } + + cd := compactDef{ + elog: trace.New(fmt.Sprintf("Badger.L%d", l.level), "Compact"), + thisLevel: l, + nextLevel: l, + top: []*table.Table{}, + bot: tables, + dropPrefix: prefix, + } + if err := s.runCompactDef(l.level, cd); err != nil { + opt.Warningf("While running compact def: %+v. Error: %v", cd, err) + return err + } + } + return nil +} + func (s *levelsController) startCompact(lc *y.Closer) { n := s.kv.opt.NumCompactors lc.AddRunning(n - 1) @@ -311,8 +378,9 @@ func (l *levelHandler) isCompactable(delSize int64) bool { } type compactionPriority struct { - level int - score float64 + level int + score float64 + dropPrefix []byte } // pickCompactLevel determines which level to compact. @@ -350,7 +418,7 @@ func (s *levelsController) pickCompactLevels() (prios []compactionPriority) { // compactBuildTables merge topTables and botTables to form a list of new tables. func (s *levelsController) compactBuildTables( - l int, cd compactDef) ([]*table.Table, func() error, error) { + lev int, cd compactDef) ([]*table.Table, func() error, error) { topTables := cd.top botTables := cd.bot @@ -358,7 +426,7 @@ func (s *levelsController) compactBuildTables( { kr := getKeyRange(cd.top) for i, lh := range s.levels { - if i <= l { // Skip upper levels. + if i <= lev { // Skip upper levels. continue } lh.RLock() @@ -369,7 +437,6 @@ func (s *levelsController) compactBuildTables( break } } - cd.elog.LazyPrintf("Key range overlaps with lower levels: %v", hasOverlap) } // Try to collect stats so that we can inform value log about GC. That would help us find which @@ -385,15 +452,26 @@ func (s *levelsController) compactBuildTables( // Create iterators across all the tables involved first. var iters []y.Iterator - if l == 0 { + if lev == 0 { iters = appendIteratorsReversed(iters, topTables, false) - } else { + } else if len(topTables) > 0 { y.AssertTrue(len(topTables) == 1) iters = []y.Iterator{topTables[0].NewIterator(false)} } // Next level has level>=1 and we can use ConcatIterator as key ranges do not overlap. - iters = append(iters, table.NewConcatIterator(botTables, false)) + var valid []*table.Table + for _, table := range botTables { + if len(cd.dropPrefix) > 0 && + bytes.HasPrefix(table.Smallest(), cd.dropPrefix) && + bytes.HasPrefix(table.Biggest(), cd.dropPrefix) { + // All the keys in this table have the dropPrefix. So, this table does not need to be + // in the iterator and can be dropped immediately. + continue + } + valid = append(valid, table) + } + iters = append(iters, table.NewConcatIterator(valid, false)) it := y.NewMergeIterator(iters, false) defer it.Close() // Important to close the iterator to do ref counting. @@ -417,6 +495,13 @@ func (s *levelsController) compactBuildTables( builder := table.NewTableBuilder() var numKeys, numSkips uint64 for ; it.Valid(); it.Next() { + // See if we need to skip the prefix. + if len(cd.dropPrefix) > 0 && bytes.HasPrefix(it.Key(), cd.dropPrefix) { + numSkips++ + updateStats(it.Value()) + continue + } + // See if we need to skip this key. if len(skipKey) > 0 { if y.SameKey(it.Key(), skipKey) { @@ -441,7 +526,8 @@ func (s *levelsController) compactBuildTables( vs := it.Value() version := y.ParseTs(it.Key()) - if version <= discardTs { + // Do not discard entries inserted by merge operator. These entries will be discarded once they're merged + if version <= discardTs && vs.Meta&bitMergeEntry == 0 { // Keep track of the number of versions encountered for this key. Only consider the // versions which are below the minReadTs, otherwise, we might end up discarding the // only valid version for a running transaction. @@ -474,8 +560,8 @@ func (s *levelsController) compactBuildTables( } // It was true that it.Valid() at least once in the loop above, which means we // called Add() at least once, and builder is not Empty(). - cd.elog.LazyPrintf("Added %d keys. Skipped %d keys.", numKeys, numSkips) - cd.elog.LazyPrintf("LOG Compact. Iteration took: %v\n", time.Since(timeStart)) + s.kv.opt.Debugf("LOG Compact. Added %d keys. Skipped %d keys. Iteration took: %v", + numKeys, numSkips, time.Since(timeStart)) if !builder.Empty() { numBuilds++ fileID := s.reserveFileID() @@ -533,8 +619,8 @@ func (s *levelsController) compactBuildTables( sort.Slice(newTables, func(i, j int) bool { return y.CompareKeys(newTables[i].Biggest(), newTables[j].Biggest()) < 0 }) - s.kv.vlog.updateGCStats(discardStats) - cd.elog.LazyPrintf("Discard stats: %v", discardStats) + s.kv.vlog.updateDiscardStats(discardStats) + s.kv.opt.Debugf("Discard stats: %v", discardStats) return newTables, func() error { return decrRefs(newTables) }, nil } @@ -566,6 +652,8 @@ type compactDef struct { nextRange keyRange thisSize int64 + + dropPrefix []byte } func (cd *compactDef) lockLevels() { @@ -689,7 +777,7 @@ func (s *levelsController) runCompactDef(l int, cd compactDef) (err error) { // See comment earlier in this function about the ordering of these ops, and the order in which // we access levels when reading. - if err := nextLevel.replaceTables(newTables); err != nil { + if err := nextLevel.replaceTables(cd.bot, newTables); err != nil { return err } if err := thisLevel.deleteTables(cd.top); err != nil { @@ -699,8 +787,9 @@ func (s *levelsController) runCompactDef(l int, cd compactDef) (err error) { // Note: For level 0, while doCompact is running, it is possible that new tables are added. // However, the tables are added only to the end, so it is ok to just delete the first table. - cd.elog.LazyPrintf("LOG Compact %d->%d, del %d tables, add %d tables, took %v\n", - l, l+1, len(cd.top)+len(cd.bot), len(newTables), time.Since(timeStart)) + s.kv.opt.Infof("LOG Compact %d->%d, del %d tables, add %d tables, took %v\n", + thisLevel.level, nextLevel.level, len(cd.top)+len(cd.bot), + len(newTables), time.Since(timeStart)) return nil } @@ -712,41 +801,40 @@ func (s *levelsController) doCompact(p compactionPriority) error { y.AssertTrue(l+1 < s.kv.opt.MaxLevels) // Sanity check. cd := compactDef{ - elog: trace.New(fmt.Sprintf("Badger.L%d", l), "Compact"), - thisLevel: s.levels[l], - nextLevel: s.levels[l+1], + elog: trace.New(fmt.Sprintf("Badger.L%d", l), "Compact"), + thisLevel: s.levels[l], + nextLevel: s.levels[l+1], + dropPrefix: p.dropPrefix, } cd.elog.SetMaxEvents(100) defer cd.elog.Finish() - cd.elog.LazyPrintf("Got compaction priority: %+v", p) + s.kv.opt.Infof("Got compaction priority: %+v", p) // While picking tables to be compacted, both levels' tables are expected to // remain unchanged. if l == 0 { if !s.fillTablesL0(&cd) { - cd.elog.LazyPrintf("fillTables failed for level: %d\n", l) return errFillTables } } else { if !s.fillTables(&cd) { - cd.elog.LazyPrintf("fillTables failed for level: %d\n", l) return errFillTables } } defer s.cstatus.delete(cd) // Remove the ranges from compaction status. - cd.elog.LazyPrintf("Running for level: %d\n", cd.thisLevel.level) + s.kv.opt.Infof("Running for level: %d\n", cd.thisLevel.level) s.cstatus.toLog(cd.elog) if err := s.runCompactDef(l, cd); err != nil { // This compaction couldn't be done successfully. - cd.elog.LazyPrintf("\tLOG Compact FAILED with error: %+v: %+v", err, cd) + s.kv.opt.Warningf("LOG Compact FAILED with error: %+v: %+v", err, cd) return err } s.cstatus.toLog(cd.elog) - cd.elog.LazyPrintf("Compaction for level: %d DONE", cd.thisLevel.level) + s.kv.opt.Infof("Compaction for level: %d DONE", cd.thisLevel.level) return nil } @@ -858,23 +946,35 @@ func (s *levelsController) appendIterators( // TableInfo represents the information about a table. type TableInfo struct { - ID uint64 - Level int - Left []byte - Right []byte + ID uint64 + Level int + Left []byte + Right []byte + KeyCount uint64 // Number of keys in the table } -func (s *levelsController) getTableInfo() (result []TableInfo) { +func (s *levelsController) getTableInfo(withKeysCount bool) (result []TableInfo) { for _, l := range s.levels { + l.RLock() for _, t := range l.tables { + var count uint64 + if withKeysCount { + it := t.NewIterator(false) + for it.Rewind(); it.Valid(); it.Next() { + count++ + } + } + info := TableInfo{ - ID: t.ID(), - Level: l.level, - Left: t.Smallest(), - Right: t.Biggest(), + ID: t.ID(), + Level: l.level, + Left: t.Smallest(), + Right: t.Biggest(), + KeyCount: count, } result = append(result, info) } + l.RUnlock() } sort.Slice(result, func(i, j int) bool { if result[i].Level != result[j].Level { diff --git a/vendor/github.com/dgraph-io/badger/logger.go b/vendor/github.com/dgraph-io/badger/logger.go index c8e64c21..3a9b8a33 100644 --- a/vendor/github.com/dgraph-io/badger/logger.go +++ b/vendor/github.com/dgraph-io/badger/logger.go @@ -24,8 +24,9 @@ import ( // Logger is implemented by any logging system that is used for standard logs. type Logger interface { Errorf(string, ...interface{}) - Infof(string, ...interface{}) Warningf(string, ...interface{}) + Infof(string, ...interface{}) + Debugf(string, ...interface{}) } // Errorf logs an ERROR log message to the logger specified in opts or to the @@ -53,6 +54,14 @@ func (opt *Options) Warningf(format string, v ...interface{}) { opt.Logger.Warningf(format, v...) } +// Debugf logs a DEBUG message to the logger specified in opts. +func (opt *Options) Debugf(format string, v ...interface{}) { + if opt.Logger == nil { + return + } + opt.Logger.Debugf(format, v...) +} + type defaultLog struct { *log.Logger } @@ -63,10 +72,14 @@ func (l *defaultLog) Errorf(f string, v ...interface{}) { l.Printf("ERROR: "+f, v...) } +func (l *defaultLog) Warningf(f string, v ...interface{}) { + l.Printf("WARNING: "+f, v...) +} + func (l *defaultLog) Infof(f string, v ...interface{}) { l.Printf("INFO: "+f, v...) } -func (l *defaultLog) Warningf(f string, v ...interface{}) { - l.Printf("WARNING: "+f, v...) +func (l *defaultLog) Debugf(f string, v ...interface{}) { + l.Printf("DEBUG: "+f, v...) } diff --git a/vendor/github.com/dgraph-io/badger/logger_test.go b/vendor/github.com/dgraph-io/badger/logger_test.go deleted file mode 100644 index 321c379a..00000000 --- a/vendor/github.com/dgraph-io/badger/logger_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -type mockLogger struct { - output string -} - -func (l *mockLogger) Errorf(f string, v ...interface{}) { - l.output = fmt.Sprintf("ERROR: "+f, v...) -} - -func (l *mockLogger) Infof(f string, v ...interface{}) { - l.output = fmt.Sprintf("INFO: "+f, v...) -} - -func (l *mockLogger) Warningf(f string, v ...interface{}) { - l.output = fmt.Sprintf("WARNING: "+f, v...) -} - -// Test that the DB-specific log is used instead of the global log. -func TestDbLog(t *testing.T) { - l := &mockLogger{} - opt := Options{Logger: l} - - opt.Errorf("test") - require.Equal(t, "ERROR: test", l.output) - opt.Infof("test") - require.Equal(t, "INFO: test", l.output) - opt.Warningf("test") - require.Equal(t, "WARNING: test", l.output) -} - -// Test that the global logger is used when no logger is specified in Options. -func TestNoDbLog(t *testing.T) { - l := &mockLogger{} - opt := Options{} - opt.Logger = l - - opt.Errorf("test") - require.Equal(t, "ERROR: test", l.output) - opt.Infof("test") - require.Equal(t, "INFO: test", l.output) - opt.Warningf("test") - require.Equal(t, "WARNING: test", l.output) -} diff --git a/vendor/github.com/dgraph-io/badger/managed_db_test.go b/vendor/github.com/dgraph-io/badger/managed_db_test.go deleted file mode 100644 index dbb84c82..00000000 --- a/vendor/github.com/dgraph-io/badger/managed_db_test.go +++ /dev/null @@ -1,353 +0,0 @@ -package badger - -import ( - "io/ioutil" - "math" - "math/rand" - "os" - "runtime" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/stretchr/testify/require" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -func val(large bool) []byte { - var buf []byte - if large { - buf = make([]byte, 8192) - } else { - buf = make([]byte, 16) - } - rand.Read(buf) - return buf -} - -func numKeys(db *DB) int { - var count int - err := db.View(func(txn *Txn) error { - itr := txn.NewIterator(DefaultIteratorOptions) - defer itr.Close() - - for itr.Rewind(); itr.Valid(); itr.Next() { - count++ - } - return nil - }) - y.Check(err) - return count -} - -func numKeysManaged(db *DB, readTs uint64) int { - txn := db.NewTransactionAt(readTs, false) - defer txn.Discard() - - itr := txn.NewIterator(DefaultIteratorOptions) - defer itr.Close() - - var count int - for itr.Rewind(); itr.Valid(); itr.Next() { - count++ - } - return count -} - -func TestDropAllManaged(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - opts.managedTxns = true - opts.ValueLogFileSize = 5 << 20 - db, err := Open(opts) - require.NoError(t, err) - - N := uint64(10000) - populate := func(db *DB, start uint64) { - var wg sync.WaitGroup - for i := start; i < start+N; i++ { - wg.Add(1) - txn := db.NewTransactionAt(math.MaxUint64, true) - require.NoError(t, txn.Set([]byte(key("key", int(i))), val(true))) - require.NoError(t, txn.CommitAt(uint64(i), func(err error) { - require.NoError(t, err) - wg.Done() - })) - } - wg.Wait() - } - - populate(db, N) - require.Equal(t, int(N), numKeysManaged(db, math.MaxUint64)) - - require.NoError(t, db.DropAll()) - require.NoError(t, db.DropAll()) // Just call it twice, for fun. - require.Equal(t, 0, numKeysManaged(db, math.MaxUint64)) - - // Check that we can still write to mdb, and using lower timestamps. - populate(db, 1) - require.Equal(t, int(N), numKeysManaged(db, math.MaxUint64)) - db.Close() - - // Ensure that value log is correctly replayed, that we are preserving badgerHead. - opts.managedTxns = true - db2, err := Open(opts) - require.NoError(t, err) - require.Equal(t, int(N), numKeysManaged(db2, math.MaxUint64)) - db2.Close() -} - -func TestDropAll(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - opts.ValueLogFileSize = 5 << 20 - db, err := Open(opts) - require.NoError(t, err) - - N := uint64(10000) - populate := func(db *DB) { - writer := db.NewWriteBatch() - for i := uint64(0); i < N; i++ { - require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0)) - } - require.NoError(t, writer.Flush()) - } - - populate(db) - require.Equal(t, int(N), numKeys(db)) - - require.NoError(t, db.DropAll()) - require.Equal(t, 0, numKeys(db)) - - // Check that we can still write to mdb, and using lower timestamps. - populate(db) - require.Equal(t, int(N), numKeys(db)) - db.Close() - - // Ensure that value log is correctly replayed. - db2, err := Open(opts) - require.NoError(t, err) - require.Equal(t, int(N), numKeys(db2)) - db2.Close() -} - -func TestDropAllTwice(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - opts.ValueLogFileSize = 5 << 20 - db, err := Open(opts) - require.NoError(t, err) - - N := uint64(10000) - populate := func(db *DB) { - writer := db.NewWriteBatch() - for i := uint64(0); i < N; i++ { - require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0)) - } - require.NoError(t, writer.Flush()) - } - - populate(db) - require.Equal(t, int(N), numKeys(db)) - - require.NoError(t, db.DropAll()) - require.Equal(t, 0, numKeys(db)) - - // Call DropAll again. - require.NoError(t, db.DropAll()) -} - -func TestDropAllWithPendingTxn(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - opts.ValueLogFileSize = 5 << 20 - db, err := Open(opts) - require.NoError(t, err) - - N := uint64(10000) - populate := func(db *DB) { - writer := db.NewWriteBatch() - for i := uint64(0); i < N; i++ { - require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0)) - } - require.NoError(t, writer.Flush()) - } - - populate(db) - require.Equal(t, int(N), numKeys(db)) - - txn := db.NewTransaction(true) - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - itr := txn.NewIterator(DefaultIteratorOptions) - defer itr.Close() - - var keys []string - for { - var count int - for itr.Rewind(); itr.Valid(); itr.Next() { - count++ - item := itr.Item() - keys = append(keys, string(item.KeyCopy(nil))) - _, err := item.ValueCopy(nil) - if err != nil { - t.Logf("Got error during value copy: %v", err) - return - } - } - t.Logf("Got number of keys: %d\n", count) - for _, key := range keys { - item, err := txn.Get([]byte(key)) - if err != nil { - t.Logf("Got error during key lookup: %v", err) - return - } - if _, err := item.ValueCopy(nil); err != nil { - t.Logf("Got error during second value copy: %v", err) - return - } - } - } - }() - // Do not cancel txn. - - go func() { - time.Sleep(2 * time.Second) - require.NoError(t, db.DropAll()) - }() - wg.Wait() -} - -func TestDropReadOnly(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - opts.ValueLogFileSize = 5 << 20 - db, err := Open(opts) - require.NoError(t, err) - N := uint64(1000) - populate := func(db *DB) { - writer := db.NewWriteBatch() - for i := uint64(0); i < N; i++ { - require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0)) - } - require.NoError(t, writer.Flush()) - } - - populate(db) - require.Equal(t, int(N), numKeys(db)) - require.NoError(t, db.Close()) - - opts.ReadOnly = true - db2, err := Open(opts) - // acquireDirectoryLock returns ErrWindowsNotSupported on Windows. It can be ignored safely. - if runtime.GOOS == "windows" { - require.Equal(t, err, ErrWindowsNotSupported) - } else { - require.NoError(t, err) - } - require.Panics(t, func() { db2.DropAll() }) -} - -func TestWriteAfterClose(t *testing.T) { - dir, err := ioutil.TempDir(".", "badger-test") - require.NoError(t, err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - opts.ValueLogFileSize = 5 << 20 - db, err := Open(opts) - require.NoError(t, err) - N := uint64(1000) - populate := func(db *DB) { - writer := db.NewWriteBatch() - for i := uint64(0); i < N; i++ { - require.NoError(t, writer.Set([]byte(key("key", int(i))), val(true), 0)) - } - require.NoError(t, writer.Flush()) - } - - populate(db) - require.Equal(t, int(N), numKeys(db)) - require.NoError(t, db.Close()) - err = db.Update(func(txn *Txn) error { - return txn.Set([]byte("a"), []byte("b")) - }) - require.Equal(t, ErrBlockedWrites, err) -} - -func TestDropAllRace(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opts := getTestOptions(dir) - opts.managedTxns = true - db, err := Open(opts) - require.NoError(t, err) - - N := 10000 - // Start a goroutine to keep trying to write to DB while DropAll happens. - closer := y.NewCloser(1) - go func() { - defer closer.Done() - ticker := time.NewTicker(time.Millisecond) - defer ticker.Stop() - - i := N + 1 // Writes would happen above N. - var errors int32 - for { - select { - case <-ticker.C: - i++ - txn := db.NewTransactionAt(math.MaxUint64, true) - require.NoError(t, txn.Set([]byte(key("key", i)), val(false))) - if err := txn.CommitAt(uint64(i), func(err error) { - if err != nil { - atomic.AddInt32(&errors, 1) - } - }); err != nil { - atomic.AddInt32(&errors, 1) - } - case <-closer.HasBeenClosed(): - // The following causes a data race. - // t.Logf("i: %d. Number of (expected) write errors: %d.\n", i, errors) - return - } - } - }() - - var wg sync.WaitGroup - for i := 1; i <= N; i++ { - wg.Add(1) - txn := db.NewTransactionAt(math.MaxUint64, true) - require.NoError(t, txn.Set([]byte(key("key", i)), val(false))) - require.NoError(t, txn.CommitAt(uint64(i), func(err error) { - require.NoError(t, err) - wg.Done() - })) - } - wg.Wait() - - before := numKeysManaged(db, math.MaxUint64) - require.True(t, before > N) - - require.NoError(t, db.DropAll()) - closer.SignalAndWait() - - after := numKeysManaged(db, math.MaxUint64) - t.Logf("Before: %d. After dropall: %d\n", before, after) - require.True(t, after < before) - db.Close() -} diff --git a/vendor/github.com/dgraph-io/badger/manifest.go b/vendor/github.com/dgraph-io/badger/manifest.go index 06a5b96b..34ce1217 100644 --- a/vendor/github.com/dgraph-io/badger/manifest.go +++ b/vendor/github.com/dgraph-io/badger/manifest.go @@ -27,9 +27,9 @@ import ( "path/filepath" "sync" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" ) // Manifest represents the contents of the MANIFEST file in a Badger store. diff --git a/vendor/github.com/dgraph-io/badger/manifest_test.go b/vendor/github.com/dgraph-io/badger/manifest_test.go deleted file mode 100644 index 7bf55a68..00000000 --- a/vendor/github.com/dgraph-io/badger/manifest_test.go +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "fmt" - "io/ioutil" - "math/rand" - "os" - "path/filepath" - "sort" - "testing" - - "gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace" - - "github.com/stretchr/testify/require" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -func TestManifestBasic(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opt := getTestOptions(dir) - { - kv, err := Open(opt) - require.NoError(t, err) - n := 5000 - for i := 0; i < n; i++ { - if (i % 10000) == 0 { - fmt.Printf("Putting i=%d\n", i) - } - k := []byte(fmt.Sprintf("%16x", rand.Int63())) - txnSet(t, kv, k, k, 0x00) - } - txnSet(t, kv, []byte("testkey"), []byte("testval"), 0x05) - kv.validate() - require.NoError(t, kv.Close()) - } - - kv, err := Open(opt) - require.NoError(t, err) - - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get([]byte("testkey")) - require.NoError(t, err) - require.EqualValues(t, "testval", string(getItemValue(t, item))) - require.EqualValues(t, byte(0x05), item.UserMeta()) - return nil - })) - require.NoError(t, kv.Close()) -} - -func helpTestManifestFileCorruption(t *testing.T, off int64, errorContent string) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opt := getTestOptions(dir) - { - kv, err := Open(opt) - require.NoError(t, err) - require.NoError(t, kv.Close()) - } - fp, err := os.OpenFile(filepath.Join(dir, ManifestFilename), os.O_RDWR, 0) - require.NoError(t, err) - // Mess with magic value or version to force error - _, err = fp.WriteAt([]byte{'X'}, off) - require.NoError(t, err) - require.NoError(t, fp.Close()) - kv, err := Open(opt) - defer func() { - if kv != nil { - kv.Close() - } - }() - require.Error(t, err) - require.Contains(t, err.Error(), errorContent) -} - -func TestManifestMagic(t *testing.T) { - helpTestManifestFileCorruption(t, 3, "bad magic") -} - -func TestManifestVersion(t *testing.T) { - helpTestManifestFileCorruption(t, 4, "unsupported version") -} - -func key(prefix string, i int) string { - return prefix + fmt.Sprintf("%04d", i) -} - -func buildTestTable(t *testing.T, prefix string, n int) *os.File { - y.AssertTrue(n <= 10000) - keyValues := make([][]string, n) - for i := 0; i < n; i++ { - k := key(prefix, i) - v := fmt.Sprintf("%d", i) - keyValues[i] = []string{k, v} - } - return buildTable(t, keyValues) -} - -// TODO - Move these to somewhere where table package can also use it. -// keyValues is n by 2 where n is number of pairs. -func buildTable(t *testing.T, keyValues [][]string) *os.File { - b := table.NewTableBuilder() - defer b.Close() - // TODO: Add test for file garbage collection here. No files should be left after the tests here. - - filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63()) - f, err := y.OpenSyncedFile(filename, true) - if t != nil { - require.NoError(t, err) - } else { - y.Check(err) - } - - sort.Slice(keyValues, func(i, j int) bool { - return keyValues[i][0] < keyValues[j][0] - }) - for _, kv := range keyValues { - y.AssertTrue(len(kv) == 2) - err := b.Add(y.KeyWithTs([]byte(kv[0]), 10), y.ValueStruct{ - Value: []byte(kv[1]), - Meta: 'A', - UserMeta: 0, - }) - if t != nil { - require.NoError(t, err) - } else { - y.Check(err) - } - } - f.Write(b.Finish()) - f.Close() - f, _ = y.OpenSyncedFile(filename, true) - return f -} - -func TestOverlappingKeyRangeError(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - kv, err := Open(opt) - require.NoError(t, err) - - lh0 := newLevelHandler(kv, 0) - lh1 := newLevelHandler(kv, 1) - f := buildTestTable(t, "k", 2) - t1, err := table.OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer t1.DecrRef() - - done := lh0.tryAddLevel0Table(t1) - require.Equal(t, true, done) - - cd := compactDef{ - thisLevel: lh0, - nextLevel: lh1, - elog: trace.New("Badger", "Compact"), - } - - manifest := createManifest() - lc, err := newLevelsController(kv, &manifest) - require.NoError(t, err) - done = lc.fillTablesL0(&cd) - require.Equal(t, true, done) - lc.runCompactDef(0, cd) - - f = buildTestTable(t, "l", 2) - t2, err := table.OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer t2.DecrRef() - done = lh0.tryAddLevel0Table(t2) - require.Equal(t, true, done) - - cd = compactDef{ - thisLevel: lh0, - nextLevel: lh1, - elog: trace.New("Badger", "Compact"), - } - lc.fillTablesL0(&cd) - lc.runCompactDef(0, cd) -} - -func TestManifestRewrite(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - deletionsThreshold := 10 - mf, m, err := helpOpenOrCreateManifestFile(dir, false, deletionsThreshold) - defer func() { - if mf != nil { - mf.close() - } - }() - require.NoError(t, err) - require.Equal(t, 0, m.Creations) - require.Equal(t, 0, m.Deletions) - - err = mf.addChanges([]*pb.ManifestChange{ - newCreateChange(0, 0, nil), - }) - require.NoError(t, err) - - for i := uint64(0); i < uint64(deletionsThreshold*3); i++ { - ch := []*pb.ManifestChange{ - newCreateChange(i+1, 0, nil), - newDeleteChange(i), - } - err := mf.addChanges(ch) - require.NoError(t, err) - } - err = mf.close() - require.NoError(t, err) - mf = nil - mf, m, err = helpOpenOrCreateManifestFile(dir, false, deletionsThreshold) - require.NoError(t, err) - require.Equal(t, map[uint64]TableManifest{ - uint64(deletionsThreshold * 3): {Level: 0, Checksum: []byte{}}, - }, m.Tables) -} diff --git a/vendor/github.com/dgraph-io/badger/merge.go b/vendor/github.com/dgraph-io/badger/merge.go index af35e1f4..7bca447a 100644 --- a/vendor/github.com/dgraph-io/badger/merge.go +++ b/vendor/github.com/dgraph-io/badger/merge.go @@ -20,8 +20,8 @@ import ( "sync" "time" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" ) // MergeOperator represents a Badger merge operator. @@ -63,11 +63,11 @@ var errNoMerge = errors.New("No need for merge") func (op *MergeOperator) iterateAndMerge(txn *Txn) (val []byte, err error) { opt := DefaultIteratorOptions opt.AllVersions = true - it := txn.NewIterator(opt) + it := txn.NewKeyIterator(op.key, opt) defer it.Close() var numVersions int - for it.Rewind(); it.ValidForPrefix(op.key); it.Next() { + for it.Rewind(); it.Valid(); it.Next() { item := it.Item() numVersions++ if numVersions == 1 { @@ -107,8 +107,8 @@ func (op *MergeOperator) compact() error { if err != nil { return err } - - // Write value back to db + // Write value back to the DB. It is important that we do not set the bitMergeEntry bit + // here. When compaction happens, all the older merged entries will be removed. return txn.SetWithDiscard(op.key, val, 0) }) @@ -144,7 +144,7 @@ func (op *MergeOperator) runCompactions(dur time.Duration) { // routine into the values that were recorded by previous invocations to Add(). func (op *MergeOperator) Add(val []byte) error { return op.db.Update(func(txn *Txn) error { - return txn.Set(op.key, val) + return txn.setMergeEntry(op.key, val) }) } diff --git a/vendor/github.com/dgraph-io/badger/options.go b/vendor/github.com/dgraph-io/badger/options.go index cdfddbe7..560b65b2 100644 --- a/vendor/github.com/dgraph-io/badger/options.go +++ b/vendor/github.com/dgraph-io/badger/options.go @@ -17,7 +17,7 @@ package badger import ( - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" + "github.com/dgraph-io/badger/options" ) // NOTE: Keep the comments in the following to 75 chars width, so they @@ -53,6 +53,18 @@ type Options struct { // How many versions to keep per key. NumVersionsToKeep int + // Open the DB as read-only. With this set, multiple processes can + // open the same Badger DB. Note: if the DB being opened had crashed + // before and has vlog data to be replayed, ReadOnly will cause Open + // to fail with an appropriate message. + ReadOnly bool + + // Truncate value log to delete corrupt data, if any. Would not truncate if ReadOnly is set. + Truncate bool + + // DB-specific logger which will override the global logger. + Logger Logger + // 3. Flags that user might want to review // ---------------------------------------- // The following affect all levels of LSM tree. @@ -89,6 +101,13 @@ type Options struct { // efficient when the DB is opened later. CompactL0OnClose bool + // After this many number of value log file rotates, there would be a force flushing of memtable + // to disk. This is useful in write loads with fewer keys and larger values. This work load + // would fill up the value logs quickly, while not filling up the Memtables. Thus, on a crash + // and restart, the value log head could cause the replay of a good number of value log files + // which can slow things on start. + LogRotatesToFlush int32 + // Transaction start and commit timestamps are managed by end-user. // This is only useful for databases built on top of Badger (like Dgraph). // Not recommended for most users. @@ -99,17 +118,6 @@ type Options struct { maxBatchCount int64 // max entries in batch maxBatchSize int64 // max batch size in bytes - // Open the DB as read-only. With this set, multiple processes can - // open the same Badger DB. Note: if the DB being opened had crashed - // before and has vlog data to be replayed, ReadOnly will cause Open - // to fail with an appropriate message. - ReadOnly bool - - // Truncate value log to delete corrupt data, if any. Would not truncate if ReadOnly is set. - Truncate bool - - // DB-specific logger which will override the global logger. - Logger Logger } // DefaultOptions sets a list of recommended options for good performance. @@ -117,7 +125,7 @@ type Options struct { var DefaultOptions = Options{ LevelOneSize: 256 << 20, LevelSizeMultiplier: 10, - TableLoadingMode: options.LoadToRAM, + TableLoadingMode: options.MemoryMap, ValueLogLoadingMode: options.MemoryMap, // table.MemoryMap to mmap() the tables. // table.Nothing to not preload the tables. @@ -140,6 +148,7 @@ var DefaultOptions = Options{ ValueThreshold: 32, Truncate: false, Logger: defaultLogger, + LogRotatesToFlush: 2, } // LSMOnlyOptions follows from DefaultOptions, but sets a higher ValueThreshold diff --git a/vendor/github.com/dgraph-io/badger/package.json b/vendor/github.com/dgraph-io/badger/package.json deleted file mode 100644 index 86883fbb..00000000 --- a/vendor/github.com/dgraph-io/badger/package.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "author": "dgraph-io", - "bugs": { - "url": "https://github.com/dgraph-io/badger" - }, - "gx": { - "dvcsimport": "github.com/dgraph-io/badger" - }, - "gxDependencies": [ - { - "author": "whyrusleeping", - "hash": "QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy", - "name": "errors", - "version": "0.0.1" - }, - { - "author": "magik6k", - "hash": "Qmbvv2urkn5Wtwws4yzjE85qRjB293EodchZofJsrTRuvN", - "name": "go-lz4", - "version": "1.0.0" - }, - { - "author": "kubuxu", - "hash": "QmWaLViWQF8jgyoLLqqcSrnp6dJpHESiJfzor1vrfDyTZf", - "name": "bbloom", - "version": "0.1.2" - }, - { - "author": "kubuxu", - "hash": "QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu", - "name": "sys", - "version": "0.2.0" - }, - { - "author": "whyrusleeping", - "hash": "QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn", - "name": "go-net", - "version": "0.2.0" - }, - { - "author": "magik6k", - "hash": "QmRFFHk2jw9tgjxv12bCuuTnSbVXxEvYQkuNCLMEv9eUwP", - "name": "go-farm", - "version": "1.0.0" - }, - { - "author": "magik6k", - "hash": "QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy", - "name": "go-humanize", - "version": "1.0.1" - }, - { - "author": "GoGo", - "hash": "QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB", - "name": "gogo-protobuf", - "version": "1.2.1" - }, - { - "author": "magik6k", - "hash": "QmXj63M2w2Pq7mnBpcrs7Va8prmfhvfMUNqVhJ9TgjiMbT", - "name": "cobra", - "version": "0.0.1" - } - ], - "gxVersion": "0.10.0", - "language": "go", - "license": "Apache 2.0", - "name": "badger", - "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", - "version": "2.11.4" -} - diff --git a/vendor/github.com/dgraph-io/badger/pb/gen.sh b/vendor/github.com/dgraph-io/badger/pb/gen.sh old mode 100644 new mode 100755 index bb446f26..49b44ff4 --- a/vendor/github.com/dgraph-io/badger/pb/gen.sh +++ b/vendor/github.com/dgraph-io/badger/pb/gen.sh @@ -4,4 +4,4 @@ protos=${GOPATH-$HOME/go}/src/github.com/dgraph-io/badger/pb pushd $protos > /dev/null -protoc --gogofaster_out=. -I=. pb.proto +protoc --gofast_out=plugins=grpc:. -I=. pb.proto diff --git a/vendor/github.com/dgraph-io/badger/pb/pb.pb.go b/vendor/github.com/dgraph-io/badger/pb/pb.pb.go index 147c581c..f9a2c6ee 100644 --- a/vendor/github.com/dgraph-io/badger/pb/pb.pb.go +++ b/vendor/github.com/dgraph-io/badger/pb/pb.pb.go @@ -5,7 +5,7 @@ package pb import ( fmt "fmt" - proto "gx/ipfs/QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB/gogo-protobuf/proto" + proto "github.com/golang/protobuf/proto" io "io" math "math" ) @@ -19,7 +19,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type ManifestChange_Operation int32 @@ -53,6 +53,11 @@ type KV struct { Version uint64 `protobuf:"varint,4,opt,name=version,proto3" json:"version,omitempty"` ExpiresAt uint64 `protobuf:"varint,5,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` Meta []byte `protobuf:"bytes,6,opt,name=meta,proto3" json:"meta,omitempty"` + // Stream id is used to identify which stream the KV came from. + StreamId uint32 `protobuf:"varint,10,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *KV) Reset() { *m = KV{} } @@ -130,8 +135,18 @@ func (m *KV) GetMeta() []byte { return nil } +func (m *KV) GetStreamId() uint32 { + if m != nil { + return m.StreamId + } + return 0 +} + type KVList struct { - Kv []*KV `protobuf:"bytes,1,rep,name=kv,proto3" json:"kv,omitempty"` + Kv []*KV `protobuf:"bytes,1,rep,name=kv,proto3" json:"kv,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *KVList) Reset() { *m = KVList{} } @@ -176,7 +191,10 @@ func (m *KVList) GetKv() []*KV { type ManifestChangeSet struct { // A set of changes that are applied atomically. - Changes []*ManifestChange `protobuf:"bytes,1,rep,name=changes,proto3" json:"changes,omitempty"` + Changes []*ManifestChange `protobuf:"bytes,1,rep,name=changes,proto3" json:"changes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ManifestChangeSet) Reset() { *m = ManifestChangeSet{} } @@ -220,10 +238,13 @@ func (m *ManifestChangeSet) GetChanges() []*ManifestChange { } type ManifestChange struct { - Id uint64 `protobuf:"varint,1,opt,name=Id,proto3" json:"Id,omitempty"` - Op ManifestChange_Operation `protobuf:"varint,2,opt,name=Op,proto3,enum=pb.ManifestChange_Operation" json:"Op,omitempty"` - Level uint32 `protobuf:"varint,3,opt,name=Level,proto3" json:"Level,omitempty"` - Checksum []byte `protobuf:"bytes,4,opt,name=Checksum,proto3" json:"Checksum,omitempty"` + Id uint64 `protobuf:"varint,1,opt,name=Id,proto3" json:"Id,omitempty"` + Op ManifestChange_Operation `protobuf:"varint,2,opt,name=Op,proto3,enum=pb.ManifestChange_Operation" json:"Op,omitempty"` + Level uint32 `protobuf:"varint,3,opt,name=Level,proto3" json:"Level,omitempty"` + Checksum []byte `protobuf:"bytes,4,opt,name=Checksum,proto3" json:"Checksum,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ManifestChange) Reset() { *m = ManifestChange{} } @@ -298,29 +319,30 @@ func init() { func init() { proto.RegisterFile("pb.proto", fileDescriptor_f80abaa17e25ccc8) } var fileDescriptor_f80abaa17e25ccc8 = []byte{ - // 349 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x91, 0x4b, 0x4e, 0xc2, 0x40, - 0x18, 0xc7, 0x3b, 0x43, 0x29, 0xf0, 0x89, 0x04, 0x27, 0xc6, 0x4c, 0x7c, 0x34, 0x4d, 0xdd, 0xb0, - 0x20, 0x5d, 0xe0, 0x09, 0x10, 0xbb, 0x20, 0x40, 0x48, 0x46, 0xc2, 0x96, 0xb4, 0x30, 0x4a, 0x53, - 0x68, 0x27, 0xed, 0xd0, 0xe8, 0x2d, 0x5c, 0x79, 0x03, 0xef, 0xe2, 0x92, 0xa5, 0x4b, 0x03, 0x17, - 0x31, 0x1d, 0x1e, 0x09, 0x71, 0xf7, 0x7f, 0xcc, 0xf7, 0x5f, 0xfc, 0x06, 0xca, 0xc2, 0x77, 0x44, - 0x12, 0xcb, 0x98, 0x60, 0xe1, 0xdb, 0x9f, 0x08, 0x70, 0x6f, 0x4c, 0xea, 0x50, 0x08, 0xf9, 0x3b, - 0x45, 0x16, 0x6a, 0x54, 0x59, 0x2e, 0xc9, 0x25, 0x14, 0x33, 0x6f, 0xb1, 0xe2, 0x14, 0xab, 0x6c, - 0x67, 0xc8, 0x0d, 0x54, 0x56, 0x29, 0x4f, 0x26, 0x4b, 0x2e, 0x3d, 0x5a, 0x50, 0x4d, 0x39, 0x0f, - 0x06, 0x5c, 0x7a, 0x84, 0x42, 0x29, 0xe3, 0x49, 0x1a, 0xc4, 0x11, 0xd5, 0x2d, 0xd4, 0xd0, 0xd9, - 0xc1, 0x92, 0x3b, 0x00, 0xfe, 0x26, 0x82, 0x84, 0xa7, 0x13, 0x4f, 0xd2, 0xa2, 0x2a, 0x2b, 0xfb, - 0xa4, 0x2d, 0x09, 0x01, 0x5d, 0x0d, 0x1a, 0x6a, 0x50, 0x69, 0xdb, 0x02, 0xa3, 0x37, 0xee, 0x07, - 0xa9, 0x24, 0x57, 0x80, 0xc3, 0x8c, 0x22, 0xab, 0xd0, 0x38, 0x6b, 0x19, 0x8e, 0xf0, 0x9d, 0xde, - 0x98, 0xe1, 0x30, 0xb3, 0xdb, 0x70, 0x31, 0xf0, 0xa2, 0xe0, 0x85, 0xa7, 0xb2, 0x33, 0xf7, 0xa2, - 0x57, 0xfe, 0xcc, 0x25, 0x69, 0x42, 0x69, 0xaa, 0x4c, 0xba, 0xbf, 0x20, 0xf9, 0xc5, 0xe9, 0x3b, - 0x76, 0x78, 0x62, 0x7f, 0x21, 0xa8, 0x9d, 0x76, 0xa4, 0x06, 0xb8, 0x3b, 0x53, 0x20, 0x74, 0x86, - 0xbb, 0x33, 0xd2, 0x04, 0x3c, 0x14, 0x0a, 0x42, 0xad, 0x75, 0xfb, 0x7f, 0xcb, 0x19, 0x0a, 0x9e, - 0x78, 0x32, 0x88, 0x23, 0x86, 0x87, 0x22, 0xa7, 0xd6, 0xe7, 0x19, 0x5f, 0x28, 0x36, 0xe7, 0x6c, - 0x67, 0xc8, 0x35, 0x94, 0x3b, 0x73, 0x3e, 0x0d, 0xd3, 0xd5, 0x52, 0x91, 0xa9, 0xb2, 0xa3, 0xb7, - 0xef, 0xa1, 0x72, 0x9c, 0x20, 0x00, 0x46, 0x87, 0xb9, 0xed, 0x91, 0x5b, 0xd7, 0x72, 0xfd, 0xe4, - 0xf6, 0xdd, 0x91, 0x5b, 0x47, 0x8f, 0xf4, 0x7b, 0x63, 0xa2, 0xf5, 0xc6, 0x44, 0xbf, 0x1b, 0x13, - 0x7d, 0x6c, 0x4d, 0x6d, 0xbd, 0x35, 0xb5, 0x9f, 0xad, 0xa9, 0xf9, 0x86, 0xfa, 0xca, 0x87, 0xbf, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xc9, 0xf9, 0xca, 0x14, 0xd6, 0x01, 0x00, 0x00, + // 365 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x91, 0x4f, 0x8a, 0xdb, 0x30, + 0x14, 0xc6, 0x47, 0x8a, 0xc7, 0xe3, 0xbc, 0xce, 0x04, 0x57, 0x94, 0x22, 0xfa, 0xc7, 0x18, 0x77, + 0xe3, 0xc5, 0xe0, 0xc5, 0xf4, 0x04, 0x69, 0xea, 0x45, 0x48, 0x42, 0x40, 0x0d, 0xd9, 0x06, 0x39, + 0x7e, 0x6d, 0x8c, 0x13, 0x5b, 0x58, 0x8a, 0x69, 0x6f, 0xd2, 0x0b, 0xf4, 0x04, 0xbd, 0x44, 0x97, + 0x3d, 0x42, 0x49, 0x2f, 0x52, 0xac, 0xfc, 0x81, 0xd0, 0xdd, 0xfb, 0xbe, 0xef, 0xbd, 0x4f, 0xf0, + 0x13, 0x78, 0x2a, 0x4b, 0x54, 0x53, 0x9b, 0x9a, 0x51, 0x95, 0x45, 0x3f, 0x09, 0xd0, 0xc9, 0x92, + 0xf9, 0xd0, 0x2b, 0xf1, 0x1b, 0x27, 0x21, 0x89, 0xef, 0x45, 0x37, 0xb2, 0x17, 0x70, 0xdb, 0xca, + 0xed, 0x1e, 0x39, 0xb5, 0xde, 0x51, 0xb0, 0xd7, 0xd0, 0xdf, 0x6b, 0x6c, 0x56, 0x3b, 0x34, 0x92, + 0xf7, 0x6c, 0xe2, 0x75, 0xc6, 0x0c, 0x8d, 0x64, 0x1c, 0xee, 0x5a, 0x6c, 0x74, 0x51, 0x57, 0xdc, + 0x09, 0x49, 0xec, 0x88, 0xb3, 0x64, 0x6f, 0x01, 0xf0, 0xab, 0x2a, 0x1a, 0xd4, 0x2b, 0x69, 0xf8, + 0xad, 0x0d, 0xfb, 0x27, 0x67, 0x68, 0x18, 0x03, 0xc7, 0x16, 0xba, 0xb6, 0xd0, 0xce, 0xdd, 0x4b, + 0xda, 0x34, 0x28, 0x77, 0xab, 0x22, 0xe7, 0x10, 0x92, 0xf8, 0x41, 0x78, 0x47, 0x63, 0x9c, 0x47, + 0x21, 0xb8, 0x93, 0xe5, 0xb4, 0xd0, 0x86, 0xbd, 0x04, 0x5a, 0xb6, 0x9c, 0x84, 0xbd, 0xf8, 0xd9, + 0x93, 0x9b, 0xa8, 0x2c, 0x99, 0x2c, 0x05, 0x2d, 0xdb, 0x68, 0x08, 0xcf, 0x67, 0xb2, 0x2a, 0x3e, + 0xa3, 0x36, 0xa3, 0x8d, 0xac, 0xbe, 0xe0, 0x27, 0x34, 0xec, 0x11, 0xee, 0xd6, 0x56, 0xe8, 0xd3, + 0x05, 0xeb, 0x2e, 0xae, 0xf7, 0xc4, 0x79, 0x25, 0xfa, 0x41, 0x60, 0x70, 0x9d, 0xb1, 0x01, 0xd0, + 0x71, 0x6e, 0x29, 0x39, 0x82, 0x8e, 0x73, 0xf6, 0x08, 0x74, 0xae, 0x2c, 0xa1, 0xc1, 0xd3, 0x9b, + 0xff, 0xbb, 0x92, 0xb9, 0xc2, 0x46, 0x9a, 0xa2, 0xae, 0x04, 0x9d, 0xab, 0x0e, 0xe9, 0x14, 0x5b, + 0xdc, 0x5a, 0x70, 0x0f, 0xe2, 0x28, 0xd8, 0x2b, 0xf0, 0x46, 0x1b, 0x5c, 0x97, 0x7a, 0xbf, 0xb3, + 0xd8, 0xee, 0xc5, 0x45, 0x47, 0xef, 0xa0, 0x7f, 0xa9, 0x60, 0x00, 0xee, 0x48, 0xa4, 0xc3, 0x45, + 0xea, 0xdf, 0x74, 0xf3, 0xc7, 0x74, 0x9a, 0x2e, 0x52, 0x9f, 0x7c, 0xf0, 0x7f, 0x1d, 0x02, 0xf2, + 0xfb, 0x10, 0x90, 0x3f, 0x87, 0x80, 0x7c, 0xff, 0x1b, 0xdc, 0x64, 0xae, 0xfd, 0xdf, 0xf7, 0xff, + 0x02, 0x00, 0x00, 0xff, 0xff, 0xeb, 0x28, 0x5d, 0xcf, 0xeb, 0x01, 0x00, 0x00, } func (m *KV) Marshal() (dAtA []byte, err error) { @@ -372,6 +394,14 @@ func (m *KV) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPb(dAtA, i, uint64(len(m.Meta))) i += copy(dAtA[i:], m.Meta) } + if m.StreamId != 0 { + dAtA[i] = 0x50 + i++ + i = encodeVarintPb(dAtA, i, uint64(m.StreamId)) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } return i, nil } @@ -402,6 +432,9 @@ func (m *KVList) MarshalTo(dAtA []byte) (int, error) { i += n } } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } return i, nil } @@ -432,6 +465,9 @@ func (m *ManifestChangeSet) MarshalTo(dAtA []byte) (int, error) { i += n } } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } return i, nil } @@ -471,6 +507,9 @@ func (m *ManifestChange) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPb(dAtA, i, uint64(len(m.Checksum))) i += copy(dAtA[i:], m.Checksum) } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } return i, nil } @@ -511,6 +550,12 @@ func (m *KV) Size() (n int) { if l > 0 { n += 1 + l + sovPb(uint64(l)) } + if m.StreamId != 0 { + n += 1 + sovPb(uint64(m.StreamId)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } return n } @@ -526,6 +571,9 @@ func (m *KVList) Size() (n int) { n += 1 + l + sovPb(uint64(l)) } } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } return n } @@ -541,6 +589,9 @@ func (m *ManifestChangeSet) Size() (n int) { n += 1 + l + sovPb(uint64(l)) } } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } return n } @@ -563,6 +614,9 @@ func (m *ManifestChange) Size() (n int) { if l > 0 { n += 1 + l + sovPb(uint64(l)) } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } return n } @@ -782,6 +836,25 @@ func (m *KV) Unmarshal(dAtA []byte) error { m.Meta = []byte{} } iNdEx = postIndex + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StreamId", wireType) + } + m.StreamId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StreamId |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPb(dAtA[iNdEx:]) @@ -797,6 +870,7 @@ func (m *KV) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -884,6 +958,7 @@ func (m *KVList) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -971,6 +1046,7 @@ func (m *ManifestChangeSet) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -1115,6 +1191,7 @@ func (m *ManifestChange) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } diff --git a/vendor/github.com/dgraph-io/badger/pb/pb.proto b/vendor/github.com/dgraph-io/badger/pb/pb.proto index b790cf69..c6e7f413 100644 --- a/vendor/github.com/dgraph-io/badger/pb/pb.proto +++ b/vendor/github.com/dgraph-io/badger/pb/pb.proto @@ -26,6 +26,9 @@ message KV { uint64 version = 4; uint64 expires_at = 5; bytes meta = 6; + + // Stream id is used to identify which stream the KV came from. + uint32 stream_id = 10; } message KVList { diff --git a/vendor/github.com/dgraph-io/badger/publisher.go b/vendor/github.com/dgraph-io/badger/publisher.go new file mode 100644 index 00000000..60f4fc90 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/publisher.go @@ -0,0 +1,151 @@ +/* + * Copyright 2019 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package badger + +import ( + "bytes" + "sync" + + "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/y" +) + +type subscriber struct { + prefixes [][]byte + sendCh chan<- *pb.KVList + subCloser *y.Closer +} + +type publisher struct { + sync.Mutex + pubCh chan requests + subscribers map[uint64]subscriber + nextID uint64 +} + +func newPublisher() *publisher { + return &publisher{ + pubCh: make(chan requests, 1000), + subscribers: make(map[uint64]subscriber), + nextID: 0, + } +} + +func (p *publisher) listenForUpdates(c *y.Closer) { + defer func() { + p.cleanSubscribers() + c.Done() + }() + slurp := func(batch []*request) { + for { + select { + case reqs := <-p.pubCh: + batch = append(batch, reqs...) + default: + p.publishUpdates(batch) + return + } + } + } + for { + select { + case <-c.HasBeenClosed(): + return + case reqs := <-p.pubCh: + slurp(reqs) + } + } +} + +func (p *publisher) publishUpdates(reqs requests) { + kvs := &pb.KVList{} + p.Lock() + defer func() { + p.Unlock() + // Release all the request. + reqs.DecrRef() + }() + for _, s := range p.subscribers { + for _, prefix := range s.prefixes { + for _, req := range reqs { + for _, e := range req.Entries { + // TODO: Use trie to find subscribers. + if bytes.HasPrefix(e.Key, prefix) { + k := y.SafeCopy(nil, e.Key) + kv := &pb.KV{ + Key: y.ParseKey(k), + Value: y.SafeCopy(nil, e.Value), + Meta: []byte{e.UserMeta}, + ExpiresAt: e.ExpiresAt, + Version: y.ParseTs(k), + } + kvs.Kv = append(kvs.Kv, kv) + } + } + } + } + if len(kvs.GetKv()) > 0 { + s.sendCh <- kvs + } + } +} + +func (p *publisher) newSubscriber(c *y.Closer, prefixes ...[]byte) (<-chan *pb.KVList, uint64) { + p.Lock() + defer p.Unlock() + ch := make(chan *pb.KVList, 1000) + id := p.nextID + // Increment next ID. + p.nextID++ + p.subscribers[id] = subscriber{ + prefixes: prefixes, + sendCh: ch, + subCloser: c, + } + return ch, id +} + +// cleanSubscribers stops all the subscribers. Ideally, It should be called while closing DB. +func (p *publisher) cleanSubscribers() { + p.Lock() + defer p.Unlock() + for id, s := range p.subscribers { + delete(p.subscribers, id) + s.subCloser.SignalAndWait() + } +} + +func (p *publisher) deleteSubscriber(id uint64) { + p.Lock() + defer p.Unlock() + if _, ok := p.subscribers[id]; !ok { + return + } + delete(p.subscribers, id) +} + +func (p *publisher) sendUpdates(reqs []*request) { + // TODO: Prefix check before pushing into pubCh. + if p.noOfSubscribers() != 0 { + p.pubCh <- reqs + } +} + +func (p *publisher) noOfSubscribers() int { + p.Lock() + defer p.Unlock() + return len(p.subscribers) +} diff --git a/vendor/github.com/dgraph-io/badger/skl/arena.go b/vendor/github.com/dgraph-io/badger/skl/arena.go index 2decb75c..def55071 100644 --- a/vendor/github.com/dgraph-io/badger/skl/arena.go +++ b/vendor/github.com/dgraph-io/badger/skl/arena.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "unsafe" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" + "github.com/dgraph-io/badger/y" ) const ( diff --git a/vendor/github.com/dgraph-io/badger/skl/skl.go b/vendor/github.com/dgraph-io/badger/skl/skl.go index 81a0506e..b465b09e 100644 --- a/vendor/github.com/dgraph-io/badger/skl/skl.go +++ b/vendor/github.com/dgraph-io/badger/skl/skl.go @@ -38,7 +38,7 @@ import ( "sync/atomic" "unsafe" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" + "github.com/dgraph-io/badger/y" ) const ( diff --git a/vendor/github.com/dgraph-io/badger/skl/skl_test.go b/vendor/github.com/dgraph-io/badger/skl/skl_test.go deleted file mode 100644 index cc695fbf..00000000 --- a/vendor/github.com/dgraph-io/badger/skl/skl_test.go +++ /dev/null @@ -1,475 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package skl - -import ( - "encoding/binary" - "fmt" - "math/rand" - "strconv" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -const arenaSize = 1 << 20 - -func newValue(v int) []byte { - return []byte(fmt.Sprintf("%05d", v)) -} - -// length iterates over skiplist to give exact size. -func length(s *Skiplist) int { - x := s.getNext(s.head, 0) - count := 0 - for x != nil { - count++ - x = s.getNext(x, 0) - } - return count -} - -func TestEmpty(t *testing.T) { - key := []byte("aaa") - l := NewSkiplist(arenaSize) - - v := l.Get(key) - require.True(t, v.Value == nil) // Cannot use require.Nil for unsafe.Pointer nil. - - for _, less := range []bool{true, false} { - for _, allowEqual := range []bool{true, false} { - n, found := l.findNear(key, less, allowEqual) - require.Nil(t, n) - require.False(t, found) - } - } - - it := l.NewIterator() - require.False(t, it.Valid()) - - it.SeekToFirst() - require.False(t, it.Valid()) - - it.SeekToLast() - require.False(t, it.Valid()) - - it.Seek(key) - require.False(t, it.Valid()) - - l.DecrRef() - require.True(t, l.valid()) // Check the reference counting. - - it.Close() - require.False(t, l.valid()) // Check the reference counting. -} - -// TestBasic tests single-threaded inserts and updates and gets. -func TestBasic(t *testing.T) { - l := NewSkiplist(arenaSize) - val1 := newValue(42) - val2 := newValue(52) - val3 := newValue(62) - val4 := newValue(72) - - // Try inserting values. - // Somehow require.Nil doesn't work when checking for unsafe.Pointer(nil). - l.Put(y.KeyWithTs([]byte("key1"), 0), y.ValueStruct{Value: val1, Meta: 55, UserMeta: 0}) - l.Put(y.KeyWithTs([]byte("key2"), 2), y.ValueStruct{Value: val2, Meta: 56, UserMeta: 0}) - l.Put(y.KeyWithTs([]byte("key3"), 0), y.ValueStruct{Value: val3, Meta: 57, UserMeta: 0}) - - v := l.Get(y.KeyWithTs([]byte("key"), 0)) - require.True(t, v.Value == nil) - - v = l.Get(y.KeyWithTs([]byte("key1"), 0)) - require.True(t, v.Value != nil) - require.EqualValues(t, "00042", string(v.Value)) - require.EqualValues(t, 55, v.Meta) - - v = l.Get(y.KeyWithTs([]byte("key2"), 0)) - require.True(t, v.Value == nil) - - v = l.Get(y.KeyWithTs([]byte("key3"), 0)) - require.True(t, v.Value != nil) - require.EqualValues(t, "00062", string(v.Value)) - require.EqualValues(t, 57, v.Meta) - - l.Put(y.KeyWithTs([]byte("key3"), 1), y.ValueStruct{Value: val4, Meta: 12, UserMeta: 0}) - v = l.Get(y.KeyWithTs([]byte("key3"), 1)) - require.True(t, v.Value != nil) - require.EqualValues(t, "00072", string(v.Value)) - require.EqualValues(t, 12, v.Meta) -} - -// TestConcurrentBasic tests concurrent writes followed by concurrent reads. -func TestConcurrentBasic(t *testing.T) { - const n = 1000 - l := NewSkiplist(arenaSize) - var wg sync.WaitGroup - key := func(i int) []byte { - return y.KeyWithTs([]byte(fmt.Sprintf("%05d", i)), 0) - } - for i := 0; i < n; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - l.Put(key(i), - y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0}) - }(i) - } - wg.Wait() - // Check values. Concurrent reads. - for i := 0; i < n; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - v := l.Get(key(i)) - require.True(t, v.Value != nil) - require.EqualValues(t, newValue(i), v.Value) - }(i) - } - wg.Wait() - require.EqualValues(t, n, length(l)) -} - -// TestOneKey will read while writing to one single key. -func TestOneKey(t *testing.T) { - const n = 100 - key := y.KeyWithTs([]byte("thekey"), 0) - l := NewSkiplist(arenaSize) - defer l.DecrRef() - - var wg sync.WaitGroup - for i := 0; i < n; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - l.Put(key, y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0}) - }(i) - } - // We expect that at least some write made it such that some read returns a value. - var sawValue int32 - for i := 0; i < n; i++ { - wg.Add(1) - go func() { - defer wg.Done() - p := l.Get(key) - if p.Value == nil { - return - } - atomic.AddInt32(&sawValue, 1) - v, err := strconv.Atoi(string(p.Value)) - require.NoError(t, err) - require.True(t, 0 <= v && v < n, fmt.Sprintf("invalid value %d", v)) - }() - } - wg.Wait() - require.True(t, sawValue > 0) - require.EqualValues(t, 1, length(l)) -} - -func TestFindNear(t *testing.T) { - l := NewSkiplist(arenaSize) - defer l.DecrRef() - for i := 0; i < 1000; i++ { - key := fmt.Sprintf("%05d", i*10+5) - l.Put(y.KeyWithTs([]byte(key), 0), y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0}) - } - - n, eq := l.findNear(y.KeyWithTs([]byte("00001"), 0), false, false) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("00005"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("00001"), 0), false, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("00005"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("00001"), 0), true, false) - require.Nil(t, n) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("00001"), 0), true, true) - require.Nil(t, n) - require.False(t, eq) - - n, eq = l.findNear(y.KeyWithTs([]byte("00005"), 0), false, false) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("00015"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("00005"), 0), false, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("00005"), 0), string(n.key(l.arena))) - require.True(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("00005"), 0), true, false) - require.Nil(t, n) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("00005"), 0), true, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("00005"), 0), string(n.key(l.arena))) - require.True(t, eq) - - n, eq = l.findNear(y.KeyWithTs([]byte("05555"), 0), false, false) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("05565"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("05555"), 0), false, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("05555"), 0), string(n.key(l.arena))) - require.True(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("05555"), 0), true, false) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("05545"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("05555"), 0), true, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("05555"), 0), string(n.key(l.arena))) - require.True(t, eq) - - n, eq = l.findNear(y.KeyWithTs([]byte("05558"), 0), false, false) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("05565"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("05558"), 0), false, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("05565"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("05558"), 0), true, false) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("05555"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("05558"), 0), true, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("05555"), 0), string(n.key(l.arena))) - require.False(t, eq) - - n, eq = l.findNear(y.KeyWithTs([]byte("09995"), 0), false, false) - require.Nil(t, n) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("09995"), 0), false, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("09995"), 0), string(n.key(l.arena))) - require.True(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("09995"), 0), true, false) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("09985"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("09995"), 0), true, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("09995"), 0), string(n.key(l.arena))) - require.True(t, eq) - - n, eq = l.findNear(y.KeyWithTs([]byte("59995"), 0), false, false) - require.Nil(t, n) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("59995"), 0), false, true) - require.Nil(t, n) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("59995"), 0), true, false) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("09995"), 0), string(n.key(l.arena))) - require.False(t, eq) - n, eq = l.findNear(y.KeyWithTs([]byte("59995"), 0), true, true) - require.NotNil(t, n) - require.EqualValues(t, y.KeyWithTs([]byte("09995"), 0), string(n.key(l.arena))) - require.False(t, eq) -} - -// TestIteratorNext tests a basic iteration over all nodes from the beginning. -func TestIteratorNext(t *testing.T) { - const n = 100 - l := NewSkiplist(arenaSize) - defer l.DecrRef() - it := l.NewIterator() - defer it.Close() - require.False(t, it.Valid()) - it.SeekToFirst() - require.False(t, it.Valid()) - for i := n - 1; i >= 0; i-- { - l.Put(y.KeyWithTs([]byte(fmt.Sprintf("%05d", i)), 0), - y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0}) - } - it.SeekToFirst() - for i := 0; i < n; i++ { - require.True(t, it.Valid()) - v := it.Value() - require.EqualValues(t, newValue(i), v.Value) - it.Next() - } - require.False(t, it.Valid()) -} - -// TestIteratorPrev tests a basic iteration over all nodes from the end. -func TestIteratorPrev(t *testing.T) { - const n = 100 - l := NewSkiplist(arenaSize) - defer l.DecrRef() - it := l.NewIterator() - defer it.Close() - require.False(t, it.Valid()) - it.SeekToFirst() - require.False(t, it.Valid()) - for i := 0; i < n; i++ { - l.Put(y.KeyWithTs([]byte(fmt.Sprintf("%05d", i)), 0), - y.ValueStruct{Value: newValue(i), Meta: 0, UserMeta: 0}) - } - it.SeekToLast() - for i := n - 1; i >= 0; i-- { - require.True(t, it.Valid()) - v := it.Value() - require.EqualValues(t, newValue(i), v.Value) - it.Prev() - } - require.False(t, it.Valid()) -} - -// TestIteratorSeek tests Seek and SeekForPrev. -func TestIteratorSeek(t *testing.T) { - const n = 100 - l := NewSkiplist(arenaSize) - defer l.DecrRef() - - it := l.NewIterator() - defer it.Close() - - require.False(t, it.Valid()) - it.SeekToFirst() - require.False(t, it.Valid()) - // 1000, 1010, 1020, ..., 1990. - for i := n - 1; i >= 0; i-- { - v := i*10 + 1000 - l.Put(y.KeyWithTs([]byte(fmt.Sprintf("%05d", i*10+1000)), 0), - y.ValueStruct{Value: newValue(v), Meta: 0, UserMeta: 0}) - } - it.SeekToFirst() - require.True(t, it.Valid()) - v := it.Value() - require.EqualValues(t, "01000", v.Value) - - it.Seek(y.KeyWithTs([]byte("01000"), 0)) - require.True(t, it.Valid()) - v = it.Value() - require.EqualValues(t, "01000", v.Value) - - it.Seek(y.KeyWithTs([]byte("01005"), 0)) - require.True(t, it.Valid()) - v = it.Value() - require.EqualValues(t, "01010", v.Value) - - it.Seek(y.KeyWithTs([]byte("01010"), 0)) - require.True(t, it.Valid()) - v = it.Value() - require.EqualValues(t, "01010", v.Value) - - it.Seek(y.KeyWithTs([]byte("99999"), 0)) - require.False(t, it.Valid()) - - // Try SeekForPrev. - it.SeekForPrev(y.KeyWithTs([]byte("00"), 0)) - require.False(t, it.Valid()) - - it.SeekForPrev(y.KeyWithTs([]byte("01000"), 0)) - require.True(t, it.Valid()) - v = it.Value() - require.EqualValues(t, "01000", v.Value) - - it.SeekForPrev(y.KeyWithTs([]byte("01005"), 0)) - require.True(t, it.Valid()) - v = it.Value() - require.EqualValues(t, "01000", v.Value) - - it.SeekForPrev(y.KeyWithTs([]byte("01010"), 0)) - require.True(t, it.Valid()) - v = it.Value() - require.EqualValues(t, "01010", v.Value) - - it.SeekForPrev(y.KeyWithTs([]byte("99999"), 0)) - require.True(t, it.Valid()) - v = it.Value() - require.EqualValues(t, "01990", v.Value) -} - -func randomKey(rng *rand.Rand) []byte { - b := make([]byte, 8) - key := rng.Uint32() - key2 := rng.Uint32() - binary.LittleEndian.PutUint32(b, key) - binary.LittleEndian.PutUint32(b[4:], key2) - return y.KeyWithTs(b, 0) -} - -// Standard test. Some fraction is read. Some fraction is write. Writes have -// to go through mutex lock. -func BenchmarkReadWrite(b *testing.B) { - value := newValue(123) - for i := 0; i <= 10; i++ { - readFrac := float32(i) / 10.0 - b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) { - l := NewSkiplist(int64((b.N + 1) * MaxNodeSize)) - defer l.DecrRef() - b.ResetTimer() - var count int - b.RunParallel(func(pb *testing.PB) { - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - for pb.Next() { - if rng.Float32() < readFrac { - v := l.Get(randomKey(rng)) - if v.Value != nil { - count++ - } - } else { - l.Put(randomKey(rng), y.ValueStruct{Value: value, Meta: 0, UserMeta: 0}) - } - } - }) - }) - } -} - -// Standard test. Some fraction is read. Some fraction is write. Writes have -// to go through mutex lock. -func BenchmarkReadWriteMap(b *testing.B) { - value := newValue(123) - for i := 0; i <= 10; i++ { - readFrac := float32(i) / 10.0 - b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) { - m := make(map[string][]byte) - var mutex sync.RWMutex - b.ResetTimer() - var count int - b.RunParallel(func(pb *testing.PB) { - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - for pb.Next() { - if rand.Float32() < readFrac { - mutex.RLock() - _, ok := m[string(randomKey(rng))] - mutex.RUnlock() - if ok { - count++ - } - } else { - mutex.Lock() - m[string(randomKey(rng))] = value - mutex.Unlock() - } - } - }) - }) - } -} diff --git a/vendor/github.com/dgraph-io/badger/stream.go b/vendor/github.com/dgraph-io/badger/stream.go index f40e5a2e..f0841a6a 100644 --- a/vendor/github.com/dgraph-io/badger/stream.go +++ b/vendor/github.com/dgraph-io/badger/stream.go @@ -19,12 +19,14 @@ package badger import ( "bytes" "context" + "math" "sync" + "sync/atomic" "time" - humanize "gx/ipfs/QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy/go-humanize" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" + "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/y" + humanize "github.com/dustin/go-humanize" ) const pageSize = 4 << 20 // 4MB @@ -66,10 +68,11 @@ type Stream struct { // single goroutine, i.e. logic within Send method can expect single threaded execution. Send func(*pb.KVList) error - readTs uint64 - db *DB - rangeCh chan keyRange - kvChan chan *pb.KVList + readTs uint64 + db *DB + rangeCh chan keyRange + kvChan chan *pb.KVList + nextStreamId uint32 } // ToList is a default implementation of KeyToList. It picks up all valid versions of the key, @@ -113,6 +116,23 @@ func (st *Stream) ToList(key []byte, itr *Iterator) (*pb.KVList, error) { // end byte slices are owned by keyRange struct. func (st *Stream) produceRanges(ctx context.Context) { splits := st.db.KeySplits(st.Prefix) + + // We don't need to create more key ranges than NumGo goroutines. This way, we will have limited + // number of "streams" coming out, which then helps limit the memory used by SSWriter. + { + pickEvery := int(math.Floor(float64(len(splits)) / float64(st.NumGo))) + if pickEvery < 1 { + pickEvery = 1 + } + filtered := splits[:0] + for i, split := range splits { + if (i+1)%pickEvery == 0 { + filtered = append(filtered, split) + } + } + splits = filtered + } + start := y.SafeCopy(nil, st.Prefix) for _, key := range splits { st.rangeCh <- keyRange{left: start, right: y.SafeCopy(nil, []byte(key))} @@ -143,6 +163,9 @@ func (st *Stream) produceKVs(ctx context.Context) error { itr := txn.NewIterator(iterOpts) defer itr.Close() + // This unique stream id is used to identify all the keys from this iteration. + streamId := atomic.AddUint32(&st.nextStreamId, 1) + outList := new(pb.KVList) var prevKey []byte for itr.Seek(kr.left); itr.Valid(); { @@ -174,13 +197,28 @@ func (st *Stream) produceKVs(ctx context.Context) error { outList.Kv = append(outList.Kv, list.Kv...) size += list.Size() if size >= pageSize { - st.kvChan <- outList + for _, kv := range outList.Kv { + kv.StreamId = streamId + } + select { + case st.kvChan <- outList: + case <-ctx.Done(): + return ctx.Err() + } outList = new(pb.KVList) size = 0 } } if len(outList.Kv) > 0 { - st.kvChan <- outList + for _, kv := range outList.Kv { + kv.StreamId = streamId + } + // TODO: Think of a way to indicate that a stream is over. + select { + case st.kvChan <- outList: + case <-ctx.Done(): + return ctx.Err() + } } return nil } @@ -278,8 +316,8 @@ func (st *Stream) Orchestrate(ctx context.Context) error { // kvChan should only have a small capacity to ensure that we don't buffer up too much data if // sending is slow. Page size is set to 4MB, which is used to lazily cap the size of each - // KVList. To get around 64MB buffer, we can set the channel size to 16. - st.kvChan = make(chan *pb.KVList, 16) + // KVList. To get 128MB buffer, we can set the channel size to 32. + st.kvChan = make(chan *pb.KVList, 32) if st.KeyToList == nil { st.KeyToList = st.ToList diff --git a/vendor/github.com/dgraph-io/badger/stream_test.go b/vendor/github.com/dgraph-io/badger/stream_test.go deleted file mode 100644 index cdb1ec56..00000000 --- a/vendor/github.com/dgraph-io/badger/stream_test.go +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2018 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "context" - "fmt" - "io/ioutil" - "math" - "os" - "strconv" - "strings" - "testing" - - "github.com/stretchr/testify/require" - bpb "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/pb" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -func openManaged(dir string) (*DB, error) { - opt := DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - - return OpenManaged(opt) -} - -func keyWithPrefix(prefix string, k int) []byte { - return []byte(fmt.Sprintf("%s-%d", prefix, k)) -} - -func keyToInt(k []byte) (string, int) { - splits := strings.Split(string(k), "-") - key, err := strconv.Atoi(splits[1]) - y.Check(err) - return splits[0], key -} - -func value(k int) []byte { - return []byte(fmt.Sprintf("%08d", k)) -} - -type collector struct { - kv []*bpb.KV -} - -func (c *collector) Send(list *bpb.KVList) error { - c.kv = append(c.kv, list.Kv...) - return nil -} - -var ctxb = context.Background() - -func TestStream(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - db, err := openManaged(dir) - require.NoError(t, err) - - var count int - for _, prefix := range []string{"p0", "p1", "p2"} { - txn := db.NewTransactionAt(math.MaxUint64, true) - for i := 1; i <= 100; i++ { - require.NoError(t, txn.Set(keyWithPrefix(prefix, i), value(i))) - count++ - } - require.NoError(t, txn.CommitAt(5, nil)) - } - - stream := db.NewStreamAt(math.MaxUint64) - stream.LogPrefix = "Testing" - c := &collector{} - stream.Send = func(list *bpb.KVList) error { - return c.Send(list) - } - - // Test case 1. Retrieve everything. - err = stream.Orchestrate(ctxb) - require.NoError(t, err) - require.Equal(t, 300, len(c.kv), "Expected 300. Got: %d", len(c.kv)) - - m := make(map[string]int) - for _, kv := range c.kv { - prefix, ki := keyToInt(kv.Key) - expected := value(ki) - require.Equal(t, expected, kv.Value) - m[prefix]++ - } - require.Equal(t, 3, len(m)) - for pred, count := range m { - require.Equal(t, 100, count, "Count mismatch for pred: %s", pred) - } - - // Test case 2. Retrieve only 1 predicate. - stream.Prefix = []byte("p1") - c.kv = c.kv[:0] - err = stream.Orchestrate(ctxb) - require.NoError(t, err) - require.Equal(t, 100, len(c.kv), "Expected 100. Got: %d", len(c.kv)) - - m = make(map[string]int) - for _, kv := range c.kv { - prefix, ki := keyToInt(kv.Key) - expected := value(ki) - require.Equal(t, expected, kv.Value) - m[prefix]++ - } - require.Equal(t, 1, len(m)) - for pred, count := range m { - require.Equal(t, 100, count, "Count mismatch for pred: %s", pred) - } - - // Test case 3. Retrieve select keys within the predicate. - c.kv = c.kv[:0] - stream.ChooseKey = func(item *Item) bool { - _, k := keyToInt(item.Key()) - return k%2 == 0 - } - err = stream.Orchestrate(ctxb) - require.NoError(t, err) - require.Equal(t, 50, len(c.kv), "Expected 50. Got: %d", len(c.kv)) - - m = make(map[string]int) - for _, kv := range c.kv { - prefix, ki := keyToInt(kv.Key) - expected := value(ki) - require.Equal(t, expected, kv.Value) - m[prefix]++ - } - require.Equal(t, 1, len(m)) - for pred, count := range m { - require.Equal(t, 50, count, "Count mismatch for pred: %s", pred) - } - - // Test case 4. Retrieve select keys from all predicates. - c.kv = c.kv[:0] - stream.Prefix = []byte{} - err = stream.Orchestrate(ctxb) - require.NoError(t, err) - require.Equal(t, 150, len(c.kv), "Expected 150. Got: %d", len(c.kv)) - - m = make(map[string]int) - for _, kv := range c.kv { - prefix, ki := keyToInt(kv.Key) - expected := value(ki) - require.Equal(t, expected, kv.Value) - m[prefix]++ - } - require.Equal(t, 3, len(m)) - for pred, count := range m { - require.Equal(t, 50, count, "Count mismatch for pred: %s", pred) - } -} diff --git a/vendor/github.com/dgraph-io/badger/stream_writer.go b/vendor/github.com/dgraph-io/badger/stream_writer.go new file mode 100644 index 00000000..cdf8849c --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/stream_writer.go @@ -0,0 +1,311 @@ +/* + * Copyright 2019 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package badger + +import ( + "bytes" + "math" + + "github.com/dgraph-io/badger/pb" + "github.com/dgraph-io/badger/table" + "github.com/dgraph-io/badger/y" + humanize "github.com/dustin/go-humanize" + "github.com/pkg/errors" +) + +const headStreamId uint32 = math.MaxUint32 + +// StreamWriter is used to write data coming from multiple streams. The streams must not have any +// overlapping key ranges. Within each stream, the keys must be sorted. Badger Stream framework is +// capable of generating such an output. So, this StreamWriter can be used at the other end to build +// BadgerDB at a much faster pace by writing SSTables (and value logs) directly to LSM tree levels +// without causing any compactions at all. This is way faster than using batched writer or using +// transactions, but only applicable in situations where the keys are pre-sorted and the DB is being +// bootstrapped. Existing data would get deleted when using this writer. So, this is only useful +// when restoring from backup or replicating DB across servers. +// +// StreamWriter should not be called on in-use DB instances. It is designed only to bootstrap new +// DBs. +type StreamWriter struct { + db *DB + done func() + throttle *y.Throttle + head valuePointer + maxVersion uint64 + writers map[uint32]*sortedWriter +} + +// NewStreamWriter creates a StreamWriter. Right after creating StreamWriter, Prepare must be +// called. The memory usage of a StreamWriter is directly proportional to the number of streams +// possible. So, efforts must be made to keep the number of streams low. Stream framework would +// typically use 16 goroutines and hence create 16 streams. +func (db *DB) NewStreamWriter() *StreamWriter { + return &StreamWriter{ + db: db, + // throttle shouldn't make much difference. Memory consumption is based on the number of + // concurrent streams being processed. + throttle: y.NewThrottle(16), + writers: make(map[uint32]*sortedWriter), + } +} + +// Prepare should be called before writing any entry to StreamWriter. It deletes all data present in +// existing DB, stops compactions and any writes being done by other means. Be very careful when +// calling Prepare, because it could result in permanent data loss. Not calling Prepare would result +// in a corrupt Badger instance. +func (sw *StreamWriter) Prepare() error { + var err error + sw.done, err = sw.db.dropAll() + return err +} + +// Write writes KVList to DB. Each KV within the list contains the stream id which StreamWriter +// would use to demux the writes. +func (sw *StreamWriter) Write(kvs *pb.KVList) error { + var entries []*Entry + for _, kv := range kvs.Kv { + var meta, userMeta byte + if len(kv.Meta) > 0 { + meta = kv.Meta[0] + } + if len(kv.UserMeta) > 0 { + userMeta = kv.UserMeta[0] + } + if sw.maxVersion < kv.Version { + sw.maxVersion = kv.Version + } + e := &Entry{ + Key: y.KeyWithTs(kv.Key, kv.Version), + Value: kv.Value, + UserMeta: userMeta, + ExpiresAt: kv.ExpiresAt, + meta: meta, + } + // If the value can be colocated with the key in LSM tree, we can skip + // writing the value to value log. + e.skipVlog = sw.db.shouldWriteValueToLSM(*e) + entries = append(entries, e) + } + req := &request{ + Entries: entries, + } + y.AssertTrue(len(kvs.Kv) == len(req.Entries)) + if err := sw.db.vlog.write([]*request{req}); err != nil { + return err + } + + for i, kv := range kvs.Kv { + e := req.Entries[i] + vptr := req.Ptrs[i] + if !vptr.IsZero() { + y.AssertTrue(sw.head.Less(vptr)) + sw.head = vptr + } + + writer, ok := sw.writers[kv.StreamId] + if !ok { + writer = sw.newWriter(kv.StreamId) + sw.writers[kv.StreamId] = writer + } + + var vs y.ValueStruct + if e.skipVlog { + vs = y.ValueStruct{ + Value: e.Value, + Meta: e.meta, + UserMeta: e.UserMeta, + ExpiresAt: e.ExpiresAt, + } + } else { + vbuf := make([]byte, vptrSize) + vs = y.ValueStruct{ + Value: vptr.Encode(vbuf), + Meta: e.meta | bitValuePointer, + UserMeta: e.UserMeta, + ExpiresAt: e.ExpiresAt, + } + } + if err := writer.Add(e.Key, vs); err != nil { + return err + } + } + return nil +} + +// Flush is called once we are done writing all the entries. It syncs DB directories. It also +// updates Oracle with maxVersion found in all entries (if DB is not managed). +func (sw *StreamWriter) Flush() error { + defer sw.done() + for _, writer := range sw.writers { + if err := writer.Done(); err != nil { + return err + } + } + + // Encode and write the value log head into a new table. + data := make([]byte, vptrSize) + sw.head.Encode(data) + headWriter := sw.newWriter(headStreamId) + if err := headWriter.Add( + y.KeyWithTs(head, sw.maxVersion), + y.ValueStruct{Value: data}); err != nil { + return err + } + if err := headWriter.Done(); err != nil { + return err + } + + if !sw.db.opt.managedTxns { + sw.db.orc = newOracle(sw.db.opt) + sw.db.orc.nextTxnTs = sw.maxVersion + sw.db.orc.txnMark.Done(sw.maxVersion) + sw.db.orc.readMark.Done(sw.maxVersion) + sw.db.orc.incrementNextTs() + } + + // Wait for all files to be written. + if err := sw.throttle.Finish(); err != nil { + return err + } + + // Now sync the directories, so all the files are registered. + if sw.db.opt.ValueDir != sw.db.opt.Dir { + if err := syncDir(sw.db.opt.ValueDir); err != nil { + return err + } + } + return syncDir(sw.db.opt.Dir) +} + +type sortedWriter struct { + db *DB + throttle *y.Throttle + + builder *table.Builder + lastKey []byte + streamId uint32 +} + +func (sw *StreamWriter) newWriter(streamId uint32) *sortedWriter { + return &sortedWriter{ + db: sw.db, + streamId: streamId, + throttle: sw.throttle, + builder: table.NewTableBuilder(), + } +} + +// ErrUnsortedKey is returned when any out of order key arrives at sortedWriter during call to Add. +var ErrUnsortedKey = errors.New("Keys not in sorted order") + +// Add adds key and vs to sortedWriter. +func (w *sortedWriter) Add(key []byte, vs y.ValueStruct) error { + if bytes.Compare(key, w.lastKey) <= 0 { + return ErrUnsortedKey + } + sameKey := y.SameKey(key, w.lastKey) + w.lastKey = y.SafeCopy(w.lastKey, key) + + if err := w.builder.Add(key, vs); err != nil { + return err + } + // Same keys should go into the same SSTable. + if !sameKey && w.builder.ReachedCapacity(w.db.opt.MaxTableSize) { + return w.send() + } + return nil +} + +func (w *sortedWriter) send() error { + if err := w.throttle.Do(); err != nil { + return err + } + go func(builder *table.Builder) { + data := builder.Finish() + err := w.createTable(data) + w.throttle.Done(err) + }(w.builder) + w.builder = table.NewTableBuilder() + return nil +} + +// Done is called once we are done writing all keys and valueStructs +// to sortedWriter. It completes writing current SST to disk. +func (w *sortedWriter) Done() error { + if w.builder.Empty() { + return nil + } + return w.send() +} + +func (w *sortedWriter) createTable(data []byte) error { + if len(data) == 0 { + return nil + } + fileID := w.db.lc.reserveFileID() + fd, err := y.CreateSyncedFile(table.NewFilename(fileID, w.db.opt.Dir), true) + if err != nil { + return err + } + if _, err := fd.Write(data); err != nil { + return err + } + tbl, err := table.OpenTable(fd, w.db.opt.TableLoadingMode, nil) + if err != nil { + return err + } + lc := w.db.lc + + var lhandler *levelHandler + // We should start the levels from 1, because we need level 0 to set the !badger!head key. We + // cannot mix up this key with other keys from the DB, otherwise we would introduce a range + // overlap violation. + y.AssertTrue(len(lc.levels) > 1) + for _, l := range lc.levels[1:] { + ratio := float64(l.getTotalSize()) / float64(l.maxTotalSize) + if ratio < 1.0 { + lhandler = l + break + } + } + if lhandler == nil { + // If we're exceeding the size of the lowest level, shove it in the lowest level. Can't do + // better than that. + lhandler = lc.levels[len(lc.levels)-1] + } + if w.streamId == headStreamId { + // This is a special !badger!head key. We should store it at level 0, separate from all the + // other keys to avoid an overlap. + lhandler = lc.levels[0] + } + // Now that table can be opened successfully, let's add this to the MANIFEST. + change := &pb.ManifestChange{ + Id: tbl.ID(), + Op: pb.ManifestChange_CREATE, + Level: uint32(lhandler.level), + Checksum: tbl.Checksum, + } + if err := w.db.manifest.addChanges([]*pb.ManifestChange{change}); err != nil { + return err + } + if err := lhandler.replaceTables([]*table.Table{}, []*table.Table{tbl}); err != nil { + return err + } + w.db.opt.Infof("Table created: %d at level: %d for stream: %d. Size: %s\n", + fileID, lhandler.level, w.streamId, humanize.Bytes(uint64(tbl.Size()))) + return nil +} diff --git a/vendor/github.com/dgraph-io/badger/structs.go b/vendor/github.com/dgraph-io/badger/structs.go index eaeeb1d4..2161b7f3 100644 --- a/vendor/github.com/dgraph-io/badger/structs.go +++ b/vendor/github.com/dgraph-io/badger/structs.go @@ -6,7 +6,7 @@ import ( "fmt" "hash/crc32" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" + "github.com/dgraph-io/badger/y" ) type valuePointer struct { @@ -85,7 +85,8 @@ type Entry struct { meta byte // Fields maintained internally. - offset uint32 + offset uint32 + skipVlog bool } func (e *Entry) estimateSize(threshold int) int { diff --git a/vendor/github.com/dgraph-io/badger/table/builder.go b/vendor/github.com/dgraph-io/badger/table/builder.go index a19c6589..43e65622 100644 --- a/vendor/github.com/dgraph-io/badger/table/builder.go +++ b/vendor/github.com/dgraph-io/badger/table/builder.go @@ -22,8 +22,8 @@ import ( "io" "math" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmWaLViWQF8jgyoLLqqcSrnp6dJpHESiJfzor1vrfDyTZf/bbloom" + "github.com/AndreasBriese/bbloom" + "github.com/dgraph-io/badger/y" ) var ( @@ -201,7 +201,7 @@ func (b *Builder) blockIndex() []byte { // Finish finishes the table by appending the index. func (b *Builder) Finish() []byte { - bf, _ := bbloom.New(float64(b.keyCount), 0.01) + bf := bbloom.New(float64(b.keyCount), 0.01) var klen [2]byte key := make([]byte, 1024) for { @@ -224,7 +224,7 @@ func (b *Builder) Finish() []byte { b.buf.Write(index) // Write bloom filter. - bdata, _ := bf.JSONMarshal() + bdata := bf.JSONMarshal() n, err := b.buf.Write(bdata) y.Check(err) var buf [4]byte diff --git a/vendor/github.com/dgraph-io/badger/table/iterator.go b/vendor/github.com/dgraph-io/badger/table/iterator.go index 28feb99e..0eb5ed01 100644 --- a/vendor/github.com/dgraph-io/badger/table/iterator.go +++ b/vendor/github.com/dgraph-io/badger/table/iterator.go @@ -22,8 +22,8 @@ import ( "math" "sort" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" ) type blockIterator struct { diff --git a/vendor/github.com/dgraph-io/badger/table/table.go b/vendor/github.com/dgraph-io/badger/table/table.go index f8119346..9650c08e 100644 --- a/vendor/github.com/dgraph-io/badger/table/table.go +++ b/vendor/github.com/dgraph-io/badger/table/table.go @@ -30,10 +30,10 @@ import ( "sync" "sync/atomic" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" - "gx/ipfs/QmWaLViWQF8jgyoLLqqcSrnp6dJpHESiJfzor1vrfDyTZf/bbloom" + "github.com/AndreasBriese/bbloom" + "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" ) const fileSuffix = ".sst" @@ -59,7 +59,7 @@ type Table struct { tableSize int // Initialized in OpenTable, using fd.Stat(). blockIndex []keyOffset - ref int32 // For file garbage collection. Atomic. + ref int32 // For file garbage collection. Atomic. loadingMode options.FileLoadingMode mmap []byte // Memory mapped. @@ -197,7 +197,7 @@ func (t *Table) Close() error { return t.fd.Close() } -func (t *Table) read(off int, sz int) ([]byte, error) { +func (t *Table) read(off, sz int) ([]byte, error) { if len(t.mmap) > 0 { if len(t.mmap[off:]) < sz { return nil, y.ErrEOF @@ -212,7 +212,7 @@ func (t *Table) read(off int, sz int) ([]byte, error) { return res, err } -func (t *Table) readNoFail(off int, sz int) []byte { +func (t *Table) readNoFail(off, sz int) []byte { res, err := t.read(off, sz) y.Check(err) return res @@ -230,7 +230,7 @@ func (t *Table) readIndex() error { bloomLen := int(binary.BigEndian.Uint32(buf)) readPos -= bloomLen data := t.readNoFail(readPos, bloomLen) - t.bf = *bbloom.JSONUnmarshal(data) + t.bf = bbloom.JSONUnmarshal(data) readPos -= 4 buf = t.readNoFail(readPos, 4) diff --git a/vendor/github.com/dgraph-io/badger/table/table_test.go b/vendor/github.com/dgraph-io/badger/table/table_test.go deleted file mode 100644 index d16e4083..00000000 --- a/vendor/github.com/dgraph-io/badger/table/table_test.go +++ /dev/null @@ -1,729 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package table - -import ( - "fmt" - "math/rand" - "os" - "sort" - "testing" - - "github.com/stretchr/testify/require" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -func key(prefix string, i int) string { - return prefix + fmt.Sprintf("%04d", i) -} - -func buildTestTable(t *testing.T, prefix string, n int) *os.File { - y.AssertTrue(n <= 10000) - keyValues := make([][]string, n) - for i := 0; i < n; i++ { - k := key(prefix, i) - v := fmt.Sprintf("%d", i) - keyValues[i] = []string{k, v} - } - return buildTable(t, keyValues) -} - -// keyValues is n by 2 where n is number of pairs. -func buildTable(t *testing.T, keyValues [][]string) *os.File { - b := NewTableBuilder() - defer b.Close() - // TODO: Add test for file garbage collection here. No files should be left after the tests here. - - filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63()) - f, err := y.OpenSyncedFile(filename, true) - if t != nil { - require.NoError(t, err) - } else { - y.Check(err) - } - - sort.Slice(keyValues, func(i, j int) bool { - return keyValues[i][0] < keyValues[j][0] - }) - for _, kv := range keyValues { - y.AssertTrue(len(kv) == 2) - err := b.Add(y.KeyWithTs([]byte(kv[0]), 0), y.ValueStruct{Value: []byte(kv[1]), Meta: 'A', UserMeta: 0}) - if t != nil { - require.NoError(t, err) - } else { - y.Check(err) - } - } - f.Write(b.Finish()) - f.Close() - f, _ = y.OpenSyncedFile(filename, true) - return f -} - -func TestTableIterator(t *testing.T) { - for _, n := range []int{99, 100, 101} { - t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) { - f := buildTestTable(t, "key", n) - table, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer table.DecrRef() - it := table.NewIterator(false) - defer it.Close() - count := 0 - for it.Rewind(); it.Valid(); it.Next() { - v := it.Value() - k := y.KeyWithTs([]byte(key("key", count)), 0) - require.EqualValues(t, k, it.Key()) - require.EqualValues(t, fmt.Sprintf("%d", count), string(v.Value)) - count++ - } - require.Equal(t, count, n) - }) - } -} - -func TestSeekToFirst(t *testing.T) { - for _, n := range []int{99, 100, 101, 199, 200, 250, 9999, 10000} { - t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) { - f := buildTestTable(t, "key", n) - table, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer table.DecrRef() - it := table.NewIterator(false) - defer it.Close() - it.seekToFirst() - require.True(t, it.Valid()) - v := it.Value() - require.EqualValues(t, "0", string(v.Value)) - require.EqualValues(t, 'A', v.Meta) - }) - } -} - -func TestSeekToLast(t *testing.T) { - for _, n := range []int{99, 100, 101, 199, 200, 250, 9999, 10000} { - t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) { - f := buildTestTable(t, "key", n) - table, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer table.DecrRef() - it := table.NewIterator(false) - defer it.Close() - it.seekToLast() - require.True(t, it.Valid()) - v := it.Value() - require.EqualValues(t, fmt.Sprintf("%d", n-1), string(v.Value)) - require.EqualValues(t, 'A', v.Meta) - it.prev() - require.True(t, it.Valid()) - v = it.Value() - require.EqualValues(t, fmt.Sprintf("%d", n-2), string(v.Value)) - require.EqualValues(t, 'A', v.Meta) - }) - } -} - -func TestSeek(t *testing.T) { - f := buildTestTable(t, "k", 10000) - table, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer table.DecrRef() - - it := table.NewIterator(false) - defer it.Close() - - var data = []struct { - in string - valid bool - out string - }{ - {"abc", true, "k0000"}, - {"k0100", true, "k0100"}, - {"k0100b", true, "k0101"}, // Test case where we jump to next block. - {"k1234", true, "k1234"}, - {"k1234b", true, "k1235"}, - {"k9999", true, "k9999"}, - {"z", false, ""}, - } - - for _, tt := range data { - it.seek(y.KeyWithTs([]byte(tt.in), 0)) - if !tt.valid { - require.False(t, it.Valid()) - continue - } - require.True(t, it.Valid()) - k := it.Key() - require.EqualValues(t, tt.out, string(y.ParseKey(k))) - } -} - -func TestSeekForPrev(t *testing.T) { - f := buildTestTable(t, "k", 10000) - table, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer table.DecrRef() - - it := table.NewIterator(false) - defer it.Close() - - var data = []struct { - in string - valid bool - out string - }{ - {"abc", false, ""}, - {"k0100", true, "k0100"}, - {"k0100b", true, "k0100"}, // Test case where we jump to next block. - {"k1234", true, "k1234"}, - {"k1234b", true, "k1234"}, - {"k9999", true, "k9999"}, - {"z", true, "k9999"}, - } - - for _, tt := range data { - it.seekForPrev(y.KeyWithTs([]byte(tt.in), 0)) - if !tt.valid { - require.False(t, it.Valid()) - continue - } - require.True(t, it.Valid()) - k := it.Key() - require.EqualValues(t, tt.out, string(y.ParseKey(k))) - } -} - -func TestIterateFromStart(t *testing.T) { - // Vary the number of elements added. - for _, n := range []int{99, 100, 101, 199, 200, 250, 9999, 10000} { - t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) { - f := buildTestTable(t, "key", n) - table, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer table.DecrRef() - ti := table.NewIterator(false) - defer ti.Close() - ti.reset() - ti.seekToFirst() - require.True(t, ti.Valid()) - // No need to do a Next. - // ti.Seek brings us to the first key >= "". Essentially a SeekToFirst. - var count int - for ; ti.Valid(); ti.next() { - v := ti.Value() - require.EqualValues(t, fmt.Sprintf("%d", count), string(v.Value)) - require.EqualValues(t, 'A', v.Meta) - count++ - } - require.EqualValues(t, n, count) - }) - } -} - -func TestIterateFromEnd(t *testing.T) { - // Vary the number of elements added. - for _, n := range []int{99, 100, 101, 199, 200, 250, 9999, 10000} { - t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) { - f := buildTestTable(t, "key", n) - table, err := OpenTable(f, options.FileIO, nil) - require.NoError(t, err) - defer table.DecrRef() - ti := table.NewIterator(false) - defer ti.Close() - ti.reset() - ti.seek(y.KeyWithTs([]byte("zzzzzz"), 0)) // Seek to end, an invalid element. - require.False(t, ti.Valid()) - for i := n - 1; i >= 0; i-- { - ti.prev() - require.True(t, ti.Valid()) - v := ti.Value() - require.EqualValues(t, fmt.Sprintf("%d", i), string(v.Value)) - require.EqualValues(t, 'A', v.Meta) - } - ti.prev() - require.False(t, ti.Valid()) - }) - } -} - -func TestTable(t *testing.T) { - f := buildTestTable(t, "key", 10000) - table, err := OpenTable(f, options.FileIO, nil) - require.NoError(t, err) - defer table.DecrRef() - ti := table.NewIterator(false) - defer ti.Close() - kid := 1010 - seek := y.KeyWithTs([]byte(key("key", kid)), 0) - for ti.seek(seek); ti.Valid(); ti.next() { - k := ti.Key() - require.EqualValues(t, string(y.ParseKey(k)), key("key", kid)) - kid++ - } - if kid != 10000 { - t.Errorf("Expected kid: 10000. Got: %v", kid) - } - - ti.seek(y.KeyWithTs([]byte(key("key", 99999)), 0)) - require.False(t, ti.Valid()) - - ti.seek(y.KeyWithTs([]byte(key("key", -1)), 0)) - require.True(t, ti.Valid()) - k := ti.Key() - require.EqualValues(t, string(y.ParseKey(k)), key("key", 0)) -} - -func TestIterateBackAndForth(t *testing.T) { - f := buildTestTable(t, "key", 10000) - table, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer table.DecrRef() - - seek := y.KeyWithTs([]byte(key("key", 1010)), 0) - it := table.NewIterator(false) - defer it.Close() - it.seek(seek) - require.True(t, it.Valid()) - k := it.Key() - require.EqualValues(t, seek, k) - - it.prev() - it.prev() - require.True(t, it.Valid()) - k = it.Key() - require.EqualValues(t, key("key", 1008), string(y.ParseKey(k))) - - it.next() - it.next() - require.True(t, it.Valid()) - k = it.Key() - require.EqualValues(t, key("key", 1010), y.ParseKey(k)) - - it.seek(y.KeyWithTs([]byte(key("key", 2000)), 0)) - require.True(t, it.Valid()) - k = it.Key() - require.EqualValues(t, key("key", 2000), y.ParseKey(k)) - - it.prev() - require.True(t, it.Valid()) - k = it.Key() - require.EqualValues(t, key("key", 1999), y.ParseKey(k)) - - it.seekToFirst() - k = it.Key() - require.EqualValues(t, key("key", 0), y.ParseKey(k)) -} - -func TestUniIterator(t *testing.T) { - f := buildTestTable(t, "key", 10000) - table, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer table.DecrRef() - { - it := table.NewIterator(false) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - v := it.Value() - require.EqualValues(t, fmt.Sprintf("%d", count), string(v.Value)) - require.EqualValues(t, 'A', v.Meta) - count++ - } - require.EqualValues(t, 10000, count) - } - { - it := table.NewIterator(true) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - v := it.Value() - require.EqualValues(t, fmt.Sprintf("%d", 10000-1-count), string(v.Value)) - require.EqualValues(t, 'A', v.Meta) - count++ - } - require.EqualValues(t, 10000, count) - } -} - -// Try having only one table. -func TestConcatIteratorOneTable(t *testing.T) { - f := buildTable(t, [][]string{ - {"k1", "a1"}, - {"k2", "a2"}, - }) - - tbl, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer tbl.DecrRef() - - it := NewConcatIterator([]*Table{tbl}, false) - defer it.Close() - - it.Rewind() - require.True(t, it.Valid()) - k := it.Key() - require.EqualValues(t, "k1", string(y.ParseKey(k))) - vs := it.Value() - require.EqualValues(t, "a1", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) -} - -func TestConcatIterator(t *testing.T) { - f := buildTestTable(t, "keya", 10000) - f2 := buildTestTable(t, "keyb", 10000) - f3 := buildTestTable(t, "keyc", 10000) - tbl, err := OpenTable(f, options.MemoryMap, nil) - require.NoError(t, err) - defer tbl.DecrRef() - tbl2, err := OpenTable(f2, options.LoadToRAM, nil) - require.NoError(t, err) - defer tbl2.DecrRef() - tbl3, err := OpenTable(f3, options.LoadToRAM, nil) - require.NoError(t, err) - defer tbl3.DecrRef() - - { - it := NewConcatIterator([]*Table{tbl, tbl2, tbl3}, false) - defer it.Close() - it.Rewind() - require.True(t, it.Valid()) - var count int - for ; it.Valid(); it.Next() { - vs := it.Value() - require.EqualValues(t, fmt.Sprintf("%d", count%10000), string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - count++ - } - require.EqualValues(t, 30000, count) - - it.Seek(y.KeyWithTs([]byte("a"), 0)) - require.EqualValues(t, "keya0000", string(y.ParseKey(it.Key()))) - vs := it.Value() - require.EqualValues(t, "0", string(vs.Value)) - - it.Seek(y.KeyWithTs([]byte("keyb"), 0)) - require.EqualValues(t, "keyb0000", string(y.ParseKey(it.Key()))) - vs = it.Value() - require.EqualValues(t, "0", string(vs.Value)) - - it.Seek(y.KeyWithTs([]byte("keyb9999b"), 0)) - require.EqualValues(t, "keyc0000", string(y.ParseKey(it.Key()))) - vs = it.Value() - require.EqualValues(t, "0", string(vs.Value)) - - it.Seek(y.KeyWithTs([]byte("keyd"), 0)) - require.False(t, it.Valid()) - } - { - it := NewConcatIterator([]*Table{tbl, tbl2, tbl3}, true) - defer it.Close() - it.Rewind() - require.True(t, it.Valid()) - var count int - for ; it.Valid(); it.Next() { - vs := it.Value() - require.EqualValues(t, fmt.Sprintf("%d", 10000-(count%10000)-1), string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - count++ - } - require.EqualValues(t, 30000, count) - - it.Seek(y.KeyWithTs([]byte("a"), 0)) - require.False(t, it.Valid()) - - it.Seek(y.KeyWithTs([]byte("keyb"), 0)) - require.EqualValues(t, "keya9999", string(y.ParseKey(it.Key()))) - vs := it.Value() - require.EqualValues(t, "9999", string(vs.Value)) - - it.Seek(y.KeyWithTs([]byte("keyb9999b"), 0)) - require.EqualValues(t, "keyb9999", string(y.ParseKey(it.Key()))) - vs = it.Value() - require.EqualValues(t, "9999", string(vs.Value)) - - it.Seek(y.KeyWithTs([]byte("keyd"), 0)) - require.EqualValues(t, "keyc9999", string(y.ParseKey(it.Key()))) - vs = it.Value() - require.EqualValues(t, "9999", string(vs.Value)) - } -} - -func TestMergingIterator(t *testing.T) { - f1 := buildTable(t, [][]string{ - {"k1", "a1"}, - {"k2", "a2"}, - }) - f2 := buildTable(t, [][]string{ - {"k1", "b1"}, - {"k2", "b2"}, - }) - tbl1, err := OpenTable(f1, options.LoadToRAM, nil) - require.NoError(t, err) - defer tbl1.DecrRef() - tbl2, err := OpenTable(f2, options.LoadToRAM, nil) - require.NoError(t, err) - defer tbl2.DecrRef() - it1 := tbl1.NewIterator(false) - it2 := NewConcatIterator([]*Table{tbl2}, false) - it := y.NewMergeIterator([]y.Iterator{it1, it2}, false) - defer it.Close() - - it.Rewind() - require.True(t, it.Valid()) - k := it.Key() - require.EqualValues(t, "k1", string(y.ParseKey(k))) - vs := it.Value() - require.EqualValues(t, "a1", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - it.Next() - - require.True(t, it.Valid()) - k = it.Key() - require.EqualValues(t, "k2", string(y.ParseKey(k))) - vs = it.Value() - require.EqualValues(t, "a2", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - it.Next() - - require.False(t, it.Valid()) -} - -func TestMergingIteratorReversed(t *testing.T) { - f1 := buildTable(t, [][]string{ - {"k1", "a1"}, - {"k2", "a2"}, - }) - f2 := buildTable(t, [][]string{ - {"k1", "b1"}, - {"k2", "b2"}, - }) - tbl1, err := OpenTable(f1, options.LoadToRAM, nil) - require.NoError(t, err) - defer tbl1.DecrRef() - tbl2, err := OpenTable(f2, options.LoadToRAM, nil) - require.NoError(t, err) - defer tbl2.DecrRef() - it1 := tbl1.NewIterator(true) - it2 := NewConcatIterator([]*Table{tbl2}, true) - it := y.NewMergeIterator([]y.Iterator{it1, it2}, true) - defer it.Close() - - it.Rewind() - require.True(t, it.Valid()) - k := it.Key() - require.EqualValues(t, "k2", string(y.ParseKey(k))) - vs := it.Value() - require.EqualValues(t, "a2", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - it.Next() - - require.True(t, it.Valid()) - k = it.Key() - require.EqualValues(t, "k1", string(y.ParseKey(k))) - vs = it.Value() - require.EqualValues(t, "a1", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - it.Next() - - require.False(t, it.Valid()) -} - -// Take only the first iterator. -func TestMergingIteratorTakeOne(t *testing.T) { - f1 := buildTable(t, [][]string{ - {"k1", "a1"}, - {"k2", "a2"}, - }) - f2 := buildTable(t, [][]string{}) - - t1, err := OpenTable(f1, options.LoadToRAM, nil) - require.NoError(t, err) - defer t1.DecrRef() - t2, err := OpenTable(f2, options.LoadToRAM, nil) - require.NoError(t, err) - defer t2.DecrRef() - - it1 := NewConcatIterator([]*Table{t1}, false) - it2 := NewConcatIterator([]*Table{t2}, false) - it := y.NewMergeIterator([]y.Iterator{it1, it2}, false) - defer it.Close() - - it.Rewind() - require.True(t, it.Valid()) - k := it.Key() - require.EqualValues(t, "k1", string(y.ParseKey(k))) - vs := it.Value() - require.EqualValues(t, "a1", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - it.Next() - - require.True(t, it.Valid()) - k = it.Key() - require.EqualValues(t, "k2", string(y.ParseKey(k))) - vs = it.Value() - require.EqualValues(t, "a2", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - it.Next() - - require.False(t, it.Valid()) -} - -// Take only the second iterator. -func TestMergingIteratorTakeTwo(t *testing.T) { - f1 := buildTable(t, [][]string{}) - f2 := buildTable(t, [][]string{ - {"k1", "a1"}, - {"k2", "a2"}, - }) - - t1, err := OpenTable(f1, options.LoadToRAM, nil) - require.NoError(t, err) - defer t1.DecrRef() - t2, err := OpenTable(f2, options.LoadToRAM, nil) - require.NoError(t, err) - defer t2.DecrRef() - - it1 := NewConcatIterator([]*Table{t1}, false) - it2 := NewConcatIterator([]*Table{t2}, false) - it := y.NewMergeIterator([]y.Iterator{it1, it2}, false) - defer it.Close() - - it.Rewind() - require.True(t, it.Valid()) - k := it.Key() - require.EqualValues(t, "k1", string(y.ParseKey(k))) - vs := it.Value() - require.EqualValues(t, "a1", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - it.Next() - - require.True(t, it.Valid()) - k = it.Key() - require.EqualValues(t, "k2", string(y.ParseKey(k))) - vs = it.Value() - require.EqualValues(t, "a2", string(vs.Value)) - require.EqualValues(t, 'A', vs.Meta) - it.Next() - - require.False(t, it.Valid()) -} - -func BenchmarkRead(b *testing.B) { - n := 5 << 20 - builder := NewTableBuilder() - filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63()) - f, err := y.OpenSyncedFile(filename, true) - y.Check(err) - for i := 0; i < n; i++ { - k := fmt.Sprintf("%016x", i) - v := fmt.Sprintf("%d", i) - y.Check(builder.Add([]byte(k), y.ValueStruct{Value: []byte(v), Meta: 123, UserMeta: 0})) - } - - f.Write(builder.Finish()) - tbl, err := OpenTable(f, options.MemoryMap, nil) - y.Check(err) - defer tbl.DecrRef() - - // y.Printf("Size of table: %d\n", tbl.Size()) - b.ResetTimer() - // Iterate b.N times over the entire table. - for i := 0; i < b.N; i++ { - func() { - it := tbl.NewIterator(false) - defer it.Close() - for it.seekToFirst(); it.Valid(); it.next() { - } - }() - } -} - -func BenchmarkReadAndBuild(b *testing.B) { - n := 5 << 20 - builder := NewTableBuilder() - filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63()) - f, err := y.OpenSyncedFile(filename, true) - y.Check(err) - for i := 0; i < n; i++ { - k := fmt.Sprintf("%016x", i) - v := fmt.Sprintf("%d", i) - y.Check(builder.Add([]byte(k), y.ValueStruct{Value: []byte(v), Meta: 123, UserMeta: 0})) - } - - f.Write(builder.Finish()) - tbl, err := OpenTable(f, options.MemoryMap, nil) - y.Check(err) - defer tbl.DecrRef() - - // y.Printf("Size of table: %d\n", tbl.Size()) - b.ResetTimer() - // Iterate b.N times over the entire table. - for i := 0; i < b.N; i++ { - func() { - newBuilder := NewTableBuilder() - it := tbl.NewIterator(false) - defer it.Close() - for it.seekToFirst(); it.Valid(); it.next() { - vs := it.Value() - newBuilder.Add(it.Key(), vs) - } - newBuilder.Finish() - }() - } -} - -func BenchmarkReadMerged(b *testing.B) { - n := 5 << 20 - m := 5 // Number of tables. - y.AssertTrue((n % m) == 0) - tableSize := n / m - var tables []*Table - for i := 0; i < m; i++ { - filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63()) - builder := NewTableBuilder() - f, err := y.OpenSyncedFile(filename, true) - y.Check(err) - for j := 0; j < tableSize; j++ { - id := j*m + i // Arrays are interleaved. - // id := i*tableSize+j (not interleaved) - k := fmt.Sprintf("%016x", id) - v := fmt.Sprintf("%d", id) - y.Check(builder.Add([]byte(k), y.ValueStruct{Value: []byte(v), Meta: 123, UserMeta: 0})) - } - f.Write(builder.Finish()) - tbl, err := OpenTable(f, options.MemoryMap, nil) - y.Check(err) - tables = append(tables, tbl) - defer tbl.DecrRef() - } - - b.ResetTimer() - // Iterate b.N times over the entire table. - for i := 0; i < b.N; i++ { - func() { - var iters []y.Iterator - for _, tbl := range tables { - iters = append(iters, tbl.NewIterator(false)) - } - it := y.NewMergeIterator(iters, false) - defer it.Close() - for it.Rewind(); it.Valid(); it.Next() { - } - }() - } -} diff --git a/vendor/github.com/dgraph-io/badger/test.sh b/vendor/github.com/dgraph-io/badger/test.sh old mode 100644 new mode 100755 index 2216ecbd..e2df230e --- a/vendor/github.com/dgraph-io/badger/test.sh +++ b/vendor/github.com/dgraph-io/badger/test.sh @@ -17,8 +17,15 @@ go test -v --manual=true -run='TestTruncateVlogNoClose$' . truncate --size=4096 p/000000.vlog go test -v --manual=true -run='TestTruncateVlogNoClose2$' . go test -v --manual=true -run='TestTruncateVlogNoClose3$' . -rm -R p +rm -R p || true # Then the normal tests. +echo +echo "==> Starting tests with value log mmapped..." +sleep 5 go test -v --vlog_mmap=true -race ./... + +echo +echo "==> Starting tests with value log not mmapped..." +sleep 5 go test -v --vlog_mmap=false -race ./... diff --git a/vendor/github.com/dgraph-io/badger/txn.go b/vendor/github.com/dgraph-io/badger/txn.go index 1164108a..f6faa926 100644 --- a/vendor/github.com/dgraph-io/badger/txn.go +++ b/vendor/github.com/dgraph-io/badger/txn.go @@ -27,9 +27,9 @@ import ( "sync/atomic" "time" - farm "gx/ipfs/QmRFFHk2jw9tgjxv12bCuuTnSbVXxEvYQkuNCLMEv9eUwP/go-farm" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/y" + farm "github.com/dgryski/go-farm" + "github.com/pkg/errors" ) type oracle struct { @@ -127,6 +127,12 @@ func (o *oracle) nextTs() uint64 { return o.nextTxnTs } +func (o *oracle) incrementNextTs() { + o.Lock() + defer o.Unlock() + o.nextTxnTs++ +} + // Any deleted or invalid versions at or below ts would be discarded during // compaction to reclaim disk space in LSM tree and thence value log. func (o *oracle) setDiscardTs(ts uint64) { @@ -353,7 +359,7 @@ func (txn *Txn) SetWithDiscard(key, val []byte, meta byte) error { // SetWithTTL adds a key-value pair to the database, along with a time-to-live // (TTL) setting. A key stored with a TTL would automatically expire after the -// time has elapsed , and be eligible for garbage collection. +// time has elapsed, and be eligible for garbage collection. // // The current transaction keeps a reference to the key and val byte slice // arguments. Users must not modify key and val until the end of the @@ -364,6 +370,12 @@ func (txn *Txn) SetWithTTL(key, val []byte, dur time.Duration) error { return txn.SetEntry(e) } +// setMergeEntry is similar to SetEntry but it sets the bitMergeEntry flag +func (txn *Txn) setMergeEntry(key, val []byte) error { + e := &Entry{Key: key, Value: val, meta: bitMergeEntry} + return txn.SetEntry(e) +} + func exceedsSize(prefix string, max int64, key []byte) error { return errors.Errorf("%s with size %d exceeded %d limit. %s:\n%s", prefix, len(key), max, prefix, hex.Dump(key[:1<<10])) diff --git a/vendor/github.com/dgraph-io/badger/txn_test.go b/vendor/github.com/dgraph-io/badger/txn_test.go deleted file mode 100644 index 557a5bdf..00000000 --- a/vendor/github.com/dgraph-io/badger/txn_test.go +++ /dev/null @@ -1,845 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "fmt" - "io/ioutil" - "math/rand" - "os" - "strconv" - "sync" - "testing" - "time" - - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - - "github.com/stretchr/testify/require" -) - -func TestTxnSimple(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - - txn := db.NewTransaction(true) - - for i := 0; i < 10; i++ { - k := []byte(fmt.Sprintf("key=%d", i)) - v := []byte(fmt.Sprintf("val=%d", i)) - txn.Set(k, v) - } - - item, err := txn.Get([]byte("key=8")) - require.NoError(t, err) - - require.NoError(t, item.Value(func(val []byte) error { - require.Equal(t, []byte("val=8"), val) - return nil - })) - - require.Panics(t, func() { txn.CommitAt(100, nil) }) - require.NoError(t, txn.Commit()) - }) -} - -func TestTxnReadAfterWrite(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - var wg sync.WaitGroup - N := 100 - wg.Add(N) - for i := 0; i < N; i++ { - go func(i int) { - defer wg.Done() - key := []byte(fmt.Sprintf("key%d", i)) - err := db.Update(func(tx *Txn) error { - return tx.Set(key, key) - }) - require.NoError(t, err) - err = db.View(func(tx *Txn) error { - item, err := tx.Get(key) - require.NoError(t, err) - val, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, val, key) - return nil - }) - require.NoError(t, err) - }(i) - } - wg.Wait() - }) -} - -func TestTxnCommitAsync(t *testing.T) { - key := func(i int) []byte { - return []byte(fmt.Sprintf("key=%d", i)) - } - - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - txn := db.NewTransaction(true) - for i := 0; i < 40; i++ { - err := txn.Set(key(i), []byte(strconv.Itoa(100))) - require.NoError(t, err) - } - require.NoError(t, txn.Commit()) - txn.Discard() - - closer := y.NewCloser(1) - go func() { - defer closer.Done() - for { - select { - case <-closer.HasBeenClosed(): - return - default: - } - // Keep checking balance variant - txn := db.NewTransaction(false) - totalBalance := 0 - for i := 0; i < 40; i++ { - item, err := txn.Get(key(i)) - require.NoError(t, err) - val, err := item.ValueCopy(nil) - require.NoError(t, err) - bal, err := strconv.Atoi(string(val)) - require.NoError(t, err) - totalBalance += bal - } - require.Equal(t, totalBalance, 4000) - txn.Discard() - } - }() - - var wg sync.WaitGroup - wg.Add(100) - for i := 0; i < 100; i++ { - go func() { - txn := db.NewTransaction(true) - delta := rand.Intn(100) - for i := 0; i < 20; i++ { - err := txn.Set(key(i), []byte(strconv.Itoa(100-delta))) - require.NoError(t, err) - } - for i := 20; i < 40; i++ { - err := txn.Set(key(i), []byte(strconv.Itoa(100+delta))) - require.NoError(t, err) - } - // We are only doing writes, so there won't be any conflicts. - txn.CommitWith(func(err error) {}) - txn.Discard() - wg.Done() - }() - } - wg.Wait() - closer.SignalAndWait() - time.Sleep(time.Millisecond * 10) // allow goroutine to complete. - }) -} - -func TestTxnVersions(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - k := []byte("key") - for i := 1; i < 10; i++ { - txn := db.NewTransaction(true) - - txn.Set(k, []byte(fmt.Sprintf("valversion=%d", i))) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(i), db.orc.readTs()) - } - - checkIterator := func(itr *Iterator, i int) { - defer itr.Close() - count := 0 - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - require.Equal(t, k, item.Key()) - - val, err := item.ValueCopy(nil) - require.NoError(t, err) - exp := fmt.Sprintf("valversion=%d", i) - require.Equal(t, exp, string(val), "i=%d", i) - count++ - } - require.Equal(t, 1, count, "i=%d", i) // Should only loop once. - } - - checkAllVersions := func(itr *Iterator, i int) { - var version uint64 - if itr.opt.Reverse { - version = 1 - } else { - version = uint64(i) - } - - count := 0 - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - require.Equal(t, k, item.Key()) - require.Equal(t, version, item.Version()) - - val, err := item.ValueCopy(nil) - require.NoError(t, err) - exp := fmt.Sprintf("valversion=%d", version) - require.Equal(t, exp, string(val), "v=%d", version) - count++ - - if itr.opt.Reverse { - version++ - } else { - version-- - } - } - require.Equal(t, i, count, "i=%d", i) // Should loop as many times as i. - } - - for i := 1; i < 10; i++ { - txn := db.NewTransaction(true) - txn.readTs = uint64(i) // Read version at i. - - item, err := txn.Get(k) - require.NoError(t, err) - - val, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, []byte(fmt.Sprintf("valversion=%d", i)), val, - "Expected versions to match up at i=%d", i) - - // Try retrieving the latest version forward and reverse. - itr := txn.NewIterator(DefaultIteratorOptions) - checkIterator(itr, i) - - opt := DefaultIteratorOptions - opt.Reverse = true - itr = txn.NewIterator(opt) - checkIterator(itr, i) - - // Now try retrieving all versions forward and reverse. - opt = DefaultIteratorOptions - opt.AllVersions = true - itr = txn.NewIterator(opt) - checkAllVersions(itr, i) - itr.Close() - - opt = DefaultIteratorOptions - opt.AllVersions = true - opt.Reverse = true - itr = txn.NewIterator(opt) - checkAllVersions(itr, i) - itr.Close() - - txn.Discard() - } - txn := db.NewTransaction(true) - defer txn.Discard() - item, err := txn.Get(k) - require.NoError(t, err) - - val, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, []byte("valversion=9"), val) - }) -} - -func TestTxnWriteSkew(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // Accounts - ax := []byte("x") - ay := []byte("y") - - // Set balance to $100 in each account. - txn := db.NewTransaction(true) - defer txn.Discard() - val := []byte(strconv.Itoa(100)) - txn.Set(ax, val) - txn.Set(ay, val) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(1), db.orc.readTs()) - - getBal := func(txn *Txn, key []byte) (bal int) { - item, err := txn.Get(key) - require.NoError(t, err) - - val, err := item.ValueCopy(nil) - require.NoError(t, err) - bal, err = strconv.Atoi(string(val)) - require.NoError(t, err) - return bal - } - - // Start two transactions, each would read both accounts and deduct from one account. - txn1 := db.NewTransaction(true) - - sum := getBal(txn1, ax) - sum += getBal(txn1, ay) - require.Equal(t, 200, sum) - txn1.Set(ax, []byte("0")) // Deduct 100 from ax. - - // Let's read this back. - sum = getBal(txn1, ax) - require.Equal(t, 0, sum) - sum += getBal(txn1, ay) - require.Equal(t, 100, sum) - // Don't commit yet. - - txn2 := db.NewTransaction(true) - - sum = getBal(txn2, ax) - sum += getBal(txn2, ay) - require.Equal(t, 200, sum) - txn2.Set(ay, []byte("0")) // Deduct 100 from ay. - - // Let's read this back. - sum = getBal(txn2, ax) - require.Equal(t, 100, sum) - sum += getBal(txn2, ay) - require.Equal(t, 100, sum) - - // Commit both now. - require.NoError(t, txn1.Commit()) - require.Error(t, txn2.Commit()) // This should fail. - - require.Equal(t, uint64(2), db.orc.readTs()) - }) -} - -// a3, a2, b4 (del), b3, c2, c1 -// Read at ts=4 -> a3, c2 -// Read at ts=4(Uncommitted) -> a3, b4 -// Read at ts=3 -> a3, b3, c2 -// Read at ts=2 -> a2, c2 -// Read at ts=1 -> c1 -func TestTxnIterationEdgeCase(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - ka := []byte("a") - kb := []byte("b") - kc := []byte("c") - - // c1 - txn := db.NewTransaction(true) - txn.Set(kc, []byte("c1")) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(1), db.orc.readTs()) - - // a2, c2 - txn = db.NewTransaction(true) - txn.Set(ka, []byte("a2")) - txn.Set(kc, []byte("c2")) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(2), db.orc.readTs()) - - // b3 - txn = db.NewTransaction(true) - txn.Set(ka, []byte("a3")) - txn.Set(kb, []byte("b3")) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(3), db.orc.readTs()) - - // b4, c4(del) (Uncommitted) - txn4 := db.NewTransaction(true) - require.NoError(t, txn4.Set(kb, []byte("b4"))) - require.NoError(t, txn4.Delete(kc)) - require.Equal(t, uint64(3), db.orc.readTs()) - - // b4 (del) - txn = db.NewTransaction(true) - txn.Delete(kb) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(4), db.orc.readTs()) - - checkIterator := func(itr *Iterator, expected []string) { - defer itr.Close() - var i int - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - val, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, expected[i], string(val), "readts=%d", itr.readTs) - i++ - } - require.Equal(t, len(expected), i) - } - - txn = db.NewTransaction(true) - defer txn.Discard() - itr := txn.NewIterator(DefaultIteratorOptions) - itr5 := txn4.NewIterator(DefaultIteratorOptions) - checkIterator(itr, []string{"a3", "c2"}) - checkIterator(itr5, []string{"a3", "b4"}) - - rev := DefaultIteratorOptions - rev.Reverse = true - itr = txn.NewIterator(rev) - itr5 = txn4.NewIterator(rev) - checkIterator(itr, []string{"c2", "a3"}) - checkIterator(itr5, []string{"b4", "a3"}) - - txn.readTs = 3 - itr = txn.NewIterator(DefaultIteratorOptions) - checkIterator(itr, []string{"a3", "b3", "c2"}) - itr = txn.NewIterator(rev) - checkIterator(itr, []string{"c2", "b3", "a3"}) - - txn.readTs = 2 - itr = txn.NewIterator(DefaultIteratorOptions) - checkIterator(itr, []string{"a2", "c2"}) - itr = txn.NewIterator(rev) - checkIterator(itr, []string{"c2", "a2"}) - - txn.readTs = 1 - itr = txn.NewIterator(DefaultIteratorOptions) - checkIterator(itr, []string{"c1"}) - itr = txn.NewIterator(rev) - checkIterator(itr, []string{"c1"}) - }) -} - -// a2, a3, b4 (del), b3, c2, c1 -// Read at ts=4 -> a3, c2 -// Read at ts=3 -> a3, b3, c2 -// Read at ts=2 -> a2, c2 -// Read at ts=1 -> c1 -func TestTxnIterationEdgeCase2(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - ka := []byte("a") - kb := []byte("aa") - kc := []byte("aaa") - - // c1 - txn := db.NewTransaction(true) - txn.Set(kc, []byte("c1")) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(1), db.orc.readTs()) - - // a2, c2 - txn = db.NewTransaction(true) - txn.Set(ka, []byte("a2")) - txn.Set(kc, []byte("c2")) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(2), db.orc.readTs()) - - // b3 - txn = db.NewTransaction(true) - txn.Set(ka, []byte("a3")) - txn.Set(kb, []byte("b3")) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(3), db.orc.readTs()) - - // b4 (del) - txn = db.NewTransaction(true) - txn.Delete(kb) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(4), db.orc.readTs()) - - checkIterator := func(itr *Iterator, expected []string) { - defer itr.Close() - var i int - for itr.Rewind(); itr.Valid(); itr.Next() { - item := itr.Item() - val, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, expected[i], string(val), "readts=%d", itr.readTs) - i++ - } - require.Equal(t, len(expected), i) - } - txn = db.NewTransaction(true) - defer txn.Discard() - rev := DefaultIteratorOptions - rev.Reverse = true - - itr := txn.NewIterator(DefaultIteratorOptions) - checkIterator(itr, []string{"a3", "c2"}) - itr = txn.NewIterator(rev) - checkIterator(itr, []string{"c2", "a3"}) - - txn.readTs = 5 - itr = txn.NewIterator(DefaultIteratorOptions) - itr.Seek(ka) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), ka) - itr.Seek(kc) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kc) - itr.Close() - - itr = txn.NewIterator(rev) - itr.Seek(ka) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), ka) - itr.Seek(kc) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kc) - itr.Close() - - txn.readTs = 3 - itr = txn.NewIterator(DefaultIteratorOptions) - checkIterator(itr, []string{"a3", "b3", "c2"}) - itr = txn.NewIterator(rev) - checkIterator(itr, []string{"c2", "b3", "a3"}) - - txn.readTs = 2 - itr = txn.NewIterator(DefaultIteratorOptions) - checkIterator(itr, []string{"a2", "c2"}) - itr = txn.NewIterator(rev) - checkIterator(itr, []string{"c2", "a2"}) - - txn.readTs = 1 - itr = txn.NewIterator(DefaultIteratorOptions) - checkIterator(itr, []string{"c1"}) - itr = txn.NewIterator(rev) - checkIterator(itr, []string{"c1"}) - }) -} - -func TestTxnIterationEdgeCase3(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - kb := []byte("abc") - kc := []byte("acd") - kd := []byte("ade") - - // c1 - txn := db.NewTransaction(true) - txn.Set(kc, []byte("c1")) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(1), db.orc.readTs()) - - // b2 - txn = db.NewTransaction(true) - txn.Set(kb, []byte("b2")) - require.NoError(t, txn.Commit()) - require.Equal(t, uint64(2), db.orc.readTs()) - - txn2 := db.NewTransaction(true) - require.NoError(t, txn2.Set(kd, []byte("d2"))) - require.NoError(t, txn2.Delete(kc)) - - txn = db.NewTransaction(true) - defer txn.Discard() - rev := DefaultIteratorOptions - rev.Reverse = true - - itr := txn.NewIterator(DefaultIteratorOptions) - itr.Seek([]byte("ab")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Seek([]byte("ac")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kc) - itr.Seek(nil) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Seek([]byte("ac")) - itr.Rewind() - itr.Seek(nil) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Seek([]byte("ac")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kc) - itr.Close() - - // Keys: "abc", "ade" - // Read pending writes. - itr = txn2.NewIterator(DefaultIteratorOptions) - itr.Seek([]byte("ab")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Seek([]byte("ac")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kd) - itr.Seek(nil) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Seek([]byte("ac")) - itr.Rewind() - itr.Seek(nil) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Seek([]byte("ad")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kd) - itr.Close() - - itr = txn.NewIterator(rev) - itr.Seek([]byte("ac")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Seek([]byte("ad")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kc) - itr.Seek(nil) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kc) - itr.Seek([]byte("ac")) - itr.Rewind() - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kc) - itr.Seek([]byte("ad")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kc) - itr.Close() - - // Keys: "abc", "ade" - itr = txn2.NewIterator(rev) - itr.Seek([]byte("ad")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Seek([]byte("ae")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kd) - itr.Seek(nil) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kd) - itr.Seek([]byte("ab")) - itr.Rewind() - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kd) - itr.Seek([]byte("ac")) - require.True(t, itr.Valid()) - require.Equal(t, itr.item.Key(), kb) - itr.Close() - }) -} - -func TestIteratorAllVersionsWithDeleted(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // Write two keys - err := db.Update(func(txn *Txn) error { - txn.Set([]byte("answer1"), []byte("42")) - txn.Set([]byte("answer2"), []byte("43")) - return nil - }) - require.NoError(t, err) - - // Delete the specific key version from underlying db directly - err = db.View(func(txn *Txn) error { - item, err := txn.Get([]byte("answer1")) - require.NoError(t, err) - err = txn.db.batchSet([]*Entry{ - { - Key: y.KeyWithTs(item.key, item.version), - meta: bitDelete, - }, - }) - require.NoError(t, err) - return err - }) - require.NoError(t, err) - - opts := DefaultIteratorOptions - opts.AllVersions = true - opts.PrefetchValues = false - - // Verify that deleted shows up when AllVersions is set. - err = db.View(func(txn *Txn) error { - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - count++ - item := it.Item() - if count == 1 { - require.Equal(t, []byte("answer1"), item.Key()) - require.True(t, item.meta&bitDelete > 0) - } else { - require.Equal(t, []byte("answer2"), item.Key()) - } - } - require.Equal(t, 2, count) - return nil - }) - require.NoError(t, err) - }) -} - -func TestIteratorAllVersionsWithDeleted2(t *testing.T) { - runBadgerTest(t, nil, func(t *testing.T, db *DB) { - // Set and delete alternatively - for i := 0; i < 4; i++ { - err := db.Update(func(txn *Txn) error { - if i%2 == 0 { - txn.Set([]byte("key"), []byte("value")) - return nil - } - txn.Delete([]byte("key")) - return nil - }) - require.NoError(t, err) - } - - opts := DefaultIteratorOptions - opts.AllVersions = true - opts.PrefetchValues = false - - // Verify that deleted shows up when AllVersions is set. - err := db.View(func(txn *Txn) error { - it := txn.NewIterator(opts) - defer it.Close() - var count int - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - require.Equal(t, []byte("key"), item.Key()) - if count%2 != 0 { - val, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, val, []byte("value")) - } else { - require.True(t, item.meta&bitDelete > 0) - } - count++ - } - require.Equal(t, 4, count) - return nil - }) - require.NoError(t, err) - }) -} - -func TestManagedDB(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opt := getTestOptions(dir) - opt.managedTxns = true - db, err := Open(opt) - require.NoError(t, err) - defer db.Close() - - key := func(i int) []byte { - return []byte(fmt.Sprintf("key-%02d", i)) - } - - val := func(i int) []byte { - return []byte(fmt.Sprintf("val-%d", i)) - } - - require.Panics(t, func() { - db.Update(func(tx *Txn) error { return nil }) - }) - - err = db.View(func(tx *Txn) error { return nil }) - require.NoError(t, err) - - // Write data at t=3. - txn := db.NewTransactionAt(3, true) - for i := 0; i <= 3; i++ { - require.NoError(t, txn.Set(key(i), val(i))) - } - require.Panics(t, func() { txn.Commit() }) - require.NoError(t, txn.CommitAt(3, nil)) - - // Read data at t=2. - txn = db.NewTransactionAt(2, false) - for i := 0; i <= 3; i++ { - _, err := txn.Get(key(i)) - require.Equal(t, ErrKeyNotFound, err) - } - txn.Discard() - - // Read data at t=3. - txn = db.NewTransactionAt(3, false) - for i := 0; i <= 3; i++ { - item, err := txn.Get(key(i)) - require.NoError(t, err) - require.Equal(t, uint64(3), item.Version()) - v, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, val(i), v) - } - txn.Discard() - - // Write data at t=7. - txn = db.NewTransactionAt(6, true) - for i := 0; i <= 7; i++ { - _, err := txn.Get(key(i)) - if err == nil { - continue // Don't overwrite existing keys. - } - require.NoError(t, txn.Set(key(i), val(i))) - } - require.NoError(t, txn.CommitAt(7, nil)) - - // Read data at t=9. - txn = db.NewTransactionAt(9, false) - for i := 0; i <= 9; i++ { - item, err := txn.Get(key(i)) - if i <= 7 { - require.NoError(t, err) - } else { - require.Equal(t, ErrKeyNotFound, err) - } - - if i <= 3 { - require.Equal(t, uint64(3), item.Version()) - } else if i <= 7 { - require.Equal(t, uint64(7), item.Version()) - } - if i <= 7 { - v, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, val(i), v) - } - } - txn.Discard() -} - -func TestArmV7Issue311Fix(t *testing.T) { - dir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - config := DefaultOptions - config.TableLoadingMode = options.MemoryMap - config.ValueLogFileSize = 16 << 20 - config.LevelOneSize = 8 << 20 - config.MaxTableSize = 2 << 20 - config.Dir = dir - config.ValueDir = dir - config.SyncWrites = false - - db, err := Open(config) - if err != nil { - t.Fatalf("cannot open db at location %s: %v", dir, err) - } - - err = db.View(func(txn *Txn) error { return nil }) - if err != nil { - t.Fatal(err) - } - - err = db.Update(func(txn *Txn) error { - return txn.Set([]byte{0x11}, []byte{0x22}) - }) - if err != nil { - t.Fatal(err) - } - - err = db.Update(func(txn *Txn) error { - return txn.Set([]byte{0x11}, []byte{0x22}) - }) - - if err != nil { - t.Fatal(err) - } - - if err = db.Close(); err != nil { - t.Fatal(err) - } -} diff --git a/vendor/github.com/dgraph-io/badger/util.go b/vendor/github.com/dgraph-io/badger/util.go index e74fc87f..02952a80 100644 --- a/vendor/github.com/dgraph-io/badger/util.go +++ b/vendor/github.com/dgraph-io/badger/util.go @@ -23,9 +23,9 @@ import ( "sync/atomic" "time" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/table" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/table" + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" ) // summary is produced when DB is closed. Currently it is used only for testing. diff --git a/vendor/github.com/dgraph-io/badger/value.go b/vendor/github.com/dgraph-io/badger/value.go index 3c7511b2..77cdc185 100644 --- a/vendor/github.com/dgraph-io/badger/value.go +++ b/vendor/github.com/dgraph-io/badger/value.go @@ -20,6 +20,7 @@ import ( "bufio" "bytes" "encoding/binary" + "encoding/json" "fmt" "hash/crc32" "io" @@ -34,10 +35,10 @@ import ( "sync/atomic" "time" - "gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/y" + "github.com/pkg/errors" + "golang.org/x/net/trace" ) // Values have their first byte being byteData or byteDelete. This helps us distinguish between @@ -46,7 +47,8 @@ const ( bitDelete byte = 1 << 0 // Set if the key has been deleted. bitValuePointer byte = 1 << 1 // Set if the value is NOT stored directly next to key. bitDiscardEarlierVersions byte = 1 << 2 // Set if earlier versions can be discarded. - + // Set if item shouldn't be discarded via compactions (used by merge operator) + bitMergeEntry byte = 1 << 3 // The MSB 2 bits are for transactions. bitTxn byte = 1 << 6 // Set if the entry is part of a txn. bitFinTxn byte = 1 << 7 // Set if the entry is to indicate end of txn in value log. @@ -555,6 +557,9 @@ func (vlog *valueLog) decrIteratorCount() error { } func (vlog *valueLog) deleteLogFile(lf *logFile) error { + if lf == nil { + return nil + } path := vlog.fpath(lf.fid) if err := lf.munmap(); err != nil { _ = lf.fd.Close() @@ -702,11 +707,15 @@ func errFile(err error, path string, msg string) error { } func (vlog *valueLog) replayLog(lf *logFile, offset uint32, replayFn logEntry) error { - // We should open the file in RW mode, so it can be truncated. var err error - lf.fd, err = os.OpenFile(lf.path, os.O_RDWR, 0) + mode := os.O_RDONLY + if vlog.opt.Truncate { + // We should open the file in RW mode, so it can be truncated. + mode = os.O_RDWR + } + lf.fd, err = os.OpenFile(lf.path, mode, 0) if err != nil { - return errFile(err, lf.path, "Open file in RW mode") + return errFile(err, lf.path, "Open file") } defer lf.fd.Close() @@ -745,7 +754,10 @@ func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry) error { vlog.db = db vlog.elog = trace.NewEventLog("Badger", "Valuelog") vlog.garbageCh = make(chan struct{}, 1) // Only allow one GC at a time. - vlog.lfDiscardStats = &lfDiscardStats{m: make(map[uint32]int64)} + + if err := vlog.populateDiscardStats(); err != nil { + return err + } if err := vlog.populateFilesMap(); err != nil { return err @@ -877,40 +889,68 @@ type request struct { Ptrs []valuePointer Wg sync.WaitGroup Err error + ref int32 +} + +func (req *request) IncrRef() { + atomic.AddInt32(&req.ref, 1) +} + +func (req *request) DecrRef() { + nRef := atomic.AddInt32(&req.ref, -1) + if nRef > 0 { + return + } + req.Entries = nil + requestPool.Put(req) } func (req *request) Wait() error { req.Wg.Wait() - req.Entries = nil err := req.Err - requestPool.Put(req) + req.DecrRef() // DecrRef after writing to DB. return err } -// sync is thread-unsafe and should not be called concurrently with write. -func (vlog *valueLog) sync() error { +type requests []*request + +func (reqs requests) DecrRef() { + for _, req := range reqs { + req.DecrRef() + } +} + +// sync function syncs content of latest value log file to disk. Syncing of value log directory is +// not required here as it happens every time a value log file rotation happens(check createVlogFile +// function). During rotation, previous value log file also gets synced to disk. It only syncs file +// if fid >= vlog.maxFid. In some cases such as replay(while openning db), it might be called with +// fid < vlog.maxFid. To sync irrespective of file id just call it with math.MaxUint32. +func (vlog *valueLog) sync(fid uint32) error { if vlog.opt.SyncWrites { return nil } vlog.filesLock.RLock() - if len(vlog.filesMap) == 0 { + maxFid := atomic.LoadUint32(&vlog.maxFid) + // During replay it is possible to get sync call with fid less than maxFid. + // Because older file has already been synced, we can return from here. + if fid < maxFid || len(vlog.filesMap) == 0 { vlog.filesLock.RUnlock() return nil } - maxFid := atomic.LoadUint32(&vlog.maxFid) curlf := vlog.filesMap[maxFid] + // Sometimes it is possible that vlog.maxFid has been increased but file creation + // with same id is still in progress and this function is called. In those cases + // entry for the file might not be present in vlog.filesMap. + if curlf == nil { + vlog.filesLock.RUnlock() + return nil + } curlf.lock.RLock() vlog.filesLock.RUnlock() - dirSyncCh := make(chan error) - go func() { dirSyncCh <- syncDir(vlog.opt.ValueDir) }() err := curlf.sync() curlf.lock.RUnlock() - dirSyncErr := <-dirSyncCh - if err != nil { - err = dirSyncErr - } return err } @@ -955,6 +995,7 @@ func (vlog *valueLog) write(reqs []*request) error { return err } curlf = newlf + atomic.AddInt32(&vlog.db.logRotates, 1) } return nil } @@ -962,8 +1003,13 @@ func (vlog *valueLog) write(reqs []*request) error { for i := range reqs { b := reqs[i] b.Ptrs = b.Ptrs[:0] + var written int for j := range b.Entries { e := b.Entries[j] + if e.skipVlog { + b.Ptrs = append(b.Ptrs, valuePointer{}) + continue + } var p valuePointer p.Fid = curlf.fid @@ -975,8 +1021,9 @@ func (vlog *valueLog) write(reqs []*request) error { } p.Len = uint32(plen) b.Ptrs = append(b.Ptrs, p) + written++ } - vlog.numEntriesWritten += uint32(len(b.Entries)) + vlog.numEntriesWritten += uint32(written) // We write to disk here so that all entries that are part of the same transaction are // written to the same vlog file. writeNow := @@ -1157,6 +1204,7 @@ func (vlog *valueLog) doRunGC(lf *logFile, discardRatio float64, tr trace.Trace) // Set up the sampling window sizes. sizeWindow := float64(fi.Size()) * 0.1 // 10% of the file as window. + sizeWindowM := sizeWindow / (1 << 20) // in MBs. countWindow := int(float64(vlog.opt.ValueLogMaxEntries) * 0.01) // 1% of num entries. tr.LazyPrintf("Size window: %5.2f. Count window: %d.", sizeWindow, countWindow) @@ -1185,7 +1233,7 @@ func (vlog *valueLog) doRunGC(lf *logFile, discardRatio float64, tr trace.Trace) tr.LazyPrintf("Stopping sampling after %d entries.", countWindow) return errStop } - if r.total > sizeWindow { + if r.total > sizeWindowM { tr.LazyPrintf("Stopping sampling after reaching window size.") return errStop } @@ -1250,7 +1298,7 @@ func (vlog *valueLog) doRunGC(lf *logFile, discardRatio float64, tr trace.Trace) // If we couldn't sample at least a 1000 KV pairs or at least 75% of the window size, // and what we can discard is below the threshold, we should skip the rewrite. - if (r.count < countWindow && r.total < sizeWindow*0.75) || r.discard < discardRatio*r.total { + if (r.count < countWindow && r.total < sizeWindowM*0.75) || r.discard < discardRatio*r.total { tr.LazyPrintf("Skipping GC on fid: %d", lf.fid) return ErrNoRewrite } @@ -1305,10 +1353,44 @@ func (vlog *valueLog) runGC(discardRatio float64, head valuePointer) error { } } -func (vlog *valueLog) updateGCStats(stats map[uint32]int64) { +func (vlog *valueLog) updateDiscardStats(stats map[uint32]int64) { vlog.lfDiscardStats.Lock() for fid, sz := range stats { vlog.lfDiscardStats.m[fid] += sz } vlog.lfDiscardStats.Unlock() } + +// encodedDiscardStats returns []byte representation of lfDiscardStats +// This will be called while storing stats in BadgerDB +func (vlog *valueLog) encodedDiscardStats() []byte { + vlog.lfDiscardStats.Lock() + defer vlog.lfDiscardStats.Unlock() + + encodedStats, _ := json.Marshal(vlog.lfDiscardStats.m) + return encodedStats +} + +// populateDiscardStats populates vlog.lfDiscardStats +// This function will be called while initializing valueLog +func (vlog *valueLog) populateDiscardStats() error { + discardStatsKey := y.KeyWithTs(lfDiscardStatsKey, math.MaxUint64) + vs, err := vlog.db.get(discardStatsKey) + if err != nil { + return err + } + + // check if value is Empty + if vs.Value == nil || len(vs.Value) == 0 { + vlog.lfDiscardStats = &lfDiscardStats{m: make(map[uint32]int64)} + return nil + } + + var statsMap map[uint32]int64 + if err := json.Unmarshal(vs.Value, &statsMap); err != nil { + return err + } + vlog.opt.Debugf("Value Log Discard stats: %v", statsMap) + vlog.lfDiscardStats = &lfDiscardStats{m: statsMap} + return nil +} diff --git a/vendor/github.com/dgraph-io/badger/value_test.go b/vendor/github.com/dgraph-io/badger/value_test.go deleted file mode 100644 index b32d583c..00000000 --- a/vendor/github.com/dgraph-io/badger/value_test.go +++ /dev/null @@ -1,880 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package badger - -import ( - "fmt" - "io/ioutil" - "math/rand" - "os" - "sync" - "testing" - - "github.com/stretchr/testify/require" - humanize "gx/ipfs/QmQMxG9D52TirZd9eLA37nxiNspnMRkKbyPWrVAa1gvtSy/go-humanize" - "gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/options" - "gx/ipfs/QmU4emVTYFKnoJ5yK3pPEN9joyEx6U7y892PDx26ZtNxQd/badger/y" -) - -func TestValueBasic(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - y.Check(err) - defer os.RemoveAll(dir) - - kv, _ := Open(getTestOptions(dir)) - defer kv.Close() - log := &kv.vlog - - // Use value big enough that the value log writes them even if SyncWrites is false. - const val1 = "sampleval012345678901234567890123" - const val2 = "samplevalb012345678901234567890123" - require.True(t, len(val1) >= kv.opt.ValueThreshold) - - e := &Entry{ - Key: []byte("samplekey"), - Value: []byte(val1), - meta: bitValuePointer, - } - e2 := &Entry{ - Key: []byte("samplekeyb"), - Value: []byte(val2), - meta: bitValuePointer, - } - - b := new(request) - b.Entries = []*Entry{e, e2} - - log.write([]*request{b}) - require.Len(t, b.Ptrs, 2) - t.Logf("Pointer written: %+v %+v\n", b.Ptrs[0], b.Ptrs[1]) - - s := new(y.Slice) - buf1, cb1, err1 := log.readValueBytes(b.Ptrs[0], s) - buf2, cb2, err2 := log.readValueBytes(b.Ptrs[1], s) - require.NoError(t, err1) - require.NoError(t, err2) - defer runCallback(cb1) - defer runCallback(cb2) - - readEntries := []Entry{valueBytesToEntry(buf1), valueBytesToEntry(buf2)} - require.EqualValues(t, []Entry{ - { - Key: []byte("samplekey"), - Value: []byte(val1), - meta: bitValuePointer, - }, - { - Key: []byte("samplekeyb"), - Value: []byte(val2), - meta: bitValuePointer, - }, - }, readEntries) - -} - -func TestValueGCManaged(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - N := 10000 - opt := getTestOptions(dir) - opt.ValueLogMaxEntries = uint32(N / 10) - opt.managedTxns = true - db, err := Open(opt) - require.NoError(t, err) - defer db.Close() - - var ts uint64 - newTs := func() uint64 { - ts++ - return ts - } - - sz := 64 << 10 - var wg sync.WaitGroup - for i := 0; i < N; i++ { - v := make([]byte, sz) - rand.Read(v[:rand.Intn(sz)]) - - wg.Add(1) - txn := db.NewTransactionAt(newTs(), true) - require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v)) - require.NoError(t, txn.CommitAt(newTs(), func(err error) { - wg.Done() - require.NoError(t, err) - })) - } - - for i := 0; i < N; i++ { - wg.Add(1) - txn := db.NewTransactionAt(newTs(), true) - require.NoError(t, txn.Delete([]byte(fmt.Sprintf("key%d", i)))) - require.NoError(t, txn.CommitAt(newTs(), func(err error) { - wg.Done() - require.NoError(t, err) - })) - } - wg.Wait() - files, err := ioutil.ReadDir(dir) - require.NoError(t, err) - for _, fi := range files { - t.Logf("File: %s. Size: %s\n", fi.Name(), humanize.Bytes(uint64(fi.Size()))) - } - - for i := 0; i < 100; i++ { - // Try at max 100 times to GC even a single value log file. - if err := db.RunValueLogGC(0.0001); err == nil { - return // Done - } - } - require.Fail(t, "Unable to GC even a single value log file.") -} - -func TestValueGC(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opt := getTestOptions(dir) - opt.ValueLogFileSize = 1 << 20 - - kv, _ := Open(opt) - defer kv.Close() - - sz := 32 << 10 - txn := kv.NewTransaction(true) - for i := 0; i < 100; i++ { - v := make([]byte, sz) - rand.Read(v[:rand.Intn(sz)]) - require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v)) - if i%20 == 0 { - require.NoError(t, txn.Commit()) - txn = kv.NewTransaction(true) - } - } - require.NoError(t, txn.Commit()) - - for i := 0; i < 45; i++ { - txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i))) - } - - kv.vlog.filesLock.RLock() - lf := kv.vlog.filesMap[kv.vlog.sortedFids()[0]] - kv.vlog.filesLock.RUnlock() - - // lf.iterate(0, func(e Entry) bool { - // e.print("lf") - // return true - // }) - - tr := trace.New("Test", "Test") - defer tr.Finish() - kv.vlog.rewrite(lf, tr) - for i := 45; i < 100; i++ { - key := []byte(fmt.Sprintf("key%d", i)) - - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get(key) - require.NoError(t, err) - val := getItemValue(t, item) - require.NotNil(t, val) - require.True(t, len(val) == sz, "Size found: %d", len(val)) - return nil - })) - } -} - -func TestValueGC2(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opt := getTestOptions(dir) - opt.ValueLogFileSize = 1 << 20 - - kv, _ := Open(opt) - defer kv.Close() - - sz := 32 << 10 - txn := kv.NewTransaction(true) - for i := 0; i < 100; i++ { - v := make([]byte, sz) - rand.Read(v[:rand.Intn(sz)]) - require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v)) - if i%20 == 0 { - require.NoError(t, txn.Commit()) - txn = kv.NewTransaction(true) - } - } - require.NoError(t, txn.Commit()) - - for i := 0; i < 5; i++ { - txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i))) - } - - for i := 5; i < 10; i++ { - v := []byte(fmt.Sprintf("value%d", i)) - txnSet(t, kv, []byte(fmt.Sprintf("key%d", i)), v, 0) - } - - kv.vlog.filesLock.RLock() - lf := kv.vlog.filesMap[kv.vlog.sortedFids()[0]] - kv.vlog.filesLock.RUnlock() - - // lf.iterate(0, func(e Entry) bool { - // e.print("lf") - // return true - // }) - - tr := trace.New("Test", "Test") - defer tr.Finish() - kv.vlog.rewrite(lf, tr) - for i := 0; i < 5; i++ { - key := []byte(fmt.Sprintf("key%d", i)) - require.NoError(t, kv.View(func(txn *Txn) error { - _, err := txn.Get(key) - require.Equal(t, ErrKeyNotFound, err) - return nil - })) - } - for i := 5; i < 10; i++ { - key := []byte(fmt.Sprintf("key%d", i)) - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get(key) - require.NoError(t, err) - val := getItemValue(t, item) - require.NotNil(t, val) - require.Equal(t, string(val), fmt.Sprintf("value%d", i)) - return nil - })) - } - for i := 10; i < 100; i++ { - key := []byte(fmt.Sprintf("key%d", i)) - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get(key) - require.NoError(t, err) - val := getItemValue(t, item) - require.NotNil(t, val) - require.True(t, len(val) == sz, "Size found: %d", len(val)) - return nil - })) - } -} - -func TestValueGC3(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opt := getTestOptions(dir) - opt.ValueLogFileSize = 1 << 20 - - kv, err := Open(opt) - require.NoError(t, err) - defer kv.Close() - - // We want to test whether an iterator can continue through a value log GC. - - valueSize := 32 << 10 - - var value3 []byte - txn := kv.NewTransaction(true) - for i := 0; i < 100; i++ { - v := make([]byte, valueSize) // 32K * 100 will take >=3'276'800 B. - if i == 3 { - value3 = v - } - rand.Read(v[:]) - // Keys key000, key001, key002, such that sorted order matches insertion order - require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%03d", i)), v)) - if i%20 == 0 { - require.NoError(t, txn.Commit()) - txn = kv.NewTransaction(true) - } - } - require.NoError(t, txn.Commit()) - - // Start an iterator to keys in the first value log file - itOpt := IteratorOptions{ - PrefetchValues: false, - PrefetchSize: 0, - Reverse: false, - } - - txn = kv.NewTransaction(true) - it := txn.NewIterator(itOpt) - defer it.Close() - // Walk a few keys - it.Rewind() - require.True(t, it.Valid()) - item := it.Item() - require.Equal(t, []byte("key000"), item.Key()) - it.Next() - require.True(t, it.Valid()) - item = it.Item() - require.Equal(t, []byte("key001"), item.Key()) - it.Next() - require.True(t, it.Valid()) - item = it.Item() - require.Equal(t, []byte("key002"), item.Key()) - - // Like other tests, we pull out a logFile to rewrite it directly - - kv.vlog.filesLock.RLock() - logFile := kv.vlog.filesMap[kv.vlog.sortedFids()[0]] - kv.vlog.filesLock.RUnlock() - - tr := trace.New("Test", "Test") - defer tr.Finish() - kv.vlog.rewrite(logFile, tr) - it.Next() - require.True(t, it.Valid()) - item = it.Item() - require.Equal(t, []byte("key003"), item.Key()) - - v3, err := item.ValueCopy(nil) - require.NoError(t, err) - require.Equal(t, value3, v3) -} - -func TestValueGC4(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opt := getTestOptions(dir) - opt.ValueLogFileSize = 1 << 20 - opt.Truncate = true - - kv, err := Open(opt) - require.NoError(t, err) - defer kv.Close() - - sz := 128 << 10 // 5 entries per value log file. - txn := kv.NewTransaction(true) - for i := 0; i < 24; i++ { - v := make([]byte, sz) - rand.Read(v[:rand.Intn(sz)]) - require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v)) - if i%3 == 0 { - require.NoError(t, txn.Commit()) - txn = kv.NewTransaction(true) - } - } - require.NoError(t, txn.Commit()) - - for i := 0; i < 8; i++ { - txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i))) - } - - for i := 8; i < 16; i++ { - v := []byte(fmt.Sprintf("value%d", i)) - txnSet(t, kv, []byte(fmt.Sprintf("key%d", i)), v, 0) - } - - kv.vlog.filesLock.RLock() - lf0 := kv.vlog.filesMap[kv.vlog.sortedFids()[0]] - lf1 := kv.vlog.filesMap[kv.vlog.sortedFids()[1]] - kv.vlog.filesLock.RUnlock() - - // lf.iterate(0, func(e Entry) bool { - // e.print("lf") - // return true - // }) - - tr := trace.New("Test", "Test") - defer tr.Finish() - kv.vlog.rewrite(lf0, tr) - kv.vlog.rewrite(lf1, tr) - - err = kv.vlog.Close() - require.NoError(t, err) - - err = kv.vlog.open(kv, valuePointer{Fid: 2}, kv.replayFunction()) - require.NoError(t, err) - - for i := 0; i < 8; i++ { - key := []byte(fmt.Sprintf("key%d", i)) - require.NoError(t, kv.View(func(txn *Txn) error { - _, err := txn.Get(key) - require.Equal(t, ErrKeyNotFound, err) - return nil - })) - } - for i := 8; i < 16; i++ { - key := []byte(fmt.Sprintf("key%d", i)) - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get(key) - require.NoError(t, err) - val := getItemValue(t, item) - require.NotNil(t, val) - require.Equal(t, string(val), fmt.Sprintf("value%d", i)) - return nil - })) - } -} - -func TestChecksums(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - // Set up SST with K1=V1 - opts := getTestOptions(dir) - opts.Truncate = true - opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb - kv, err := Open(opts) - require.NoError(t, err) - require.NoError(t, kv.Close()) - - var ( - k0 = []byte("k0") - k1 = []byte("k1") - k2 = []byte("k2") - k3 = []byte("k3") - v0 = []byte("value0-012345678901234567890123012345678901234567890123") - v1 = []byte("value1-012345678901234567890123012345678901234567890123") - v2 = []byte("value2-012345678901234567890123012345678901234567890123") - v3 = []byte("value3-012345678901234567890123012345678901234567890123") - ) - // Make sure the value log would actually store the item - require.True(t, len(v0) >= kv.opt.ValueThreshold) - - // Use a vlog with K0=V0 and a (corrupted) second transaction(k1,k2) - buf := createVlog(t, []*Entry{ - {Key: k0, Value: v0}, - {Key: k1, Value: v1}, - {Key: k2, Value: v2}, - }) - buf[len(buf)-1]++ // Corrupt last byte - require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777)) - - // K1 should exist, but K2 shouldn't. - kv, err = Open(opts) - require.NoError(t, err) - - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get(k0) - require.NoError(t, err) - require.Equal(t, getItemValue(t, item), v0) - - _, err = txn.Get(k1) - require.Equal(t, ErrKeyNotFound, err) - - _, err = txn.Get(k2) - require.Equal(t, ErrKeyNotFound, err) - return nil - })) - - // Write K3 at the end of the vlog. - txnSet(t, kv, k3, v3, 0) - require.NoError(t, kv.Close()) - - // The vlog should contain K0 and K3 (K1 and k2 was lost when Badger started up - // last due to checksum failure). - kv, err = Open(opts) - require.NoError(t, err) - - { - txn := kv.NewTransaction(false) - - iter := txn.NewIterator(DefaultIteratorOptions) - iter.Seek(k0) - require.True(t, iter.Valid()) - it := iter.Item() - require.Equal(t, it.Key(), k0) - require.Equal(t, getItemValue(t, it), v0) - iter.Next() - require.True(t, iter.Valid()) - it = iter.Item() - require.Equal(t, it.Key(), k3) - require.Equal(t, getItemValue(t, it), v3) - - iter.Close() - txn.Discard() - } - - require.NoError(t, kv.Close()) -} - -func TestPartialAppendToValueLog(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - // Create skeleton files. - opts := getTestOptions(dir) - opts.Truncate = true - opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb - kv, err := Open(opts) - require.NoError(t, err) - require.NoError(t, kv.Close()) - - var ( - k0 = []byte("k0") - k1 = []byte("k1") - k2 = []byte("k2") - k3 = []byte("k3") - v0 = []byte("value0-01234567890123456789012012345678901234567890123") - v1 = []byte("value1-01234567890123456789012012345678901234567890123") - v2 = []byte("value2-01234567890123456789012012345678901234567890123") - v3 = []byte("value3-01234567890123456789012012345678901234567890123") - ) - // Values need to be long enough to actually get written to value log. - require.True(t, len(v3) >= kv.opt.ValueThreshold) - - // Create truncated vlog to simulate a partial append. - // k0 - single transaction, k1 and k2 in another transaction - buf := createVlog(t, []*Entry{ - {Key: k0, Value: v0}, - {Key: k1, Value: v1}, - {Key: k2, Value: v2}, - }) - buf = buf[:len(buf)-6] - require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777)) - - // Badger should now start up - kv, err = Open(opts) - require.NoError(t, err) - - require.NoError(t, kv.View(func(txn *Txn) error { - item, err := txn.Get(k0) - require.NoError(t, err) - require.Equal(t, v0, getItemValue(t, item)) - - _, err = txn.Get(k1) - require.Equal(t, ErrKeyNotFound, err) - _, err = txn.Get(k2) - require.Equal(t, ErrKeyNotFound, err) - return nil - })) - - // When K3 is set, it should be persisted after a restart. - txnSet(t, kv, k3, v3, 0) - require.NoError(t, kv.Close()) - kv, err = Open(opts) - require.NoError(t, err) - checkKeys(t, kv, [][]byte{k3}) - - // Replay value log from beginning, badger head is past k2. - require.NoError(t, kv.vlog.Close()) - require.NoError(t, - kv.vlog.open(kv, valuePointer{Fid: 0}, kv.replayFunction())) - require.NoError(t, kv.Close()) -} - -func TestReadOnlyOpenWithPartialAppendToValueLog(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - // Create skeleton files. - opts := getTestOptions(dir) - opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb - kv, err := Open(opts) - require.NoError(t, err) - require.NoError(t, kv.Close()) - - var ( - k0 = []byte("k0") - k1 = []byte("k1") - k2 = []byte("k2") - v0 = []byte("value0-012345678901234567890123") - v1 = []byte("value1-012345678901234567890123") - v2 = []byte("value2-012345678901234567890123") - ) - - // Create truncated vlog to simulate a partial append. - // k0 - single transaction, k1 and k2 in another transaction - buf := createVlog(t, []*Entry{ - {Key: k0, Value: v0}, - {Key: k1, Value: v1}, - {Key: k2, Value: v2}, - }) - buf = buf[:len(buf)-6] - require.NoError(t, ioutil.WriteFile(vlogFilePath(dir, 0), buf, 0777)) - - opts.ReadOnly = true - // Badger should fail a read-only open with values to replay - kv, err = Open(opts) - require.Error(t, err) - require.Regexp(t, "Database was not properly closed, cannot open read-only|Read-only mode is not supported on Windows", err.Error()) -} - -func TestValueLogTrigger(t *testing.T) { - t.Skip("Difficult to trigger compaction, so skipping. Re-enable after fixing #226") - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opt := getTestOptions(dir) - opt.ValueLogFileSize = 1 << 20 - kv, err := Open(opt) - require.NoError(t, err) - - // Write a lot of data, so it creates some work for valug log GC. - sz := 32 << 10 - txn := kv.NewTransaction(true) - for i := 0; i < 100; i++ { - v := make([]byte, sz) - rand.Read(v[:rand.Intn(sz)]) - require.NoError(t, txn.Set([]byte(fmt.Sprintf("key%d", i)), v)) - if i%20 == 0 { - require.NoError(t, txn.Commit()) - txn = kv.NewTransaction(true) - } - } - require.NoError(t, txn.Commit()) - - for i := 0; i < 45; i++ { - txnDelete(t, kv, []byte(fmt.Sprintf("key%d", i))) - } - - require.NoError(t, kv.RunValueLogGC(0.5)) - - require.NoError(t, kv.Close()) - - err = kv.RunValueLogGC(0.5) - require.Equal(t, ErrRejected, err, "Error should be returned after closing DB.") -} - -func createVlog(t *testing.T, entries []*Entry) []byte { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - - opts := getTestOptions(dir) - opts.ValueLogFileSize = 100 * 1024 * 1024 // 100Mb - kv, err := Open(opts) - require.NoError(t, err) - txnSet(t, kv, entries[0].Key, entries[0].Value, entries[0].meta) - entries = entries[1:] - txn := kv.NewTransaction(true) - for _, entry := range entries { - require.NoError(t, txn.SetWithMeta(entry.Key, entry.Value, entry.meta)) - } - require.NoError(t, txn.Commit()) - require.NoError(t, kv.Close()) - - filename := vlogFilePath(dir, 0) - buf, err := ioutil.ReadFile(filename) - require.NoError(t, err) - return buf -} - -func TestPenultimateLogCorruption(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - require.NoError(t, err) - defer os.RemoveAll(dir) - opt := getTestOptions(dir) - opt.ValueLogLoadingMode = options.FileIO - // Each txn generates at least two entries. 3 txns will fit each file. - opt.ValueLogMaxEntries = 5 - - db0, err := Open(opt) - require.NoError(t, err) - - h := testHelper{db: db0, t: t} - h.writeRange(0, 7) - h.readRange(0, 7) - - for i := 2; i >= 0; i-- { - fpath := vlogFilePath(dir, uint32(i)) - fi, err := os.Stat(fpath) - require.NoError(t, err) - require.True(t, fi.Size() > 0, "Empty file at log=%d", i) - if i == 0 { - err := os.Truncate(fpath, fi.Size()-1) - require.NoError(t, err) - } - } - // Simulate a crash by not closing db0, but releasing the locks. - if db0.dirLockGuard != nil { - require.NoError(t, db0.dirLockGuard.release()) - } - if db0.valueDirGuard != nil { - require.NoError(t, db0.valueDirGuard.release()) - } - - opt.Truncate = true - db1, err := Open(opt) - require.NoError(t, err) - h.db = db1 - h.readRange(0, 1) // Only 2 should be gone, because it is at the end of logfile 0. - h.readRange(3, 7) - err = db1.View(func(txn *Txn) error { - _, err := txn.Get(h.key(2)) // Verify that 2 is gone. - require.Equal(t, ErrKeyNotFound, err) - return nil - }) - require.NoError(t, err) - require.NoError(t, db1.Close()) -} - -func checkKeys(t *testing.T, kv *DB, keys [][]byte) { - i := 0 - txn := kv.NewTransaction(false) - iter := txn.NewIterator(IteratorOptions{}) - for iter.Seek(keys[0]); iter.Valid(); iter.Next() { - require.Equal(t, iter.Item().Key(), keys[i]) - i++ - } - require.Equal(t, i, len(keys)) -} - -type testHelper struct { - db *DB - t *testing.T - val []byte -} - -func (th *testHelper) key(i int) []byte { - return []byte(fmt.Sprintf("%010d", i)) -} -func (th *testHelper) value() []byte { - if len(th.val) > 0 { - return th.val - } - th.val = make([]byte, 100) - y.Check2(rand.Read(th.val)) - return th.val -} - -// writeRange [from, to]. -func (th *testHelper) writeRange(from, to int) { - for i := from; i <= to; i++ { - err := th.db.Update(func(txn *Txn) error { - return txn.Set(th.key(i), th.value()) - }) - require.NoError(th.t, err) - } -} - -func (th *testHelper) readRange(from, to int) { - for i := from; i <= to; i++ { - err := th.db.View(func(txn *Txn) error { - item, err := txn.Get(th.key(i)) - if err != nil { - return err - } - return item.Value(func(val []byte) error { - require.Equal(th.t, val, th.value(), "key=%q", th.key(i)) - return nil - - }) - }) - require.NoError(th.t, err, "key=%q", th.key(i)) - } -} - -// Test Bug #578, which showed that if a value is moved during value log GC, an -// older version can end up at a higher level in the LSM tree than a newer -// version, causing the data to not be returned. -func TestBug578(t *testing.T) { - dir, err := ioutil.TempDir("", "badger") - y.Check(err) - defer os.RemoveAll(dir) - - opts := DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - opts.ValueLogMaxEntries = 64 - opts.MaxTableSize = 1 << 13 - - db, err := Open(opts) - require.NoError(t, err) - - h := testHelper{db: db, t: t} - - // Let's run this whole thing a few times. - for j := 0; j < 10; j++ { - t.Logf("Cycle: %d\n", j) - h.writeRange(0, 32) - h.writeRange(0, 10) - h.writeRange(50, 72) - h.writeRange(40, 72) - h.writeRange(40, 72) - - // Run value log GC a few times. - for i := 0; i < 5; i++ { - db.RunValueLogGC(0.5) - } - h.readRange(0, 10) - } -} - -func BenchmarkReadWrite(b *testing.B) { - rwRatio := []float32{ - 0.1, 0.2, 0.5, 1.0, - } - valueSize := []int{ - 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, - } - - for _, vsz := range valueSize { - for _, rw := range rwRatio { - b.Run(fmt.Sprintf("%3.1f,%04d", rw, vsz), func(b *testing.B) { - dir, err := ioutil.TempDir("", "vlog-benchmark") - y.Check(err) - defer os.RemoveAll(dir) - - db, err := Open(getTestOptions(dir)) - y.Check(err) - - vl := &db.vlog - b.ResetTimer() - - for i := 0; i < b.N; i++ { - e := new(Entry) - e.Key = make([]byte, 16) - e.Value = make([]byte, vsz) - bl := new(request) - bl.Entries = []*Entry{e} - - var ptrs []valuePointer - - vl.write([]*request{bl}) - ptrs = append(ptrs, bl.Ptrs...) - - f := rand.Float32() - if f < rw { - vl.write([]*request{bl}) - - } else { - ln := len(ptrs) - if ln == 0 { - b.Fatalf("Zero length of ptrs") - } - idx := rand.Intn(ln) - s := new(y.Slice) - buf, cb, err := vl.readValueBytes(ptrs[idx], s) - if err != nil { - b.Fatalf("Benchmark Read: %v", err) - } - - e := valueBytesToEntry(buf) - if len(e.Key) != 16 { - b.Fatalf("Key is invalid") - } - if len(e.Value) != vsz { - b.Fatalf("Value is invalid") - } - cb() - } - } - }) - } - } -} diff --git a/vendor/github.com/dgraph-io/badger/y/error.go b/vendor/github.com/dgraph-io/badger/y/error.go index 4f341cab..59bb2835 100644 --- a/vendor/github.com/dgraph-io/badger/y/error.go +++ b/vendor/github.com/dgraph-io/badger/y/error.go @@ -32,7 +32,7 @@ import ( "fmt" "log" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/pkg/errors" ) var debugMode = true diff --git a/vendor/github.com/dgraph-io/badger/y/file_dsync.go b/vendor/github.com/dgraph-io/badger/y/file_dsync.go index 10b8c9ca..3f3445e2 100644 --- a/vendor/github.com/dgraph-io/badger/y/file_dsync.go +++ b/vendor/github.com/dgraph-io/badger/y/file_dsync.go @@ -18,7 +18,7 @@ package y -import "gx/ipfs/QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu/sys/unix" +import "golang.org/x/sys/unix" func init() { datasyncFileFlag = unix.O_DSYNC diff --git a/vendor/github.com/dgraph-io/badger/y/iterator.go b/vendor/github.com/dgraph-io/badger/y/iterator.go index c6eb9f0b..719e8ec8 100644 --- a/vendor/github.com/dgraph-io/badger/y/iterator.go +++ b/vendor/github.com/dgraph-io/badger/y/iterator.go @@ -21,7 +21,7 @@ import ( "container/heap" "encoding/binary" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/pkg/errors" ) // ValueStruct represents the value info that can be associated with a key, but also the internal diff --git a/vendor/github.com/dgraph-io/badger/y/iterator_test.go b/vendor/github.com/dgraph-io/badger/y/iterator_test.go deleted file mode 100644 index cff88be0..00000000 --- a/vendor/github.com/dgraph-io/badger/y/iterator_test.go +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright 2017 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package y - -import ( - "sort" - "testing" - - "github.com/stretchr/testify/require" -) - -type SimpleIterator struct { - keys [][]byte - vals [][]byte - idx int - reversed bool -} - -var ( - closeCount int -) - -func (s *SimpleIterator) Close() error { closeCount++; return nil } - -func (s *SimpleIterator) Next() { - if !s.reversed { - s.idx++ - } else { - s.idx-- - } -} - -func (s *SimpleIterator) Rewind() { - if !s.reversed { - s.idx = 0 - } else { - s.idx = len(s.keys) - 1 - } -} - -func (s *SimpleIterator) Seek(key []byte) { - key = KeyWithTs(key, 0) - if !s.reversed { - s.idx = sort.Search(len(s.keys), func(i int) bool { - return CompareKeys(s.keys[i], key) >= 0 - }) - } else { - n := len(s.keys) - s.idx = n - 1 - sort.Search(n, func(i int) bool { - return CompareKeys(s.keys[n-1-i], key) <= 0 - }) - } -} - -func (s *SimpleIterator) Key() []byte { return s.keys[s.idx] } -func (s *SimpleIterator) Value() ValueStruct { - return ValueStruct{ - Value: s.vals[s.idx], - UserMeta: 55, - Meta: 0, - } -} -func (s *SimpleIterator) Valid() bool { - return s.idx >= 0 && s.idx < len(s.keys) -} - -func newSimpleIterator(keys []string, vals []string, reversed bool) *SimpleIterator { - k := make([][]byte, len(keys)) - v := make([][]byte, len(vals)) - AssertTrue(len(keys) == len(vals)) - for i := 0; i < len(keys); i++ { - k[i] = KeyWithTs([]byte(keys[i]), 0) - v[i] = []byte(vals[i]) - } - return &SimpleIterator{ - keys: k, - vals: v, - idx: -1, - reversed: reversed, - } -} - -func getAll(it Iterator) ([]string, []string) { - var keys, vals []string - for ; it.Valid(); it.Next() { - k := it.Key() - keys = append(keys, string(ParseKey(k))) - v := it.Value() - vals = append(vals, string(v.Value)) - } - return keys, vals -} - -func closeAndCheck(t *testing.T, it Iterator, expected int) { - closeCount = 0 - it.Close() - require.EqualValues(t, expected, closeCount) -} - -func TestSimpleIterator(t *testing.T) { - keys := []string{"1", "2", "3"} - vals := []string{"v1", "v2", "v3"} - it := newSimpleIterator(keys, vals, false) - it.Rewind() - k, v := getAll(it) - require.EqualValues(t, keys, k) - require.EqualValues(t, vals, v) - - closeAndCheck(t, it, 1) -} - -func reversed(a []string) []string { - var out []string - for i := len(a) - 1; i >= 0; i-- { - out = append(out, a[i]) - } - return out -} - -func TestMergeSingle(t *testing.T) { - keys := []string{"1", "2", "3"} - vals := []string{"v1", "v2", "v3"} - it := newSimpleIterator(keys, vals, false) - mergeIt := NewMergeIterator([]Iterator{it}, false) - mergeIt.Rewind() - k, v := getAll(mergeIt) - require.EqualValues(t, keys, k) - require.EqualValues(t, vals, v) - closeAndCheck(t, mergeIt, 1) -} - -func TestMergeSingleReversed(t *testing.T) { - keys := []string{"1", "2", "3"} - vals := []string{"v1", "v2", "v3"} - it := newSimpleIterator(keys, vals, true) - mergeIt := NewMergeIterator([]Iterator{it}, true) - mergeIt.Rewind() - k, v := getAll(mergeIt) - require.EqualValues(t, reversed(keys), k) - require.EqualValues(t, reversed(vals), v) - closeAndCheck(t, mergeIt, 1) -} - -func TestMergeMore(t *testing.T) { - it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, false) - it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, false) - it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, false) - it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, false) - - mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, false) - expectedKeys := []string{"1", "2", "3", "5", "7", "9"} - expectedVals := []string{"a1", "b2", "a3", "b5", "a7", "d9"} - mergeIt.Rewind() - k, v := getAll(mergeIt) - require.EqualValues(t, expectedKeys, k) - require.EqualValues(t, expectedVals, v) - closeAndCheck(t, mergeIt, 4) -} - -// Ensure MergeIterator satisfies the Iterator interface -func TestMergeIteratorNested(t *testing.T) { - keys := []string{"1", "2", "3"} - vals := []string{"v1", "v2", "v3"} - it := newSimpleIterator(keys, vals, false) - mergeIt := NewMergeIterator([]Iterator{it}, false) - mergeIt2 := NewMergeIterator([]Iterator{mergeIt}, false) - mergeIt2.Rewind() - k, v := getAll(mergeIt2) - require.EqualValues(t, keys, k) - require.EqualValues(t, vals, v) - closeAndCheck(t, mergeIt2, 1) -} - -func TestMergeIteratorSeek(t *testing.T) { - it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, false) - it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, false) - it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, false) - it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, false) - mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, false) - mergeIt.Seek([]byte("4")) - k, v := getAll(mergeIt) - require.EqualValues(t, []string{"5", "7", "9"}, k) - require.EqualValues(t, []string{"b5", "a7", "d9"}, v) - closeAndCheck(t, mergeIt, 4) -} - -func TestMergeIteratorSeekReversed(t *testing.T) { - it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, true) - it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, true) - it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, true) - it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, true) - mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, true) - mergeIt.Seek([]byte("5")) - k, v := getAll(mergeIt) - require.EqualValues(t, []string{"5", "3", "2", "1"}, k) - require.EqualValues(t, []string{"b5", "a3", "b2", "a1"}, v) - closeAndCheck(t, mergeIt, 4) -} - -func TestMergeIteratorSeekInvalid(t *testing.T) { - it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, false) - it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, false) - it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, false) - it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, false) - mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, false) - mergeIt.Seek([]byte("f")) - require.False(t, mergeIt.Valid()) - closeAndCheck(t, mergeIt, 4) -} - -func TestMergeIteratorSeekInvalidReversed(t *testing.T) { - it := newSimpleIterator([]string{"1", "3", "7"}, []string{"a1", "a3", "a7"}, true) - it2 := newSimpleIterator([]string{"2", "3", "5"}, []string{"b2", "b3", "b5"}, true) - it3 := newSimpleIterator([]string{"1"}, []string{"c1"}, true) - it4 := newSimpleIterator([]string{"1", "7", "9"}, []string{"d1", "d7", "d9"}, true) - mergeIt := NewMergeIterator([]Iterator{it, it2, it3, it4}, true) - mergeIt.Seek([]byte("0")) - require.False(t, mergeIt.Valid()) - closeAndCheck(t, mergeIt, 4) -} diff --git a/vendor/github.com/dgraph-io/badger/y/mmap_unix.go b/vendor/github.com/dgraph-io/badger/y/mmap_unix.go index 0d6d70ea..f9203a01 100644 --- a/vendor/github.com/dgraph-io/badger/y/mmap_unix.go +++ b/vendor/github.com/dgraph-io/badger/y/mmap_unix.go @@ -23,7 +23,7 @@ import ( "syscall" "unsafe" - "gx/ipfs/QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu/sys/unix" + "golang.org/x/sys/unix" ) // Mmap uses the mmap system call to memory-map a file. If writable is true, diff --git a/vendor/github.com/dgraph-io/badger/y/watermark.go b/vendor/github.com/dgraph-io/badger/y/watermark.go index 6d17c3cb..c0bbb194 100644 --- a/vendor/github.com/dgraph-io/badger/y/watermark.go +++ b/vendor/github.com/dgraph-io/badger/y/watermark.go @@ -21,15 +21,15 @@ import ( "context" "sync/atomic" - "gx/ipfs/QmRvYNctevGUW52urgmoFZscT6buMKqhHezLUS64WepGWn/go-net/trace" + "golang.org/x/net/trace" ) type uint64Heap []uint64 -func (u uint64Heap) Len() int { return len(u) } -func (u uint64Heap) Less(i int, j int) bool { return u[i] < u[j] } -func (u uint64Heap) Swap(i int, j int) { u[i], u[j] = u[j], u[i] } -func (u *uint64Heap) Push(x interface{}) { *u = append(*u, x.(uint64)) } +func (u uint64Heap) Len() int { return len(u) } +func (u uint64Heap) Less(i, j int) bool { return u[i] < u[j] } +func (u uint64Heap) Swap(i, j int) { u[i], u[j] = u[j], u[i] } +func (u *uint64Heap) Push(x interface{}) { *u = append(*u, x.(uint64)) } func (u *uint64Heap) Pop() interface{} { old := *u n := len(old) diff --git a/vendor/github.com/dgraph-io/badger/y/y.go b/vendor/github.com/dgraph-io/badger/y/y.go index 47cb2922..4948315a 100644 --- a/vendor/github.com/dgraph-io/badger/y/y.go +++ b/vendor/github.com/dgraph-io/badger/y/y.go @@ -26,7 +26,7 @@ import ( "sync" "time" - "gx/ipfs/QmVmDhyTTUcQXFD1rRQ64fGLMSAoaQvNH3hwuaCFAPq2hy/errors" + "github.com/pkg/errors" ) // ErrEOF indicates an end of file when trying to read from a memory mapped file @@ -48,6 +48,9 @@ var ( // CastagnoliCrcTable is a CRC32 polynomial table CastagnoliCrcTable = crc32.MakeTable(crc32.Castagnoli) + + // Dummy channel for nil closers. + dummyCloserChan = make(chan struct{}) ) // OpenExistingFile opens an existing file, errors if it doesn't exist. @@ -91,7 +94,7 @@ func OpenTruncFile(filename string, sync bool) (*os.File, error) { } // SafeCopy does append(a[:0], src...). -func SafeCopy(a []byte, src []byte) []byte { +func SafeCopy(a, src []byte) []byte { return append(a[:0], src...) } @@ -122,7 +125,7 @@ func ParseTs(key []byte) uint64 { // is same. // a would be sorted higher than aa if we use bytes.compare // All keys should have timestamp. -func CompareKeys(key1 []byte, key2 []byte) int { +func CompareKeys(key1, key2 []byte) int { AssertTrue(len(key1) > 8 && len(key2) > 8) if cmp := bytes.Compare(key1[:len(key1)-8], key2[:len(key2)-8]); cmp != 0 { return cmp @@ -203,11 +206,17 @@ func (lc *Closer) Signal() { // HasBeenClosed gets signaled when Signal() is called. func (lc *Closer) HasBeenClosed() <-chan struct{} { + if lc == nil { + return dummyCloserChan + } return lc.closed } // Done calls Done() on the WaitGroup. func (lc *Closer) Done() { + if lc == nil { + return + } lc.waiting.Done() } @@ -227,9 +236,11 @@ func (lc *Closer) SignalAndWait() { // provides a mechanism to check for errors encountered by workers and wait for // them to finish. type Throttle struct { - wg sync.WaitGroup - ch chan struct{} - errCh chan error + once sync.Once + wg sync.WaitGroup + ch chan struct{} + errCh chan error + finishErr error } // NewThrottle creates a new throttle with a max number of workers. @@ -271,16 +282,21 @@ func (t *Throttle) Done(err error) { t.wg.Done() } -// Finish waits until all workers have finished working. It would return any -// error passed by Done. +// Finish waits until all workers have finished working. It would return any error passed by Done. +// If Finish is called multiple time, it will wait for workers to finish only once(first time). +// From next calls, it will return same error as found on first call. func (t *Throttle) Finish() error { - t.wg.Wait() - close(t.ch) - close(t.errCh) - for err := range t.errCh { - if err != nil { - return err + t.once.Do(func() { + t.wg.Wait() + close(t.ch) + close(t.errCh) + for err := range t.errCh { + if err != nil { + t.finishErr = err + return + } } - } - return nil + }) + + return t.finishErr }