326 lines
8.0 KiB
Go
326 lines
8.0 KiB
Go
/*
|
|
* Copyright 2018 Dgraph Labs, Inc. and Contributors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package badger
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"math/rand"
|
|
"os"
|
|
"path"
|
|
"regexp"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestTruncateVlogWithClose(t *testing.T) {
|
|
key := func(i int) []byte {
|
|
return []byte(fmt.Sprintf("%d%10d", i, i))
|
|
}
|
|
data := func(l int) []byte {
|
|
m := make([]byte, l)
|
|
_, err := rand.Read(m)
|
|
require.NoError(t, err)
|
|
return m
|
|
}
|
|
|
|
dir, err := ioutil.TempDir("", "badger")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(dir)
|
|
|
|
opt := getTestOptions(dir)
|
|
opt.SyncWrites = true
|
|
opt.Truncate = true
|
|
opt.ValueThreshold = 1 // Force all reads from value log.
|
|
|
|
db, err := Open(opt)
|
|
require.NoError(t, err)
|
|
|
|
err = db.Update(func(txn *Txn) error {
|
|
return txn.Set(key(0), data(4055))
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Close the DB.
|
|
require.NoError(t, db.Close())
|
|
require.NoError(t, os.Truncate(path.Join(dir, "000000.vlog"), 4096))
|
|
|
|
// Reopen and write some new data.
|
|
db, err = Open(opt)
|
|
require.NoError(t, err)
|
|
for i := 0; i < 32; i++ {
|
|
err := db.Update(func(txn *Txn) error {
|
|
return txn.Set(key(i), data(10))
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
// Read it back to ensure that we can read it now.
|
|
for i := 0; i < 32; i++ {
|
|
err := db.View(func(txn *Txn) error {
|
|
item, err := txn.Get(key(i))
|
|
require.NoError(t, err)
|
|
val := getItemValue(t, item)
|
|
require.Equal(t, 10, len(val))
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, db.Close())
|
|
|
|
// Reopen and read the data again.
|
|
db, err = Open(opt)
|
|
require.NoError(t, err)
|
|
for i := 0; i < 32; i++ {
|
|
err := db.View(func(txn *Txn) error {
|
|
item, err := txn.Get(key(i))
|
|
require.NoError(t, err)
|
|
val := getItemValue(t, item)
|
|
require.Equal(t, 10, len(val))
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, db.Close())
|
|
}
|
|
|
|
var manual = flag.Bool("manual", false, "Set when manually running some tests.")
|
|
|
|
// The following 3 TruncateVlogNoClose tests should be run one after another.
|
|
// None of these close the DB, simulating a crash. They should be run with a
|
|
// script, which truncates the value log to 4096, lining up with the end of the
|
|
// first entry in the txn. At <4096, it would cause the entry to be truncated
|
|
// immediately, at >4096, same thing.
|
|
func TestTruncateVlogNoClose(t *testing.T) {
|
|
if !*manual {
|
|
t.Skip("Skipping test meant to be run manually.")
|
|
return
|
|
}
|
|
fmt.Println("running")
|
|
dir := "p"
|
|
opts := getTestOptions(dir)
|
|
opts.SyncWrites = true
|
|
opts.Truncate = true
|
|
|
|
kv, err := Open(opts)
|
|
require.NoError(t, err)
|
|
key := func(i int) string {
|
|
return fmt.Sprintf("%d%10d", i, i)
|
|
}
|
|
data := fmt.Sprintf("%4055d", 1)
|
|
err = kv.Update(func(txn *Txn) error {
|
|
return txn.Set([]byte(key(0)), []byte(data))
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
func TestTruncateVlogNoClose2(t *testing.T) {
|
|
if !*manual {
|
|
t.Skip("Skipping test meant to be run manually.")
|
|
return
|
|
}
|
|
dir := "p"
|
|
opts := getTestOptions(dir)
|
|
opts.SyncWrites = true
|
|
opts.Truncate = true
|
|
|
|
kv, err := Open(opts)
|
|
require.NoError(t, err)
|
|
key := func(i int) string {
|
|
return fmt.Sprintf("%d%10d", i, i)
|
|
}
|
|
data := fmt.Sprintf("%10d", 1)
|
|
for i := 32; i < 64; i++ {
|
|
err := kv.Update(func(txn *Txn) error {
|
|
return txn.Set([]byte(key(i)), []byte(data))
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
for i := 32; i < 64; i++ {
|
|
require.NoError(t, kv.View(func(txn *Txn) error {
|
|
item, err := txn.Get([]byte(key(i)))
|
|
require.NoError(t, err)
|
|
val := getItemValue(t, item)
|
|
require.NotNil(t, val)
|
|
require.True(t, len(val) > 0)
|
|
return nil
|
|
}))
|
|
}
|
|
}
|
|
func TestTruncateVlogNoClose3(t *testing.T) {
|
|
if !*manual {
|
|
t.Skip("Skipping test meant to be run manually.")
|
|
return
|
|
}
|
|
fmt.Print("Running")
|
|
dir := "p"
|
|
opts := getTestOptions(dir)
|
|
opts.SyncWrites = true
|
|
opts.Truncate = true
|
|
|
|
kv, err := Open(opts)
|
|
require.NoError(t, err)
|
|
key := func(i int) string {
|
|
return fmt.Sprintf("%d%10d", i, i)
|
|
}
|
|
for i := 32; i < 64; i++ {
|
|
require.NoError(t, kv.View(func(txn *Txn) error {
|
|
item, err := txn.Get([]byte(key(i)))
|
|
require.NoError(t, err)
|
|
val := getItemValue(t, item)
|
|
require.NotNil(t, val)
|
|
require.True(t, len(val) > 0)
|
|
return nil
|
|
}))
|
|
}
|
|
}
|
|
|
|
func TestBigKeyValuePairs(t *testing.T) {
|
|
// This test takes too much memory. So, run separately.
|
|
if !*manual {
|
|
t.Skip("Skipping test meant to be run manually.")
|
|
return
|
|
}
|
|
opts := DefaultOptions
|
|
opts.MaxTableSize = 1 << 20
|
|
opts.ValueLogMaxEntries = 64
|
|
runBadgerTest(t, &opts, func(t *testing.T, db *DB) {
|
|
bigK := make([]byte, 65001)
|
|
bigV := make([]byte, db.opt.ValueLogFileSize+1)
|
|
small := make([]byte, 65000)
|
|
|
|
txn := db.NewTransaction(true)
|
|
require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, small))
|
|
require.Regexp(t, regexp.MustCompile("Value.*exceeded"), txn.Set(small, bigV))
|
|
|
|
require.NoError(t, txn.Set(small, small))
|
|
require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, bigV))
|
|
|
|
require.NoError(t, db.View(func(txn *Txn) error {
|
|
_, err := txn.Get(small)
|
|
require.Equal(t, ErrKeyNotFound, err)
|
|
return nil
|
|
}))
|
|
|
|
// Now run a longer test, which involves value log GC.
|
|
data := fmt.Sprintf("%100d", 1)
|
|
key := func(i int) string {
|
|
return fmt.Sprintf("%65000d", i)
|
|
}
|
|
|
|
saveByKey := func(key string, value []byte) error {
|
|
return db.Update(func(txn *Txn) error {
|
|
return txn.Set([]byte(key), value)
|
|
})
|
|
}
|
|
|
|
getByKey := func(key string) error {
|
|
return db.View(func(txn *Txn) error {
|
|
item, err := txn.Get([]byte(key))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return item.Value(func(val []byte) error {
|
|
if len(val) == 0 {
|
|
log.Fatalf("key not found %q", len(key))
|
|
}
|
|
return nil
|
|
})
|
|
})
|
|
}
|
|
|
|
for i := 0; i < 32; i++ {
|
|
if i < 30 {
|
|
require.NoError(t, saveByKey(key(i), []byte(data)))
|
|
} else {
|
|
require.NoError(t, saveByKey(key(i), []byte(fmt.Sprintf("%100d", i))))
|
|
}
|
|
}
|
|
|
|
for j := 0; j < 5; j++ {
|
|
for i := 0; i < 32; i++ {
|
|
if i < 30 {
|
|
require.NoError(t, saveByKey(key(i), []byte(data)))
|
|
} else {
|
|
require.NoError(t, saveByKey(key(i), []byte(fmt.Sprintf("%100d", i))))
|
|
}
|
|
}
|
|
}
|
|
|
|
for i := 0; i < 32; i++ {
|
|
require.NoError(t, getByKey(key(i)))
|
|
}
|
|
|
|
var loops int
|
|
var err error
|
|
for err == nil {
|
|
err = db.RunValueLogGC(0.5)
|
|
require.NotRegexp(t, regexp.MustCompile("truncate"), err)
|
|
loops++
|
|
}
|
|
t.Logf("Ran value log GC %d times. Last error: %v\n", loops, err)
|
|
})
|
|
}
|
|
|
|
// The following test checks for issue #585.
|
|
func TestPushValueLogLimit(t *testing.T) {
|
|
// This test takes too much memory. So, run separately.
|
|
if !*manual {
|
|
t.Skip("Skipping test meant to be run manually.")
|
|
return
|
|
}
|
|
opt := DefaultOptions
|
|
opt.ValueLogMaxEntries = 64
|
|
opt.ValueLogFileSize = 2 << 30
|
|
runBadgerTest(t, &opt, func(t *testing.T, db *DB) {
|
|
data := []byte(fmt.Sprintf("%30d", 1))
|
|
key := func(i int) string {
|
|
return fmt.Sprintf("%100d", i)
|
|
}
|
|
|
|
for i := 0; i < 32; i++ {
|
|
if i == 4 {
|
|
v := make([]byte, 2<<30)
|
|
err := db.Update(func(txn *Txn) error {
|
|
return txn.Set([]byte(key(i)), v)
|
|
})
|
|
require.NoError(t, err)
|
|
} else {
|
|
err := db.Update(func(txn *Txn) error {
|
|
return txn.Set([]byte(key(i)), data)
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
}
|
|
|
|
for i := 0; i < 32; i++ {
|
|
err := db.View(func(txn *Txn) error {
|
|
item, err := txn.Get([]byte(key(i)))
|
|
require.NoError(t, err, "Getting key: %s", key(i))
|
|
err = item.Value(func(v []byte) error {
|
|
_ = v
|
|
return nil
|
|
})
|
|
require.NoError(t, err, "Getting value: %s", key(i))
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|