core/rawdb: enforce readonly in freezer instantiation (#24119)
* freezer: add readonly flag to table * freezer: enforce readonly in table repair * freezer: enforce readonly in newFreezer * minor fix * minor * core/rawdb: test that writing during readonly fails * rm unused log * check readonly on batch append * minor * Revert "check readonly on batch append" This reverts commit 2ddb5ec4ba7534bf6edbdfec158ea99a2eed5036. * review fixes * minor test refactor * attempt at fixing windows issue * add comment re windows sync issue * k->kind * open readonly db for genesis check Co-authored-by: Martin Holst Swende <martin@swende.se>
This commit is contained in:
parent
f80ce141a1
commit
4aab440ee2
@ -539,7 +539,7 @@ func freezerInspect(ctx *cli.Context) error {
|
|||||||
defer stack.Close()
|
defer stack.Close()
|
||||||
path := filepath.Join(stack.ResolvePath("chaindata"), "ancient")
|
path := filepath.Join(stack.ResolvePath("chaindata"), "ancient")
|
||||||
log.Info("Opening freezer", "location", path, "name", kind)
|
log.Info("Opening freezer", "location", path, "name", kind)
|
||||||
if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy); err != nil {
|
if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil {
|
||||||
return err
|
return err
|
||||||
} else {
|
} else {
|
||||||
f.DumpIndex(start, end)
|
f.DumpIndex(start, end)
|
||||||
|
@ -1675,7 +1675,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
|
|||||||
if ctx.GlobalIsSet(DataDirFlag.Name) {
|
if ctx.GlobalIsSet(DataDirFlag.Name) {
|
||||||
// Check if we have an already initialized chain and fall back to
|
// Check if we have an already initialized chain and fall back to
|
||||||
// that if so. Otherwise we need to generate a new genesis spec.
|
// that if so. Otherwise we need to generate a new genesis spec.
|
||||||
chaindb := MakeChainDatabase(ctx, stack, false) // TODO (MariusVanDerWijden) make this read only
|
chaindb := MakeChainDatabase(ctx, stack, true)
|
||||||
if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
|
if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
|
||||||
cfg.Genesis = nil // fallback to db content
|
cfg.Genesis = nil // fallback to db content
|
||||||
}
|
}
|
||||||
|
@ -133,7 +133,7 @@ func newFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
|
|||||||
|
|
||||||
// Create the tables.
|
// Create the tables.
|
||||||
for name, disableSnappy := range tables {
|
for name, disableSnappy := range tables {
|
||||||
table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy)
|
table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy, readonly)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for _, table := range freezer.tables {
|
for _, table := range freezer.tables {
|
||||||
table.Close()
|
table.Close()
|
||||||
@ -144,8 +144,15 @@ func newFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
|
|||||||
freezer.tables[name] = table
|
freezer.tables[name] = table
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if freezer.readonly {
|
||||||
|
// In readonly mode only validate, don't truncate.
|
||||||
|
// validate also sets `freezer.frozen`.
|
||||||
|
err = freezer.validate()
|
||||||
|
} else {
|
||||||
// Truncate all tables to common length.
|
// Truncate all tables to common length.
|
||||||
if err := freezer.repair(); err != nil {
|
err = freezer.repair()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
for _, table := range freezer.tables {
|
for _, table := range freezer.tables {
|
||||||
table.Close()
|
table.Close()
|
||||||
}
|
}
|
||||||
@ -308,6 +315,33 @@ func (f *freezer) Sync() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// validate checks that every table has the same length.
|
||||||
|
// Used instead of `repair` in readonly mode.
|
||||||
|
func (f *freezer) validate() error {
|
||||||
|
if len(f.tables) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var (
|
||||||
|
length uint64
|
||||||
|
name string
|
||||||
|
)
|
||||||
|
// Hack to get length of any table
|
||||||
|
for kind, table := range f.tables {
|
||||||
|
length = atomic.LoadUint64(&table.items)
|
||||||
|
name = kind
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// Now check every table against that length
|
||||||
|
for kind, table := range f.tables {
|
||||||
|
items := atomic.LoadUint64(&table.items)
|
||||||
|
if length != items {
|
||||||
|
return fmt.Errorf("freezer tables %s and %s have differing lengths: %d != %d", kind, name, items, length)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
atomic.StoreUint64(&f.frozen, length)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// repair truncates all data tables to the same length.
|
// repair truncates all data tables to the same length.
|
||||||
func (f *freezer) repair() error {
|
func (f *freezer) repair() error {
|
||||||
min := uint64(math.MaxUint64)
|
min := uint64(math.MaxUint64)
|
||||||
|
@ -95,6 +95,7 @@ type freezerTable struct {
|
|||||||
items uint64 // Number of items stored in the table (including items removed from tail)
|
items uint64 // Number of items stored in the table (including items removed from tail)
|
||||||
|
|
||||||
noCompression bool // if true, disables snappy compression. Note: does not work retroactively
|
noCompression bool // if true, disables snappy compression. Note: does not work retroactively
|
||||||
|
readonly bool
|
||||||
maxFileSize uint32 // Max file size for data-files
|
maxFileSize uint32 // Max file size for data-files
|
||||||
name string
|
name string
|
||||||
path string
|
path string
|
||||||
@ -119,8 +120,8 @@ type freezerTable struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewFreezerTable opens the given path as a freezer table.
|
// NewFreezerTable opens the given path as a freezer table.
|
||||||
func NewFreezerTable(path, name string, disableSnappy bool) (*freezerTable, error) {
|
func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) {
|
||||||
return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy)
|
return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly)
|
||||||
}
|
}
|
||||||
|
|
||||||
// openFreezerFileForAppend opens a freezer table file and seeks to the end
|
// openFreezerFileForAppend opens a freezer table file and seeks to the end
|
||||||
@ -164,7 +165,7 @@ func truncateFreezerFile(file *os.File, size int64) error {
|
|||||||
// newTable opens a freezer table, creating the data and index files if they are
|
// newTable opens a freezer table, creating the data and index files if they are
|
||||||
// non existent. Both files are truncated to the shortest common length to ensure
|
// non existent. Both files are truncated to the shortest common length to ensure
|
||||||
// they don't go out of sync.
|
// they don't go out of sync.
|
||||||
func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression bool) (*freezerTable, error) {
|
func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression, readonly bool) (*freezerTable, error) {
|
||||||
// Ensure the containing directory exists and open the indexEntry file
|
// Ensure the containing directory exists and open the indexEntry file
|
||||||
if err := os.MkdirAll(path, 0755); err != nil {
|
if err := os.MkdirAll(path, 0755); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -177,7 +178,16 @@ func newTable(path string, name string, readMeter metrics.Meter, writeMeter metr
|
|||||||
// Compressed idx
|
// Compressed idx
|
||||||
idxName = fmt.Sprintf("%s.cidx", name)
|
idxName = fmt.Sprintf("%s.cidx", name)
|
||||||
}
|
}
|
||||||
offsets, err := openFreezerFileForAppend(filepath.Join(path, idxName))
|
var (
|
||||||
|
err error
|
||||||
|
offsets *os.File
|
||||||
|
)
|
||||||
|
if readonly {
|
||||||
|
// Will fail if table doesn't exist
|
||||||
|
offsets, err = openFreezerFileForReadOnly(filepath.Join(path, idxName))
|
||||||
|
} else {
|
||||||
|
offsets, err = openFreezerFileForAppend(filepath.Join(path, idxName))
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -192,6 +202,7 @@ func newTable(path string, name string, readMeter metrics.Meter, writeMeter metr
|
|||||||
path: path,
|
path: path,
|
||||||
logger: log.New("database", path, "table", name),
|
logger: log.New("database", path, "table", name),
|
||||||
noCompression: noCompression,
|
noCompression: noCompression,
|
||||||
|
readonly: readonly,
|
||||||
maxFileSize: maxFilesize,
|
maxFileSize: maxFilesize,
|
||||||
}
|
}
|
||||||
if err := tab.repair(); err != nil {
|
if err := tab.repair(); err != nil {
|
||||||
@ -252,7 +263,11 @@ func (t *freezerTable) repair() error {
|
|||||||
|
|
||||||
t.index.ReadAt(buffer, offsetsSize-indexEntrySize)
|
t.index.ReadAt(buffer, offsetsSize-indexEntrySize)
|
||||||
lastIndex.unmarshalBinary(buffer)
|
lastIndex.unmarshalBinary(buffer)
|
||||||
|
if t.readonly {
|
||||||
|
t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForReadOnly)
|
||||||
|
} else {
|
||||||
t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForAppend)
|
t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForAppend)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -301,6 +316,8 @@ func (t *freezerTable) repair() error {
|
|||||||
contentExp = int64(lastIndex.offset)
|
contentExp = int64(lastIndex.offset)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Sync() fails for read-only files on windows.
|
||||||
|
if !t.readonly {
|
||||||
// Ensure all reparation changes have been written to disk
|
// Ensure all reparation changes have been written to disk
|
||||||
if err := t.index.Sync(); err != nil {
|
if err := t.index.Sync(); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -308,6 +325,7 @@ func (t *freezerTable) repair() error {
|
|||||||
if err := t.head.Sync(); err != nil {
|
if err := t.head.Sync(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
// Update the item and byte counters and return
|
// Update the item and byte counters and return
|
||||||
t.items = uint64(t.itemOffset) + uint64(offsetsSize/indexEntrySize-1) // last indexEntry points to the end of the data file
|
t.items = uint64(t.itemOffset) + uint64(offsetsSize/indexEntrySize-1) // last indexEntry points to the end of the data file
|
||||||
t.headBytes = contentSize
|
t.headBytes = contentSize
|
||||||
@ -334,8 +352,12 @@ func (t *freezerTable) preopen() (err error) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if t.readonly {
|
||||||
|
t.head, err = t.openFile(t.headId, openFreezerFileForReadOnly)
|
||||||
|
} else {
|
||||||
// Open head in read/write
|
// Open head in read/write
|
||||||
t.head, err = t.openFile(t.headId, openFreezerFileForAppend)
|
t.head, err = t.openFile(t.headId, openFreezerFileForAppend)
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -40,7 +40,7 @@ func TestFreezerBasics(t *testing.T) {
|
|||||||
// set cutoff at 50 bytes
|
// set cutoff at 50 bytes
|
||||||
f, err := newTable(os.TempDir(),
|
f, err := newTable(os.TempDir(),
|
||||||
fmt.Sprintf("unittest-%d", rand.Uint64()),
|
fmt.Sprintf("unittest-%d", rand.Uint64()),
|
||||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true)
|
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -85,7 +85,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
|
|||||||
f *freezerTable
|
f *freezerTable
|
||||||
err error
|
err error
|
||||||
)
|
)
|
||||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -99,7 +99,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
|
|||||||
require.NoError(t, batch.commit())
|
require.NoError(t, batch.commit())
|
||||||
f.Close()
|
f.Close()
|
||||||
|
|
||||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -116,7 +116,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
|
|||||||
t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
|
t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
|
||||||
}
|
}
|
||||||
f.Close()
|
f.Close()
|
||||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -131,7 +131,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
|
|||||||
|
|
||||||
// Fill table
|
// Fill table
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -160,7 +160,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
|
|||||||
|
|
||||||
// Now open it again
|
// Now open it again
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -183,7 +183,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
|||||||
|
|
||||||
// Fill a table and close it
|
// Fill a table and close it
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -209,7 +209,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
|||||||
|
|
||||||
// Now open it again
|
// Now open it again
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -232,7 +232,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
|||||||
|
|
||||||
// And if we open it, we should now be able to read all of them (new values)
|
// And if we open it, we should now be able to read all of them (new values)
|
||||||
{
|
{
|
||||||
f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
for y := 1; y < 255; y++ {
|
for y := 1; y < 255; y++ {
|
||||||
exp := getChunk(15, ^y)
|
exp := getChunk(15, ^y)
|
||||||
got, err := f.Retrieve(uint64(y))
|
got, err := f.Retrieve(uint64(y))
|
||||||
@ -254,7 +254,7 @@ func TestSnappyDetection(t *testing.T) {
|
|||||||
|
|
||||||
// Open with snappy
|
// Open with snappy
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -265,7 +265,7 @@ func TestSnappyDetection(t *testing.T) {
|
|||||||
|
|
||||||
// Open without snappy
|
// Open without snappy
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -277,7 +277,7 @@ func TestSnappyDetection(t *testing.T) {
|
|||||||
|
|
||||||
// Open with snappy
|
// Open with snappy
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -309,7 +309,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
|
|||||||
|
|
||||||
// Fill a table and close it
|
// Fill a table and close it
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -345,7 +345,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
|
|||||||
// 45, 45, 15
|
// 45, 45, 15
|
||||||
// with 3+3+1 items
|
// with 3+3+1 items
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -366,7 +366,7 @@ func TestFreezerTruncate(t *testing.T) {
|
|||||||
|
|
||||||
// Fill table
|
// Fill table
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -382,7 +382,7 @@ func TestFreezerTruncate(t *testing.T) {
|
|||||||
|
|
||||||
// Reopen, truncate
|
// Reopen, truncate
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -407,7 +407,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
|
|||||||
|
|
||||||
// Fill table
|
// Fill table
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -440,7 +440,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
|
|||||||
|
|
||||||
// Reopen
|
// Reopen
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -475,7 +475,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
|
|||||||
|
|
||||||
// Fill table
|
// Fill table
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -491,7 +491,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
|
|||||||
|
|
||||||
// Reopen and read all files
|
// Reopen and read all files
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -523,7 +523,7 @@ func TestFreezerOffset(t *testing.T) {
|
|||||||
|
|
||||||
// Fill table
|
// Fill table
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -584,7 +584,7 @@ func TestFreezerOffset(t *testing.T) {
|
|||||||
|
|
||||||
// Now open again
|
// Now open again
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -638,7 +638,7 @@ func TestFreezerOffset(t *testing.T) {
|
|||||||
|
|
||||||
// Check that existing items have been moved to index 1M.
|
// Check that existing items have been moved to index 1M.
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -726,7 +726,7 @@ func TestSequentialRead(t *testing.T) {
|
|||||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||||
fname := fmt.Sprintf("batchread-%d", rand.Uint64())
|
fname := fmt.Sprintf("batchread-%d", rand.Uint64())
|
||||||
{ // Fill table
|
{ // Fill table
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -736,7 +736,7 @@ func TestSequentialRead(t *testing.T) {
|
|||||||
f.Close()
|
f.Close()
|
||||||
}
|
}
|
||||||
{ // Open it, iterate, verify iteration
|
{ // Open it, iterate, verify iteration
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -757,7 +757,7 @@ func TestSequentialRead(t *testing.T) {
|
|||||||
}
|
}
|
||||||
{ // Open it, iterate, verify byte limit. The byte limit is less than item
|
{ // Open it, iterate, verify byte limit. The byte limit is less than item
|
||||||
// size, so each lookup should only return one item
|
// size, so each lookup should only return one item
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -786,7 +786,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
|
|||||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||||
fname := fmt.Sprintf("batchread-2-%d", rand.Uint64())
|
fname := fmt.Sprintf("batchread-2-%d", rand.Uint64())
|
||||||
{ // Fill table
|
{ // Fill table
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -808,7 +808,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
|
|||||||
{100, 109, 10},
|
{100, 109, 10},
|
||||||
} {
|
} {
|
||||||
{
|
{
|
||||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true)
|
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -829,3 +829,89 @@ func TestSequentialReadByteLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFreezerReadonly(t *testing.T) {
|
||||||
|
tmpdir := os.TempDir()
|
||||||
|
// Case 1: Check it fails on non-existent file.
|
||||||
|
_, err := newTable(tmpdir,
|
||||||
|
fmt.Sprintf("readonlytest-%d", rand.Uint64()),
|
||||||
|
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("readonly table instantiation should fail for non-existent table")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Case 2: Check that it fails on invalid index length.
|
||||||
|
fname := fmt.Sprintf("readonlytest-%d", rand.Uint64())
|
||||||
|
idxFile, err := openFreezerFileForAppend(filepath.Join(tmpdir, fmt.Sprintf("%s.ridx", fname)))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Failed to open index file: %v\n", err)
|
||||||
|
}
|
||||||
|
// size should not be a multiple of indexEntrySize.
|
||||||
|
idxFile.Write(make([]byte, 17))
|
||||||
|
idxFile.Close()
|
||||||
|
_, err = newTable(tmpdir, fname,
|
||||||
|
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("readonly table instantiation should fail for invalid index size")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Case 3: Open table non-readonly table to write some data.
|
||||||
|
// Then corrupt the head file and make sure opening the table
|
||||||
|
// again in readonly triggers an error.
|
||||||
|
fname = fmt.Sprintf("readonlytest-%d", rand.Uint64())
|
||||||
|
f, err := newTable(tmpdir, fname,
|
||||||
|
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to instantiate table: %v", err)
|
||||||
|
}
|
||||||
|
writeChunks(t, f, 8, 32)
|
||||||
|
// Corrupt table file
|
||||||
|
if _, err := f.head.Write([]byte{1, 1}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = newTable(tmpdir, fname,
|
||||||
|
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("readonly table instantiation should fail for corrupt table file")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Case 4: Write some data to a table and later re-open it as readonly.
|
||||||
|
// Should be successful.
|
||||||
|
fname = fmt.Sprintf("readonlytest-%d", rand.Uint64())
|
||||||
|
f, err = newTable(tmpdir, fname,
|
||||||
|
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to instantiate table: %v\n", err)
|
||||||
|
}
|
||||||
|
writeChunks(t, f, 32, 128)
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f, err = newTable(tmpdir, fname,
|
||||||
|
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
v, err := f.Retrieve(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
exp := getChunk(128, 10)
|
||||||
|
if !bytes.Equal(v, exp) {
|
||||||
|
t.Errorf("retrieved value is incorrect")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Case 5: Now write some data via a batch.
|
||||||
|
// This should fail either during AppendRaw or Commit
|
||||||
|
batch := f.newBatch()
|
||||||
|
writeErr := batch.AppendRaw(32, make([]byte, 1))
|
||||||
|
if writeErr == nil {
|
||||||
|
writeErr = batch.commit()
|
||||||
|
}
|
||||||
|
if writeErr == nil {
|
||||||
|
t.Fatalf("Writing to readonly table should fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -253,6 +253,44 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFreezerReadonlyValidate(t *testing.T) {
|
||||||
|
tables := map[string]bool{"a": true, "b": true}
|
||||||
|
dir, err := ioutil.TempDir("", "freezer")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(dir)
|
||||||
|
// Open non-readonly freezer and fill individual tables
|
||||||
|
// with different amount of data.
|
||||||
|
f, err := newFreezer(dir, "", false, 2049, tables)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("can't open freezer", err)
|
||||||
|
}
|
||||||
|
var item = make([]byte, 1024)
|
||||||
|
aBatch := f.tables["a"].newBatch()
|
||||||
|
require.NoError(t, aBatch.AppendRaw(0, item))
|
||||||
|
require.NoError(t, aBatch.AppendRaw(1, item))
|
||||||
|
require.NoError(t, aBatch.AppendRaw(2, item))
|
||||||
|
require.NoError(t, aBatch.commit())
|
||||||
|
bBatch := f.tables["b"].newBatch()
|
||||||
|
require.NoError(t, bBatch.AppendRaw(0, item))
|
||||||
|
require.NoError(t, bBatch.commit())
|
||||||
|
if f.tables["a"].items != 3 {
|
||||||
|
t.Fatalf("unexpected number of items in table")
|
||||||
|
}
|
||||||
|
if f.tables["b"].items != 1 {
|
||||||
|
t.Fatalf("unexpected number of items in table")
|
||||||
|
}
|
||||||
|
require.NoError(t, f.Close())
|
||||||
|
|
||||||
|
// Re-openening as readonly should fail when validating
|
||||||
|
// table lengths.
|
||||||
|
f, err = newFreezer(dir, "", true, 2049, tables)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("readonly freezer should fail with differing table lengths")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func newFreezerForTesting(t *testing.T, tables map[string]bool) (*freezer, string) {
|
func newFreezerForTesting(t *testing.T, tables map[string]bool) (*freezer, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user