From 46ec63b84932705219aad80174f8968357373b69 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Thu, 22 Aug 2019 05:47:24 -0400 Subject: [PATCH] ethdb/dbtest: addd test suite for ethdb backends (#19960) - Move the existing tests from memorydb into a generalized testsuite that can be run by any ethdb backend implementation. - Add several more test cases to clarify some non-obvious nuances when implementing a custom ethdb backend, such as the behaviour of NewIteratorWithPrefix vs NewIteratorWithStart. - Add leveldb to the testsuite using in-memory storage for fast execution. --- ethdb/dbtest/testsuite.go | 315 ++++++++++++++++++++++++++++++++ ethdb/leveldb/leveldb_test.go | 40 ++++ ethdb/memorydb/memorydb_test.go | 86 +-------- 3 files changed, 364 insertions(+), 77 deletions(-) create mode 100644 ethdb/dbtest/testsuite.go create mode 100644 ethdb/leveldb/leveldb_test.go diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go new file mode 100644 index 000000000..dce2ba2a1 --- /dev/null +++ b/ethdb/dbtest/testsuite.go @@ -0,0 +1,315 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package dbtest + +import ( + "bytes" + "reflect" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/ethdb" +) + +// TestDatabaseSuite runs a suite of tests against a KeyValueStore database +// implementation. +func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { + t.Run("Iterator", func(t *testing.T) { + tests := []struct { + content map[string]string + prefix string + order []string + }{ + // Empty databases should be iterable + {map[string]string{}, "", nil}, + {map[string]string{}, "non-existent-prefix", nil}, + + // Single-item databases should be iterable + {map[string]string{"key": "val"}, "", []string{"key"}}, + {map[string]string{"key": "val"}, "k", []string{"key"}}, + {map[string]string{"key": "val"}, "l", nil}, + + // Multi-item databases should be fully iterable + { + map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, + "", + []string{"k1", "k2", "k3", "k4", "k5"}, + }, + { + map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, + "k", + []string{"k1", "k2", "k3", "k4", "k5"}, + }, + { + map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, + "l", + nil, + }, + // Multi-item databases should be prefix-iterable + { + map[string]string{ + "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", + "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", + }, + "ka", + []string{"ka1", "ka2", "ka3", "ka4", "ka5"}, + }, + { + map[string]string{ + "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", + "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", + }, + "kc", + nil, + }, + } + for i, tt := range tests { + // Create the key-value data store + db := New() + for key, val := range tt.content { + if err := db.Put([]byte(key), []byte(val)); err != nil { + t.Fatalf("test %d: failed to insert item %s:%s into database: %v", i, key, val, err) + } + } + // Iterate over the database with the given configs and verify the results + it, idx := db.NewIteratorWithPrefix([]byte(tt.prefix)), 0 + for it.Next() { + if len(tt.order) <= idx { + t.Errorf("test %d: prefix=%q more items than expected: checking idx=%d (key %q), expecting len=%d", i, tt.prefix, idx, it.Key(), len(tt.order)) + break + } + if !bytes.Equal(it.Key(), []byte(tt.order[idx])) { + t.Errorf("test %d: item %d: key mismatch: have %s, want %s", i, idx, string(it.Key()), tt.order[idx]) + } + if !bytes.Equal(it.Value(), []byte(tt.content[tt.order[idx]])) { + t.Errorf("test %d: item %d: value mismatch: have %s, want %s", i, idx, string(it.Value()), tt.content[tt.order[idx]]) + } + idx++ + } + if err := it.Error(); err != nil { + t.Errorf("test %d: iteration failed: %v", i, err) + } + if idx != len(tt.order) { + t.Errorf("test %d: iteration terminated prematurely: have %d, want %d", i, idx, len(tt.order)) + } + db.Close() + } + }) + + t.Run("IteratorWith", func(t *testing.T) { + db := New() + defer db.Close() + + keys := []string{"1", "2", "3", "4", "6", "10", "11", "12", "20", "21", "22"} + sort.Strings(keys) // 1, 10, 11, etc + + for _, k := range keys { + if err := db.Put([]byte(k), nil); err != nil { + t.Fatal(err) + } + } + + { + it := db.NewIterator() + got, want := iterateKeys(it), keys + if err := it.Error(); err != nil { + t.Fatal(err) + } + it.Release() + if !reflect.DeepEqual(got, want) { + t.Errorf("Iterator: got: %s; want: %s", got, want) + } + } + + { + it := db.NewIteratorWithPrefix([]byte("1")) + got, want := iterateKeys(it), []string{"1", "10", "11", "12"} + if err := it.Error(); err != nil { + t.Fatal(err) + } + it.Release() + if !reflect.DeepEqual(got, want) { + t.Errorf("IteratorWithPrefix(1): got: %s; want: %s", got, want) + } + } + + { + it := db.NewIteratorWithPrefix([]byte("5")) + got, want := iterateKeys(it), []string{} + if err := it.Error(); err != nil { + t.Fatal(err) + } + it.Release() + if !reflect.DeepEqual(got, want) { + t.Errorf("IteratorWithPrefix(1): got: %s; want: %s", got, want) + } + } + + { + it := db.NewIteratorWithStart([]byte("2")) + got, want := iterateKeys(it), []string{"2", "20", "21", "22", "3", "4", "6"} + if err := it.Error(); err != nil { + t.Fatal(err) + } + it.Release() + if !reflect.DeepEqual(got, want) { + t.Errorf("IteratorWithStart(2): got: %s; want: %s", got, want) + } + } + + { + it := db.NewIteratorWithStart([]byte("5")) + got, want := iterateKeys(it), []string{"6"} + if err := it.Error(); err != nil { + t.Fatal(err) + } + it.Release() + if !reflect.DeepEqual(got, want) { + t.Errorf("IteratorWithStart(2): got: %s; want: %s", got, want) + } + } + }) + + t.Run("KeyValueOperations", func(t *testing.T) { + db := New() + defer db.Close() + + key := []byte("foo") + + if got, err := db.Has(key); err != nil { + t.Error(err) + } else if got { + t.Errorf("wrong value: %t", got) + } + + value := []byte("hello world") + if err := db.Put(key, value); err != nil { + t.Error(err) + } + + if got, err := db.Has(key); err != nil { + t.Error(err) + } else if !got { + t.Errorf("wrong value: %t", got) + } + + if got, err := db.Get(key); err != nil { + t.Error(err) + } else if !bytes.Equal(got, value) { + t.Errorf("wrong value: %q", got) + } + + if err := db.Delete(key); err != nil { + t.Error(err) + } + + if got, err := db.Has(key); err != nil { + t.Error(err) + } else if got { + t.Errorf("wrong value: %t", got) + } + }) + + t.Run("Batch", func(t *testing.T) { + db := New() + defer db.Close() + + b := db.NewBatch() + for _, k := range []string{"1", "2", "3", "4"} { + if err := b.Put([]byte(k), nil); err != nil { + t.Fatal(err) + } + } + + if has, err := db.Has([]byte("1")); err != nil { + t.Fatal(err) + } else if has { + t.Error("db contains element before batch write") + } + + if err := b.Write(); err != nil { + t.Fatal(err) + } + + { + it := db.NewIterator() + if got, want := iterateKeys(it), []string{"1", "2", "3", "4"}; !reflect.DeepEqual(got, want) { + t.Errorf("got: %s; want: %s", got, want) + } + it.Release() + } + + b.Reset() + + // Mix writes and deletes in batch + b.Put([]byte("5"), nil) + b.Delete([]byte("1")) + b.Put([]byte("6"), nil) + b.Delete([]byte("3")) + b.Put([]byte("3"), nil) + + if err := b.Write(); err != nil { + t.Fatal(err) + } + + { + it := db.NewIterator() + if got, want := iterateKeys(it), []string{"2", "3", "4", "5", "6"}; !reflect.DeepEqual(got, want) { + t.Errorf("got: %s; want: %s", got, want) + } + it.Release() + } + }) + + t.Run("BatchReplay", func(t *testing.T) { + db := New() + defer db.Close() + + want := []string{"1", "2", "3", "4"} + b := db.NewBatch() + for _, k := range want { + if err := b.Put([]byte(k), nil); err != nil { + t.Fatal(err) + } + } + + b2 := db.NewBatch() + if err := b.Replay(b2); err != nil { + t.Fatal(err) + } + + if err := b2.Replay(db); err != nil { + t.Fatal(err) + } + + it := db.NewIterator() + if got := iterateKeys(it); !reflect.DeepEqual(got, want) { + t.Errorf("got: %s; want: %s", got, want) + } + it.Release() + }) + +} + +func iterateKeys(it ethdb.Iterator) []string { + keys := []string{} + for it.Next() { + keys = append(keys, string(it.Key())) + } + sort.Strings(keys) + return keys +} diff --git a/ethdb/leveldb/leveldb_test.go b/ethdb/leveldb/leveldb_test.go new file mode 100644 index 000000000..421d9b469 --- /dev/null +++ b/ethdb/leveldb/leveldb_test.go @@ -0,0 +1,40 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package leveldb + +import ( + "testing" + + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/dbtest" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/storage" +) + +func TestLevelDB(t *testing.T) { + t.Run("DatabaseSuite", func(t *testing.T) { + dbtest.TestDatabaseSuite(t, func() ethdb.KeyValueStore { + db, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + t.Fatal(err) + } + return &Database{ + db: db, + } + }) + }) +} diff --git a/ethdb/memorydb/memorydb_test.go b/ethdb/memorydb/memorydb_test.go index 325c065c1..dba18ad30 100644 --- a/ethdb/memorydb/memorydb_test.go +++ b/ethdb/memorydb/memorydb_test.go @@ -17,84 +17,16 @@ package memorydb import ( - "bytes" "testing" + + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/dbtest" ) -// Tests that key-value iteration on top of a memory database works. -func TestMemoryDBIterator(t *testing.T) { - tests := []struct { - content map[string]string - prefix string - order []string - }{ - // Empty databases should be iterable - {map[string]string{}, "", nil}, - {map[string]string{}, "non-existent-prefix", nil}, - - // Single-item databases should be iterable - {map[string]string{"key": "val"}, "", []string{"key"}}, - {map[string]string{"key": "val"}, "k", []string{"key"}}, - {map[string]string{"key": "val"}, "l", nil}, - - // Multi-item databases should be fully iterable - { - map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "", - []string{"k1", "k2", "k3", "k4", "k5"}, - }, - { - map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "k", - []string{"k1", "k2", "k3", "k4", "k5"}, - }, - { - map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "l", - nil, - }, - // Multi-item databases should be prefix-iterable - { - map[string]string{ - "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", - "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", - }, - "ka", - []string{"ka1", "ka2", "ka3", "ka4", "ka5"}, - }, - { - map[string]string{ - "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", - "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", - }, - "kc", - nil, - }, - } - for i, tt := range tests { - // Create the key-value data store - db := New() - for key, val := range tt.content { - if err := db.Put([]byte(key), []byte(val)); err != nil { - t.Fatalf("test %d: failed to insert item %s:%s into database: %v", i, key, val, err) - } - } - // Iterate over the database with the given configs and verify the results - it, idx := db.NewIteratorWithPrefix([]byte(tt.prefix)), 0 - for it.Next() { - if !bytes.Equal(it.Key(), []byte(tt.order[idx])) { - t.Errorf("test %d: item %d: key mismatch: have %s, want %s", i, idx, string(it.Key()), tt.order[idx]) - } - if !bytes.Equal(it.Value(), []byte(tt.content[tt.order[idx]])) { - t.Errorf("test %d: item %d: value mismatch: have %s, want %s", i, idx, string(it.Value()), tt.content[tt.order[idx]]) - } - idx++ - } - if err := it.Error(); err != nil { - t.Errorf("test %d: iteration failed: %v", i, err) - } - if idx != len(tt.order) { - t.Errorf("test %d: iteration terminated prematurely: have %d, want %d", i, idx, len(tt.order)) - } - } +func TestMemoryDB(t *testing.T) { + t.Run("DatabaseSuite", func(t *testing.T) { + dbtest.TestDatabaseSuite(t, func() ethdb.KeyValueStore { + return New() + }) + }) }