feat(store/v2): snapshot manager (#18458)

This commit is contained in:
cool-developer
2023-12-07 21:50:40 +00:00
committed by GitHub
parent 14bb52ad92
commit f6df368d8d
26 changed files with 776 additions and 260 deletions
+6 -24
View File
@@ -11,6 +11,7 @@ import (
_ "github.com/mattn/go-sqlite3"
"cosmossdk.io/store/v2"
"cosmossdk.io/store/v2/storage"
)
const (
@@ -40,7 +41,7 @@ const (
`
)
var _ store.VersionedDatabase = (*Database)(nil)
var _ storage.Database = (*Database)(nil)
type Database struct {
storage *sql.DB
@@ -91,6 +92,10 @@ func (db *Database) Close() error {
return err
}
func (db *Database) NewBatch(version uint64) (store.Batch, error) {
return NewBatch(db.storage, version)
}
func (db *Database) GetLatestVersion() (uint64, error) {
stmt, err := db.storage.Prepare("SELECT value FROM state_storage WHERE store_key = ? AND key = ?")
if err != nil {
@@ -168,29 +173,6 @@ func (db *Database) Get(storeKey string, targetVersion uint64, key []byte) ([]by
return nil, nil
}
func (db *Database) ApplyChangeset(version uint64, cs *store.Changeset) error {
b, err := NewBatch(db.storage, version)
if err != nil {
return err
}
for storeKey, pairs := range cs.Pairs {
for _, kvPair := range pairs {
if kvPair.Value == nil {
if err := b.Delete(storeKey, kvPair.Key); err != nil {
return err
}
} else {
if err := b.Set(storeKey, kvPair.Key, kvPair.Value); err != nil {
return err
}
}
}
}
return b.Write()
}
// Prune removes all versions of all keys that are <= the given version. It keeps
// the latest (non-tombstoned) version of each key/value tuple to handle queries
// above the prune version. This is analogous to RocksDB full_history_ts_low.
+14 -10
View File
@@ -19,7 +19,8 @@ const (
func TestStorageTestSuite(t *testing.T) {
s := &storage.StorageTestSuite{
NewDB: func(dir string) (store.VersionedDatabase, error) {
return New(dir)
db, err := New(dir)
return storage.NewStorageStore(db), err
},
EmptyBatchSize: 0,
}
@@ -31,15 +32,16 @@ func TestDatabase_ReverseIterator(t *testing.T) {
require.NoError(t, err)
defer db.Close()
cs := store.NewChangeset(map[string]store.KVPairs{storeKey1: {}})
batch, err := db.NewBatch(1)
require.NoError(t, err)
for i := 0; i < 100; i++ {
key := fmt.Sprintf("key%03d", i) // key000, key001, ..., key099
val := fmt.Sprintf("val%03d", i) // val000, val001, ..., val099
cs.AddKVPair(storeKey1, store.KVPair{Key: []byte(key), Value: []byte(val)})
require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val)))
}
require.NoError(t, db.ApplyChangeset(1, cs))
require.NoError(t, batch.Write())
// reverse iterator without an end key
iter, err := db.ReverseIterator(storeKey1, 1, []byte("key000"), nil)
@@ -106,15 +108,16 @@ func TestParallelWrites(t *testing.T) {
go func(i int) {
<-triggerStartCh
defer wg.Done()
cs := store.NewChangeset(map[string]store.KVPairs{storeKey1: {}})
batch, err := db.NewBatch(uint64(i + 1))
require.NoError(t, err)
for j := 0; j < kvCount; j++ {
key := fmt.Sprintf("key-%d-%03d", i, j)
val := fmt.Sprintf("val-%d-%03d", i, j)
cs.AddKVPair(storeKey1, store.KVPair{Key: []byte(key), Value: []byte(val)})
require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val)))
}
require.NoError(t, db.ApplyChangeset(uint64(i+1), cs))
require.NoError(t, batch.Write())
}(i)
}
@@ -155,15 +158,16 @@ func TestParallelWriteAndPruning(t *testing.T) {
<-triggerStartCh
defer wg.Done()
for i := 0; i < latestVersion; i++ {
cs := store.NewChangeset(map[string]store.KVPairs{storeKey1: {}})
batch, err := db.NewBatch(uint64(i + 1))
require.NoError(t, err)
for j := 0; j < kvCount; j++ {
key := fmt.Sprintf("key-%d-%03d", i, j)
val := fmt.Sprintf("val-%d-%03d", i, j)
cs.AddKVPair(storeKey1, store.KVPair{Key: []byte(key), Value: []byte(val)})
require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val)))
}
require.NoError(t, db.ApplyChangeset(uint64(i+1), cs))
require.NoError(t, batch.Write())
}
}()
// start a goroutine that prunes the database