diff --git a/store/commitment/db.go b/store/commitment/db.go deleted file mode 100644 index 85da3e5de0..0000000000 --- a/store/commitment/db.go +++ /dev/null @@ -1,80 +0,0 @@ -package commitment - -import ( - "sync" - - ics23 "github.com/cosmos/ics23/go" - - "cosmossdk.io/store/v2" -) - -// Database represents a state commitment store. It is designed to securely store -// and manage the most recent state information, crucial for achieving consensus. -// Each module creates its own instance of Database for managing its specific state. -type Database struct { - mu sync.Mutex - tree store.Tree -} - -// NewDatabase creates a new Database instance. -func NewDatabase(tree store.Tree) *Database { - return &Database{ - tree: tree, - } -} - -// WriteBatch writes a batch of key-value pairs to the database. -func (db *Database) WriteBatch(cs *store.Changeset) error { - db.mu.Lock() - defer db.mu.Unlock() - - return db.tree.WriteBatch(cs) -} - -// WorkingHash returns the working hash of the database. -func (db *Database) WorkingHash() []byte { - db.mu.Lock() - defer db.mu.Unlock() - - return db.tree.WorkingHash() -} - -// LoadVersion loads the state at the given version. -func (db *Database) LoadVersion(version uint64) error { - db.mu.Lock() - defer db.mu.Unlock() - - return db.tree.LoadVersion(version) -} - -// Commit commits the current state to the database. -func (db *Database) Commit() ([]byte, error) { - db.mu.Lock() - defer db.mu.Unlock() - - return db.tree.Commit() -} - -// GetProof returns a proof for the given key and version. -func (db *Database) GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) { - db.mu.Lock() - defer db.mu.Unlock() - - return db.tree.GetProof(version, key) -} - -// GetLatestVersion returns the latest version of the database. -func (db *Database) GetLatestVersion() uint64 { - db.mu.Lock() - defer db.mu.Unlock() - - return db.tree.GetLatestVersion() -} - -// Close closes the database and releases all resources. -func (db *Database) Close() error { - db.mu.Lock() - defer db.mu.Unlock() - - return db.tree.Close() -} diff --git a/store/commitment/iavl/tree.go b/store/commitment/iavl/tree.go index 671e60b128..411eb708c1 100644 --- a/store/commitment/iavl/tree.go +++ b/store/commitment/iavl/tree.go @@ -11,7 +11,7 @@ import ( "cosmossdk.io/store/v2" ) -var _ store.Tree = (*IavlTree)(nil) +var _ store.Committer = (*IavlTree)(nil) // IavlTree is a wrapper around iavl.MutableTree. type IavlTree struct { @@ -78,6 +78,11 @@ func (t *IavlTree) GetLatestVersion() uint64 { return uint64(t.tree.Version()) } +// Prune prunes all versions up to and including the provided version. +func (t *IavlTree) Prune(version uint64) error { + return t.tree.DeleteVersionsTo(int64(version)) +} + // Close closes the iavl tree. func (t *IavlTree) Close() error { return nil diff --git a/store/commitment/db_test.go b/store/commitment/iavl/tree_test.go similarity index 67% rename from store/commitment/db_test.go rename to store/commitment/iavl/tree_test.go index 546ed1df01..7946523939 100644 --- a/store/commitment/db_test.go +++ b/store/commitment/iavl/tree_test.go @@ -1,4 +1,4 @@ -package commitment +package iavl import ( "testing" @@ -8,19 +8,12 @@ import ( "cosmossdk.io/log" "cosmossdk.io/store/v2" - "cosmossdk.io/store/v2/commitment/iavl" ) -func generateTree(treeType string) store.Tree { - if treeType == "iavl" { - cfg := iavl.DefaultConfig() - db := dbm.NewMemDB() - tree := iavl.NewIavlTree(db, log.NewNopLogger(), cfg) - - return tree - } - - return nil +func generateTree(treeType string) *IavlTree { + cfg := DefaultConfig() + db := dbm.NewMemDB() + return NewIavlTree(db, log.NewNopLogger(), cfg) } func TestIavlTree(t *testing.T) { @@ -49,7 +42,6 @@ func TestIavlTree(t *testing.T) { require.NoError(t, err) require.Equal(t, workingHash, commitHash) require.Equal(t, uint64(1), tree.GetLatestVersion()) - version1Hash := tree.WorkingHash() // write a batch of version 2 cs2 := store.NewChangeset() @@ -59,11 +51,11 @@ func TestIavlTree(t *testing.T) { cs2.Add([]byte("key1"), nil) // delete key1 err = tree.WriteBatch(cs2) require.NoError(t, err) - workingHash = tree.WorkingHash() - require.NotNil(t, workingHash) + version2Hash := tree.WorkingHash() + require.NotNil(t, version2Hash) commitHash, err = tree.Commit() require.NoError(t, err) - require.Equal(t, workingHash, commitHash) + require.Equal(t, version2Hash, commitHash) // get proof for key1 proof, err := tree.GetProof(1, []byte("key1")) @@ -74,10 +66,26 @@ func TestIavlTree(t *testing.T) { require.NoError(t, err) require.NotNil(t, proof.GetNonexist()) - // load version 1 - err = tree.LoadVersion(1) + // write a batch of version 3 + cs3 := store.NewChangeset() + cs3.Add([]byte("key7"), []byte("value7")) + cs3.Add([]byte("key8"), []byte("value8")) + err = tree.WriteBatch(cs3) require.NoError(t, err) - require.Equal(t, version1Hash, tree.WorkingHash()) + _, err = tree.Commit() + require.NoError(t, err) + + // prune version 1 + err = tree.Prune(1) + require.NoError(t, err) + require.Equal(t, uint64(3), tree.GetLatestVersion()) + err = tree.LoadVersion(1) + require.Error(t, err) + + // load version 2 + err = tree.LoadVersion(2) + require.NoError(t, err) + require.Equal(t, version2Hash, tree.WorkingHash()) // close the db require.NoError(t, tree.Close()) diff --git a/store/database.go b/store/database.go index df5ad19706..08fcaf348e 100644 --- a/store/database.go +++ b/store/database.go @@ -2,6 +2,8 @@ package store import ( "io" + + ics23 "github.com/cosmos/ics23/go" ) // Reader wraps the Has and Get method of a backing data store. @@ -63,7 +65,21 @@ type VersionedDatabase interface { io.Closer } -// Committer defines a contract for committing state. +// Committer defines an API for committing state. type Committer interface { - Commit() error + WriteBatch(cs *Changeset) error + WorkingHash() []byte + GetLatestVersion() uint64 + LoadVersion(targetVersion uint64) error + Commit() ([]byte, error) + GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) + + // Prune attempts to prune all versions up to and including the provided + // version argument. The operation should be idempotent. An error should be + // returned upon failure. + Prune(version uint64) error + + // Close releases associated resources. It should NOT be idempotent. It must + // only be called once and any call after may panic. + io.Closer } diff --git a/store/pruning/README.md b/store/pruning/README.md new file mode 100644 index 0000000000..60d83ca8ef --- /dev/null +++ b/store/pruning/README.md @@ -0,0 +1,15 @@ +# Pruning + +## Overview + +Pruning is the mechanism for deleting old versions data from both state storage and commitment. The pruning operation is triggered periodically. + +## Pruning Options + +Generally, there are three configurable parameters for pruning options: + +- `pruning-keep-recent`: the number of recent versions to keep. +- `pruning-interval`: the interval between two pruning operations. +- `pruning-sync`: the flag to sync/async the pruning operation. + +Different options will be applied to the state storage and commitment. The pruning option have an effect on the snapshot operation, but it will not manage the conflict resolution in SDK, it is the responsibility of the dedicated backend. diff --git a/store/pruning/manager.go b/store/pruning/manager.go new file mode 100644 index 0000000000..ea94589cf8 --- /dev/null +++ b/store/pruning/manager.go @@ -0,0 +1,156 @@ +package pruning + +import ( + "sync" + + "cosmossdk.io/log" + "cosmossdk.io/store/v2" +) + +// Manager is an abstraction to handle pruning of SS and SC backends. +type Manager struct { + mtx sync.Mutex + isStarted bool + + stateStorage store.VersionedDatabase + stateCommitment store.Committer + + logger log.Logger + storageOpts Options + commitmentOpts Options + + chStorage chan struct{} + chCommitment chan struct{} +} + +// NewManager creates a new Manager instance. +func NewManager( + logger log.Logger, + ss store.VersionedDatabase, + sc store.Committer, +) *Manager { + return &Manager{ + stateStorage: ss, + stateCommitment: sc, + logger: logger, + storageOpts: DefaultOptions(), + commitmentOpts: DefaultOptions(), + } +} + +// SetStorageOptions sets the state storage options. +func (m *Manager) SetStorageOptions(opts Options) { + m.storageOpts = opts +} + +// SetCommitmentOptions sets the state commitment options. +func (m *Manager) SetCommitmentOptions(opts Options) { + m.commitmentOpts = opts +} + +// Start starts the manager. +func (m *Manager) Start() { + m.mtx.Lock() + defer m.mtx.Unlock() + + if m.isStarted { + return + } + m.isStarted = true + + if !m.storageOpts.Sync { + m.chStorage = make(chan struct{}, 1) + m.chStorage <- struct{}{} + } + if !m.commitmentOpts.Sync { + m.chCommitment = make(chan struct{}, 1) + m.chCommitment <- struct{}{} + } +} + +// Stop stops the manager and waits for all goroutines to finish. +func (m *Manager) Stop() { + m.mtx.Lock() + defer m.mtx.Unlock() + + if !m.isStarted { + return + } + m.isStarted = false + + if !m.storageOpts.Sync { + <-m.chStorage + close(m.chStorage) + } + if !m.commitmentOpts.Sync { + <-m.chCommitment + close(m.chCommitment) + } +} + +// Prune prunes the state storage and state commitment. +// It will check the pruning conditions and prune if necessary. +func (m *Manager) Prune(height uint64) { + m.mtx.Lock() + defer m.mtx.Unlock() + + if !m.isStarted { + return + } + + // storage pruning + if m.storageOpts.Interval > 0 && height > m.storageOpts.KeepRecent && height%m.storageOpts.Interval == 0 { + pruneHeight := height - m.storageOpts.KeepRecent - 1 + if m.storageOpts.Sync { + m.pruneStorage(pruneHeight) + } else { + // it will not block if the previous pruning is still running + select { + case _, stillOpen := <-m.chStorage: + if stillOpen { + go func() { + m.pruneStorage(pruneHeight) + m.chStorage <- struct{}{} + }() + } + default: + } + } + } + + // commitment pruning + if m.commitmentOpts.Interval > 0 && height > m.commitmentOpts.KeepRecent && height%m.commitmentOpts.Interval == 0 { + pruneHeight := height - m.commitmentOpts.KeepRecent - 1 + if m.commitmentOpts.Sync { + m.pruneCommitment(pruneHeight) + } else { + // it will not block if the previous pruning is still running + select { + case _, stillOpen := <-m.chCommitment: + if stillOpen { + go func() { + m.pruneCommitment(pruneHeight) + m.chCommitment <- struct{}{} + }() + } + default: + } + } + } +} + +func (m *Manager) pruneStorage(height uint64) { + m.logger.Debug("pruning state storage", "height", height) + + if err := m.stateStorage.Prune(height); err != nil { + m.logger.Error("failed to prune state storage", "err", err) + } +} + +func (m *Manager) pruneCommitment(height uint64) { + m.logger.Debug("pruning state commitment", "height", height) + + if err := m.stateCommitment.Prune(height); err != nil { + m.logger.Error("failed to prune state commitment", "err", err) + } +} diff --git a/store/pruning/manager_test.go b/store/pruning/manager_test.go new file mode 100644 index 0000000000..10c5966241 --- /dev/null +++ b/store/pruning/manager_test.go @@ -0,0 +1,85 @@ +package pruning + +import ( + "fmt" + "testing" + + dbm "github.com/cosmos/cosmos-db" + "github.com/stretchr/testify/suite" + + "cosmossdk.io/log" + "cosmossdk.io/store/v2" + "cosmossdk.io/store/v2/commitment/iavl" + "cosmossdk.io/store/v2/storage/sqlite" +) + +type PruningTestSuite struct { + suite.Suite + + manager *Manager + ss store.VersionedDatabase + sc store.Committer +} + +func TestPruningTestSuite(t *testing.T) { + suite.Run(t, &PruningTestSuite{}) +} + +func (s *PruningTestSuite) SetupTest() { + noopLog := log.NewNopLogger() + + ss, err := sqlite.New(s.T().TempDir()) + s.Require().NoError(err) + + sc := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig()) + + s.manager = NewManager(noopLog, ss, sc) + s.ss = ss + s.sc = sc +} + +func (s *PruningTestSuite) TearDownTest() { + s.manager.Start() + s.manager.Stop() +} + +func (s *PruningTestSuite) TestPruning() { + s.manager.SetCommitmentOptions(Options{4, 2, false}) + s.manager.SetStorageOptions(Options{3, 3, true}) + s.manager.Start() + + // write 10 batches + for i := 0; i < 10; i++ { + version := uint64(i + 1) + cs := store.NewChangeset() + cs.Add([]byte("key"), []byte(fmt.Sprintf("value%d", version))) + err := s.sc.WriteBatch(cs) + s.Require().NoError(err) + _, err = s.sc.Commit() + s.Require().NoError(err) + err = s.ss.ApplyChangeset(version, cs) + s.Require().NoError(err) + s.manager.Prune(uint64(i + 1)) + } + + // wait for pruning to finish + s.manager.Stop() + + // check the store for the version 6 + val, err := s.ss.Get("", 6, []byte("key")) + s.Require().NoError(err) + s.Require().Equal([]byte("value6"), val) + // check the store for the version 5 + val, err = s.ss.Get("", 5, []byte("key")) + s.Require().NoError(err) + s.Require().Nil(val) + + // check the commitment for the version 6 + proof, err := s.sc.GetProof(6, []byte("key")) + s.Require().NoError(err) + s.Require().NotNil(proof.GetExist()) + // check the commitment for the version 5 + proof, err = s.sc.GetProof(5, []byte("key")) + s.Require().Error(err) + s.Require().Nil(proof) +} diff --git a/store/pruning/options.go b/store/pruning/options.go new file mode 100644 index 0000000000..fc1245180c --- /dev/null +++ b/store/pruning/options.go @@ -0,0 +1,25 @@ +package pruning + +// Options defines the pruning configuration. +type Options struct { + // KeepRecent sets the number of recent versions to keep. + KeepRecent uint64 + + // Interval sets the number of how often to prune. + // If set to 0, no pruning will be done. + Interval uint64 + + // Sync when set to true ensure that pruning will be performed + // synchronously, otherwise by default it will be done asynchronously. + Sync bool +} + +// DefaultOptions returns the default pruning options. +// Interval is set to 0, which means no pruning will be done. +func DefaultOptions() Options { + return Options{ + KeepRecent: 0, + Interval: 0, + Sync: false, + } +} diff --git a/store/root/store.go b/store/root/store.go index e938810bae..52c586a95a 100644 --- a/store/root/store.go +++ b/store/root/store.go @@ -10,9 +10,9 @@ import ( "cosmossdk.io/log" "cosmossdk.io/store/v2" - "cosmossdk.io/store/v2/commitment" "cosmossdk.io/store/v2/kv/branch" "cosmossdk.io/store/v2/kv/trace" + "cosmossdk.io/store/v2/pruning" ) // defaultStoreKey defines the default store key used for the single SC backend. @@ -34,7 +34,7 @@ type Store struct { stateStore store.VersionedDatabase // stateCommitment reflects the state commitment (SC) backend - stateCommitment *commitment.Database + stateCommitment store.Committer // rootKVStore reflects the root BranchedKVStore that is used to accumulate writes // and branch off of. @@ -54,25 +54,31 @@ type Store struct { // traceContext defines the tracing context, if any, for trace operations traceContext store.TraceContext + + // pruningManager manages pruning of the SS and SC backends + pruningManager *pruning.Manager } func New( logger log.Logger, initVersion uint64, ss store.VersionedDatabase, - sc *commitment.Database, + sc store.Committer, ) (store.RootStore, error) { rootKVStore, err := branch.New(defaultStoreKey, ss) if err != nil { return nil, err } + pruningManager := pruning.NewManager(logger, ss, sc) + return &Store{ logger: logger.With("module", "root_store"), initialVersion: initVersion, stateStore: ss, stateCommitment: sc, rootKVStore: rootKVStore, + pruningManager: pruningManager, }, nil } @@ -87,17 +93,28 @@ func (s *Store) Close() (err error) { s.lastCommitInfo = nil s.commitHeader = nil + s.pruningManager.Stop() + return err } +// SetPruningOptions sets the pruning options on the SS and SC backends. +// NOTE: It will also start the pruning manager. +func (s *Store) SetPruningOptions(ssOpts, scOpts pruning.Options) { + s.pruningManager.SetStorageOptions(ssOpts) + s.pruningManager.SetCommitmentOptions(scOpts) + + s.pruningManager.Start() +} + // MountSCStore performs a no-op as a SC backend must be provided at initialization. -func (s *Store) MountSCStore(_ string, _ store.Tree) error { +func (s *Store) MountSCStore(_ string, _ store.Committer) error { return errors.New("cannot mount SC store; SC must be provided on initialization") } // GetSCStore returns the store's state commitment (SC) backend. Note, the store // key is ignored as there exists only a single SC tree. -func (s *Store) GetSCStore(_ string) store.Tree { +func (s *Store) GetSCStore(_ string) store.Committer { return s.stateCommitment } @@ -317,6 +334,9 @@ func (s *Store) Commit() ([]byte, error) { s.workingHash = nil + // prune SS and SC + s.pruningManager.Prune(version) + return s.lastCommitInfo.Hash(), nil } diff --git a/store/root/store_test.go b/store/root/store_test.go index b0838fe4bf..ba372f965e 100644 --- a/store/root/store_test.go +++ b/store/root/store_test.go @@ -10,7 +10,6 @@ import ( "cosmossdk.io/log" "cosmossdk.io/store/v2" - "cosmossdk.io/store/v2/commitment" "cosmossdk.io/store/v2/commitment/iavl" "cosmossdk.io/store/v2/storage/sqlite" ) @@ -31,8 +30,7 @@ func (s *RootStoreTestSuite) SetupTest() { ss, err := sqlite.New(s.T().TempDir()) s.Require().NoError(err) - tree := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig()) - sc := commitment.NewDatabase(tree) + sc := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig()) rs, err := New(noopLog, 1, ss, sc) s.Require().NoError(err) diff --git a/store/store.go b/store/store.go index 60c2d9fa28..741a4c49b4 100644 --- a/store/store.go +++ b/store/store.go @@ -22,11 +22,11 @@ type RootStore interface { // GetSCStore should return the SC backend for the given store key. A RootStore // implementation may choose to ignore the store key in cases where only a single // SC backend is used. - GetSCStore(storeKey string) Tree + GetSCStore(storeKey string) Committer // MountSCStore should mount the given SC backend for the given store key. For // implementations that utilize a single SC backend, this method may be optional // or a no-op. - MountSCStore(storeKey string, sc Tree) error + MountSCStore(storeKey string, sc Committer) error // GetKVStore returns the KVStore for the given store key. If an implementation // chooses to have a single SS backend, the store key may be ignored. GetKVStore(storeKey string) KVStore diff --git a/store/tree.go b/store/tree.go deleted file mode 100644 index 6fce611490..0000000000 --- a/store/tree.go +++ /dev/null @@ -1,16 +0,0 @@ -package store - -import ( - ics23 "github.com/cosmos/ics23/go" -) - -// Tree is an interface for a commitment layer to support multiple backends. -type Tree interface { - WriteBatch(cs *Changeset) error - WorkingHash() []byte - GetLatestVersion() uint64 - LoadVersion(targetVersion uint64) error - Commit() ([]byte, error) - GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) - Close() error -}