From 76529d768036f4f11a5bb063613db0f01045d3f1 Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 16 Oct 2024 17:55:23 +0200 Subject: [PATCH] feat(store/v2): add version exists (#22235) Co-authored-by: Cool Developer --- runtime/v2/store.go | 2 +- store/v2/commitment/store.go | 1 + store/v2/database.go | 28 +++-- store/v2/mock/db_mock.go | 15 +++ store/v2/mock/types.go | 2 +- store/v2/root/store.go | 20 ++-- store/v2/root/store_mock_test.go | 2 +- store/v2/root/store_test.go | 2 +- store/v2/storage/database.go | 1 + store/v2/storage/pebbledb/db.go | 9 ++ store/v2/storage/rocksdb/db.go | 9 ++ store/v2/storage/rocksdb/db_noflag.go | 4 + store/v2/storage/sqlite/db.go | 15 ++- store/v2/storage/storage_bench_test.go | 8 +- store/v2/storage/storage_test_suite.go | 142 ++++++++++++++++++++++++- store/v2/storage/store.go | 7 +- store/v2/store.go | 2 +- 17 files changed, 237 insertions(+), 32 deletions(-) diff --git a/runtime/v2/store.go b/runtime/v2/store.go index 40912ea41f..5d37c321c1 100644 --- a/runtime/v2/store.go +++ b/runtime/v2/store.go @@ -43,7 +43,7 @@ type Store interface { Query(storeKey []byte, version uint64, key []byte, prove bool) (storev2.QueryResult, error) // GetStateStorage returns the SS backend. - GetStateStorage() storev2.VersionedDatabase + GetStateStorage() storev2.VersionedWriter // GetStateCommitment returns the SC backend. GetStateCommitment() storev2.Committer diff --git a/store/v2/commitment/store.go b/store/v2/commitment/store.go index da04409875..e1f6df9d20 100644 --- a/store/v2/commitment/store.go +++ b/store/v2/commitment/store.go @@ -275,6 +275,7 @@ func (c *CommitStore) GetProof(storeKey []byte, version uint64, key []byte) ([]p return []proof.CommitmentOp{commitOp, *storeCommitmentOp}, nil } +// Get implements store.VersionedReader. func (c *CommitStore) Get(storeKey []byte, version uint64, key []byte) ([]byte, error) { tree, ok := c.multiTrees[conv.UnsafeBytesToStr(storeKey)] if !ok { diff --git a/store/v2/database.go b/store/v2/database.go index 58a3ee7ef6..c4f00defe8 100644 --- a/store/v2/database.go +++ b/store/v2/database.go @@ -7,19 +7,29 @@ import ( "cosmossdk.io/store/v2/proof" ) -// VersionedDatabase defines an API for a versioned database that allows reads, +// VersionedWriter defines an API for a versioned database that allows reads, // writes, iteration and commitment over a series of versions. -type VersionedDatabase interface { +type VersionedWriter interface { + VersionedReader + + SetLatestVersion(version uint64) error + ApplyChangeset(version uint64, cs *corestore.Changeset) error + + // Close releases associated resources. It should NOT be idempotent. It must + // only be called once and any call after may panic. + io.Closer +} + +type VersionedReader interface { Has(storeKey []byte, version uint64, key []byte) (bool, error) Get(storeKey []byte, version uint64, key []byte) ([]byte, error) + GetLatestVersion() (uint64, error) - SetLatestVersion(version uint64) error + VersionExists(v uint64) (bool, error) Iterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) ReverseIterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) - ApplyChangeset(version uint64, cs *corestore.Changeset) error - // Close releases associated resources. It should NOT be idempotent. It must // only be called once and any call after may panic. io.Closer @@ -53,18 +63,14 @@ type Committer interface { // GetProof returns the proof of existence or non-existence for the given key. GetProof(storeKey []byte, version uint64, key []byte) ([]proof.CommitmentOp, error) - // Get returns the value for the given key at the given version. - // - // NOTE: This method only exists to support migration from IAVL v0/v1 to v2. - // Once migration is complete, this method should be removed and/or not used. - Get(storeKey []byte, version uint64, key []byte) ([]byte, error) - // SetInitialVersion sets the initial version of the committer. SetInitialVersion(version uint64) error // GetCommitInfo returns the CommitInfo for the given version. GetCommitInfo(version uint64) (*proof.CommitInfo, error) + Get(storeKey []byte, version uint64, key []byte) ([]byte, error) + // Close releases associated resources. It should NOT be idempotent. It must // only be called once and any call after may panic. io.Closer diff --git a/store/v2/mock/db_mock.go b/store/v2/mock/db_mock.go index 8014e48514..60f6606079 100644 --- a/store/v2/mock/db_mock.go +++ b/store/v2/mock/db_mock.go @@ -404,3 +404,18 @@ func (mr *MockStateStorageMockRecorder) SetLatestVersion(version any) *gomock.Ca mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLatestVersion", reflect.TypeOf((*MockStateStorage)(nil).SetLatestVersion), version) } + +// VersionExists mocks base method. +func (m *MockStateStorage) VersionExists(v uint64) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VersionExists", v) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VersionExists indicates an expected call of VersionExists. +func (mr *MockStateStorageMockRecorder) VersionExists(v any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VersionExists", reflect.TypeOf((*MockStateStorage)(nil).VersionExists), v) +} diff --git a/store/v2/mock/types.go b/store/v2/mock/types.go index 95d623c839..a8c2a196d5 100644 --- a/store/v2/mock/types.go +++ b/store/v2/mock/types.go @@ -12,7 +12,7 @@ type StateCommitter interface { // StateStorage is a mock of store.VersionedDatabase type StateStorage interface { - store.VersionedDatabase + store.VersionedWriter store.UpgradableDatabase store.Pruner store.PausablePruner diff --git a/store/v2/root/store.go b/store/v2/root/store.go index 7447c60ae7..a96c65bafe 100644 --- a/store/v2/root/store.go +++ b/store/v2/root/store.go @@ -38,7 +38,7 @@ type Store struct { dbCloser io.Closer // stateStorage reflects the state storage backend - stateStorage store.VersionedDatabase + stateStorage store.VersionedWriter // stateCommitment reflects the state commitment (SC) backend stateCommitment store.Committer @@ -74,7 +74,7 @@ type Store struct { func New( dbCloser io.Closer, logger corelog.Logger, - ss store.VersionedDatabase, + ss store.VersionedWriter, sc store.Committer, pm *pruning.Manager, mm *migration.Manager, @@ -127,19 +127,21 @@ func (s *Store) StateLatest() (uint64, corestore.ReaderMap, error) { return v, NewReaderMap(v, s), nil } +// StateAt checks if the requested version is present in ss. func (s *Store) StateAt(v uint64) (corestore.ReaderMap, error) { - // TODO(bez): We may want to avoid relying on the SC metadata here. Instead, - // we should add a VersionExists() method to the VersionedDatabase interface. - // - // Ref: https://github.com/cosmos/cosmos-sdk/issues/19091 - if cInfo, err := s.stateCommitment.GetCommitInfo(v); err != nil || cInfo == nil { - return nil, fmt.Errorf("failed to get commit info for version %d: %w", v, err) + // check if version is present in state storage + isExist, err := s.stateStorage.VersionExists(v) + if err != nil { + return nil, err + } + if !isExist { + return nil, fmt.Errorf("version %d does not exist", v) } return NewReaderMap(v, s), nil } -func (s *Store) GetStateStorage() store.VersionedDatabase { +func (s *Store) GetStateStorage() store.VersionedWriter { return s.stateStorage } diff --git a/store/v2/root/store_mock_test.go b/store/v2/root/store_mock_test.go index 68956ab479..3a14054cd7 100644 --- a/store/v2/root/store_mock_test.go +++ b/store/v2/root/store_mock_test.go @@ -16,7 +16,7 @@ import ( "cosmossdk.io/store/v2/pruning" ) -func newTestRootStore(ss store.VersionedDatabase, sc store.Committer) *Store { +func newTestRootStore(ss store.VersionedWriter, sc store.Committer) *Store { noopLog := coretesting.NewNopLogger() pm := pruning.NewManager(sc.(store.Pruner), ss.(store.Pruner), nil, nil) return &Store{ diff --git a/store/v2/root/store_test.go b/store/v2/root/store_test.go index 9f925b098b..2d3f5a5b57 100644 --- a/store/v2/root/store_test.go +++ b/store/v2/root/store_test.go @@ -90,7 +90,7 @@ func (s *RootStoreTestSuite) newStoreWithPruneConfig(config *store.PruningOption s.rootStore = rs } -func (s *RootStoreTestSuite) newStoreWithBackendMount(ss store.VersionedDatabase, sc store.Committer, pm *pruning.Manager) { +func (s *RootStoreTestSuite) newStoreWithBackendMount(ss store.VersionedWriter, sc store.Committer, pm *pruning.Manager) { noopLog := coretesting.NewNopLogger() rs, err := New(dbm.NewMemDB(), noopLog, ss, sc, pm, nil, nil) diff --git a/store/v2/storage/database.go b/store/v2/storage/database.go index fba5998395..e969a9ee63 100644 --- a/store/v2/storage/database.go +++ b/store/v2/storage/database.go @@ -16,6 +16,7 @@ type Database interface { Get(storeKey []byte, version uint64, key []byte) ([]byte, error) GetLatestVersion() (uint64, error) SetLatestVersion(version uint64) error + VersionExists(version uint64) (bool, error) Iterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) ReverseIterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) diff --git a/store/v2/storage/pebbledb/db.go b/store/v2/storage/pebbledb/db.go index e5fd49ca3e..1547dba102 100644 --- a/store/v2/storage/pebbledb/db.go +++ b/store/v2/storage/pebbledb/db.go @@ -137,6 +137,15 @@ func (db *Database) GetLatestVersion() (uint64, error) { return binary.LittleEndian.Uint64(bz), closer.Close() } +func (db *Database) VersionExists(version uint64) (bool, error) { + latestVersion, err := db.GetLatestVersion() + if err != nil { + return false, err + } + + return latestVersion >= version && version >= db.earliestVersion, nil +} + func (db *Database) setPruneHeight(pruneVersion uint64) error { db.earliestVersion = pruneVersion + 1 diff --git a/store/v2/storage/rocksdb/db.go b/store/v2/storage/rocksdb/db.go index afac4cc9ff..248b014f7b 100644 --- a/store/v2/storage/rocksdb/db.go +++ b/store/v2/storage/rocksdb/db.go @@ -131,6 +131,15 @@ func (db *Database) GetLatestVersion() (uint64, error) { return binary.LittleEndian.Uint64(bz), nil } +func (db *Database) VersionExists(version uint64) (bool, error) { + latestVersion, err := db.GetLatestVersion() + if err != nil { + return false, err + } + + return latestVersion >= version && version >= db.tsLow, nil +} + func (db *Database) Has(storeKey []byte, version uint64, key []byte) (bool, error) { slice, err := db.getSlice(storeKey, version, key) if err != nil { diff --git a/store/v2/storage/rocksdb/db_noflag.go b/store/v2/storage/rocksdb/db_noflag.go index def31e437a..93bc3090f2 100644 --- a/store/v2/storage/rocksdb/db_noflag.go +++ b/store/v2/storage/rocksdb/db_noflag.go @@ -36,6 +36,10 @@ func (db *Database) GetLatestVersion() (uint64, error) { panic("rocksdb requires a build flag") } +func (db *Database) VersionExists(version uint64) (bool, error) { + panic("rocksdb requires a build flag") +} + func (db *Database) Has(storeKey []byte, version uint64, key []byte) (bool, error) { panic("rocksdb requires a build flag") } diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go index 38750a0ac9..9256489286 100644 --- a/store/v2/storage/sqlite/db.go +++ b/store/v2/storage/sqlite/db.go @@ -103,7 +103,11 @@ func (db *Database) NewBatch(version uint64) (store.Batch, error) { } func (db *Database) GetLatestVersion() (uint64, error) { - stmt, err := db.storage.Prepare("SELECT value FROM state_storage WHERE store_key = ? AND key = ?") + stmt, err := db.storage.Prepare(` + SELECT value + FROM state_storage + WHERE store_key = ? AND key = ? + `) if err != nil { return 0, fmt.Errorf("failed to prepare SQL statement: %w", err) } @@ -123,6 +127,15 @@ func (db *Database) GetLatestVersion() (uint64, error) { return latestHeight, nil } +func (db *Database) VersionExists(v uint64) (bool, error) { + latestVersion, err := db.GetLatestVersion() + if err != nil { + return false, err + } + + return latestVersion >= v && v >= db.earliestVersion, nil +} + func (db *Database) SetLatestVersion(version uint64) error { _, err := db.storage.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, version, 0, version) if err != nil { diff --git a/store/v2/storage/storage_bench_test.go b/store/v2/storage/storage_bench_test.go index 960c144782..35ffbbf5f8 100644 --- a/store/v2/storage/storage_bench_test.go +++ b/store/v2/storage/storage_bench_test.go @@ -24,12 +24,12 @@ import ( var storeKey1 = []byte("store1") var ( - backends = map[string]func(dataDir string) (store.VersionedDatabase, error){ - "rocksdb_versiondb_opts": func(dataDir string) (store.VersionedDatabase, error) { + backends = map[string]func(dataDir string) (store.VersionedWriter, error){ + "rocksdb_versiondb_opts": func(dataDir string) (store.VersionedWriter, error) { db, err := rocksdb.New(dataDir) return storage.NewStorageStore(db, coretesting.NewNopLogger()), err }, - "pebbledb_default_opts": func(dataDir string) (store.VersionedDatabase, error) { + "pebbledb_default_opts": func(dataDir string) (store.VersionedWriter, error) { db, err := pebbledb.New(dataDir) if err == nil && db != nil { db.SetSync(false) @@ -37,7 +37,7 @@ var ( return storage.NewStorageStore(db, coretesting.NewNopLogger()), err }, - "btree_sqlite": func(dataDir string) (store.VersionedDatabase, error) { + "btree_sqlite": func(dataDir string) (store.VersionedWriter, error) { db, err := sqlite.New(dataDir) return storage.NewStorageStore(db, coretesting.NewNopLogger()), err }, diff --git a/store/v2/storage/storage_test_suite.go b/store/v2/storage/storage_test_suite.go index 08f455ca12..fb5ff24a0c 100644 --- a/store/v2/storage/storage_test_suite.go +++ b/store/v2/storage/storage_test_suite.go @@ -880,9 +880,149 @@ func (s *StorageTestSuite) TestRemovingOldStoreKey() { } } +// TestVersionExists tests the VersionExists method of the Database struct. +func (s *StorageTestSuite) TestVersionExists() { + // Define test cases + testCases := []struct { + name string + setup func(t *testing.T, db *StorageStore) + version uint64 + expectedExists bool + expectError bool + }{ + { + name: "Fresh database: version 0 exists", + setup: func(t *testing.T, db *StorageStore) { + t.Helper() + // No setup needed for fresh database + }, + version: 0, + expectedExists: true, + expectError: false, + }, + { + name: "Fresh database: version 1 exists", + setup: func(t *testing.T, db *StorageStore) { + t.Helper() + // No setup needed for fresh database + }, + version: 1, + expectedExists: false, + expectError: false, + }, + { + name: "After setting latest version to 10, version 5 exists", + setup: func(t *testing.T, db *StorageStore) { + t.Helper() + err := db.SetLatestVersion(10) + if err != nil { + t.Fatalf("Setting latest version should not error: %v", err) + } + }, + version: 5, + expectedExists: true, // Since pruning hasn't occurred, earliestVersion is still 0 + expectError: false, + }, + { + name: "After setting latest version to 10 and pruning to 5, version 4 does not exist", + setup: func(t *testing.T, db *StorageStore) { + t.Helper() + err := db.SetLatestVersion(10) + if err != nil { + t.Fatalf("Setting latest version should not error: %v", err) + } + + err = db.Prune(5) + if err != nil { + t.Fatalf("Pruning to version 5 should not error: %v", err) + } + }, + version: 4, + expectedExists: false, + expectError: false, + }, + { + name: "After setting latest version to 10 and pruning to 5, version 5 does not exist", + setup: func(t *testing.T, db *StorageStore) { + t.Helper() + err := db.SetLatestVersion(10) + if err != nil { + t.Fatalf("Setting latest version should not error: %v", err) + } + + err = db.Prune(5) + if err != nil { + t.Fatalf("Pruning to version 5 should not error: %v", err) + } + }, + version: 5, + expectedExists: false, + expectError: false, + }, + { + name: "After setting latest version to 10 and pruning to 5, version 6 exists", + setup: func(t *testing.T, db *StorageStore) { + t.Helper() + err := db.SetLatestVersion(10) + if err != nil { + t.Fatalf("Setting latest version should not error: %v", err) + } + + err = db.Prune(5) + if err != nil { + t.Fatalf("Pruning to version 5 should not error: %v", err) + } + }, + version: 6, + expectedExists: true, + expectError: false, + }, + { + name: "After pruning to 0, all versions >=1 exist", + setup: func(t *testing.T, db *StorageStore) { + t.Helper() + err := db.SetLatestVersion(10) + if err != nil { + t.Fatalf("Setting latest version should not error: %v", err) + } + // Prune to version 0 + err = db.Prune(0) + if err != nil { + t.Fatalf("Pruning to version 0 should not error: %v", err) + } + }, + version: 1, + expectedExists: true, + expectError: false, + }, + } + + // Iterate over each test case + for _, tc := range testCases { + s.T().Run(tc.name, func(t *testing.T) { + // Initialize the database for each test + db, err := s.NewDB(t.TempDir()) + require.NoError(t, err, "Failed to initialize the database") + defer db.Close() + + // Setup test environment + tc.setup(t, db) + + // Call VersionExists and check the result + exists, err := db.VersionExists(tc.version) + if tc.expectError { + require.Error(t, err, "Expected error but got none") + } else { + require.NoError(t, err, "Did not expect an error but got one") + require.Equal(t, tc.expectedExists, exists, "Version existence mismatch") + } + }) + } +} + func dbApplyChangeset( t *testing.T, - db store.VersionedDatabase, + db store.VersionedWriter, version uint64, storeKey string, keys, vals [][]byte, diff --git a/store/v2/storage/store.go b/store/v2/storage/store.go index 9ea8395628..b16ee9e7ea 100644 --- a/store/v2/storage/store.go +++ b/store/v2/storage/store.go @@ -16,7 +16,7 @@ const ( ) var ( - _ store.VersionedDatabase = (*StorageStore)(nil) + _ store.VersionedWriter = (*StorageStore)(nil) _ snapshots.StorageSnapshotter = (*StorageStore)(nil) _ store.Pruner = (*StorageStore)(nil) _ store.UpgradableDatabase = (*StorageStore)(nil) @@ -84,6 +84,11 @@ func (ss *StorageStore) SetLatestVersion(version uint64) error { return ss.db.SetLatestVersion(version) } +// VersionExists returns true if the given version exists in the store. +func (ss *StorageStore) VersionExists(version uint64) (bool, error) { + return ss.db.VersionExists(version) +} + // Iterator returns an iterator over the specified domain and prefix. func (ss *StorageStore) Iterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) { return ss.db.Iterator(storeKey, version, start, end) diff --git a/store/v2/store.go b/store/v2/store.go index 1adf44f0b8..24ba636e10 100644 --- a/store/v2/store.go +++ b/store/v2/store.go @@ -70,7 +70,7 @@ type RootStore interface { // Backend defines the interface for the RootStore backends. type Backend interface { // GetStateStorage returns the SS backend. - GetStateStorage() VersionedDatabase + GetStateStorage() VersionedWriter // GetStateCommitment returns the SC backend. GetStateCommitment() Committer