LaconicNetwork/cosmos-sdk
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

feat(store/v2): add support for iavl/v2 (#22424)

Browse Source
This commit is contained in:
Matt Kocubinski 2024-11-29 13:02:47 -06:00 committed by GitHub
parent 6fb8299a67
commit 24d456db40
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
34 changed files with 310 additions and 953 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
runtime/v2/go.mod
View File

@ -53,7 +53,7 @@ require (
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-metrics v0.5.3 // indirect
@ -65,7 +65,6 @@ require (
github.com/linxGnu/grocksdb v1.9.3 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/onsi/gomega v1.28.1 // indirect
github.com/pkg/errors v0.9.1 // indirect

6
runtime/v2/go.sum
View File

@ -107,8 +107,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@ -164,8 +164,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

22
server/v2/cometbft/abci_test.go
View File

@ -2,18 +2,16 @@ package cometbft
import (
"context"
"cosmossdk.io/core/server"
"crypto/sha256"
"encoding/json"
"errors"
abci "github.com/cometbft/cometbft/abci/types"
"io"
"strings"
"sync"
"testing"
"time"
"cosmossdk.io/server/v2/cometbft/oe"
abci "github.com/cometbft/cometbft/abci/types"
abciproto "github.com/cometbft/cometbft/api/cometbft/abci/v1"
v1 "github.com/cometbft/cometbft/api/cometbft/types/v1"
"github.com/cosmos/gogoproto/proto"
@ -21,6 +19,7 @@ import (
"github.com/stretchr/testify/require"
appmodulev2 "cosmossdk.io/core/appmodule/v2"
"cosmossdk.io/core/server"
"cosmossdk.io/core/store"
"cosmossdk.io/core/transaction"
"cosmossdk.io/log"
@ -28,6 +27,7 @@ import (
"cosmossdk.io/server/v2/cometbft/handlers"
cometmock "cosmossdk.io/server/v2/cometbft/internal/mock"
"cosmossdk.io/server/v2/cometbft/mempool"
"cosmossdk.io/server/v2/cometbft/oe"
"cosmossdk.io/server/v2/cometbft/types"
"cosmossdk.io/server/v2/stf"
"cosmossdk.io/server/v2/stf/branch"
@ -60,10 +60,10 @@ func getQueryRouterBuilder[T any, PT interface {
*T
proto.Message
},
U any, UT interface {
*U
proto.Message
}](
U any, UT interface {
*U
proto.Message
}](
t *testing.T,
handler func(ctx context.Context, msg PT) (UT, error),
) *stf.MsgRouterBuilder {
@ -90,10 +90,10 @@ func getMsgRouterBuilder[T any, PT interface {
*T
transaction.Msg
},
U any, UT interface {
*U
transaction.Msg
}](
U any, UT interface {
*U
transaction.Msg
}](
t *testing.T,
handler func(ctx context.Context, msg PT) (UT, error),
) *stf.MsgRouterBuilder {

1
server/v2/cometbft/go.mod
View File

@ -132,7 +132,6 @@ require (
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/minio/highwayhash v1.0.3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mtibben/percent v0.2.1 // indirect

2
server/v2/cometbft/go.sum
View File

@ -345,8 +345,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q=
github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=

4
server/v2/cometbft/internal/mock/mock_store.go
View File

@ -12,7 +12,7 @@ import (
dbm "cosmossdk.io/store/v2/db"
"cosmossdk.io/store/v2/proof"
"cosmossdk.io/store/v2/storage"
"cosmossdk.io/store/v2/storage/sqlite"
"cosmossdk.io/store/v2/storage/pebbledb"
)
type MockStore struct {
@ -21,7 +21,7 @@ type MockStore struct {
}
func NewMockStorage(logger log.Logger, dir string) storev2.VersionedWriter {
storageDB, _ := sqlite.New(dir)
storageDB, _ := pebbledb.New(dir)
ss := storage.NewStorageStore(storageDB, logger)
return ss
}

5
server/v2/cometbft/oe/optimistic_execution_test.go
View File

@ -5,12 +5,13 @@ import (
"errors"
"testing"
abci "github.com/cometbft/cometbft/api/cometbft/abci/v1"
"github.com/stretchr/testify/assert"
"cosmossdk.io/core/server"
"cosmossdk.io/core/store"
"cosmossdk.io/core/transaction"
"cosmossdk.io/log"
abci "github.com/cometbft/cometbft/api/cometbft/abci/v1"
"github.com/stretchr/testify/assert"
)
func testFinalizeBlock[T transaction.Tx](context.Context, *abci.FinalizeBlockRequest) (*server.BlockResponse, store.WriterMap, []T, error) {

2
server/v2/cometbft/server.go
View File

@ -2,7 +2,6 @@ package cometbft
import (
"context"
"cosmossdk.io/server/v2/cometbft/oe"
"crypto/sha256"
"encoding/json"
"fmt"
@ -35,6 +34,7 @@ import (
"cosmossdk.io/server/v2/appmanager"
cometlog "cosmossdk.io/server/v2/cometbft/log"
"cosmossdk.io/server/v2/cometbft/mempool"
"cosmossdk.io/server/v2/cometbft/oe"
"cosmossdk.io/server/v2/cometbft/types"
"cosmossdk.io/store/v2/snapshots"

3
server/v2/go.mod
View File

@ -67,7 +67,7 @@ require (
github.com/gogo/googleapis v1.4.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
@ -83,7 +83,6 @@ require (
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/onsi/gomega v1.28.1 // indirect

6
server/v2/go.sum
View File

@ -146,8 +146,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@ -227,8 +227,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=

4
server/v2/testdata/app.toml vendored
View File

@ -25,8 +25,8 @@ minimum-gas-prices = '0stake'
app-db-backend = 'goleveldb'
[store.options]
# State storage database type. Currently we support: "sqlite", "pebble" and "rocksdb"
ss-type = 'sqlite'
# State storage database type. Currently we support: "pebble" and "rocksdb"
ss-type = 'pebble'
# State commitment database type. Currently we support: "iavl" and "iavl-v2"
sc-type = 'iavl'

1
simapp/v2/go.mod
View File

@ -170,7 +170,6 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/mdp/qrterminal/v3 v3.2.0 // indirect
github.com/minio/highwayhash v1.0.3 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect

2
simapp/v2/go.sum
View File

@ -662,8 +662,6 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mdp/qrterminal/v3 v3.2.0 h1:qteQMXO3oyTK4IHwj2mWsKYYRBOp1Pj2WRYFYYNTCdk=
github.com/mdp/qrterminal/v3 v3.2.0/go.mod h1:XGGuua4Lefrl7TLEsSONiD+UEjQXJZ4mPzF+gWYIJkk=

7
store/v2/commitment/iavl/tree_test.go
View File

@ -16,7 +16,12 @@ import (
func TestCommitterSuite(t *testing.T) {
s := &commitment.CommitStoreTestSuite{
NewStore: func(db corestore.KVStoreWithBatch, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*commitment.CommitStore, error) {
NewStore: func(
db corestore.KVStoreWithBatch,
_ string,
storeKeys, oldStoreKeys []string,
logger corelog.Logger,
) (*commitment.CommitStore, error) {
multiTrees := make(map[string]commitment.Tree)
cfg := DefaultConfig()
mountTreeFn := func(storeKey string) (commitment.Tree, error) {

147
store/v2/commitment/iavlv2/tree.go Normal file
View File

@ -0,0 +1,147 @@
package iavlv2
import (
"errors"
"fmt"
"github.com/cosmos/iavl/v2"
ics23 "github.com/cosmos/ics23/go"
corestore "cosmossdk.io/core/store"
"cosmossdk.io/store/v2"
"cosmossdk.io/store/v2/commitment"
)
var (
_ commitment.Tree = (*Tree)(nil)
_ commitment.Reader = (*Tree)(nil)
_ store.PausablePruner = (*Tree)(nil)
)
type Tree struct {
tree *iavl.Tree
}
func NewTree(treeOptions iavl.TreeOptions, dbOptions iavl.SqliteDbOptions, pool *iavl.NodePool) (*Tree, error) {
sql, err := iavl.NewSqliteDb(pool, dbOptions)
if err != nil {
return nil, err
}
tree := iavl.NewTree(sql, pool, treeOptions)
return &Tree{tree: tree}, nil
}
func (t *Tree) Set(key, value []byte) error {
_, err := t.tree.Set(key, value)
return err
}
func (t *Tree) Remove(key []byte) error {
_, _, err := t.tree.Remove(key)
return err
}
func (t *Tree) GetLatestVersion() (uint64, error) {
return uint64(t.tree.Version()), nil
}
func (t *Tree) Hash() []byte {
return t.tree.Hash()
}
func (t *Tree) Version() uint64 {
return uint64(t.tree.Version())
}
func (t *Tree) LoadVersion(version uint64) error {
if err := isHighBitSet(version); err != nil {
return err
}
if version == 0 {
return nil
}
return t.tree.LoadVersion(int64(version))
}
func (t *Tree) Commit() ([]byte, uint64, error) {
h, v, err := t.tree.SaveVersion()
return h, uint64(v), err
}
func (t *Tree) SetInitialVersion(version uint64) error {
if err := isHighBitSet(version); err != nil {
return err
}
t.tree.SetShouldCheckpoint()
return t.tree.SetInitialVersion(int64(version))
}
func (t *Tree) GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) {
if err := isHighBitSet(version); err != nil {
return nil, err
}
return t.tree.GetProof(int64(version), key)
}
func (t *Tree) Get(version uint64, key []byte) ([]byte, error) {
if err := isHighBitSet(version); err != nil {
return nil, err
}
if int64(version) != t.tree.Version() {
cloned, err := t.tree.ReadonlyClone()
if err != nil {
return nil, err
}
if err = cloned.LoadVersion(int64(version)); err != nil {
return nil, err
}
return cloned.Get(key)
} else {
return t.tree.Get(key)
}
}
func (t *Tree) Iterator(version uint64, start, end []byte, ascending bool) (corestore.Iterator, error) {
if err := isHighBitSet(version); err != nil {
return nil, err
}
if int64(version) != t.tree.Version() {
return nil, fmt.Errorf("loading past version not yet supported")
}
return t.tree.Iterator(start, end, ascending)
}
func (t *Tree) Export(version uint64) (commitment.Exporter, error) {
return nil, errors.New("snapshot import/export not yet supported")
}
func (t *Tree) Import(version uint64) (commitment.Importer, error) {
return nil, errors.New("snapshot import/export not yet supported")
}
func (t *Tree) Close() error {
return t.tree.Close()
}
func (t *Tree) Prune(version uint64) error {
if err := isHighBitSet(version); err != nil {
return err
}
return t.tree.DeleteVersionsTo(int64(version))
}
// PausePruning is unnecessary in IAVL v2 due to the advanced pruning mechanism
func (t *Tree) PausePruning(bool) {}
func (t *Tree) WorkingHash() []byte {
return t.tree.Hash()
}
func isHighBitSet(version uint64) error {
if version&(1<<63) != 0 {
return fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version)
}
return nil
}

46
store/v2/commitment/iavlv2/tree_test.go Normal file
View File

@ -0,0 +1,46 @@
package iavlv2
import (
"fmt"
"testing"
"github.com/cosmos/iavl/v2"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
corelog "cosmossdk.io/core/log"
corestore "cosmossdk.io/core/store"
"cosmossdk.io/store/v2/commitment"
)
func TestCommitterSuite(t *testing.T) {
nodePool := iavl.NewNodePool()
s := &commitment.CommitStoreTestSuite{
TreeType: "iavlv2",
NewStore: func(
db corestore.KVStoreWithBatch,
dbDir string,
storeKeys, oldStoreKeys []string,
logger corelog.Logger,
) (*commitment.CommitStore, error) {
multiTrees := make(map[string]commitment.Tree)
mountTreeFn := func(storeKey string) (commitment.Tree, error) {
path := fmt.Sprintf("%s/%s", dbDir, storeKey)
tree, err := NewTree(iavl.DefaultTreeOptions(), iavl.SqliteDbOptions{Path: path}, nodePool)
require.NoError(t, err)
return tree, nil
}
for _, storeKey := range storeKeys {
multiTrees[storeKey], _ = mountTreeFn(storeKey)
}
oldTrees := make(map[string]commitment.Tree)
for _, storeKey := range oldStoreKeys {
oldTrees[storeKey], _ = mountTreeFn(storeKey)
}
return commitment.NewCommitStore(multiTrees, oldTrees, db, logger)
},
}
suite.Run(t, s)
}

51
store/v2/commitment/store_test_suite.go
View File

@ -28,12 +28,16 @@ const (
type CommitStoreTestSuite struct {
suite.Suite
NewStore func(db corestore.KVStoreWithBatch, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*CommitStore, error)
NewStore func(db corestore.KVStoreWithBatch, dbDir string, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*CommitStore, error)
TreeType string
}
func (s *CommitStoreTestSuite) TestStore_Snapshotter() {
if s.TreeType == "iavlv2" {
s.T().Skip("FIXME: iavlv2 does not yet support snapshots")
}
storeKeys := []string{storeKey1, storeKey2}
commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger())
commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
latestVersion := uint64(10)
@ -67,7 +71,7 @@ func (s *CommitStoreTestSuite) TestStore_Snapshotter() {
},
}
targetStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger())
targetStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
chunks := make(chan io.ReadCloser, kvCount*int(latestVersion))
@ -130,7 +134,8 @@ func (s *CommitStoreTestSuite) TestStore_Snapshotter() {
func (s *CommitStoreTestSuite) TestStore_LoadVersion() {
storeKeys := []string{storeKey1, storeKey2}
mdb := dbm.NewMemDB()
commitStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger())
dbDir := s.T().TempDir()
commitStore, err := s.NewStore(mdb, dbDir, storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
latestVersion := uint64(10)
@ -151,7 +156,7 @@ func (s *CommitStoreTestSuite) TestStore_LoadVersion() {
}
// load the store with the latest version
targetStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger())
targetStore, err := s.NewStore(mdb, dbDir, storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
err = targetStore.LoadVersion(latestVersion)
s.Require().NoError(err)
@ -164,7 +169,7 @@ func (s *CommitStoreTestSuite) TestStore_LoadVersion() {
// rollback to a previous version
rollbackVersion := uint64(5)
rollbackStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger())
rollbackStore, err := s.NewStore(mdb, dbDir, storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
err = rollbackStore.LoadVersion(rollbackVersion)
s.Require().NoError(err)
@ -185,7 +190,7 @@ func (s *CommitStoreTestSuite) TestStore_LoadVersion() {
func (s *CommitStoreTestSuite) TestStore_Pruning() {
storeKeys := []string{storeKey1, storeKey2}
pruneOpts := store.NewPruningOptionWithCustom(10, 5)
commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger())
commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
latestVersion := uint64(100)
@ -225,7 +230,7 @@ func (s *CommitStoreTestSuite) TestStore_Pruning() {
func (s *CommitStoreTestSuite) TestStore_GetProof() {
storeKeys := []string{storeKey1, storeKey2}
commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger())
commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
toVersion := uint64(10)
@ -268,7 +273,7 @@ func (s *CommitStoreTestSuite) TestStore_GetProof() {
func (s *CommitStoreTestSuite) TestStore_Get() {
storeKeys := []string{storeKey1, storeKey2}
commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger())
commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
toVersion := uint64(10)
@ -303,7 +308,8 @@ func (s *CommitStoreTestSuite) TestStore_Get() {
func (s *CommitStoreTestSuite) TestStore_Upgrades() {
storeKeys := []string{storeKey1, storeKey2, storeKey3}
commitDB := dbm.NewMemDB()
commitStore, err := s.NewStore(commitDB, storeKeys, nil, coretesting.NewNopLogger())
commitDir := s.T().TempDir()
commitStore, err := s.NewStore(commitDB, commitDir, storeKeys, nil, coretesting.NewNopLogger())
s.Require().NoError(err)
latestVersion := uint64(10)
@ -330,14 +336,14 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() {
}
newStoreKeys := []string{storeKey1, storeKey2, storeKey3, "newStore1", "newStore2"}
realStoreKeys := []string{storeKey1, storeKey2, "newStore1", "newStore2"}
oldStoreKeys := []string{storeKey1, storeKey3}
commitStore, err = s.NewStore(commitDB, newStoreKeys, oldStoreKeys, coretesting.NewNopLogger())
oldStoreKeys := []string{storeKey3}
commitStore, err = s.NewStore(commitDB, commitDir, newStoreKeys, oldStoreKeys, coretesting.NewNopLogger())
s.Require().NoError(err)
err = commitStore.LoadVersionAndUpgrade(latestVersion, upgrades)
s.Require().NoError(err)
// GetProof should work for the old stores
for _, storeKey := range []string{storeKey1, storeKey3} {
for _, storeKey := range []string{storeKey3} {
for i := uint64(1); i <= latestVersion; i++ {
for j := 0; j < kvCount; j++ {
proof, err := commitStore.GetProof([]byte(storeKey), i, []byte(fmt.Sprintf("key-%d-%d", i, j)))
@ -391,20 +397,20 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() {
// verify existing store
for i := uint64(1); i < latestVersion*2; i++ {
for j := 0; j < kvCount; j++ {
proof, err := commitStore.GetProof([]byte(storeKey2), i, []byte(fmt.Sprintf("key-%d-%d", i, j)))
prf, err := commitStore.GetProof([]byte(storeKey2), i, []byte(fmt.Sprintf("key-%d-%d", i, j)))
s.Require().NoError(err)
s.Require().NotNil(proof)
s.Require().NotNil(prf)
}
}
// create a new commitment store with one more upgrades
upgrades = &corestore.StoreUpgrades{
Added: []string{storeKey3},
Deleted: []string{storeKey2},
Added: []string{"newStore3"},
}
newRealStoreKeys := []string{storeKey1, storeKey3, "newStore1", "newStore2"}
newRealStoreKeys := []string{storeKey1, "newStore1", "newStore2", "newStore3"}
oldStoreKeys = []string{storeKey2, storeKey3}
commitStore, err = s.NewStore(commitDB, newStoreKeys, oldStoreKeys, coretesting.NewNopLogger())
commitStore, err = s.NewStore(commitDB, commitDir, newRealStoreKeys, oldStoreKeys, coretesting.NewNopLogger())
s.Require().NoError(err)
err = commitStore.LoadVersionAndUpgrade(2*latestVersion-1, upgrades)
s.Require().NoError(err)
@ -420,7 +426,8 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() {
kvPairs[storeKey] = append(kvPairs[storeKey], corestore.KVPair{Key: key, Value: value})
}
}
s.Require().NoError(commitStore.WriteChangeset(corestore.NewChangesetWithPairs(i, kvPairs)))
err = commitStore.WriteChangeset(corestore.NewChangesetWithPairs(i, kvPairs))
s.Require().NoError(err)
commitInfo, err := commitStore.Commit(i)
s.Require().NoError(err)
s.Require().NotNil(commitInfo)
@ -432,6 +439,7 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() {
// prune the old stores
s.Require().NoError(commitStore.Prune(latestVersion))
s.T().Logf("prune to version %d", latestVersion)
// GetProof should fail for the old stores
for _, storeKey := range []string{storeKey1, storeKey3} {
for i := uint64(1); i <= latestVersion; i++ {
@ -441,6 +449,7 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() {
}
}
}
s.T().Log("GetProof should work for the new stores")
// GetProof should not fail for the newly removed store
for i := latestVersion + 1; i < latestVersion*2; i++ {
for j := 0; j < kvCount; j++ {
@ -450,6 +459,7 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() {
}
}
s.T().Logf("Prune to version %d", latestVersion*2)
s.Require().NoError(commitStore.Prune(latestVersion * 2))
// GetProof should fail for the newly deleted stores
for i := uint64(1); i < latestVersion*2; i++ {
@ -458,10 +468,11 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() {
s.Require().Error(err)
}
}
s.T().Log("GetProof should work for the new added store")
// GetProof should work for the new added store
for i := latestVersion*2 + 1; i < latestVersion*3; i++ {
for j := 0; j < kvCount; j++ {
proof, err := commitStore.GetProof([]byte(storeKey3), i, []byte(fmt.Sprintf("key-%d-%d", i, j)))
proof, err := commitStore.GetProof([]byte("newStore3"), i, []byte(fmt.Sprintf("key-%d-%d", i, j)))
s.Require().NoError(err)
s.Require().NotNil(proof)
}

10
store/v2/go.mod
View File

@ -11,11 +11,11 @@ require (
github.com/cosmos/cosmos-proto v1.0.0-beta.5
github.com/cosmos/gogoproto v1.7.0
github.com/cosmos/iavl v1.3.1
github.com/cosmos/iavl/v2 v2.0.0-20241128205019-1b18c0edbbd9
github.com/cosmos/ics23/go v0.11.0
github.com/google/btree v1.1.2
github.com/google/btree v1.1.3
github.com/hashicorp/go-metrics v0.5.3
github.com/linxGnu/grocksdb v1.9.3
github.com/mattn/go-sqlite3 v1.14.22
github.com/spf13/cast v1.7.0
github.com/stretchr/testify v1.10.0
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d
@ -26,7 +26,9 @@ require (
require (
cosmossdk.io/schema v0.3.0 // indirect
github.com/DataDog/zstd v1.5.5 // indirect
github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bvinc/go-sqlite-lite v0.6.1 // indirect
github.com/bytedance/sonic v1.12.4 // indirect
github.com/bytedance/sonic/loader v0.2.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
@ -37,6 +39,7 @@ require (
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emicklei/dot v1.6.2 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
@ -48,6 +51,7 @@ require (
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
github.com/kocubinski/costor-api v1.1.1 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
@ -71,3 +75,5 @@ require (
google.golang.org/protobuf v1.35.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
replace cosmossdk.io/core => ../../core

26
store/v2/go.sum
View File

@ -1,5 +1,3 @@
cosmossdk.io/core v1.0.0-alpha.6 h1:5ukC4JcQKmemLQXcAgu/QoOvJI50hpBkIIg4ZT2EN8E=
cosmossdk.io/core v1.0.0-alpha.6/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY=
cosmossdk.io/core/testing v0.0.0-20241108153815-606544c7be7e h1:F+ScucYxwrrDJU8guJXQXpGhdpziYSbxW6HMP2wCNxs=
cosmossdk.io/core/testing v0.0.0-20241108153815-606544c7be7e/go.mod h1:3YvVv9aJayjPhdX0DY1IMrGse4sR63hNBWx2VtDWjGQ=
cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 h1:IQNdY2kB+k+1OM2DvqFG1+UgeU1JzZrWtwuWzI3ZfwA=
@ -15,10 +13,14 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e h1:dSeuFcs4WAJJnswS8vXy7YY1+fdlbVPuEVmDAfqvFOQ=
github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e/go.mod h1:uh71c5Vc3VNIplXOFXsnDy21T1BepgT32c5X/YPrOyc=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4=
github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s=
github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k=
github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
@ -55,6 +57,10 @@ github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fr
github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0=
github.com/cosmos/iavl v1.3.1 h1:+W1G2uSUtJMqMGpwz/fKiwZxY2DDT/9/0hyNLm6Geu0=
github.com/cosmos/iavl v1.3.1/go.mod h1:T6SfBcyhulVIY2G/ZtAtQm/QiJvsuhIos52V4dWYk88=
github.com/cosmos/iavl-bench/bench v0.0.4 h1:J6zQPiBqF4CXMM3QBsLqZgQEBGY0taX85vLIZMhmAfQ=
github.com/cosmos/iavl-bench/bench v0.0.4/go.mod h1:j2rLae77EffacWcp7mmj3Uaa4AOAmZA7ymvhsuBQKKI=
github.com/cosmos/iavl/v2 v2.0.0-20241128205019-1b18c0edbbd9 h1:H+ttW6HTzezz2l3Fp/hFNNHWA+a+7qZgNDE5OFySTiY=
github.com/cosmos/iavl/v2 v2.0.0-20241128205019-1b18c0edbbd9/go.mod h1:7RSm0aeApe3S1x4TrLffvUL6pjOtMYV4glYnpAhr2lw=
github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU=
github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIGF/JHNIY0=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@ -62,6 +68,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A=
github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@ -102,8 +110,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@ -127,6 +135,8 @@ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iP
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@ -138,6 +148,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kocubinski/costor-api v1.1.1 h1:sgfJA7T/8IfZ59zxiMrED0xdjerAFuPNBTqyO90GiEE=
github.com/kocubinski/costor-api v1.1.1/go.mod h1:ESMBMDkKfN+9vvvhhNVdKLhbOmzI3O/i16iXvRM9Tuc=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@ -155,8 +167,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -221,6 +231,10 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w=
github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=

6
store/v2/pruning/manager_test.go
View File

@ -15,7 +15,7 @@ import (
"cosmossdk.io/store/v2/commitment/iavl"
dbm "cosmossdk.io/store/v2/db"
"cosmossdk.io/store/v2/storage"
"cosmossdk.io/store/v2/storage/sqlite"
"cosmossdk.io/store/v2/storage/pebbledb"
)
var storeKeys = []string{"store1", "store2", "store3"}
@ -45,9 +45,9 @@ func (s *PruningManagerTestSuite) SetupTest() {
s.sc, err = commitment.NewCommitStore(multiTrees, nil, mdb, nopLog)
s.Require().NoError(err)
sqliteDB, err := sqlite.New(s.T().TempDir())
pebbleDB, err := pebbledb.New(s.T().TempDir())
s.Require().NoError(err)
s.ss = storage.NewStorageStore(sqliteDB, nopLog)
s.ss = storage.NewStorageStore(pebbleDB, nopLog)
scPruningOption := store.NewPruningOptionWithCustom(0, 1) // prune all
ssPruningOption := store.NewPruningOptionWithCustom(5, 10) // prune some
s.manager = NewManager(s.sc, s.ss, scPruningOption, ssPruningOption)

12
store/v2/root/factory.go
View File

@ -17,7 +17,6 @@ import (
"cosmossdk.io/store/v2/storage"
"cosmossdk.io/store/v2/storage/pebbledb"
"cosmossdk.io/store/v2/storage/rocksdb"
"cosmossdk.io/store/v2/storage/sqlite"
)
type (
@ -26,7 +25,6 @@ type (
)
const (
SSTypeSQLite SSType = "sqlite"
SSTypePebble SSType = "pebble"
SSTypeRocks SSType = "rocksdb"
SCTypeIavl SCType = "iavl"
@ -35,7 +33,7 @@ const (
// Options are the options for creating a root store.
type Options struct {
SSType SSType `mapstructure:"ss-type" toml:"ss-type" comment:"State storage database type. Currently we support: \"sqlite\", \"pebble\" and \"rocksdb\""`
SSType SSType `mapstructure:"ss-type" toml:"ss-type" comment:"State storage database type. Currently we support: \"pebble\" and \"rocksdb\""`
SCType SCType `mapstructure:"sc-type" toml:"sc-type" comment:"State commitment database type. Currently we support: \"iavl\" and \"iavl-v2\""`
SSPruningOption *store.PruningOption `mapstructure:"ss-pruning-option" toml:"ss-pruning-option" comment:"Pruning options for state storage"`
SCPruningOption *store.PruningOption `mapstructure:"sc-pruning-option" toml:"sc-pruning-option" comment:"Pruning options for state commitment"`
@ -54,7 +52,7 @@ type FactoryOptions struct {
// DefaultStoreOptions returns the default options for creating a root store.
func DefaultStoreOptions() Options {
return Options{
SSType: SSTypeSQLite,
SSType: SSTypePebble,
SCType: SCTypeIavl,
SCPruningOption: &store.PruningOption{
KeepRecent: 2,
@ -91,12 +89,6 @@ func CreateRootStore(opts *FactoryOptions) (store.RootStore, error) {
storeOpts := opts.Options
switch storeOpts.SSType {
case SSTypeSQLite:
dir := fmt.Sprintf("%s/data/ss/sqlite", opts.RootDir)
if err = ensureDir(dir); err != nil {
return nil, err
}
ssDb, err = sqlite.New(dir)
case SSTypePebble:
dir := fmt.Sprintf("%s/data/ss/pebble", opts.RootDir)
if err = ensureDir(dir); err != nil {

6
store/v2/root/migrate_test.go
View File

@ -18,7 +18,7 @@ import (
"cosmossdk.io/store/v2/pruning"
"cosmossdk.io/store/v2/snapshots"
"cosmossdk.io/store/v2/storage"
"cosmossdk.io/store/v2/storage/sqlite"
"cosmossdk.io/store/v2/storage/pebbledb"
)
var storeKeys = []string{"store1", "store2", "store3"}
@ -62,9 +62,9 @@ func (s *MigrateStoreTestSuite) SetupTest() {
}
// create a new storage and commitment stores
sqliteDB, err := sqlite.New(s.T().TempDir())
pebbleDB, err := pebbledb.New(s.T().TempDir())
s.Require().NoError(err)
ss := storage.NewStorageStore(sqliteDB, testLog)
ss := storage.NewStorageStore(pebbleDB, testLog)
multiTrees1 := make(map[string]commitment.Tree)
for _, storeKey := range storeKeys {

22
store/v2/root/store_test.go
View File

@ -17,7 +17,7 @@ import (
"cosmossdk.io/store/v2/proof"
"cosmossdk.io/store/v2/pruning"
"cosmossdk.io/store/v2/storage"
"cosmossdk.io/store/v2/storage/sqlite"
"cosmossdk.io/store/v2/storage/pebbledb"
)
const (
@ -47,9 +47,9 @@ func TestStorageTestSuite(t *testing.T) {
func (s *RootStoreTestSuite) SetupTest() {
noopLog := coretesting.NewNopLogger()
sqliteDB, err := sqlite.New(s.T().TempDir())
pebbleDB, err := pebbledb.New(s.T().TempDir())
s.Require().NoError(err)
ss := storage.NewStorageStore(sqliteDB, noopLog)
ss := storage.NewStorageStore(pebbleDB, noopLog)
tree := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig())
tree2 := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig())
@ -67,9 +67,9 @@ func (s *RootStoreTestSuite) SetupTest() {
func (s *RootStoreTestSuite) newStoreWithPruneConfig(config *store.PruningOption) {
noopLog := coretesting.NewNopLogger()
sqliteDB, err := sqlite.New(s.T().TempDir())
pebbleDB, err := pebbledb.New(s.T().TempDir())
s.Require().NoError(err)
ss := storage.NewStorageStore(sqliteDB, noopLog)
ss := storage.NewStorageStore(pebbleDB, noopLog)
mdb := dbm.NewMemDB()
multiTrees := make(map[string]commitment.Tree)
@ -535,9 +535,9 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() {
mdb1 := dbm.NewMemDB()
mdb2 := dbm.NewMemDB()
sqliteDB, err := sqlite.New(s.T().TempDir())
pebbleDB, err := pebbledb.New(s.T().TempDir())
s.Require().NoError(err)
ss := storage.NewStorageStore(sqliteDB, noopLog)
ss := storage.NewStorageStore(pebbleDB, noopLog)
tree := iavl.NewIavlTree(mdb1, noopLog, iavl.DefaultConfig())
sc, err := commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree}, nil, mdb2, noopLog)
@ -566,9 +566,9 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() {
s.Require().Equal(uint64(0), actualHeightToPrune)
// "restart"
sqliteDB, err = sqlite.New(s.T().TempDir())
pebbleDB, err = pebbledb.New(s.T().TempDir())
s.Require().NoError(err)
ss = storage.NewStorageStore(sqliteDB, noopLog)
ss = storage.NewStorageStore(pebbleDB, noopLog)
tree = iavl.NewIavlTree(mdb1, noopLog, iavl.DefaultConfig())
sc, err = commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree}, nil, mdb2, noopLog)
@ -616,10 +616,10 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() {
func (s *RootStoreTestSuite) TestMultiStoreRestart() {
noopLog := coretesting.NewNopLogger()
sqliteDB, err := sqlite.New(s.T().TempDir())
pebbleDB, err := pebbledb.New(s.T().TempDir())
s.Require().NoError(err)
ss := storage.NewStorageStore(sqliteDB, noopLog)
ss := storage.NewStorageStore(pebbleDB, noopLog)
mdb1 := dbm.NewMemDB()
mdb2 := dbm.NewMemDB()

6
store/v2/root/upgrade_test.go
View File

@ -15,7 +15,7 @@ import (
dbm "cosmossdk.io/store/v2/db"
"cosmossdk.io/store/v2/pruning"
"cosmossdk.io/store/v2/storage"
"cosmossdk.io/store/v2/storage/sqlite"
"cosmossdk.io/store/v2/storage/pebbledb"
)
type UpgradeStoreTestSuite struct {
@ -44,9 +44,9 @@ func (s *UpgradeStoreTestSuite) SetupTest() {
}
// create storage and commitment stores
sqliteDB, err := sqlite.New(s.T().TempDir())
pebbleDB, err := pebbledb.New(s.T().TempDir())
s.Require().NoError(err)
ss := storage.NewStorageStore(sqliteDB, testLog)
ss := storage.NewStorageStore(pebbleDB, testLog)
sc, err := commitment.NewCommitStore(multiTrees, nil, s.commitDB, testLog)
s.Require().NoError(err)
pm := pruning.NewManager(sc, ss, nil, nil)

1
store/v2/storage/pebbledb/batch.go
View File

@ -88,6 +88,7 @@ func (b *Batch) Delete(storeKey, key []byte) error {
return b.set(storeKey, b.version, key, []byte(tombstoneVal))
}
// Write flushes any accumulated data to disk and closes the batch.
func (b *Batch) Write() (err error) {
defer func() {
err = errors.Join(err, b.batch.Close())

104
store/v2/storage/sqlite/batch.go
View File

@ -1,104 +0,0 @@
package sqlite
import (
"database/sql"
"fmt"
"cosmossdk.io/store/v2"
)
var _ store.Batch = (*Batch)(nil)
type batchAction int
const (
batchActionSet batchAction = 0
batchActionDel batchAction = 1
)
type batchOp struct {
action batchAction
storeKey []byte
key, value []byte
}
type Batch struct {
db *sql.DB
tx *sql.Tx
ops []batchOp
size int
version uint64
}
func NewBatch(db *sql.DB, version uint64) (*Batch, error) {
tx, err := db.Begin()
if err != nil {
return nil, fmt.Errorf("failed to create SQL transaction: %w", err)
}
return &Batch{
db: db,
tx: tx,
ops: make([]batchOp, 0),
version: version,
}, nil
}
func (b *Batch) Size() int {
return b.size
}
func (b *Batch) Reset() error {
b.ops = nil
b.ops = make([]batchOp, 0)
b.size = 0
tx, err := b.db.Begin()
if err != nil {
return err
}
b.tx = tx
return nil
}
func (b *Batch) Set(storeKey, key, value []byte) error {
b.size += len(key) + len(value)
b.ops = append(b.ops, batchOp{action: batchActionSet, storeKey: storeKey, key: key, value: value})
return nil
}
func (b *Batch) Delete(storeKey, key []byte) error {
b.size += len(key)
b.ops = append(b.ops, batchOp{action: batchActionDel, storeKey: storeKey, key: key})
return nil
}
func (b *Batch) Write() error {
_, err := b.tx.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, b.version, 0, b.version)
if err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
for _, op := range b.ops {
switch op.action {
case batchActionSet:
_, err := b.tx.Exec(upsertStmt, op.storeKey, op.key, op.value, b.version, op.value)
if err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
case batchActionDel:
_, err := b.tx.Exec(delStmt, b.version, op.storeKey, op.key, b.version)
if err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
}
}
if err := b.tx.Commit(); err != nil {
return fmt.Errorf("failed to write SQL transaction: %w", err)
}
return nil
}

360
store/v2/storage/sqlite/db.go
View File

@ -1,360 +0,0 @@
package sqlite
import (
"bytes"
"database/sql"
"errors"
"fmt"
"path/filepath"
"strings"
_ "github.com/mattn/go-sqlite3"
corestore "cosmossdk.io/core/store"
"cosmossdk.io/store/v2"
storeerrors "cosmossdk.io/store/v2/errors"
"cosmossdk.io/store/v2/storage"
)
const (
driverName = "sqlite3"
dbName = "ss.db?cache=shared&mode=rwc&_journal_mode=WAL"
reservedStoreKey = "_RESERVED_"
keyLatestHeight = "latest_height"
keyPruneHeight = "prune_height"
valueRemovedStore = "removed_store"
reservedUpsertStmt = `
INSERT INTO state_storage(store_key, key, value, version)
VALUES(?, ?, ?, ?)
ON CONFLICT(store_key, key, version) DO UPDATE SET
value = ?;
`
upsertStmt = `
INSERT INTO state_storage(store_key, key, value, version)
VALUES(?, ?, ?, ?)
ON CONFLICT(store_key, key, version) DO UPDATE SET
value = ?;
`
delStmt = `
UPDATE state_storage SET tombstone = ?
WHERE id = (
SELECT id FROM state_storage WHERE store_key = ? AND key = ? AND version <= ? ORDER BY version DESC LIMIT 1
) AND tombstone = 0;
`
)
var (
_ storage.Database = (*Database)(nil)
_ store.UpgradableDatabase = (*Database)(nil)
)
type Database struct {
storage *sql.DB
// earliestVersion defines the earliest version set in the database, which is
// only updated when the database is pruned.
earliestVersion uint64
}
func New(dataDir string) (*Database, error) {
storage, err := sql.Open(driverName, filepath.Join(dataDir, dbName))
if err != nil {
return nil, fmt.Errorf("failed to open sqlite DB: %w", err)
}
stmt := `
CREATE TABLE IF NOT EXISTS state_storage (
id integer not null primary key,
store_key varchar not null,
key varchar not null,
value varchar not null,
version integer unsigned not null,
tombstone integer unsigned default 0,
unique (store_key, key, version)
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_store_key_version ON state_storage (store_key, key, version);
`
_, err = storage.Exec(stmt)
if err != nil {
return nil, fmt.Errorf("failed to exec SQL statement: %w", err)
}
pruneHeight, err := getPruneHeight(storage)
if err != nil {
return nil, fmt.Errorf("failed to get prune height: %w", err)
}
return &Database{
storage: storage,
earliestVersion: pruneHeight,
}, nil
}
func (db *Database) Close() error {
err := db.storage.Close()
db.storage = nil
return err
}
func (db *Database) NewBatch(version uint64) (store.Batch, error) {
return NewBatch(db.storage, version)
}
func (db *Database) GetLatestVersion() (uint64, error) {
stmt, err := db.storage.Prepare(`
SELECT value
FROM state_storage
WHERE store_key = ? AND key = ?
`)
if err != nil {
return 0, fmt.Errorf("failed to prepare SQL statement: %w", err)
}
defer stmt.Close()
var latestHeight uint64
if err := stmt.QueryRow(reservedStoreKey, keyLatestHeight).Scan(&latestHeight); err != nil {
if errors.Is(err, sql.ErrNoRows) {
// in case of a fresh database
return 0, nil
}
return 0, fmt.Errorf("failed to query row: %w", err)
}
return latestHeight, nil
}
func (db *Database) VersionExists(v uint64) (bool, error) {
latestVersion, err := db.GetLatestVersion()
if err != nil {
return false, err
}
return latestVersion >= v && v >= db.earliestVersion, nil
}
func (db *Database) SetLatestVersion(version uint64) error {
_, err := db.storage.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, version, 0, version)
if err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
return nil
}
func (db *Database) Has(storeKey []byte, version uint64, key []byte) (bool, error) {
val, err := db.Get(storeKey, version, key)
if err != nil {
return false, err
}
return val != nil, nil
}
func (db *Database) Get(storeKey []byte, targetVersion uint64, key []byte) ([]byte, error) {
if targetVersion < db.earliestVersion {
return nil, storeerrors.ErrVersionPruned{EarliestVersion: db.earliestVersion, RequestedVersion: targetVersion}
}
stmt, err := db.storage.Prepare(`
SELECT value, tombstone FROM state_storage
WHERE store_key = ? AND key = ? AND version <= ?
ORDER BY version DESC LIMIT 1;
`)
if err != nil {
return nil, fmt.Errorf("failed to prepare SQL statement: %w", err)
}
defer stmt.Close()
var (
value []byte
tomb uint64
)
if err := stmt.QueryRow(storeKey, key, targetVersion).Scan(&value, &tomb); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return nil, fmt.Errorf("failed to query row: %w", err)
}
// A tombstone of zero or a target version that is less than the tombstone
// version means the key is not deleted at the target version.
if tomb == 0 || targetVersion < tomb {
return value, nil
}
// the value is considered deleted
return nil, nil
}
// Prune removes all versions of all keys that are <= the given version. It keeps
// the latest (non-tombstoned) version of each key/value tuple to handle queries
// above the prune version. This is analogous to RocksDB full_history_ts_low.
//
// We perform the prune by deleting all versions of a key, excluding reserved keys,
// that are <= the given version, except for the latest version of the key.
func (db *Database) Prune(version uint64) error {
tx, err := db.storage.Begin()
if err != nil {
return fmt.Errorf("failed to create SQL transaction: %w", err)
}
defer func() {
if err != nil {
err = tx.Rollback()
}
}()
// prune all keys of old versions
pruneStmt := `DELETE FROM state_storage
WHERE version < (
SELECT max(version) FROM state_storage t2 WHERE
t2.store_key = state_storage.store_key AND
t2.key = state_storage.key AND
t2.version <= ?
) AND store_key != ?;
`
if _, err := tx.Exec(pruneStmt, version, reservedStoreKey); err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
// prune removed stores
pruneRemovedStoreKeysStmt := `DELETE FROM state_storage AS s
WHERE EXISTS (
SELECT 1 FROM
(
SELECT key, MAX(version) AS max_version
FROM state_storage
WHERE store_key = ? AND value = ? AND version <= ?
GROUP BY key
) AS t
WHERE s.store_key = t.key AND s.version <= t.max_version LIMIT 1
);
`
if _, err := tx.Exec(pruneRemovedStoreKeysStmt, reservedStoreKey, valueRemovedStore, version, version); err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
// delete the removedKeys
if _, err := tx.Exec("DELETE FROM state_storage WHERE store_key = ? AND value = ? AND version <= ?", reservedStoreKey, valueRemovedStore, version); err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
// set the prune height so we can return <nil> for queries below this height
if _, err := tx.Exec(reservedUpsertStmt, reservedStoreKey, keyPruneHeight, version, 0, version); err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("failed to write SQL transaction: %w", err)
}
db.earliestVersion = version + 1
return nil
}
func (db *Database) Iterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) {
if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
return nil, storeerrors.ErrKeyEmpty
}
if start != nil && end != nil && bytes.Compare(start, end) > 0 {
return nil, storeerrors.ErrStartAfterEnd
}
return newIterator(db, storeKey, version, start, end, false)
}
func (db *Database) ReverseIterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) {
if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
return nil, storeerrors.ErrKeyEmpty
}
if start != nil && end != nil && bytes.Compare(start, end) > 0 {
return nil, storeerrors.ErrStartAfterEnd
}
return newIterator(db, storeKey, version, start, end, true)
}
func (db *Database) PruneStoreKeys(storeKeys []string, version uint64) (err error) {
tx, err := db.storage.Begin()
if err != nil {
return fmt.Errorf("failed to create SQL transaction: %w", err)
}
defer func() {
if err != nil {
err = tx.Rollback()
}
}()
// flush removed store keys
flushRemovedStoreKeyStmt := `INSERT INTO state_storage(store_key, key, value, version)
VALUES (?, ?, ?, ?)`
for _, storeKey := range storeKeys {
if _, err := tx.Exec(flushRemovedStoreKeyStmt, reservedStoreKey, []byte(storeKey), valueRemovedStore, version); err != nil {
return fmt.Errorf("failed to exec SQL statement: %w", err)
}
}
return tx.Commit()
}
func (db *Database) PrintRowsDebug() {
stmt, err := db.storage.Prepare("SELECT store_key, key, value, version, tombstone FROM state_storage")
if err != nil {
panic(fmt.Errorf("failed to prepare SQL statement: %w", err))
}
defer stmt.Close()
rows, err := stmt.Query()
if err != nil {
panic(fmt.Errorf("failed to execute SQL query: %w", err))
}
var sb strings.Builder
for rows.Next() {
var (
storeKey []byte
key []byte
value []byte
version uint64
tomb uint64
)
if err := rows.Scan(&storeKey, &key, &value, &version, &tomb); err != nil {
panic(fmt.Sprintf("failed to scan row: %s", err))
}
sb.WriteString(fmt.Sprintf("STORE_KEY: %s, KEY: %s, VALUE: %s, VERSION: %d, TOMBSTONE: %d\n", storeKey, key, value, version, tomb))
}
if err := rows.Err(); err != nil {
panic(fmt.Errorf("received unexpected error: %w", err))
}
fmt.Println(strings.TrimSpace(sb.String()))
}
func getPruneHeight(storage *sql.DB) (uint64, error) {
stmt, err := storage.Prepare(`SELECT value FROM state_storage WHERE store_key = ? AND key = ?`)
if err != nil {
return 0, fmt.Errorf("failed to prepare SQL statement: %w", err)
}
defer stmt.Close()
var value uint64
if err := stmt.QueryRow(reservedStoreKey, keyPruneHeight).Scan(&value); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return 0, nil
}
return 0, fmt.Errorf("failed to query row: %w", err)
}
return value, nil
}

200
store/v2/storage/sqlite/db_test.go
View File

@ -1,200 +0,0 @@
package sqlite
import (
"fmt"
"sync"
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
coretesting "cosmossdk.io/core/testing"
"cosmossdk.io/store/v2/storage"
)
var storeKey1 = []byte("store1")
func TestStorageTestSuite(t *testing.T) {
s := &storage.StorageTestSuite{
NewDB: func(dir string) (*storage.StorageStore, error) {
db, err := New(dir)
return storage.NewStorageStore(db, coretesting.NewNopLogger()), err
},
EmptyBatchSize: 0,
}
suite.Run(t, s)
}
func TestDatabase_ReverseIterator(t *testing.T) {
db, err := New(t.TempDir())
require.NoError(t, err)
defer db.Close()
batch, err := db.NewBatch(1)
require.NoError(t, err)
for i := 0; i < 100; i++ {
key := fmt.Sprintf("key%03d", i) // key000, key001, ..., key099
val := fmt.Sprintf("val%03d", i) // val000, val001, ..., val099
require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val)))
}
require.NoError(t, batch.Write())
// reverse iterator without an end key
iter, err := db.ReverseIterator(storeKey1, 1, []byte("key000"), nil)
require.NoError(t, err)
defer iter.Close()
i, count := 99, 0
for ; iter.Valid(); iter.Next() {
require.Equal(t, []byte(fmt.Sprintf("key%03d", i)), iter.Key())
require.Equal(t, []byte(fmt.Sprintf("val%03d", i)), iter.Value())
i--
count++
}
require.Equal(t, 100, count)
require.NoError(t, iter.Error())
// seek past domain, which should make the iterator invalid and produce an error
require.False(t, iter.Valid())
// reverse iterator with a start and end domain
iter2, err := db.ReverseIterator(storeKey1, 1, []byte("key010"), []byte("key019"))
require.NoError(t, err)
defer iter2.Close()
i, count = 18, 0
for ; iter2.Valid(); iter2.Next() {
require.Equal(t, []byte(fmt.Sprintf("key%03d", i)), iter2.Key())
require.Equal(t, []byte(fmt.Sprintf("val%03d", i)), iter2.Value())
i--
count++
}
require.Equal(t, 9, count)
require.NoError(t, iter2.Error())
// seek past domain, which should make the iterator invalid and produce an error
require.False(t, iter2.Valid())
// start must be <= end
iter3, err := db.ReverseIterator(storeKey1, 1, []byte("key020"), []byte("key019"))
require.Error(t, err)
require.Nil(t, iter3)
}
func TestParallelWrites(t *testing.T) {
db, err := New(t.TempDir())
require.NoError(t, err)
defer db.Close()
latestVersion := 10
kvCount := 100
wg := sync.WaitGroup{}
triggerStartCh := make(chan bool)
// start 10 goroutines that write to the database
for i := 0; i < latestVersion; i++ {
wg.Add(1)
go func(i int) {
<-triggerStartCh
defer wg.Done()
batch, err := db.NewBatch(uint64(i + 1))
require.NoError(t, err)
for j := 0; j < kvCount; j++ {
key := fmt.Sprintf("key-%d-%03d", i, j)
val := fmt.Sprintf("val-%d-%03d", i, j)
require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val)))
}
require.NoError(t, batch.Write())
}(i)
}
// start the goroutines
close(triggerStartCh)
wg.Wait()
// check that all the data is there
for i := 0; i < latestVersion; i++ {
for j := 0; j < kvCount; j++ {
version := uint64(i + 1)
key := fmt.Sprintf("key-%d-%03d", i, j)
val := fmt.Sprintf("val-%d-%03d", i, j)
v, err := db.Get(storeKey1, version, []byte(key))
require.NoError(t, err)
require.Equal(t, []byte(val), v)
}
}
}
func TestParallelWriteAndPruning(t *testing.T) {
db, err := New(t.TempDir())
require.NoError(t, err)
defer db.Close()
latestVersion := 100
kvCount := 100
prunePeriod := 5
wg := sync.WaitGroup{}
triggerStartCh := make(chan bool)
// start a goroutine that write to the database
wg.Add(1)
go func() {
<-triggerStartCh
defer wg.Done()
for i := 0; i < latestVersion; i++ {
batch, err := db.NewBatch(uint64(i + 1))
require.NoError(t, err)
for j := 0; j < kvCount; j++ {
key := fmt.Sprintf("key-%d-%03d", i, j)
val := fmt.Sprintf("val-%d-%03d", i, j)
require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val)))
}
require.NoError(t, batch.Write())
}
}()
// start a goroutine that prunes the database
wg.Add(1)
go func() {
<-triggerStartCh
defer wg.Done()
for i := 10; i < latestVersion; i += prunePeriod {
for {
v, err := db.GetLatestVersion()
require.NoError(t, err)
if v > uint64(i) {
require.NoError(t, db.Prune(v-1))
break
}
}
}
}()
// start the goroutines
close(triggerStartCh)
wg.Wait()
// check if the data is pruned
version := uint64(latestVersion - prunePeriod)
val, err := db.Get(storeKey1, version, []byte(fmt.Sprintf("key-%d-%03d", version-1, 0)))
require.Error(t, err)
require.Nil(t, val)
version = uint64(latestVersion)
val, err = db.Get(storeKey1, version, []byte(fmt.Sprintf("key-%d-%03d", version-1, 0)))
require.NoError(t, err)
require.Equal(t, []byte(fmt.Sprintf("val-%d-%03d", version-1, 0)), val)
}

183
store/v2/storage/sqlite/iterator.go
View File

@ -1,183 +0,0 @@
package sqlite
import (
"bytes"
"database/sql"
"fmt"
"slices"
"strings"
corestore "cosmossdk.io/core/store"
)
var _ corestore.Iterator = (*iterator)(nil)
type iterator struct {
statement *sql.Stmt
rows *sql.Rows
key, val []byte
start, end []byte
valid bool
err error
}
func newIterator(db *Database, storeKey []byte, targetVersion uint64, start, end []byte, reverse bool) (*iterator, error) {
if targetVersion < db.earliestVersion {
return &iterator{
start: start,
end: end,
valid: false,
}, nil
}
var (
keyClause = []string{"store_key = ?", "version <= ?"}
queryArgs []any
)
switch {
case len(start) > 0 && len(end) > 0:
keyClause = append(keyClause, "key >= ?", "key < ?")
queryArgs = []any{storeKey, targetVersion, start, end, targetVersion}
case len(start) > 0 && len(end) == 0:
keyClause = append(keyClause, "key >= ?")
queryArgs = []any{storeKey, targetVersion, start, targetVersion}
case len(start) == 0 && len(end) > 0:
keyClause = append(keyClause, "key < ?")
queryArgs = []any{storeKey, targetVersion, end, targetVersion}
default:
queryArgs = []any{storeKey, targetVersion, targetVersion}
}
orderBy := "ASC"
if reverse {
orderBy = "DESC"
}
// Note, this is not susceptible to SQL injection because placeholders are used
// for parts of the query outside the store's direct control.
stmt, err := db.storage.Prepare(fmt.Sprintf(`
SELECT x.key, x.value
FROM (
SELECT key, value, version, tombstone,
row_number() OVER (PARTITION BY key ORDER BY version DESC) AS _rn
FROM state_storage WHERE %s
) x
WHERE x._rn = 1 AND (x.tombstone = 0 OR x.tombstone > ?) ORDER BY x.key %s;
`, strings.Join(keyClause, " AND "), orderBy))
if err != nil {
return nil, fmt.Errorf("failed to prepare SQL statement: %w", err)
}
rows, err := stmt.Query(queryArgs...)
if err != nil {
_ = stmt.Close()
return nil, fmt.Errorf("failed to execute SQL query: %w", err)
}
itr := &iterator{
statement: stmt,
rows: rows,
start: start,
end: end,
valid: rows.Next(),
}
if !itr.valid {
itr.err = fmt.Errorf("iterator invalid: %w", sql.ErrNoRows)
return itr, nil
}
// read the first row
itr.parseRow()
if !itr.valid {
return itr, nil
}
return itr, nil
}
func (itr *iterator) Close() (err error) {
if itr.statement != nil {
err = itr.statement.Close()
}
itr.valid = false
itr.statement = nil
itr.rows = nil
return err
}
// Domain returns the domain of the iterator. The caller must not modify the
// return values.
func (itr *iterator) Domain() ([]byte, []byte) {
return itr.start, itr.end
}
func (itr *iterator) Key() []byte {
itr.assertIsValid()
return slices.Clone(itr.key)
}
func (itr *iterator) Value() []byte {
itr.assertIsValid()
return slices.Clone(itr.val)
}
func (itr *iterator) Valid() bool {
if !itr.valid || itr.rows.Err() != nil {
itr.valid = false
return itr.valid
}
// if key is at the end or past it, consider it invalid
if end := itr.end; end != nil {
if bytes.Compare(end, itr.Key()) <= 0 {
itr.valid = false
return itr.valid
}
}
return true
}
func (itr *iterator) Next() {
if itr.rows.Next() {
itr.parseRow()
return
}
itr.valid = false
}
func (itr *iterator) Error() error {
if err := itr.rows.Err(); err != nil {
return err
}
return itr.err
}
func (itr *iterator) parseRow() {
var (
key []byte
value []byte
)
if err := itr.rows.Scan(&key, &value); err != nil {
itr.err = fmt.Errorf("failed to scan row: %w", err)
itr.valid = false
return
}
itr.key = key
itr.val = value
}
func (itr *iterator) assertIsValid() {
if !itr.valid {
panic("iterator is invalid")
}
}

5
store/v2/storage/storage_bench_test.go
View File

@ -18,7 +18,6 @@ import (
"cosmossdk.io/store/v2/storage"
"cosmossdk.io/store/v2/storage/pebbledb"
"cosmossdk.io/store/v2/storage/rocksdb"
"cosmossdk.io/store/v2/storage/sqlite"
)
var storeKey1 = []byte("store1")
@ -37,10 +36,6 @@ var (
return storage.NewStorageStore(db, coretesting.NewNopLogger()), err
},
"btree_sqlite": func(dataDir string) (store.VersionedWriter, error) {
db, err := sqlite.New(dataDir)
return storage.NewStorageStore(db, coretesting.NewNopLogger()), err
},
}
rng = rand.New(rand.NewSource(567320))
)

3
store/v2/storage/store.go
View File

@ -128,7 +128,8 @@ func (ss *StorageStore) Restore(version uint64, chStorage <-chan *corestore.Stat
if err := b.Write(); err != nil {
return err
}
if err := b.Reset(); err != nil {
b, err = ss.db.NewBatch(version)
if err != nil {
return err
}
}

1
tests/go.mod
View File

@ -175,7 +175,6 @@ require (
github.com/manifoldco/promptui v0.9.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/minio/highwayhash v1.0.3 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect

2
tests/go.sum
View File

@ -660,8 +660,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q=
github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=

4
tools/confix/data/v2-app.toml
View File

@ -56,8 +56,8 @@ minimum-gas-prices = '0stake'
app-db-backend = 'goleveldb'
[store.options]
# State storage database type. Currently we support: "sqlite", "pebble" and "rocksdb"
ss-type = 'sqlite'
# State storage database type. Currently we support: "pebble" and "rocksdb"
ss-type = 'pebble'
# State commitment database type. Currently we support: "iavl" and "iavl-v2"
sc-type = 'iavl'