LaconicNetwork/cosmos-sdk
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

chore(store): upgrade iavl to v0.19.0 (#12626)

Browse Source 
## Description

Closes: #XXXX

Upgrading IAVL to v0.19. This version includes the fast index optimization and errors propagated up top.

Currently, SDK panics on any db error, leaving the proper error handling and refactoring to future work.

Similar change in Osmosis fork, ref: https://github.com/osmosis-labs/cosmos-sdk/pull/108

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [x] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [x] added `!` to the type prefix if API or client breaking change
- [x] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/main/CONTRIBUTING.md#pr-targeting))
- [x] provided a link to the relevant issue or specification
- [x] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/main/docs/building-modules)
- [x] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/main/CONTRIBUTING.md#testing)
- [x] added a changelog entry to `CHANGELOG.md`
- [x] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [x] updated the relevant documentation or specification
- [x] reviewed "Files changed" and left comments if necessary
- [x] confirmed all CI checks have passed

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed 
- [ ] reviewed state machine logic
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)
This commit is contained in:
Roman
2022-07-20 12:25:10 +00:00
committed by GitHub
parent 987959c541
commit 6f631156f8
16 changed files with 138 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
store/iavl/store.go
+58 -27
View File
@@ -9,6 +9,7 @@ import (
ics23 "github.com/confio/ics23/go"
"github.com/cosmos/iavl"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto"
dbm "github.com/tendermint/tm-db"
@@ -42,20 +43,35 @@ type Store struct {
// LoadStore returns an IAVL Store as a CommitKVStore. Internally, it will load the
// store's version (id) from the provided DB. An error is returned if the version
// fails to load, or if called with a positive version on an empty tree.
func LoadStore(db dbm.DB, id types.CommitID, lazyLoading bool, cacheSize int) (types.CommitKVStore, error) {
return LoadStoreWithInitialVersion(db, id, lazyLoading, 0, cacheSize)
func LoadStore(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, cacheSize int) (types.CommitKVStore, error) {
return LoadStoreWithInitialVersion(db, logger, key, id, lazyLoading, 0, cacheSize)
}
// LoadStoreWithInitialVersion returns an IAVL Store as a CommitKVStore setting its initialVersion
// to the one given. Internally, it will load the store's version (id) from the
// provided DB. An error is returned if the version fails to load, or if called with a positive
// version on an empty tree.
func LoadStoreWithInitialVersion(db dbm.DB, id types.CommitID, lazyLoading bool, initialVersion uint64, cacheSize int) (types.CommitKVStore, error) {
func LoadStoreWithInitialVersion(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, initialVersion uint64, cacheSize int) (types.CommitKVStore, error) {
tree, err := iavl.NewMutableTreeWithOpts(db, cacheSize, &iavl.Options{InitialVersion: initialVersion})
if err != nil {
return nil, err
}
isUpgradeable, err := tree.IsUpgradeable()
if err != nil {
return nil, err
}
if isUpgradeable && logger != nil {
logger.Info(
"Upgrading IAVL storage for faster queries + execution on live state. This may take a while",
"store_key", key.String(),
"version", initialVersion,
"commit", fmt.Sprintf("%X", id),
"is_lazy", lazyLoading,
)
}
if lazyLoading {
_, err = tree.LazyLoadVersion(id.Version)
} else {
@@ -66,6 +82,10 @@ func LoadStoreWithInitialVersion(db dbm.DB, id types.CommitID, lazyLoading bool,
return nil, err
}
if logger != nil {
logger.Debug("Finished loading IAVL tree")
}
return &Store{
tree: tree,
}, nil
@@ -121,9 +141,14 @@ func (st *Store) Commit() types.CommitID {
// LastCommitID implements Committer.
func (st *Store) LastCommitID() types.CommitID {
hash, err := st.tree.Hash()
if err != nil {
panic(err)
}
return types.CommitID{
Version: st.tree.Version(),
Hash: st.tree.Hash(),
Hash: hash,
}
}
@@ -179,14 +204,21 @@ func (st *Store) Set(key, value []byte) {
// Implements types.KVStore.
func (st *Store) Get(key []byte) []byte {
defer telemetry.MeasureSince(time.Now(), "store", "iavl", "get")
_, value := st.tree.Get(key)
value, err := st.tree.Get(key)
if err != nil {
panic(err)
}
return value
}
// Implements types.KVStore.
func (st *Store) Has(key []byte) (exists bool) {
defer telemetry.MeasureSince(time.Now(), "store", "iavl", "has")
return st.tree.Has(key)
has, err := st.tree.Has(key)
if err != nil {
panic(err)
}
return has
}
// Implements types.KVStore.
@@ -204,30 +236,20 @@ func (st *Store) DeleteVersions(versions ...int64) error {
// Implements types.KVStore.
func (st *Store) Iterator(start, end []byte) types.Iterator {
var iTree *iavl.ImmutableTree
switch tree := st.tree.(type) {
case *immutableTree:
iTree = tree.ImmutableTree
case *iavl.MutableTree:
iTree = tree.ImmutableTree
iterator, err := st.tree.Iterator(start, end, true)
if err != nil {
panic(err)
}
return newIAVLIterator(iTree, start, end, true)
return iterator
}
// Implements types.KVStore.
func (st *Store) ReverseIterator(start, end []byte) types.Iterator {
var iTree *iavl.ImmutableTree
switch tree := st.tree.(type) {
case *immutableTree:
iTree = tree.ImmutableTree
case *iavl.MutableTree:
iTree = tree.ImmutableTree
iterator, err := st.tree.Iterator(start, end, false)
if err != nil {
panic(err)
}
return newIAVLIterator(iTree, start, end, false)
return iterator
}
// SetInitialVersion sets the initial version of the IAVL tree. It is used when
@@ -302,7 +324,12 @@ func (st *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) {
break
}
_, res.Value = tree.GetVersioned(key, res.Height)
value, err := tree.GetVersioned(key, res.Height)
if err != nil {
panic(err)
}
res.Value = value
if !req.Prove {
break
}
@@ -382,7 +409,7 @@ func getProofFromTree(tree *iavl.MutableTree, key []byte, exists bool) *tmcrypto
// iavlIterator implements types.Iterator.
type iavlIterator struct {
*iavl.Iterator
dbm.Iterator
}
var _ types.Iterator = (*iavlIterator)(nil)
@@ -391,8 +418,12 @@ var _ types.Iterator = (*iavlIterator)(nil)
// CONTRACT: Caller must release the iavlIterator, as each one creates a new
// goroutine.
func newIAVLIterator(tree *iavl.ImmutableTree, start, end []byte, ascending bool) *iavlIterator {
iterator, err := tree.Iterator(start, end, ascending)
if err != nil {
panic(err)
}
iter := &iavlIterator{
Iterator: tree.Iterator(start, end, ascending),
Iterator: iterator,
}
return iter
}
store/iavl/store_test.go
+15 -7
View File
@@ -10,6 +10,7 @@ import (
"github.com/cosmos/iavl"
"github.com/stretchr/testify/require"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/store/types"
@@ -58,13 +59,17 @@ func TestLoadStore(t *testing.T) {
store := UnsafeNewStore(tree)
// Create non-pruned height H
require.True(t, tree.Set([]byte("hello"), []byte("hallo")))
updated, err := tree.Set([]byte("hello"), []byte("hallo"))
require.NoError(t, err)
require.True(t, updated)
hash, verH, err := tree.SaveVersion()
cIDH := types.CommitID{Version: verH, Hash: hash}
require.Nil(t, err)
// Create pruned height Hp
require.True(t, tree.Set([]byte("hello"), []byte("hola")))
updated, err = tree.Set([]byte("hello"), []byte("hola"))
require.NoError(t, err)
require.True(t, updated)
hash, verHp, err := tree.SaveVersion()
cIDHp := types.CommitID{Version: verHp, Hash: hash}
require.Nil(t, err)
@@ -72,7 +77,9 @@ func TestLoadStore(t *testing.T) {
// TODO: Prune this height
// Create current height Hc
require.True(t, tree.Set([]byte("hello"), []byte("ciao")))
updated, err = tree.Set([]byte("hello"), []byte("ciao"))
require.NoError(t, err)
require.True(t, updated)
hash, verHc, err := tree.SaveVersion()
cIDHc := types.CommitID{Version: verHc, Hash: hash}
require.Nil(t, err)
@@ -93,17 +100,17 @@ func TestLoadStore(t *testing.T) {
require.Equal(t, string(hcStore.Get([]byte("hello"))), "ciao")
// Querying a new store at some previous non-pruned height H
newHStore, err := LoadStore(db, cIDH, false, DefaultIAVLCacheSize)
newHStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDH, false, DefaultIAVLCacheSize)
require.NoError(t, err)
require.Equal(t, string(newHStore.Get([]byte("hello"))), "hallo")
// Querying a new store at some previous pruned height Hp
newHpStore, err := LoadStore(db, cIDHp, false, DefaultIAVLCacheSize)
newHpStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDHp, false, DefaultIAVLCacheSize)
require.NoError(t, err)
require.Equal(t, string(newHpStore.Get([]byte("hello"))), "hola")
// Querying a new store at current height H
newHcStore, err := LoadStore(db, cIDHc, false, DefaultIAVLCacheSize)
newHcStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDHc, false, DefaultIAVLCacheSize)
require.NoError(t, err)
require.Equal(t, string(newHcStore.Get([]byte("hello"))), "ciao")
}
@@ -113,7 +120,8 @@ func TestGetImmutable(t *testing.T) {
tree, cID := newAlohaTree(t, db)
store := UnsafeNewStore(tree)
require.True(t, tree.Set([]byte("hello"), []byte("adios")))
updated, err := tree.Set([]byte("hello"), []byte("adios"))
require.True(t, updated)
hash, ver, err := tree.SaveVersion()
cID = types.CommitID{Version: ver, Hash: hash}
require.Nil(t, err)
store/iavl/tree.go
+12 -10
View File
@@ -3,6 +3,7 @@ package iavl
import (
"fmt"
"github.com/cosmos/cosmos-sdk/store/types"
"github.com/cosmos/iavl"
)
@@ -17,20 +18,21 @@ type (
// implemented by an iavl.MutableTree. For an immutable IAVL tree, a wrapper
// must be made.
Tree interface {
Has(key []byte) bool
Get(key []byte) (index int64, value []byte)
Set(key, value []byte) bool
Remove(key []byte) ([]byte, bool)
Has(key []byte) (bool, error)
Get(key []byte) ([]byte, error)
Set(key, value []byte) (bool, error)
Remove(key []byte) ([]byte, bool, error)
SaveVersion() ([]byte, int64, error)
DeleteVersion(version int64) error
DeleteVersions(versions ...int64) error
Version() int64
Hash() []byte
Hash() ([]byte, error)
VersionExists(version int64) bool
GetVersioned(key []byte, version int64) (int64, []byte)
GetVersioned(key []byte, version int64) ([]byte, error)
GetVersionedWithProof(key []byte, version int64) ([]byte, *iavl.RangeProof, error)
GetImmutable(version int64) (*iavl.ImmutableTree, error)
SetInitialVersion(version uint64)
Iterator(start, end []byte, ascending bool) (types.Iterator, error)
AvailableVersions() []int
}
@@ -42,11 +44,11 @@ type (
}
)
func (it *immutableTree) Set(_, _ []byte) bool {
func (it *immutableTree) Set(_, _ []byte) (bool, error) {
panic("cannot call 'Set' on an immutable IAVL tree")
}
func (it *immutableTree) Remove(_ []byte) ([]byte, bool) {
func (it *immutableTree) Remove(_ []byte) ([]byte, bool, error) {
panic("cannot call 'Remove' on an immutable IAVL tree")
}
@@ -70,9 +72,9 @@ func (it *immutableTree) VersionExists(version int64) bool {
return it.Version() == version
}
func (it *immutableTree) GetVersioned(key []byte, version int64) (int64, []byte) {
func (it *immutableTree) GetVersioned(key []byte, version int64) ([]byte, error) {
if it.Version() != version {
return -1, nil
return nil, fmt.Errorf("version mismatch on immutable IAVL tree; got: %d, expected: %d", version, it.Version())
}
return it.Get(key)
store/iavl/tree_test.go
-4
View File
@@ -16,10 +16,6 @@ func TestImmutableTreePanics(t *testing.T) {
require.Panics(t, func() { it.Remove([]byte{}) })
require.Panics(t, func() { it.SaveVersion() }) // nolint:errcheck
require.Panics(t, func() { it.DeleteVersion(int64(1)) }) // nolint:errcheck
v, _ := it.GetVersioned([]byte{0x01}, 1)
require.Equal(t, int64(-1), v)
v, _ = it.GetVersioned([]byte{0x01}, 0)
require.Equal(t, int64(0), v)
val, proof, err := it.GetVersionedWithProof(nil, 1)
require.Error(t, err)
store/rootmulti/proof_test.go
+1 -1
View File
@@ -15,7 +15,7 @@ import (
func TestVerifyIAVLStoreQueryProof(t *testing.T) {
// Create main tree for testing.
db := dbm.NewMemDB()
iStore, err := iavl.LoadStore(db, types.CommitID{}, false, iavl.DefaultIAVLCacheSize)
iStore, err := iavl.LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), types.CommitID{}, false, iavl.DefaultIAVLCacheSize)
store := iStore.(*iavl.Store)
require.Nil(t, err)
store.Set([]byte("MYKEY"), []byte("MYVALUE"))
store/rootmulti/store.go
+2 -2
View File
@@ -865,9 +865,9 @@ func (rs *Store) loadCommitStoreFromParams(key types.StoreKey, id types.CommitID
var err error
if params.initialVersion == 0 {
store, err = iavl.LoadStore(db, id, rs.lazyLoading, rs.iavlCacheSize)
store, err = iavl.LoadStore(db, rs.logger, key, id, rs.lazyLoading, rs.iavlCacheSize)
} else {
store, err = iavl.LoadStoreWithInitialVersion(db, id, rs.lazyLoading, params.initialVersion, rs.iavlCacheSize)
store, err = iavl.LoadStoreWithInitialVersion(db, rs.logger, key, id, rs.lazyLoading, params.initialVersion, rs.iavlCacheSize)
}
if err != nil {
store/tools/ics23/go.mod
+1 -1
View File
@@ -5,7 +5,7 @@ go 1.18
require (
github.com/confio/ics23/go v0.7.0
github.com/cosmos/cosmos-sdk v0.46.0-rc3
github.com/cosmos/iavl v0.18.0
github.com/cosmos/iavl v0.19.0
github.com/lazyledger/smt v0.2.1-0.20210709230900-03ea40719554
github.com/tendermint/tendermint v0.34.20-rc1
github.com/tendermint/tm-db v0.6.7
store/tools/ics23/iavl/cmd/testgen-iavl/main.go
+10 -4
View File
@@ -75,12 +75,12 @@ func main() {
fmt.Println("Usage: testgen-iavl [exist|nonexist] [left|right|middle] <size>")
os.Exit(1)
}
root := tree.WorkingHash()
root, _ := tree.WorkingHash()
var key, value []byte
if exist {
key = helpers.GetKey(allkeys, loc)
_, value = tree.Get(key)
value, _ = tree.Get(key)
} else {
key = helpers.GetNonKey(allkeys, loc)
}
@@ -132,14 +132,20 @@ func doBatch(args []string) error {
if err != nil {
return err
}
root := tree.WorkingHash()
root, err := tree.WorkingHash()
if err != nil {
return err
}
items := []item{}
proofs := []*ics23.CommitmentProof{}
for i := 0; i < exist; i++ {
key := []byte(helpers.GetKey(allkeys, tmproofs.Middle))
_, value := tree.Get(key)
value, err := tree.Get(key)
if err != nil {
return err
}
proof, err := iavlproofs.CreateMembershipProof(tree, key)
if err != nil {
return fmt.Errorf("create proof: %+v", err)
store/tools/ics23/iavl/create.go
+12 -4
View File
@@ -47,18 +47,23 @@ If the key exists in the tree, this will return an error.
*/
func CreateNonMembershipProof(tree *iavl.MutableTree, key []byte) (*ics23.CommitmentProof, error) {
// idx is one node right of what we want....
idx, val := tree.Get(key)
idx, val, err := tree.GetWithIndex(key)
if err != nil {
return nil, err
}
if val != nil {
return nil, fmt.Errorf("Cannot create NonExistanceProof when Key in State")
}
var err error
nonexist := &ics23.NonExistenceProof{
Key: key,
}
if idx >= 1 {
leftkey, _ := tree.GetByIndex(idx - 1)
leftkey, _, err := tree.GetByIndex(idx - 1)
if err != nil {
return nil, err
}
nonexist.Left, err = createExistenceProof(tree, leftkey)
if err != nil {
return nil, err
@@ -66,7 +71,10 @@ func CreateNonMembershipProof(tree *iavl.MutableTree, key []byte) (*ics23.Commit
}
// this will be nil if nothing right of the queried key
rightkey, _ := tree.GetByIndex(idx)
rightkey, _, err := tree.GetByIndex(idx)
if err != nil {
return nil, err
}
if rightkey != nil {
nonexist.Right, err = createExistenceProof(tree, rightkey)
if err != nil {
store/tools/ics23/iavl/create_test.go
+3 -3
View File
@@ -29,13 +29,13 @@ func TestCreateMembership(t *testing.T) {
t.Fatalf("Creating tree: %+v", err)
}
key := helpers.GetKey(allkeys, tc.loc)
_, val := tree.Get(key)
val, err := tree.Get(key)
proof, err := CreateMembershipProof(tree, key)
if err != nil {
t.Fatalf("Creating Proof: %+v", err)
}
root := tree.WorkingHash()
root, _ := tree.WorkingHash()
valid := ics23.VerifyMembership(IavlSpec, root, proof, key, val)
if !valid {
t.Fatalf("Membership Proof Invalid")
@@ -70,7 +70,7 @@ func TestCreateNonMembership(t *testing.T) {
t.Fatalf("Creating Proof: %+v", err)
}
root := tree.WorkingHash()
root, _ := tree.WorkingHash()
valid := ics23.VerifyNonMembership(IavlSpec, root, proof, key)
if !valid {
t.Fatalf("Non Membership Proof Invalid")
store/tools/ics23/iavl/helpers/helpers.go
+1 -1
View File
@@ -45,7 +45,7 @@ func GenerateIavlResult(size int, loc tmproofs.Where) (*IavlResult, error) {
if len(proof.Leaves) != 1 {
return nil, fmt.Errorf("GetWithProof returned %d leaves", len(proof.Leaves))
}
root := tree.WorkingHash()
root, _ := tree.WorkingHash()
res := &IavlResult{
Key: key,
store/types/iterator_test.go
+2 -1
View File
@@ -4,6 +4,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/store/iavl"
@@ -12,7 +13,7 @@ import (
func newMemTestKVStore(t *testing.T) types.KVStore {
db := dbm.NewMemDB()
store, err := iavl.LoadStore(db, types.CommitID{}, false, iavl.DefaultIAVLCacheSize)
store, err := iavl.LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), types.CommitID{}, false, iavl.DefaultIAVLCacheSize)
require.NoError(t, err)
return store
}