Store Refactor 1 (#2985)

This commit is contained in:
Joon
2019-02-01 17:03:09 -08:00
committed by Jack Zampolin
parent 45d59b0792
commit 08e62fb157
58 changed files with 1758 additions and 1611 deletions
+6 -3
View File
@@ -10,6 +10,9 @@ import (
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
"github.com/cosmos/cosmos-sdk/store/gaskv"
stypes "github.com/cosmos/cosmos-sdk/store/types"
)
/*
@@ -46,7 +49,7 @@ func NewContext(ms MultiStore, header abci.Header, isCheckTx bool, logger log.Lo
c = c.WithTxBytes(nil)
c = c.WithLogger(logger)
c = c.WithVoteInfos(nil)
c = c.WithGasMeter(NewInfiniteGasMeter())
c = c.WithGasMeter(stypes.NewInfiniteGasMeter())
c = c.WithMinGasPrices(DecCoins{})
c = c.WithConsensusParams(nil)
return c
@@ -74,12 +77,12 @@ func (c Context) Value(key interface{}) interface{} {
// KVStore fetches a KVStore from the MultiStore.
func (c Context) KVStore(key StoreKey) KVStore {
return c.MultiStore().GetKVStore(key).Gas(c.GasMeter(), cachedKVGasConfig)
return gaskv.NewStore(c.MultiStore().GetKVStore(key), c.GasMeter(), stypes.KVGasConfig())
}
// TransientStore fetches a TransientStore from the MultiStore.
func (c Context) TransientStore(key StoreKey) KVStore {
return c.MultiStore().GetKVStore(key).Gas(c.GasMeter(), cachedTransientGasConfig)
return gaskv.NewStore(c.MultiStore().GetKVStore(key), c.GasMeter(), stypes.TransientGasConfig())
}
//----------------------------------------
-163
View File
@@ -1,163 +0,0 @@
package types
// Gas consumption descriptors.
const (
GasIterNextCostFlatDesc = "IterNextFlat"
GasValuePerByteDesc = "ValuePerByte"
GasWritePerByteDesc = "WritePerByte"
GasReadPerByteDesc = "ReadPerByte"
GasWriteCostFlatDesc = "WriteFlat"
GasReadCostFlatDesc = "ReadFlat"
GasHasDesc = "Has"
GasDeleteDesc = "Delete"
)
var (
cachedKVGasConfig = KVGasConfig()
cachedTransientGasConfig = TransientGasConfig()
)
// Gas measured by the SDK
type Gas = uint64
// ErrorOutOfGas defines an error thrown when an action results in out of gas.
type ErrorOutOfGas struct {
Descriptor string
}
// ErrorGasOverflow defines an error thrown when an action results gas consumption
// unsigned integer overflow.
type ErrorGasOverflow struct {
Descriptor string
}
// GasMeter interface to track gas consumption
type GasMeter interface {
GasConsumed() Gas
GasConsumedToLimit() Gas
Limit() Gas
ConsumeGas(amount Gas, descriptor string)
IsPastLimit() bool
IsOutOfGas() bool
}
type basicGasMeter struct {
limit Gas
consumed Gas
}
// NewGasMeter returns a reference to a new basicGasMeter.
func NewGasMeter(limit Gas) GasMeter {
return &basicGasMeter{
limit: limit,
consumed: 0,
}
}
func (g *basicGasMeter) GasConsumed() Gas {
return g.consumed
}
func (g *basicGasMeter) Limit() Gas {
return g.limit
}
func (g *basicGasMeter) GasConsumedToLimit() Gas {
if g.IsPastLimit() {
return g.limit
}
return g.consumed
}
func (g *basicGasMeter) ConsumeGas(amount Gas, descriptor string) {
var overflow bool
// TODO: Should we set the consumed field after overflow checking?
g.consumed, overflow = AddUint64Overflow(g.consumed, amount)
if overflow {
panic(ErrorGasOverflow{descriptor})
}
if g.consumed > g.limit {
panic(ErrorOutOfGas{descriptor})
}
}
func (g *basicGasMeter) IsPastLimit() bool {
return g.consumed > g.limit
}
func (g *basicGasMeter) IsOutOfGas() bool {
return g.consumed >= g.limit
}
type infiniteGasMeter struct {
consumed Gas
}
// NewInfiniteGasMeter returns a reference to a new infiniteGasMeter.
func NewInfiniteGasMeter() GasMeter {
return &infiniteGasMeter{
consumed: 0,
}
}
func (g *infiniteGasMeter) GasConsumed() Gas {
return g.consumed
}
func (g *infiniteGasMeter) GasConsumedToLimit() Gas {
return g.consumed
}
func (g *infiniteGasMeter) Limit() Gas {
return 0
}
func (g *infiniteGasMeter) ConsumeGas(amount Gas, descriptor string) {
var overflow bool
// TODO: Should we set the consumed field after overflow checking?
g.consumed, overflow = AddUint64Overflow(g.consumed, amount)
if overflow {
panic(ErrorGasOverflow{descriptor})
}
}
func (g *infiniteGasMeter) IsPastLimit() bool {
return false
}
func (g *infiniteGasMeter) IsOutOfGas() bool {
return false
}
// GasConfig defines gas cost for each operation on KVStores
type GasConfig struct {
HasCost Gas
DeleteCost Gas
ReadCostFlat Gas
ReadCostPerByte Gas
WriteCostFlat Gas
WriteCostPerByte Gas
IterNextCostFlat Gas
}
// KVGasConfig returns a default gas config for KVStores.
func KVGasConfig() GasConfig {
return GasConfig{
HasCost: 1000,
DeleteCost: 1000,
ReadCostFlat: 1000,
ReadCostPerByte: 3,
WriteCostFlat: 2000,
WriteCostPerByte: 30,
IterNextCostFlat: 30,
}
}
// TransientGasConfig returns a default gas config for TransientStores.
func TransientGasConfig() GasConfig {
// TODO: define gasconfig for transient stores
return KVGasConfig()
}
-45
View File
@@ -1,45 +0,0 @@
package types
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGasMeter(t *testing.T) {
cases := []struct {
limit Gas
usage []Gas
}{
{10, []Gas{1, 2, 3, 4}},
{1000, []Gas{40, 30, 20, 10, 900}},
{100000, []Gas{99999, 1}},
{100000000, []Gas{50000000, 40000000, 10000000}},
{65535, []Gas{32768, 32767}},
{65536, []Gas{32768, 32767, 1}},
}
for tcnum, tc := range cases {
meter := NewGasMeter(tc.limit)
used := uint64(0)
for unum, usage := range tc.usage {
used += usage
require.NotPanics(t, func() { meter.ConsumeGas(usage, "") }, "Not exceeded limit but panicked. tc #%d, usage #%d", tcnum, unum)
require.Equal(t, used, meter.GasConsumed(), "Gas consumption not match. tc #%d, usage #%d", tcnum, unum)
require.Equal(t, used, meter.GasConsumedToLimit(), "Gas consumption (to limit) not match. tc #%d, usage #%d", tcnum, unum)
require.False(t, meter.IsPastLimit(), "Not exceeded limit but got IsPastLimit() true")
if unum < len(tc.usage)-1 {
require.False(t, meter.IsOutOfGas(), "Not yet at limit but got IsOutOfGas() true")
} else {
require.True(t, meter.IsOutOfGas(), "At limit but got IsOutOfGas() false")
}
}
require.Panics(t, func() { meter.ConsumeGas(1, "") }, "Exceeded but not panicked. tc #%d", tcnum)
require.Equal(t, meter.GasConsumedToLimit(), meter.Limit(), "Gas consumption (to limit) not match limit")
require.Equal(t, meter.GasConsumed(), meter.Limit()+1, "Gas consumption not match limit+1")
break
}
}
-11
View File
@@ -2,7 +2,6 @@ package types
import (
"encoding/json"
"math"
"testing"
"math/big"
@@ -575,16 +574,6 @@ func UintOverflow(x Uint) bool {
return x.i.Sign() == -1 || x.i.Sign() == 1 && x.i.BitLen() > 256
}
// AddUint64Overflow performs the addition operation on two uint64 integers and
// returns a boolean on whether or not the result overflows.
func AddUint64Overflow(a, b uint64) (uint64, bool) {
if math.MaxUint64-a < b {
return 0, true
}
return a + b, false
}
// intended to be used with require/assert: require.True(IntEq(...))
func IntEq(t *testing.T, exp, got Int) (*testing.T, bool, string, string, string) {
return t, exp.Equal(got), "expected:\t%v\ngot:\t\t%v", exp.String(), got.String()
-25
View File
@@ -631,28 +631,3 @@ func TestSafeSub(t *testing.T) {
)
}
}
func TestAddUint64Overflow(t *testing.T) {
testCases := []struct {
a, b uint64
result uint64
overflow bool
}{
{0, 0, 0, false},
{100, 100, 200, false},
{math.MaxUint64 / 2, math.MaxUint64/2 + 1, math.MaxUint64, false},
{math.MaxUint64 / 2, math.MaxUint64/2 + 2, 0, true},
}
for i, tc := range testCases {
res, overflow := AddUint64Overflow(tc.a, tc.b)
require.Equal(
t, tc.overflow, overflow,
"invalid overflow result; tc: #%d, a: %d, b: %d", i, tc.a, tc.b,
)
require.Equal(
t, tc.result, res,
"invalid uint64 result; tc: #%d, a: %d, b: %d", i, tc.a, tc.b,
)
}
}
+77 -345
View File
@@ -1,398 +1,130 @@
package types
import (
"bytes"
"fmt"
"io"
abci "github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/cosmos/cosmos-sdk/store/types"
)
// NOTE: These are implemented in cosmos-sdk/store.
// nolint - reexport
type (
PruningOptions = types.PruningOptions
)
// PruningStrategy specifies how old states will be deleted over time where
// keepRecent can be used with keepEvery to create a pruning "strategy".
type PruningOptions struct {
keepRecent int64
keepEvery int64
}
func NewPruningOptions(keepRecent, keepEvery int64) PruningOptions {
return PruningOptions{
keepRecent: keepRecent,
keepEvery: keepEvery,
}
}
// How much recent state will be kept. Older state will be deleted.
func (po PruningOptions) KeepRecent() int64 {
return po.keepRecent
}
// Keeps every N stated, deleting others.
func (po PruningOptions) KeepEvery() int64 {
return po.keepEvery
}
type Store interface { //nolint
GetStoreType() StoreType
CacheWrapper
}
// something that can persist to disk
type Committer interface {
Commit() CommitID
LastCommitID() CommitID
SetPruning(PruningOptions)
}
// Stores of MultiStore must implement CommitStore.
type CommitStore interface {
Committer
Store
}
// Queryable allows a Store to expose internal state to the abci.Query
// interface. Multistore can route requests to the proper Store.
//
// This is an optional, but useful extension to any CommitStore
type Queryable interface {
Query(abci.RequestQuery) abci.ResponseQuery
}
//----------------------------------------
// MultiStore
type MultiStore interface { //nolint
Store
// Cache wrap MultiStore.
// NOTE: Caller should probably not call .Write() on each, but
// call CacheMultiStore.Write().
CacheMultiStore() CacheMultiStore
// Convenience for fetching substores.
// If the store does not exist, panics.
GetStore(StoreKey) Store
GetKVStore(StoreKey) KVStore
// TracingEnabled returns if tracing is enabled for the MultiStore.
TracingEnabled() bool
// WithTracer sets the tracer for the MultiStore that the underlying
// stores will utilize to trace operations. A MultiStore is returned.
WithTracer(w io.Writer) MultiStore
// WithTracingContext sets the tracing context for a MultiStore. It is
// implied that the caller should update the context when necessary between
// tracing operations. A MultiStore is returned.
WithTracingContext(TraceContext) MultiStore
// ResetTraceContext resets the current tracing context.
ResetTraceContext() MultiStore
}
// From MultiStore.CacheMultiStore()....
type CacheMultiStore interface {
MultiStore
Write() // Writes operations to underlying KVStore
}
// A non-cache MultiStore.
type CommitMultiStore interface {
Committer
MultiStore
// Mount a store of type using the given db.
// If db == nil, the new store will use the CommitMultiStore db.
MountStoreWithDB(key StoreKey, typ StoreType, db dbm.DB)
// Panics on a nil key.
GetCommitStore(key StoreKey) CommitStore
// Panics on a nil key.
GetCommitKVStore(key StoreKey) CommitKVStore
// Load the latest persisted version. Called once after all
// calls to Mount*Store() are complete.
LoadLatestVersion() error
// Load a specific persisted version. When you load an old
// version, or when the last commit attempt didn't complete,
// the next commit after loading must be idempotent (return the
// same commit id). Otherwise the behavior is undefined.
LoadVersion(ver int64) error
}
//---------subsp-------------------------------
// KVStore
// KVStore is a simple interface to get/set data
type KVStore interface {
Store
// Get returns nil iff key doesn't exist. Panics on nil key.
Get(key []byte) []byte
// Has checks if a key exists. Panics on nil key.
Has(key []byte) bool
// Set sets the key. Panics on nil key or value.
Set(key, value []byte)
// Delete deletes the key. Panics on nil key.
Delete(key []byte)
// Iterator over a domain of keys in ascending order. End is exclusive.
// Start must be less than end, or the Iterator is invalid.
// Iterator must be closed by caller.
// To iterate over entire domain, use store.Iterator(nil, nil)
// CONTRACT: No writes may happen within a domain while an iterator exists over it.
Iterator(start, end []byte) Iterator
// Iterator over a domain of keys in descending order. End is exclusive.
// Start must be less than end, or the Iterator is invalid.
// Iterator must be closed by caller.
// CONTRACT: No writes may happen within a domain while an iterator exists over it.
ReverseIterator(start, end []byte) Iterator
// TODO Not yet implemented.
// CreateSubKVStore(key *storeKey) (KVStore, error)
// TODO Not yet implemented.
// GetSubKVStore(key *storeKey) KVStore
// Prefix applied keys with the argument
// CONTRACT: when Prefix is called on a KVStore more than once,
// the concatanation of the prefixes is applied
Prefix(prefix []byte) KVStore
// Gas consuming store
// CONTRACT: when Gas is called on a KVStore more than once,
// the concatanation of the meters/configs is applied
Gas(GasMeter, GasConfig) KVStore
}
// Alias iterator to db's Iterator for convenience.
type Iterator = dbm.Iterator
// nolint - reexport
type (
Store = types.Store
Committer = types.Committer
CommitStore = types.CommitStore
Queryable = types.Queryable
MultiStore = types.MultiStore
CacheMultiStore = types.CacheMultiStore
CommitMultiStore = types.CommitMultiStore
KVStore = types.KVStore
Iterator = types.Iterator
)
// Iterator over all the keys with a certain prefix in ascending order
func KVStorePrefixIterator(kvs KVStore, prefix []byte) Iterator {
return kvs.Iterator(prefix, PrefixEndBytes(prefix))
return types.KVStorePrefixIterator(kvs, prefix)
}
// Iterator over all the keys with a certain prefix in descending order.
func KVStoreReversePrefixIterator(kvs KVStore, prefix []byte) Iterator {
return kvs.ReverseIterator(prefix, PrefixEndBytes(prefix))
return types.KVStoreReversePrefixIterator(kvs, prefix)
}
// Compare two KVstores, return either the first key/value pair
// at which they differ and whether or not they are equal, skipping
// value comparison for a set of provided prefixes
func DiffKVStores(a KVStore, b KVStore, prefixesToSkip [][]byte) (kvA cmn.KVPair, kvB cmn.KVPair, count int64, equal bool) {
iterA := a.Iterator(nil, nil)
iterB := b.Iterator(nil, nil)
count = int64(0)
for {
if !iterA.Valid() && !iterB.Valid() {
break
}
var kvA, kvB cmn.KVPair
if iterA.Valid() {
kvA = cmn.KVPair{Key: iterA.Key(), Value: iterA.Value()}
iterA.Next()
}
if iterB.Valid() {
kvB = cmn.KVPair{Key: iterB.Key(), Value: iterB.Value()}
iterB.Next()
}
if !bytes.Equal(kvA.Key, kvB.Key) {
return kvA, kvB, count, false
}
compareValue := true
for _, prefix := range prefixesToSkip {
// Skip value comparison if we matched a prefix
if bytes.Equal(kvA.Key[:len(prefix)], prefix) {
compareValue = false
}
}
if compareValue && !bytes.Equal(kvA.Value, kvB.Value) {
return kvA, kvB, count, false
}
count++
}
return cmn.KVPair{}, cmn.KVPair{}, count, true
return types.DiffKVStores(a, b, prefixesToSkip)
}
// CacheKVStore cache-wraps a KVStore. After calling .Write() on
// the CacheKVStore, all previously created CacheKVStores on the
// object expire.
type CacheKVStore interface {
KVStore
// Writes operations to underlying KVStore
Write()
}
// Stores of MultiStore must implement CommitStore.
type CommitKVStore interface {
Committer
KVStore
}
//----------------------------------------
// CacheWrap
// CacheWrap makes the most appropriate cache-wrap. For example,
// IAVLStore.CacheWrap() returns a CacheKVStore. CacheWrap should not return
// a Committer, since Commit cache-wraps make no sense. It can return KVStore,
// HeapStore, SpaceStore, etc.
type CacheWrap interface {
// Write syncs with the underlying store.
Write()
// CacheWrap recursively wraps again.
CacheWrap() CacheWrap
// CacheWrapWithTrace recursively wraps again with tracing enabled.
CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap
}
type CacheWrapper interface { //nolint
// CacheWrap cache wraps.
CacheWrap() CacheWrap
// CacheWrapWithTrace cache wraps with tracing enabled.
CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap
}
//----------------------------------------
// CommitID
// CommitID contains the tree version number and its merkle root.
type CommitID struct {
Version int64
Hash []byte
}
func (cid CommitID) IsZero() bool { //nolint
return cid.Version == 0 && len(cid.Hash) == 0
}
func (cid CommitID) String() string {
return fmt.Sprintf("CommitID{%v:%X}", cid.Hash, cid.Version)
}
//----------------------------------------
// Store types
// kind of store
type StoreType int
const (
//nolint
StoreTypeMulti StoreType = iota
StoreTypeDB
StoreTypeIAVL
StoreTypeTransient
// nolint - reexport
type (
CacheKVStore = types.CacheKVStore
CommitKVStore = types.CommitKVStore
CacheWrap = types.CacheWrap
CacheWrapper = types.CacheWrapper
CommitID = types.CommitID
)
//----------------------------------------
// Keys for accessing substores
// nolint - reexport
type StoreType = types.StoreType
// StoreKey is a key used to index stores in a MultiStore.
type StoreKey interface {
Name() string
String() string
}
// nolint - reexport
const (
StoreTypeMulti = types.StoreTypeMulti
StoreTypeDB = types.StoreTypeDB
StoreTypeIAVL = types.StoreTypeIAVL
StoreTypeTransient = types.StoreTypeTransient
)
// KVStoreKey is used for accessing substores.
// Only the pointer value should ever be used - it functions as a capabilities key.
type KVStoreKey struct {
name string
}
// nolint - reexport
type (
StoreKey = types.StoreKey
KVStoreKey = types.KVStoreKey
TransientStoreKey = types.TransientStoreKey
)
// NewKVStoreKey returns a new pointer to a KVStoreKey.
// Use a pointer so keys don't collide.
func NewKVStoreKey(name string) *KVStoreKey {
return &KVStoreKey{
name: name,
}
return types.NewKVStoreKey(name)
}
func (key *KVStoreKey) Name() string {
return key.name
}
func (key *KVStoreKey) String() string {
return fmt.Sprintf("KVStoreKey{%p, %s}", key, key.name)
// Constructs new TransientStoreKey
// Must return a pointer according to the ocap principle
func NewTransientStoreKey(name string) *TransientStoreKey {
return types.NewTransientStoreKey(name)
}
// PrefixEndBytes returns the []byte that would end a
// range query for all []byte with a certain prefix
// Deals with last byte of prefix being FF without overflowing
func PrefixEndBytes(prefix []byte) []byte {
if prefix == nil {
return nil
}
end := make([]byte, len(prefix))
copy(end, prefix)
for {
if end[len(end)-1] != byte(255) {
end[len(end)-1]++
break
} else {
end = end[:len(end)-1]
if len(end) == 0 {
end = nil
break
}
}
}
return end
return types.PrefixEndBytes(prefix)
}
// InclusiveEndBytes returns the []byte that would end a
// range query such that the input would be included
func InclusiveEndBytes(inclusiveBytes []byte) (exclusiveBytes []byte) {
exclusiveBytes = append(inclusiveBytes, byte(0x00))
return exclusiveBytes
}
// TransientStoreKey is used for indexing transient stores in a MultiStore
type TransientStoreKey struct {
name string
}
// Constructs new TransientStoreKey
// Must return a pointer according to the ocap principle
func NewTransientStoreKey(name string) *TransientStoreKey {
return &TransientStoreKey{
name: name,
}
}
// Implements StoreKey
func (key *TransientStoreKey) Name() string {
return key.name
}
// Implements StoreKey
func (key *TransientStoreKey) String() string {
return fmt.Sprintf("TransientStoreKey{%p, %s}", key, key.name)
return types.InclusiveEndBytes(inclusiveBytes)
}
//----------------------------------------
// key-value result for iterator queries
type KVPair cmn.KVPair
type KVPair = types.KVPair
//----------------------------------------
// TraceContext contains TraceKVStore context data. It will be written with
// every trace operation.
type TraceContext map[string]interface{}
type TraceContext = types.TraceContext
// --------------------------------------
// nolint - reexport
type (
Gas = types.Gas
GasMeter = types.GasMeter
GasConfig = types.GasConfig
)
// nolint - reexport
func NewGasMeter(limit Gas) GasMeter {
return types.NewGasMeter(limit)
}
// nolint - reexport
type (
ErrorOutOfGas = types.ErrorOutOfGas
ErrorGasOverflow = types.ErrorGasOverflow
)
// nolint - reexport
func NewInfiniteGasMeter() GasMeter {
return types.NewInfiniteGasMeter()
}