From 5272ca58318d21c0846e5ca5a4a3e76047c0150f Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 25 Jul 2017 18:31:05 -0400 Subject: [PATCH 01/18] Add checkpoint to middleware not app --- app/app.go | 17 ++----- docs/guide/counter/plugins/counter/counter.go | 1 + .../counter/plugins/counter/counter_test.go | 20 +++++--- modules/base/checkpoint.go | 49 +++++++++++++++++++ 4 files changed, 67 insertions(+), 20 deletions(-) create mode 100644 modules/base/checkpoint.go diff --git a/app/app.go b/app/app.go index 378b36e0b2..c5ff537396 100644 --- a/app/app.go +++ b/app/app.go @@ -71,6 +71,7 @@ func DefaultHandler(feeDenom string) basecoin.Handler { nonce.ReplayCheck{}, roles.NewMiddleware(), fee.NewSimpleFeeMiddleware(coin.Coin{feeDenom, 0}, fee.Bank), + base.Checkpoint{}, ).Use(d) } @@ -120,21 +121,18 @@ func (app *Basecoin) DeliverTx(txBytes []byte) abci.Result { return errors.Result(err) } - // TODO: can we abstract this setup and commit logic?? - cache := app.state.CacheWrap() ctx := stack.NewContext( app.state.GetChainID(), app.height, app.logger.With("call", "delivertx"), ) - res, err := app.handler.DeliverTx(ctx, cache, tx) + fmt.Printf("state: %#v\n", app.state) + res, err := app.handler.DeliverTx(ctx, app.state, tx) if err != nil { // discard the cache... return errors.Result(err) } - // commit the cache and return result - cache.CacheSync() return res.ToABCI() } @@ -145,22 +143,17 @@ func (app *Basecoin) CheckTx(txBytes []byte) abci.Result { return errors.Result(err) } - // we also need to discard error changes, so we don't increment checktx - // sequence on error, but not delivertx - cache := app.cacheState.CacheWrap() ctx := stack.NewContext( app.state.GetChainID(), app.height, app.logger.With("call", "checktx"), ) - // checktx generally shouldn't touch the state, but we don't care - // here on the framework level, since the cacheState is thrown away next block - res, err := app.handler.CheckTx(ctx, cache, tx) + fmt.Printf("state: %#v\n", app.cacheState) + res, err := app.handler.CheckTx(ctx, app.cacheState, tx) if err != nil { return errors.Result(err) } - cache.CacheSync() return res.ToABCI() } diff --git a/docs/guide/counter/plugins/counter/counter.go b/docs/guide/counter/plugins/counter/counter.go index df07195f4c..27863c5273 100644 --- a/docs/guide/counter/plugins/counter/counter.go +++ b/docs/guide/counter/plugins/counter/counter.go @@ -104,6 +104,7 @@ func NewHandler(feeDenom string) basecoin.Handler { base.Chain{}, nonce.ReplayCheck{}, fee.NewSimpleFeeMiddleware(coin.Coin{feeDenom, 0}, fee.Bank), + base.Checkpoint{}, ).Use(dispatcher) } diff --git a/docs/guide/counter/plugins/counter/counter_test.go b/docs/guide/counter/plugins/counter/counter_test.go index ff880c5099..5fa83d7353 100644 --- a/docs/guide/counter/plugins/counter/counter_test.go +++ b/docs/guide/counter/plugins/counter/counter_test.go @@ -1,7 +1,6 @@ package counter import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -25,11 +24,12 @@ func TestCounterPlugin(t *testing.T) { eyesCli := eyescli.NewLocalClient("", 0) chainID := "test_chain_id" - // logger := log.TestingLogger().With("module", "app"), - logger := log.NewTMLogger(os.Stdout).With("module", "app") - // logger = log.NewTracingLogger(logger) + logger := log.TestingLogger().With("module", "app") + // logger := log.NewTMLogger(os.Stdout).With("module", "app") + logger = log.NewTracingLogger(logger) + h := NewHandler("gold") bcApp := app.NewBasecoin( - NewHandler("gold"), + h, eyesCli, logger, ) @@ -60,11 +60,15 @@ func TestCounterPlugin(t *testing.T) { res = DeliverCounterTx(false, nil, 2) assert.True(res.IsErr(), res.String()) - // Test an invalid send, with supported fee - res = DeliverCounterTx(true, coin.Coins{{"gold", 100}}, 2) + // Test an invalid sequence + res = DeliverCounterTx(true, nil, 2) + assert.True(res.IsErr(), res.String()) + + // Test an valid send, with supported fee + res = DeliverCounterTx(true, coin.Coins{{"gold", 100}}, 3) assert.True(res.IsOK(), res.String()) // Test unsupported fee - res = DeliverCounterTx(true, coin.Coins{{"silver", 100}}, 3) + res = DeliverCounterTx(true, coin.Coins{{"silver", 100}}, 4) assert.True(res.IsErr(), res.String()) } diff --git a/modules/base/checkpoint.go b/modules/base/checkpoint.go new file mode 100644 index 0000000000..eea375834e --- /dev/null +++ b/modules/base/checkpoint.go @@ -0,0 +1,49 @@ +package base + +import ( + "fmt" + + "github.com/tendermint/basecoin" + "github.com/tendermint/basecoin/stack" + "github.com/tendermint/basecoin/state" +) + +//nolint +const ( + NameCheckpoint = "check" +) + +// Checkpoint isolates all data store below this +type Checkpoint struct { + stack.PassOption +} + +// Name of the module - fulfills Middleware interface +func (Checkpoint) Name() string { + return NameCheckpoint +} + +var _ stack.Middleware = Chain{} + +// CheckTx reverts all data changes if there was an error +func (c Checkpoint) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { + ps := state.NewKVCache(store) + res, err = next.CheckTx(ctx, ps, tx) + if err == nil { + ps.Sync() + } + return res, err +} + +// DeliverTx reverts all data changes if there was an error +func (c Checkpoint) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { + ps := state.NewKVCache(store) + res, err = next.DeliverTx(ctx, ps, tx) + if err == nil { + fmt.Println("sync") + ps.Sync() + } else { + fmt.Println("reject") + } + return res, err +} From f6e7d4b741c384c590c722bf4c3b555d0881bde1 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 25 Jul 2017 22:19:31 -0400 Subject: [PATCH 02/18] Pull in logic from merkleeyes, get it all working with trees --- app/app.go | 77 +++---- app/app_test.go | 30 +-- app/genesis_test.go | 25 ++- cmd/basecoin/commands/start.go | 29 +-- docs/guide/counter/plugins/counter/counter.go | 2 +- .../counter/plugins/counter/counter_test.go | 13 +- {modules/base => stack}/checkpoint.go | 16 +- stack/prefixstore.go | 8 + state/merkle/state.go | 83 ++++++++ state/merkle/store.go | 196 ++++++++++++++++++ state/state.go | 82 ++------ state/state_test.go | 142 ++++++------- 12 files changed, 454 insertions(+), 249 deletions(-) rename {modules/base => stack}/checkpoint.go (79%) create mode 100644 state/merkle/state.go create mode 100644 state/merkle/store.go diff --git a/app/app.go b/app/app.go index c5ff537396..e85322c5ae 100644 --- a/app/app.go +++ b/app/app.go @@ -5,11 +5,10 @@ import ( "strings" abci "github.com/tendermint/abci/types" - "github.com/tendermint/basecoin" - eyes "github.com/tendermint/merkleeyes/client" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" + "github.com/tendermint/basecoin" "github.com/tendermint/basecoin/errors" "github.com/tendermint/basecoin/modules/auth" "github.com/tendermint/basecoin/modules/base" @@ -19,6 +18,7 @@ import ( "github.com/tendermint/basecoin/modules/roles" "github.com/tendermint/basecoin/stack" sm "github.com/tendermint/basecoin/state" + "github.com/tendermint/basecoin/state/merkle" "github.com/tendermint/basecoin/version" ) @@ -30,27 +30,24 @@ const ( // Basecoin - The ABCI application type Basecoin struct { - eyesCli *eyes.Client - state *sm.State - cacheState *sm.State - handler basecoin.Handler - height uint64 - logger log.Logger + info *sm.ChainState + + state *merkle.Store + + handler basecoin.Handler + height uint64 + logger log.Logger } var _ abci.Application = &Basecoin{} // NewBasecoin - create a new instance of the basecoin application -func NewBasecoin(handler basecoin.Handler, eyesCli *eyes.Client, logger log.Logger) *Basecoin { - state := sm.NewState(eyesCli, logger.With("module", "state")) - +func NewBasecoin(handler basecoin.Handler, store *merkle.Store, logger log.Logger) *Basecoin { return &Basecoin{ - handler: handler, - eyesCli: eyesCli, - state: state, - cacheState: nil, - height: 0, - logger: logger, + handler: handler, + info: sm.NewChainState(), + state: store, + logger: logger, } } @@ -71,21 +68,23 @@ func DefaultHandler(feeDenom string) basecoin.Handler { nonce.ReplayCheck{}, roles.NewMiddleware(), fee.NewSimpleFeeMiddleware(coin.Coin{feeDenom, 0}, fee.Bank), - base.Checkpoint{}, + stack.Checkpoint{}, ).Use(d) } -// GetState - XXX For testing, not thread safe! -func (app *Basecoin) GetState() *sm.State { - return app.state.CacheWrap() +// GetChainID returns the currently stored chain +func (app *Basecoin) GetChainID() string { + return app.info.GetChainID(app.state.Committed()) +} + +// GetState is back... please kill me +func (app *Basecoin) GetState() sm.KVStore { + return app.state.Append() } // Info - ABCI func (app *Basecoin) Info() abci.ResponseInfo { - resp, err := app.eyesCli.InfoSync() - if err != nil { - cmn.PanicCrisis(err) - } + resp := app.state.Info() app.height = resp.LastBlockHeight return abci.ResponseInfo{ Data: fmt.Sprintf("Basecoin v%v", version.Version), @@ -98,16 +97,17 @@ func (app *Basecoin) Info() abci.ResponseInfo { func (app *Basecoin) SetOption(key string, value string) string { module, key := splitKey(key) + state := app.state.Append() if module == ModuleNameBase { if key == ChainKey { - app.state.SetChainID(value) + app.info.SetChainID(state, value) return "Success" } return fmt.Sprintf("Error: unknown base option: %s", key) } - log, err := app.handler.SetOption(app.logger, app.state, module, key, value) + log, err := app.handler.SetOption(app.logger, state, module, key, value) if err == nil { return log } @@ -122,12 +122,11 @@ func (app *Basecoin) DeliverTx(txBytes []byte) abci.Result { } ctx := stack.NewContext( - app.state.GetChainID(), + app.GetChainID(), app.height, app.logger.With("call", "delivertx"), ) - fmt.Printf("state: %#v\n", app.state) - res, err := app.handler.DeliverTx(ctx, app.state, tx) + res, err := app.handler.DeliverTx(ctx, app.state.Append(), tx) if err != nil { // discard the cache... @@ -144,12 +143,11 @@ func (app *Basecoin) CheckTx(txBytes []byte) abci.Result { } ctx := stack.NewContext( - app.state.GetChainID(), + app.GetChainID(), app.height, app.logger.With("call", "checktx"), ) - fmt.Printf("state: %#v\n", app.cacheState) - res, err := app.handler.CheckTx(ctx, app.cacheState, tx) + res, err := app.handler.CheckTx(ctx, app.state.Check(), tx) if err != nil { return errors.Result(err) @@ -165,24 +163,13 @@ func (app *Basecoin) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQu return } - resQuery, err := app.eyesCli.QuerySync(reqQuery) - if err != nil { - resQuery.Log = "Failed to query MerkleEyes: " + err.Error() - resQuery.Code = abci.CodeType_InternalError - return - } - return + return app.state.Query(reqQuery) } // Commit - ABCI func (app *Basecoin) Commit() (res abci.Result) { - // Commit state res = app.state.Commit() - - // Wrap the committed state in cache for CheckTx - app.cacheState = app.state.CacheWrap() - if res.IsErr() { cmn.PanicSanity("Error getting hash: " + res.Error()) } diff --git a/app/app_test.go b/app/app_test.go index 2342bf2414..6a0fb5182d 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -2,7 +2,6 @@ package app import ( "encoding/hex" - "os" "testing" "github.com/stretchr/testify/assert" @@ -17,8 +16,8 @@ import ( "github.com/tendermint/basecoin/modules/nonce" "github.com/tendermint/basecoin/stack" "github.com/tendermint/basecoin/state" + "github.com/tendermint/basecoin/state/merkle" wire "github.com/tendermint/go-wire" - eyes "github.com/tendermint/merkleeyes/client" "github.com/tendermint/tmlibs/log" ) @@ -82,14 +81,14 @@ func (at *appTest) reset() { at.acctIn = coin.NewAccountWithKey(coin.Coins{{"mycoin", 7}}) at.acctOut = coin.NewAccountWithKey(coin.Coins{{"mycoin", 7}}) - eyesCli := eyes.NewLocalClient("", 0) - // logger := log.TestingLogger().With("module", "app"), - logger := log.NewTMLogger(os.Stdout).With("module", "app") - logger = log.NewTracingLogger(logger) + // Note: switch logger if you want to get more info + logger := log.TestingLogger() + // logger := log.NewTracingLogger(log.NewTMLogger(os.Stdout)) + store := merkle.NewStore("", 0, logger.With("module", "store")) at.app = NewBasecoin( DefaultHandler("mycoin"), - eyesCli, - logger, + store, + logger.With("module", "app"), ) res := at.app.SetOption("base/chain_id", at.chainID) @@ -142,17 +141,18 @@ func TestSetOption(t *testing.T) { assert := assert.New(t) require := require.New(t) - eyesCli := eyes.NewLocalClient("", 0) + logger := log.TestingLogger() + store := merkle.NewStore("", 0, logger.With("module", "store")) app := NewBasecoin( DefaultHandler("atom"), - eyesCli, - log.TestingLogger().With("module", "app"), + store, + logger.With("module", "app"), ) //testing ChainID chainID := "testChain" res := app.SetOption("base/chain_id", chainID) - assert.EqualValues(app.GetState().GetChainID(), chainID) + assert.EqualValues(app.GetChainID(), chainID) assert.EqualValues(res, "Success") // make a nice account... @@ -162,7 +162,7 @@ func TestSetOption(t *testing.T) { require.EqualValues(res, "Success") // make sure it is set correctly, with some balance - coins, err := getBalance(acct.Actor(), app.state) + coins, err := getBalance(acct.Actor(), app.GetState()) require.Nil(err) assert.Equal(bal, coins) @@ -189,7 +189,7 @@ func TestSetOption(t *testing.T) { res = app.SetOption("coin/account", unsortAcc) require.EqualValues(res, "Success") - coins, err = getAddr(unsortAddr, app.state) + coins, err = getAddr(unsortAddr, app.GetState()) require.Nil(err) assert.True(coins.IsValid()) assert.Equal(unsortCoins, coins) @@ -213,6 +213,8 @@ func TestTx(t *testing.T) { //Bad Balance at.acctIn.Coins = coin.Coins{{"mycoin", 2}} at.initAccount(at.acctIn) + at.app.Commit() + res, _, _ := at.exec(t, at.getTx(coin.Coins{{"mycoin", 5}}, 1), true) assert.True(res.IsErr(), "ExecTx/Bad CheckTx: Expected error return from ExecTx, returned: %v", res) res, diffIn, diffOut := at.exec(t, at.getTx(coin.Coins{{"mycoin", 5}}, 1), false) diff --git a/app/genesis_test.go b/app/genesis_test.go index 57c895c8d7..50ca77f0f2 100644 --- a/app/genesis_test.go +++ b/app/genesis_test.go @@ -8,19 +8,20 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - eyescli "github.com/tendermint/merkleeyes/client" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" "github.com/tendermint/basecoin/modules/coin" + "github.com/tendermint/basecoin/state/merkle" ) const genesisFilepath = "./testdata/genesis.json" const genesisAcctFilepath = "./testdata/genesis2.json" func TestLoadGenesisDoNotFailIfAppOptionsAreMissing(t *testing.T) { - eyesCli := eyescli.NewLocalClient("", 0) - app := NewBasecoin(DefaultHandler("mycoin"), eyesCli, log.TestingLogger()) + logger := log.TestingLogger() + store := merkle.NewStore("", 0, logger) + app := NewBasecoin(DefaultHandler("mycoin"), store, logger) err := app.LoadGenesis("./testdata/genesis3.json") require.Nil(t, err, "%+v", err) } @@ -28,18 +29,19 @@ func TestLoadGenesisDoNotFailIfAppOptionsAreMissing(t *testing.T) { func TestLoadGenesis(t *testing.T) { assert, require := assert.New(t), require.New(t) - eyesCli := eyescli.NewLocalClient("", 0) - app := NewBasecoin(DefaultHandler("mycoin"), eyesCli, log.TestingLogger()) + logger := log.TestingLogger() + store := merkle.NewStore("", 0, logger) + app := NewBasecoin(DefaultHandler("mycoin"), store, logger) err := app.LoadGenesis(genesisFilepath) require.Nil(err, "%+v", err) // check the chain id - assert.Equal("foo_bar_chain", app.GetState().GetChainID()) + assert.Equal("foo_bar_chain", app.GetChainID()) // and check the account info - previously calculated values addr, _ := hex.DecodeString("eb98e0688217cfdeb70eddf4b33cdcc37fc53197") - coins, err := getAddr(addr, app.state) + coins, err := getAddr(addr, app.GetState()) require.Nil(err) assert.True(coins.IsPositive()) @@ -57,13 +59,14 @@ func TestLoadGenesis(t *testing.T) { func TestLoadGenesisAccountAddress(t *testing.T) { assert, require := assert.New(t), require.New(t) - eyesCli := eyescli.NewLocalClient("", 0) - app := NewBasecoin(DefaultHandler("mycoin"), eyesCli, log.TestingLogger()) + logger := log.TestingLogger() + store := merkle.NewStore("", 0, logger) + app := NewBasecoin(DefaultHandler("mycoin"), store, logger) err := app.LoadGenesis(genesisAcctFilepath) require.Nil(err, "%+v", err) // check the chain id - assert.Equal("addr_accounts_chain", app.GetState().GetChainID()) + assert.Equal("addr_accounts_chain", app.GetChainID()) // make sure the accounts were set properly cases := []struct { @@ -86,7 +89,7 @@ func TestLoadGenesisAccountAddress(t *testing.T) { for i, tc := range cases { addr, err := hex.DecodeString(tc.addr) require.Nil(err, tc.addr) - coins, err := getAddr(addr, app.state) + coins, err := getAddr(addr, app.GetState()) require.Nil(err, "%+v", err) if !tc.exists { assert.True(coins.IsZero(), "%d", i) diff --git a/cmd/basecoin/commands/start.go b/cmd/basecoin/commands/start.go index 4fd2f19b2c..53c6d2e9b3 100644 --- a/cmd/basecoin/commands/start.go +++ b/cmd/basecoin/commands/start.go @@ -11,8 +11,6 @@ import ( "github.com/tendermint/abci/server" "github.com/tendermint/basecoin" - eyesApp "github.com/tendermint/merkleeyes/app" - eyes "github.com/tendermint/merkleeyes/client" "github.com/tendermint/tmlibs/cli" cmn "github.com/tendermint/tmlibs/common" @@ -22,6 +20,7 @@ import ( "github.com/tendermint/tendermint/types" "github.com/tendermint/basecoin/app" + "github.com/tendermint/basecoin/state/merkle" ) // StartCmd - command to start running the basecoin node! @@ -37,7 +36,6 @@ const EyesCacheSize = 10000 //nolint const ( FlagAddress = "address" - FlagEyes = "eyes" FlagWithoutTendermint = "without-tendermint" ) @@ -50,7 +48,6 @@ var ( func init() { flags := StartCmd.Flags() flags.String(FlagAddress, "tcp://0.0.0.0:46658", "Listen address") - flags.String(FlagEyes, "local", "MerkleEyes address, or 'local' for embedded") flags.Bool(FlagWithoutTendermint, false, "Only run basecoin abci app, assume external tendermint process") // add all standard 'tendermint node' flags tcmd.AddNodeFlags(StartCmd) @@ -58,27 +55,19 @@ func init() { func startCmd(cmd *cobra.Command, args []string) error { rootDir := viper.GetString(cli.HomeFlag) - meyes := viper.GetString(FlagEyes) - // Connect to MerkleEyes - var eyesCli *eyes.Client - if meyes == "local" { - eyesApp.SetLogger(logger.With("module", "merkleeyes")) - eyesCli = eyes.NewLocalClient(path.Join(rootDir, "data", "merkleeyes.db"), EyesCacheSize) - } else { - var err error - eyesCli, err = eyes.NewClient(meyes) - if err != nil { - return errors.Errorf("Error connecting to MerkleEyes: %v\n", err) - } - } + store := merkle.NewStore( + path.Join(rootDir, "data", "merkleeyes.db"), + EyesCacheSize, + logger.With("module", "store"), + ) // Create Basecoin app - basecoinApp := app.NewBasecoin(Handler, eyesCli, logger.With("module", "app")) + basecoinApp := app.NewBasecoin(Handler, store, logger.With("module", "app")) // if chain_id has not been set yet, load the genesis. // else, assume it's been loaded - if basecoinApp.GetState().GetChainID() == "" { + if basecoinApp.GetChainID() == "" { // If genesis file exists, set key-value options genesisFile := path.Join(rootDir, "genesis.json") if _, err := os.Stat(genesisFile); err == nil { @@ -91,7 +80,7 @@ func startCmd(cmd *cobra.Command, args []string) error { } } - chainID := basecoinApp.GetState().GetChainID() + chainID := basecoinApp.GetChainID() if viper.GetBool(FlagWithoutTendermint) { logger.Info("Starting Basecoin without Tendermint", "chain_id", chainID) // run just the abci app/server diff --git a/docs/guide/counter/plugins/counter/counter.go b/docs/guide/counter/plugins/counter/counter.go index 27863c5273..fa806d519a 100644 --- a/docs/guide/counter/plugins/counter/counter.go +++ b/docs/guide/counter/plugins/counter/counter.go @@ -104,7 +104,7 @@ func NewHandler(feeDenom string) basecoin.Handler { base.Chain{}, nonce.ReplayCheck{}, fee.NewSimpleFeeMiddleware(coin.Coin{feeDenom, 0}, fee.Bank), - base.Checkpoint{}, + stack.Checkpoint{}, ).Use(dispatcher) } diff --git a/docs/guide/counter/plugins/counter/counter_test.go b/docs/guide/counter/plugins/counter/counter_test.go index 5fa83d7353..9e8149edd7 100644 --- a/docs/guide/counter/plugins/counter/counter_test.go +++ b/docs/guide/counter/plugins/counter/counter_test.go @@ -12,8 +12,8 @@ import ( "github.com/tendermint/basecoin/modules/base" "github.com/tendermint/basecoin/modules/coin" "github.com/tendermint/basecoin/modules/nonce" + "github.com/tendermint/basecoin/state/merkle" "github.com/tendermint/go-wire" - eyescli "github.com/tendermint/merkleeyes/client" "github.com/tendermint/tmlibs/log" ) @@ -21,17 +21,16 @@ func TestCounterPlugin(t *testing.T) { assert := assert.New(t) // Basecoin initialization - eyesCli := eyescli.NewLocalClient("", 0) chainID := "test_chain_id" + logger := log.TestingLogger() + // logger := log.NewTracingLogger(log.NewTMLogger(os.Stdout)) - logger := log.TestingLogger().With("module", "app") - // logger := log.NewTMLogger(os.Stdout).With("module", "app") - logger = log.NewTracingLogger(logger) + store := merkle.NewStore("", 0, logger.With("module", "store")) h := NewHandler("gold") bcApp := app.NewBasecoin( h, - eyesCli, - logger, + store, + logger.With("module", "app"), ) bcApp.SetOption("base/chain_id", chainID) diff --git a/modules/base/checkpoint.go b/stack/checkpoint.go similarity index 79% rename from modules/base/checkpoint.go rename to stack/checkpoint.go index eea375834e..a983ae88b6 100644 --- a/modules/base/checkpoint.go +++ b/stack/checkpoint.go @@ -1,10 +1,7 @@ -package base +package stack import ( - "fmt" - "github.com/tendermint/basecoin" - "github.com/tendermint/basecoin/stack" "github.com/tendermint/basecoin/state" ) @@ -15,7 +12,7 @@ const ( // Checkpoint isolates all data store below this type Checkpoint struct { - stack.PassOption + PassOption } // Name of the module - fulfills Middleware interface @@ -23,11 +20,11 @@ func (Checkpoint) Name() string { return NameCheckpoint } -var _ stack.Middleware = Chain{} +var _ Middleware = Checkpoint{} // CheckTx reverts all data changes if there was an error func (c Checkpoint) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { - ps := state.NewKVCache(store) + ps := state.NewKVCache(unwrap(store)) res, err = next.CheckTx(ctx, ps, tx) if err == nil { ps.Sync() @@ -37,13 +34,10 @@ func (c Checkpoint) CheckTx(ctx basecoin.Context, store state.KVStore, tx baseco // DeliverTx reverts all data changes if there was an error func (c Checkpoint) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { - ps := state.NewKVCache(store) + ps := state.NewKVCache(unwrap(store)) res, err = next.DeliverTx(ctx, ps, tx) if err == nil { - fmt.Println("sync") ps.Sync() - } else { - fmt.Println("reject") } return res, err } diff --git a/stack/prefixstore.go b/stack/prefixstore.go index f6934d6e73..f366b2fa26 100644 --- a/stack/prefixstore.go +++ b/stack/prefixstore.go @@ -31,6 +31,14 @@ func stateSpace(store state.KVStore, app string) state.KVStore { return PrefixedStore(app, store) } +func unwrap(store state.KVStore) state.KVStore { + // unwrap one-level if wrapped + if pstore, ok := store.(prefixStore); ok { + store = pstore.store + } + return store +} + // PrefixedStore allows one to create an isolated state-space for a given // app prefix, but it cannot easily be unwrapped // diff --git a/state/merkle/state.go b/state/merkle/state.go new file mode 100644 index 0000000000..423a9c6ad7 --- /dev/null +++ b/state/merkle/state.go @@ -0,0 +1,83 @@ +package merkle + +import ( + "github.com/tendermint/merkleeyes/iavl" + "github.com/tendermint/tmlibs/merkle" +) + +// State represents the app states, separating the commited state (for queries) +// from the working state (for CheckTx and AppendTx) +type State struct { + committed merkle.Tree + deliverTx merkle.Tree + checkTx merkle.Tree + persistent bool +} + +func NewState(tree merkle.Tree, persistent bool) State { + return State{ + committed: tree, + deliverTx: tree.Copy(), + checkTx: tree.Copy(), + persistent: persistent, + } +} + +func (s State) Committed() Bonsai { + return Bonsai{s.committed} +} + +func (s State) Append() Bonsai { + return Bonsai{s.deliverTx} +} + +func (s State) Check() Bonsai { + return Bonsai{s.checkTx} +} + +// Hash updates the tree +func (s *State) Hash() []byte { + return s.deliverTx.Hash() +} + +// BatchSet is used for some weird magic in storing the new height +func (s *State) BatchSet(key, value []byte) { + if s.persistent { + // This is in the batch with the Save, but not in the tree + tree, ok := s.deliverTx.(*iavl.IAVLTree) + if ok { + tree.BatchSet(key, value) + } + } +} + +// Commit save persistent nodes to the database and re-copies the trees +func (s *State) Commit() []byte { + var hash []byte + if s.persistent { + hash = s.deliverTx.Save() + } else { + hash = s.deliverTx.Hash() + } + + s.committed = s.deliverTx + s.deliverTx = s.committed.Copy() + s.checkTx = s.committed.Copy() + return hash +} + +// Bonsai is a deformed tree forced to fit in a small pot +type Bonsai struct { + merkle.Tree +} + +// Get matches the signature of KVStore +func (b Bonsai) Get(key []byte) []byte { + _, value, _ := b.Tree.Get(key) + return value +} + +// Set matches the signature of KVStore +func (b Bonsai) Set(key, value []byte) { + b.Tree.Set(key, value) +} diff --git a/state/merkle/store.go b/state/merkle/store.go new file mode 100644 index 0000000000..1f513e9897 --- /dev/null +++ b/state/merkle/store.go @@ -0,0 +1,196 @@ +package merkle + +import ( + "bytes" + "fmt" + "path" + "path/filepath" + "strings" + + abci "github.com/tendermint/abci/types" + "github.com/tendermint/go-wire" + cmn "github.com/tendermint/tmlibs/common" + dbm "github.com/tendermint/tmlibs/db" + "github.com/tendermint/tmlibs/log" + + "github.com/tendermint/merkleeyes/iavl" +) + +// Store contains the merkle tree, and all info to handle abci requests +type Store struct { + State + height uint64 + hash []byte + persisted bool + + logger log.Logger +} + +var stateKey = []byte("merkle:state") // Database key for merkle tree save value db values + +// MerkleState contains the latest Merkle root hash and the number of times `Commit` has been called +type MerkleState struct { + Hash []byte + Height uint64 +} + +// NewStore initializes an in-memory IAVLTree, or attempts to load a persistant +// tree from disk +func NewStore(dbName string, cacheSize int, logger log.Logger) *Store { + // start at 1 so the height returned by query is for the + // next block, ie. the one that includes the AppHash for our current state + initialHeight := uint64(1) + + // Non-persistent case + if dbName == "" { + tree := iavl.NewIAVLTree( + 0, + nil, + ) + return &Store{ + State: NewState(tree, false), + height: initialHeight, + logger: logger, + } + } + + // Expand the path fully + dbPath, err := filepath.Abs(dbName) + if err != nil { + panic(fmt.Sprintf("Invalid Database Name: %s", dbName)) + } + + // Some external calls accidently add a ".db", which is now removed + dbPath = strings.TrimSuffix(dbPath, path.Ext(dbPath)) + + // Split the database name into it's components (dir, name) + dir := path.Dir(dbPath) + name := path.Base(dbPath) + + // Make sure the path exists + empty, _ := cmn.IsDirEmpty(dbPath + ".db") + + // Open database called "dir/name.db", if it doesn't exist it will be created + db := dbm.NewDB(name, dbm.LevelDBBackendStr, dir) + tree := iavl.NewIAVLTree(cacheSize, db) + + var eyesState MerkleState + if empty { + logger.Info("no existing db, creating new db") + eyesState = MerkleState{ + Hash: tree.Save(), + Height: initialHeight, + } + db.Set(stateKey, wire.BinaryBytes(eyesState)) + } else { + logger.Info("loading existing db") + eyesStateBytes := db.Get(stateKey) + err = wire.ReadBinaryBytes(eyesStateBytes, &eyesState) + if err != nil { + logger.Error("error reading MerkleEyesState", "err", err) + panic(err) + } + tree.Load(eyesState.Hash) + } + + return &Store{ + State: NewState(tree, true), + height: eyesState.Height, + hash: eyesState.Hash, + persisted: true, + logger: logger, + } +} + +// CloseDB closes the database +// func (s *Store) CloseDB() { +// if s.db != nil { +// s.db.Close() +// } +// } + +// Info implements abci.Application. It returns the height, hash and size (in the data). +// The height is the block that holds the transactions, not the apphash itself. +func (s *Store) Info() abci.ResponseInfo { + s.logger.Info("Info synced", "height", s.height, "hash", fmt.Sprintf("%X", s.hash)) + return abci.ResponseInfo{ + Data: cmn.Fmt("size:%v", s.State.Committed().Size()), + LastBlockHeight: s.height - 1, + LastBlockAppHash: s.hash, + } +} + +// Commit implements abci.Application +func (s *Store) Commit() abci.Result { + s.hash = s.State.Hash() + s.height++ + s.logger.Debug("Commit synced", "height", s.height, "hash", fmt.Sprintf("%X", s.hash)) + + s.State.BatchSet(stateKey, wire.BinaryBytes(MerkleState{ + Hash: s.hash, + Height: s.height, + })) + + hash := s.State.Commit() + if !bytes.Equal(hash, s.hash) { + panic("AppHash is incorrect") + } + + if s.State.Committed().Size() == 0 { + return abci.NewResultOK(nil, "Empty hash for empty tree") + } + return abci.NewResultOK(s.hash, "") +} + +// Query implements abci.Application +func (s *Store) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) { + if len(reqQuery.Data) == 0 { + return + } + tree := s.State.Committed() + + if reqQuery.Height != 0 { + // TODO: support older commits + resQuery.Code = abci.CodeType_InternalError + resQuery.Log = "merkleeyes only supports queries on latest commit" + return + } + + // set the query response height to current + resQuery.Height = s.height + + switch reqQuery.Path { + case "/store", "/key": // Get by key + key := reqQuery.Data // Data holds the key bytes + resQuery.Key = key + if reqQuery.Prove { + value, proof, exists := tree.Proof(key) + if !exists { + resQuery.Log = "Key not found" + } + resQuery.Value = value + resQuery.Proof = proof + // TODO: return index too? + } else { + value := tree.Get(key) + resQuery.Value = value + } + + case "/index": // Get by Index + index := wire.GetInt64(reqQuery.Data) + key, value := tree.GetByIndex(int(index)) + resQuery.Key = key + resQuery.Index = int64(index) + resQuery.Value = value + + case "/size": // Get size + size := tree.Size() + sizeBytes := wire.BinaryBytes(size) + resQuery.Value = sizeBytes + + default: + resQuery.Code = abci.CodeType_UnknownRequest + resQuery.Log = cmn.Fmt("Unexpected Query path: %v", reqQuery.Path) + } + return +} diff --git a/state/state.go b/state/state.go index 398e18ca5e..2751833d44 100644 --- a/state/state.go +++ b/state/state.go @@ -1,84 +1,26 @@ package state -import ( - abci "github.com/tendermint/abci/types" - eyes "github.com/tendermint/merkleeyes/client" - "github.com/tendermint/tmlibs/log" -) - -// CONTRACT: State should be quick to copy. -// See CacheWrap(). -type State struct { - chainID string - store KVStore - readCache map[string][]byte // optional, for caching writes to store - writeCache *KVCache // optional, for caching writes w/o writing to store - logger log.Logger +// ChainState maintains general information for the chain +type ChainState struct { + chainID string } -func NewState(store KVStore, l log.Logger) *State { - return &State{ - chainID: "", - store: store, - readCache: make(map[string][]byte), - writeCache: nil, - logger: l, - } +// NewChainState creates a blank state +func NewChainState() *ChainState { + return &ChainState{} } -func (s *State) SetChainID(chainID string) { +// SetChainID stores the chain id in the store +func (s *ChainState) SetChainID(store KVStore, chainID string) { s.chainID = chainID - s.store.Set([]byte("base/chain_id"), []byte(chainID)) + store.Set([]byte("base/chain_id"), []byte(chainID)) } -func (s *State) GetChainID() string { +// GetChainID gets the chain id from the cache or the store +func (s *ChainState) GetChainID(store KVStore) string { if s.chainID != "" { return s.chainID } - s.chainID = string(s.store.Get([]byte("base/chain_id"))) + s.chainID = string(store.Get([]byte("base/chain_id"))) return s.chainID } - -func (s *State) Get(key []byte) (value []byte) { - if s.readCache != nil { //if not a cachewrap - value, ok := s.readCache[string(key)] - if ok { - return value - } - } - return s.store.Get(key) -} - -func (s *State) Set(key []byte, value []byte) { - if s.readCache != nil { //if not a cachewrap - s.readCache[string(key)] = value - } - s.store.Set(key, value) -} - -func (s *State) CacheWrap() *State { - cache := NewKVCache(s) - return &State{ - chainID: s.chainID, - store: cache, - readCache: nil, - writeCache: cache, - logger: s.logger, - } -} - -// NOTE: errors if s is not from CacheWrap() -func (s *State) CacheSync() { - s.writeCache.Sync() -} - -func (s *State) Commit() abci.Result { - switch s.store.(type) { - case *eyes.Client: - s.readCache = make(map[string][]byte) - return s.store.(*eyes.Client).CommitSync() - default: - return abci.NewError(abci.CodeType_InternalError, "can only use Commit if store is merkleeyes") - } - -} diff --git a/state/state_test.go b/state/state_test.go index 8cc587b3b9..096d27d197 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -1,86 +1,88 @@ package state -import ( - "bytes" - "testing" +// TODO: something similar in the merkle package... - eyes "github.com/tendermint/merkleeyes/client" - "github.com/tendermint/tmlibs/log" +// import ( +// "bytes" +// "testing" - "github.com/stretchr/testify/assert" -) +// eyes "github.com/tendermint/merkleeyes/client" +// "github.com/tendermint/tmlibs/log" -func TestState(t *testing.T) { - assert := assert.New(t) +// "github.com/stretchr/testify/assert" +// ) - //States and Stores for tests - store := NewMemKVStore() - state := NewState(store, log.TestingLogger()) - cache := state.CacheWrap() - eyesCli := eyes.NewLocalClient("", 0) +// func TestState(t *testing.T) { +// assert := assert.New(t) - //reset the store/state/cache - reset := func() { - store = NewMemKVStore() - state = NewState(store, log.TestingLogger()) - cache = state.CacheWrap() - } +// //States and Stores for tests +// store := NewMemKVStore() +// state := NewState(store, log.TestingLogger()) +// cache := state.CacheWrap() +// eyesCli := eyes.NewLocalClient("", 0) - //set the state to using the eyesCli instead of MemKVStore - useEyesCli := func() { - state = NewState(eyesCli, log.TestingLogger()) - cache = state.CacheWrap() - } +// //reset the store/state/cache +// reset := func() { +// store = NewMemKVStore() +// state = NewState(store, log.TestingLogger()) +// cache = state.CacheWrap() +// } - //key value pairs to be tested within the system - keyvalue := []struct { - key string - value string - }{ - {"foo", "snake"}, - {"bar", "mouse"}, - } +// //set the state to using the eyesCli instead of MemKVStore +// useEyesCli := func() { +// state = NewState(eyesCli, log.TestingLogger()) +// cache = state.CacheWrap() +// } - //set the kvc to have all the key value pairs - setRecords := func(kv KVStore) { - for _, n := range keyvalue { - kv.Set([]byte(n.key), []byte(n.value)) - } - } +// //key value pairs to be tested within the system +// keyvalue := []struct { +// key string +// value string +// }{ +// {"foo", "snake"}, +// {"bar", "mouse"}, +// } - //store has all the key value pairs - storeHasAll := func(kv KVStore) bool { - for _, n := range keyvalue { - if !bytes.Equal(kv.Get([]byte(n.key)), []byte(n.value)) { - return false - } - } - return true - } +// //set the kvc to have all the key value pairs +// setRecords := func(kv KVStore) { +// for _, n := range keyvalue { +// kv.Set([]byte(n.key), []byte(n.value)) +// } +// } - //test chainID - state.SetChainID("testchain") - assert.Equal(state.GetChainID(), "testchain", "ChainID is improperly stored") +// //store has all the key value pairs +// storeHasAll := func(kv KVStore) bool { +// for _, n := range keyvalue { +// if !bytes.Equal(kv.Get([]byte(n.key)), []byte(n.value)) { +// return false +// } +// } +// return true +// } - //test basic retrieve - setRecords(state) - assert.True(storeHasAll(state), "state doesn't retrieve after Set") +// //test chainID +// state.SetChainID("testchain") +// assert.Equal(state.GetChainID(), "testchain", "ChainID is improperly stored") - //Test CacheWrap with local mem store - reset() - setRecords(cache) - assert.False(storeHasAll(store), "store retrieving before CacheSync") - cache.CacheSync() - assert.True(storeHasAll(store), "store doesn't retrieve after CacheSync") +// //test basic retrieve +// setRecords(state) +// assert.True(storeHasAll(state), "state doesn't retrieve after Set") - //Test Commit on state with non-merkle store - assert.True(state.Commit().IsErr(), "Commit shouldn't work with non-merkle store") +// //Test CacheWrap with local mem store +// reset() +// setRecords(cache) +// assert.False(storeHasAll(store), "store retrieving before CacheSync") +// cache.CacheSync() +// assert.True(storeHasAll(store), "store doesn't retrieve after CacheSync") - //Test CacheWrap with merkleeyes client store - useEyesCli() - setRecords(cache) - assert.False(storeHasAll(eyesCli), "eyesCli retrieving before Commit") - cache.CacheSync() - assert.True(state.Commit().IsOK(), "Bad Commit") - assert.True(storeHasAll(eyesCli), "eyesCli doesn't retrieve after Commit") -} +// //Test Commit on state with non-merkle store +// assert.True(state.Commit().IsErr(), "Commit shouldn't work with non-merkle store") + +// //Test CacheWrap with merkleeyes client store +// useEyesCli() +// setRecords(cache) +// assert.False(storeHasAll(eyesCli), "eyesCli retrieving before Commit") +// cache.CacheSync() +// assert.True(state.Commit().IsOK(), "Bad Commit") +// assert.True(storeHasAll(eyesCli), "eyesCli doesn't retrieve after Commit") +// } From 243d767aaa0d366f0e1e93f307766df8126190b8 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 25 Jul 2017 22:31:09 -0400 Subject: [PATCH 03/18] Fixed benchmark and compared unstable to this branch --- benchmarks/app_test.go | 13 ++++++------- benchmarks/cleanup-speed.txt | 19 +++++++++++++++++++ benchmarks/unstable-speed.txt | 19 +++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 benchmarks/cleanup-speed.txt create mode 100644 benchmarks/unstable-speed.txt diff --git a/benchmarks/app_test.go b/benchmarks/app_test.go index 03e188c3f4..e1e2c98f7c 100644 --- a/benchmarks/app_test.go +++ b/benchmarks/app_test.go @@ -6,8 +6,6 @@ import ( "testing" wire "github.com/tendermint/go-wire" - eyesApp "github.com/tendermint/merkleeyes/app" - eyes "github.com/tendermint/merkleeyes/client" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -20,6 +18,7 @@ import ( "github.com/tendermint/basecoin/modules/nonce" "github.com/tendermint/basecoin/modules/roles" "github.com/tendermint/basecoin/stack" + "github.com/tendermint/basecoin/state/merkle" ) type BenchApp struct { @@ -56,18 +55,18 @@ func NewBenchApp(h basecoin.Handler, chainID string, n int, // logger = log.NewTracingLogger(logger) // TODO: disk writing - var eyesCli *eyes.Client + var store *merkle.Store + if persist { tmpDir, _ := ioutil.TempDir("", "bc-app-benchmark") - eyesCli = eyes.NewLocalClient(tmpDir, 500) + store = merkle.NewStore(tmpDir, 500, logger) } else { - eyesCli = eyes.NewLocalClient("", 0) + store = merkle.NewStore("", 0, logger) } - eyesApp.SetLogger(logger.With("module", "merkle")) app := app.NewBasecoin( h, - eyesCli, + store, logger.With("module", "app"), ) res := app.SetOption("base/chain_id", chainID) diff --git a/benchmarks/cleanup-speed.txt b/benchmarks/cleanup-speed.txt new file mode 100644 index 0000000000..7bd397bd90 --- /dev/null +++ b/benchmarks/cleanup-speed.txt @@ -0,0 +1,19 @@ +BenchmarkMakeTx-4 2000 648379 ns/op +BenchmarkSimpleTransfer/100-10-nofee-memdb-4 5000 356487 ns/op +BenchmarkSimpleTransfer/100-10-fee-memdb-4 5000 413435 ns/op +BenchmarkSimpleTransfer/100-200-nofee-memdb-4 5000 321859 ns/op +BenchmarkSimpleTransfer/100-200-fee-memdb-4 5000 393578 ns/op +BenchmarkSimpleTransfer/10000-10-nofee-memdb-4 5000 379129 ns/op +BenchmarkSimpleTransfer/10000-10-fee-memdb-4 3000 480334 ns/op +BenchmarkSimpleTransfer/10000-200-nofee-memdb-4 5000 384398 ns/op +BenchmarkSimpleTransfer/10000-200-fee-memdb-4 3000 443481 ns/op +BenchmarkSimpleTransfer/100-10-nofee-persist-4 3000 498460 ns/op +BenchmarkSimpleTransfer/100-10-fee-persist-4 3000 559034 ns/op +BenchmarkSimpleTransfer/100-200-nofee-persist-4 5000 314090 ns/op +BenchmarkSimpleTransfer/100-200-fee-persist-4 5000 397457 ns/op +BenchmarkSimpleTransfer/10000-10-nofee-persist-4 2000 845872 ns/op +BenchmarkSimpleTransfer/10000-10-fee-persist-4 2000 929205 ns/op +BenchmarkSimpleTransfer/10000-200-nofee-persist-4 5000 596601 ns/op +BenchmarkSimpleTransfer/10000-200-fee-persist-4 5000 667093 ns/op +PASS +ok github.com/tendermint/basecoin/benchmarks 97.097s diff --git a/benchmarks/unstable-speed.txt b/benchmarks/unstable-speed.txt new file mode 100644 index 0000000000..ce4ad2806f --- /dev/null +++ b/benchmarks/unstable-speed.txt @@ -0,0 +1,19 @@ +BenchmarkMakeTx-4 2000 660064 ns/op +BenchmarkSimpleTransfer/100-10-nofee-memdb-4 5000 338378 ns/op +BenchmarkSimpleTransfer/100-10-fee-memdb-4 5000 380171 ns/op +BenchmarkSimpleTransfer/100-200-nofee-memdb-4 5000 306365 ns/op +BenchmarkSimpleTransfer/100-200-fee-memdb-4 5000 359344 ns/op +BenchmarkSimpleTransfer/10000-10-nofee-memdb-4 5000 366057 ns/op +BenchmarkSimpleTransfer/10000-10-fee-memdb-4 3000 433549 ns/op +BenchmarkSimpleTransfer/10000-200-nofee-memdb-4 5000 351662 ns/op +BenchmarkSimpleTransfer/10000-200-fee-memdb-4 3000 421573 ns/op +BenchmarkSimpleTransfer/100-10-nofee-persist-4 3000 479848 ns/op +BenchmarkSimpleTransfer/100-10-fee-persist-4 3000 544164 ns/op +BenchmarkSimpleTransfer/100-200-nofee-persist-4 5000 327999 ns/op +BenchmarkSimpleTransfer/100-200-fee-persist-4 5000 385751 ns/op +BenchmarkSimpleTransfer/10000-10-nofee-persist-4 2000 852128 ns/op +BenchmarkSimpleTransfer/10000-10-fee-persist-4 2000 1055130 ns/op +BenchmarkSimpleTransfer/10000-200-nofee-persist-4 5000 642872 ns/op +BenchmarkSimpleTransfer/10000-200-fee-persist-4 3000 686337 ns/op +PASS +ok github.com/tendermint/basecoin/benchmarks 91.717s From caff0ad01b91a7003bb300d64a3236201223bdae Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 00:06:12 -0400 Subject: [PATCH 04/18] Add SimpleDB interface for merkle tree --- state/kvstore.go | 32 +++++++++++++++ state/merkle/state.go | 94 +++++++++++++++++++++++++++++++++++++++---- state/merkle/store.go | 47 +++++++++------------- 3 files changed, 136 insertions(+), 37 deletions(-) diff --git a/state/kvstore.go b/state/kvstore.go index b07d57109f..7d0534948a 100644 --- a/state/kvstore.go +++ b/state/kvstore.go @@ -4,6 +4,7 @@ import ( "container/list" "fmt" + "github.com/tendermint/go-wire/data" . "github.com/tendermint/tmlibs/common" ) @@ -14,6 +15,37 @@ type KVStore interface { //---------------------------------------- +type Model struct { + Key data.Bytes + Value data.Bytes +} + +// What I wished to have... +type SimpleDB interface { + KVStore + + Has(key []byte) (has bool) + Remove(key []byte) (value []byte) // returns old value if there was one + + List(start, end []byte, limit int) []Model + First(start, end []byte) Model + Last(start, end []byte) Model + + // Checkpoint returns the same state, but where writes + // are buffered and don't affect the parent + Checkpoint() SimpleDB + + // Commit will take all changes from the checkpoint and write + // them to the parent. + // Returns an error if this is not a child of this one + Commit(SimpleDB) error + + // Discard will remove reference to this + Discard() +} + +//---------------------------------------- + type MemKVStore struct { m map[string][]byte } diff --git a/state/merkle/state.go b/state/merkle/state.go index 423a9c6ad7..17c4ee25e0 100644 --- a/state/merkle/state.go +++ b/state/merkle/state.go @@ -1,6 +1,10 @@ package merkle import ( + "errors" + "math/rand" + + "github.com/tendermint/basecoin/state" "github.com/tendermint/merkleeyes/iavl" "github.com/tendermint/tmlibs/merkle" ) @@ -23,16 +27,16 @@ func NewState(tree merkle.Tree, persistent bool) State { } } -func (s State) Committed() Bonsai { - return Bonsai{s.committed} +func (s State) Committed() *Bonsai { + return NewBonsai(s.committed) } -func (s State) Append() Bonsai { - return Bonsai{s.deliverTx} +func (s State) Append() *Bonsai { + return NewBonsai(s.deliverTx) } -func (s State) Check() Bonsai { - return Bonsai{s.checkTx} +func (s State) Check() *Bonsai { + return NewBonsai(s.checkTx) } // Hash updates the tree @@ -66,18 +70,92 @@ func (s *State) Commit() []byte { return hash } +// store nonce as it's own type so no one can even try to fake it +type nonce int64 + // Bonsai is a deformed tree forced to fit in a small pot type Bonsai struct { + id nonce merkle.Tree } +var _ state.SimpleDB = &Bonsai{} + +func NewBonsai(tree merkle.Tree) *Bonsai { + return &Bonsai{ + id: nonce(rand.Int63()), + Tree: tree, + } +} + // Get matches the signature of KVStore -func (b Bonsai) Get(key []byte) []byte { +func (b *Bonsai) Get(key []byte) []byte { _, value, _ := b.Tree.Get(key) return value } // Set matches the signature of KVStore -func (b Bonsai) Set(key, value []byte) { +func (b *Bonsai) Set(key, value []byte) { b.Tree.Set(key, value) } + +func (b *Bonsai) Remove(key []byte) (value []byte) { + value, _ = b.Tree.Remove(key) + return +} + +func (b *Bonsai) List(start, end []byte, limit int) []state.Model { + var res []state.Model + stopAtCount := func(key []byte, value []byte) (stop bool) { + m := state.Model{key, value} + res = append(res, m) + return len(res) >= limit + } + b.Tree.IterateRange(start, end, true, stopAtCount) + return res +} + +func (b *Bonsai) First(start, end []byte) state.Model { + var m state.Model + stopAtFirst := func(key []byte, value []byte) (stop bool) { + m = state.Model{key, value} + return true + } + b.Tree.IterateRange(start, end, true, stopAtFirst) + return m +} + +func (b *Bonsai) Last(start, end []byte) state.Model { + var m state.Model + stopAtFirst := func(key []byte, value []byte) (stop bool) { + m = state.Model{key, value} + return true + } + b.Tree.IterateRange(start, end, false, stopAtFirst) + return m +} + +func (b *Bonsai) Checkpoint() state.SimpleDB { + return &Bonsai{ + id: b.id, + Tree: b.Tree.Copy(), + } +} + +// Commit will take all changes from the checkpoint and write +// them to the parent. +// Returns an error if this is not a child of this one +func (b *Bonsai) Commit(sub state.SimpleDB) error { + bb, ok := sub.(*Bonsai) + if !ok || (b.id != bb.id) { + return errors.New("Not a sub-transaction") + } + b.Tree = bb.Tree + return nil +} + +// Discard will remove reference to this +func (b *Bonsai) Discard() { + b.id = 0 + b.Tree = nil +} diff --git a/state/merkle/store.go b/state/merkle/store.go index 1f513e9897..7064af7220 100644 --- a/state/merkle/store.go +++ b/state/merkle/store.go @@ -28,8 +28,8 @@ type Store struct { var stateKey = []byte("merkle:state") // Database key for merkle tree save value db values -// MerkleState contains the latest Merkle root hash and the number of times `Commit` has been called -type MerkleState struct { +// ChainState contains the latest Merkle root hash and the number of times `Commit` has been called +type ChainState struct { Hash []byte Height uint64 } @@ -74,29 +74,29 @@ func NewStore(dbName string, cacheSize int, logger log.Logger) *Store { db := dbm.NewDB(name, dbm.LevelDBBackendStr, dir) tree := iavl.NewIAVLTree(cacheSize, db) - var eyesState MerkleState + var chainState ChainState if empty { logger.Info("no existing db, creating new db") - eyesState = MerkleState{ + chainState = ChainState{ Hash: tree.Save(), Height: initialHeight, } - db.Set(stateKey, wire.BinaryBytes(eyesState)) + db.Set(stateKey, wire.BinaryBytes(chainState)) } else { logger.Info("loading existing db") eyesStateBytes := db.Get(stateKey) - err = wire.ReadBinaryBytes(eyesStateBytes, &eyesState) + err = wire.ReadBinaryBytes(eyesStateBytes, &chainState) if err != nil { logger.Error("error reading MerkleEyesState", "err", err) panic(err) } - tree.Load(eyesState.Hash) + tree.Load(chainState.Hash) } return &Store{ State: NewState(tree, true), - height: eyesState.Height, - hash: eyesState.Hash, + height: chainState.Height, + hash: chainState.Hash, persisted: true, logger: logger, } @@ -112,7 +112,9 @@ func NewStore(dbName string, cacheSize int, logger log.Logger) *Store { // Info implements abci.Application. It returns the height, hash and size (in the data). // The height is the block that holds the transactions, not the apphash itself. func (s *Store) Info() abci.ResponseInfo { - s.logger.Info("Info synced", "height", s.height, "hash", fmt.Sprintf("%X", s.hash)) + s.logger.Info("Info synced", + "height", s.height, + "hash", fmt.Sprintf("%X", s.hash)) return abci.ResponseInfo{ Data: cmn.Fmt("size:%v", s.State.Committed().Size()), LastBlockHeight: s.height - 1, @@ -124,9 +126,11 @@ func (s *Store) Info() abci.ResponseInfo { func (s *Store) Commit() abci.Result { s.hash = s.State.Hash() s.height++ - s.logger.Debug("Commit synced", "height", s.height, "hash", fmt.Sprintf("%X", s.hash)) + s.logger.Debug("Commit synced", + "height", s.height, + "hash", fmt.Sprintf("%X", s.hash)) - s.State.BatchSet(stateKey, wire.BinaryBytes(MerkleState{ + s.State.BatchSet(stateKey, wire.BinaryBytes(ChainState{ Hash: s.hash, Height: s.height, })) @@ -144,10 +148,6 @@ func (s *Store) Commit() abci.Result { // Query implements abci.Application func (s *Store) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) { - if len(reqQuery.Data) == 0 { - return - } - tree := s.State.Committed() if reqQuery.Height != 0 { // TODO: support older commits @@ -159,6 +159,8 @@ func (s *Store) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) // set the query response height to current resQuery.Height = s.height + tree := s.State.Committed() + switch reqQuery.Path { case "/store", "/key": // Get by key key := reqQuery.Data // Data holds the key bytes @@ -170,24 +172,11 @@ func (s *Store) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) } resQuery.Value = value resQuery.Proof = proof - // TODO: return index too? } else { value := tree.Get(key) resQuery.Value = value } - case "/index": // Get by Index - index := wire.GetInt64(reqQuery.Data) - key, value := tree.GetByIndex(int(index)) - resQuery.Key = key - resQuery.Index = int64(index) - resQuery.Value = value - - case "/size": // Get size - size := tree.Size() - sizeBytes := wire.BinaryBytes(size) - resQuery.Value = sizeBytes - default: resQuery.Code = abci.CodeType_UnknownRequest resQuery.Log = cmn.Fmt("Unexpected Query path: %v", reqQuery.Path) From 28ebfd64dddb5df4db785c0ea7db431f8670038f Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 12:41:59 -0400 Subject: [PATCH 05/18] Started implementing SimpleDB for MemKVStore --- state/kvcache.go | 111 +++++++++++++++++++++++++ state/kvcache_test.go | 70 ++++++++++++++++ state/kvstore.go | 187 ++++++++++++++++++------------------------ 3 files changed, 261 insertions(+), 107 deletions(-) create mode 100644 state/kvcache.go create mode 100644 state/kvcache_test.go diff --git a/state/kvcache.go b/state/kvcache.go new file mode 100644 index 0000000000..10acf8f7c2 --- /dev/null +++ b/state/kvcache.go @@ -0,0 +1,111 @@ +package state + +import ( + "container/list" + "fmt" + + cmn "github.com/tendermint/tmlibs/common" +) + +// KVCache is a cache that enforces deterministic sync order. +type KVCache struct { + store KVStore + cache map[string]kvCacheValue + keys *list.List + logging bool + logLines []string +} + +type kvCacheValue struct { + v []byte // The value of some key + e *list.Element // The KVCache.keys element +} + +// NOTE: If store is nil, creates a new MemKVStore +func NewKVCache(store KVStore) *KVCache { + if store == nil { + store = NewMemKVStore() + } + return (&KVCache{ + store: store, + }).Reset() +} + +func (kvc *KVCache) SetLogging() { + kvc.logging = true +} + +func (kvc *KVCache) GetLogLines() []string { + return kvc.logLines +} + +func (kvc *KVCache) ClearLogLines() { + kvc.logLines = nil +} + +func (kvc *KVCache) Reset() *KVCache { + kvc.cache = make(map[string]kvCacheValue) + kvc.keys = list.New() + return kvc +} + +func (kvc *KVCache) Set(key []byte, value []byte) { + if kvc.logging { + line := fmt.Sprintf("Set %v = %v", LegibleBytes(key), LegibleBytes(value)) + kvc.logLines = append(kvc.logLines, line) + } + cacheValue, ok := kvc.cache[string(key)] + if ok { + kvc.keys.MoveToBack(cacheValue.e) + } else { + cacheValue.e = kvc.keys.PushBack(key) + } + cacheValue.v = value + kvc.cache[string(key)] = cacheValue +} + +func (kvc *KVCache) Get(key []byte) (value []byte) { + cacheValue, ok := kvc.cache[string(key)] + if ok { + if kvc.logging { + line := fmt.Sprintf("Get (hit) %v = %v", LegibleBytes(key), LegibleBytes(cacheValue.v)) + kvc.logLines = append(kvc.logLines, line) + } + return cacheValue.v + } else { + value := kvc.store.Get(key) + kvc.cache[string(key)] = kvCacheValue{ + v: value, + e: kvc.keys.PushBack(key), + } + if kvc.logging { + line := fmt.Sprintf("Get (miss) %v = %v", LegibleBytes(key), LegibleBytes(value)) + kvc.logLines = append(kvc.logLines, line) + } + return value + } +} + +//Update the store with the values from the cache +func (kvc *KVCache) Sync() { + for e := kvc.keys.Front(); e != nil; e = e.Next() { + key := e.Value.([]byte) + value := kvc.cache[string(key)] + kvc.store.Set(key, value.v) + } + kvc.Reset() +} + +//---------------------------------------- + +func LegibleBytes(data []byte) string { + s := "" + for _, b := range data { + if 0x21 <= b && b < 0x7F { + s += cmn.Green(string(b)) + } else { + s += cmn.Blue(cmn.Fmt("%02X", b)) + } + } + return s +} diff --git a/state/kvcache_test.go b/state/kvcache_test.go new file mode 100644 index 0000000000..e8ab4ab35f --- /dev/null +++ b/state/kvcache_test.go @@ -0,0 +1,70 @@ +package state + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKVCache(t *testing.T) { + assert := assert.New(t) + + //stores to be tested + ms := NewMemKVStore() + store := NewMemKVStore() + kvc := NewKVCache(store) + + //key value pairs to be tested within the system + var keyvalue = []struct { + key string + value string + }{ + {"foo", "snake"}, + {"bar", "mouse"}, + } + + //set the kvc to have all the key value pairs + setRecords := func(kv KVStore) { + for _, n := range keyvalue { + kv.Set([]byte(n.key), []byte(n.value)) + } + } + + //store has all the key value pairs + storeHasAll := func(kv KVStore) bool { + for _, n := range keyvalue { + if !bytes.Equal(kv.Get([]byte(n.key)), []byte(n.value)) { + return false + } + } + return true + } + + //test read/write for MemKVStore + setRecords(ms) + assert.True(storeHasAll(ms), "MemKVStore doesn't retrieve after Set") + + //test read/write for KVCache + setRecords(kvc) + assert.True(storeHasAll(kvc), "KVCache doesn't retrieve after Set") + + //test reset + kvc.Reset() + assert.False(storeHasAll(kvc), "KVCache retrieving after reset") + + //test sync + setRecords(kvc) + assert.False(storeHasAll(store), "store retrieving before synced") + kvc.Sync() + assert.True(storeHasAll(store), "store isn't retrieving after synced") + + //test logging + assert.Zero(len(kvc.GetLogLines()), "logging events existed before using SetLogging") + kvc.SetLogging() + setRecords(kvc) + assert.Equal(len(kvc.GetLogLines()), 2, "incorrect number of logging events recorded") + kvc.ClearLogLines() + assert.Zero(len(kvc.GetLogLines()), "logging events still exists after ClearLogLines") + +} diff --git a/state/kvstore.go b/state/kvstore.go index 7d0534948a..51aa450c0d 100644 --- a/state/kvstore.go +++ b/state/kvstore.go @@ -1,13 +1,12 @@ package state import ( - "container/list" - "fmt" + "sort" "github.com/tendermint/go-wire/data" - . "github.com/tendermint/tmlibs/common" ) +// KVStore is a simple interface to get/set data type KVStore interface { Set(key, value []byte) Get(key []byte) (value []byte) @@ -15,30 +14,34 @@ type KVStore interface { //---------------------------------------- +// Model grabs together key and value to allow easier return values type Model struct { Key data.Bytes Value data.Bytes } -// What I wished to have... +// SimpleDB allows us to do some basic range queries on a db type SimpleDB interface { KVStore Has(key []byte) (has bool) Remove(key []byte) (value []byte) // returns old value if there was one + // Start is inclusive, End is exclusive... + // Thus List ([]byte{12, 13}, []byte{12, 14}) will return anything with + // the prefix []byte{12, 13} List(start, end []byte, limit int) []Model First(start, end []byte) Model Last(start, end []byte) Model - // Checkpoint returns the same state, but where writes - // are buffered and don't affect the parent - Checkpoint() SimpleDB + // // Checkpoint returns the same state, but where writes + // // are buffered and don't affect the parent + // Checkpoint() SimpleDB - // Commit will take all changes from the checkpoint and write - // them to the parent. - // Returns an error if this is not a child of this one - Commit(SimpleDB) error + // // Commit will take all changes from the checkpoint and write + // // them to the parent. + // // Returns an error if this is not a child of this one + // Commit(SimpleDB) error // Discard will remove reference to this Discard() @@ -50,6 +53,8 @@ type MemKVStore struct { m map[string][]byte } +var _ SimpleDB = NewMemKVStore() + func NewMemKVStore() *MemKVStore { return &MemKVStore{ m: make(map[string][]byte, 0), @@ -64,107 +69,75 @@ func (mkv *MemKVStore) Get(key []byte) (value []byte) { return mkv.m[string(key)] } -//---------------------------------------- - -// A Cache that enforces deterministic sync order. -type KVCache struct { - store KVStore - cache map[string]kvCacheValue - keys *list.List - logging bool - logLines []string +func (mkv *MemKVStore) Has(key []byte) (has bool) { + _, ok := mkv.m[string(key)] + return ok } -type kvCacheValue struct { - v []byte // The value of some key - e *list.Element // The KVCache.keys element +func (mkv *MemKVStore) Remove(key []byte) (value []byte) { + val := mkv.m[string(key)] + delete(mkv.m, string(key)) + return val } -// NOTE: If store is nil, creates a new MemKVStore -func NewKVCache(store KVStore) *KVCache { - if store == nil { - store = NewMemKVStore() - } - return (&KVCache{ - store: store, - }).Reset() -} +func (mkv *MemKVStore) List(start, end []byte, limit int) []Model { + keys := mkv.keysInRange(start, end) + sort.Strings(keys) + keys = keys[:limit] -func (kvc *KVCache) SetLogging() { - kvc.logging = true -} - -func (kvc *KVCache) GetLogLines() []string { - return kvc.logLines -} - -func (kvc *KVCache) ClearLogLines() { - kvc.logLines = nil -} - -func (kvc *KVCache) Reset() *KVCache { - kvc.cache = make(map[string]kvCacheValue) - kvc.keys = list.New() - return kvc -} - -func (kvc *KVCache) Set(key []byte, value []byte) { - if kvc.logging { - line := fmt.Sprintf("Set %v = %v", LegibleBytes(key), LegibleBytes(value)) - kvc.logLines = append(kvc.logLines, line) - } - cacheValue, ok := kvc.cache[string(key)] - if ok { - kvc.keys.MoveToBack(cacheValue.e) - } else { - cacheValue.e = kvc.keys.PushBack(key) - } - cacheValue.v = value - kvc.cache[string(key)] = cacheValue -} - -func (kvc *KVCache) Get(key []byte) (value []byte) { - cacheValue, ok := kvc.cache[string(key)] - if ok { - if kvc.logging { - line := fmt.Sprintf("Get (hit) %v = %v", LegibleBytes(key), LegibleBytes(cacheValue.v)) - kvc.logLines = append(kvc.logLines, line) - } - return cacheValue.v - } else { - value := kvc.store.Get(key) - kvc.cache[string(key)] = kvCacheValue{ - v: value, - e: kvc.keys.PushBack(key), - } - if kvc.logging { - line := fmt.Sprintf("Get (miss) %v = %v", LegibleBytes(key), LegibleBytes(value)) - kvc.logLines = append(kvc.logLines, line) - } - return value - } -} - -//Update the store with the values from the cache -func (kvc *KVCache) Sync() { - for e := kvc.keys.Front(); e != nil; e = e.Next() { - key := e.Value.([]byte) - value := kvc.cache[string(key)] - kvc.store.Set(key, value.v) - } - kvc.Reset() -} - -//---------------------------------------- - -func LegibleBytes(data []byte) string { - s := "" - for _, b := range data { - if 0x21 <= b && b < 0x7F { - s += Green(string(b)) - } else { - s += Blue(Fmt("%02X", b)) + res := make([]Model, len(keys)) + for i, k := range keys { + res[i] = Model{ + Key: []byte(k), + Value: mkv.m[k], } } - return s + return res +} + +// First iterates through all keys to find the one that matches +func (mkv *MemKVStore) First(start, end []byte) Model { + key := "" + for _, k := range mkv.keysInRange(start, end) { + if key == "" || k < key { + key = k + } + } + if key == "" { + return Model{} + } + return Model{ + Key: []byte(key), + Value: mkv.m[key], + } +} + +func (mkv *MemKVStore) Last(start, end []byte) Model { + key := "" + for _, k := range mkv.keysInRange(start, end) { + if key == "" || k > key { + key = k + } + } + if key == "" { + return Model{} + } + return Model{ + Key: []byte(key), + Value: mkv.m[key], + } +} + +func (mkv *MemKVStore) Discard() { + mkv.m = make(map[string][]byte, 0) +} + +func (mkv *MemKVStore) keysInRange(start, end []byte) (res []string) { + s, e := string(start), string(end) + for k := range mkv.m { + if k >= s && k < e { + res = append(res, k) + } + } + return } From 37b5d16b73f9a004edc975e4853c5f4b765d2ec7 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 14:39:04 -0400 Subject: [PATCH 06/18] Add basic kv tests --- state/{state.go => chainstate.go} | 0 state/{state_test.go => chainstate_test.go} | 0 state/kvstore.go | 11 +- state/kvstore_test.go | 70 ---------- state/store_test.go | 136 ++++++++++++++++++++ 5 files changed, 146 insertions(+), 71 deletions(-) rename state/{state.go => chainstate.go} (100%) rename state/{state_test.go => chainstate_test.go} (100%) delete mode 100644 state/kvstore_test.go create mode 100644 state/store_test.go diff --git a/state/state.go b/state/chainstate.go similarity index 100% rename from state/state.go rename to state/chainstate.go diff --git a/state/state_test.go b/state/chainstate_test.go similarity index 100% rename from state/state_test.go rename to state/chainstate_test.go diff --git a/state/kvstore.go b/state/kvstore.go index 51aa450c0d..4471c8fd21 100644 --- a/state/kvstore.go +++ b/state/kvstore.go @@ -49,12 +49,16 @@ type SimpleDB interface { //---------------------------------------- +// MemKVStore is a simple implementation of SimpleDB. +// It is only intended for quick testing, not to be used +// in production or with large data stores. type MemKVStore struct { m map[string][]byte } var _ SimpleDB = NewMemKVStore() +// NewMemKVStore initializes a MemKVStore func NewMemKVStore() *MemKVStore { return &MemKVStore{ m: make(map[string][]byte, 0), @@ -83,7 +87,12 @@ func (mkv *MemKVStore) Remove(key []byte) (value []byte) { func (mkv *MemKVStore) List(start, end []byte, limit int) []Model { keys := mkv.keysInRange(start, end) sort.Strings(keys) - keys = keys[:limit] + if limit > 0 && len(keys) > 0 { + if limit > len(keys) { + limit = len(keys) + } + keys = keys[:limit] + } res := make([]Model, len(keys)) for i, k := range keys { diff --git a/state/kvstore_test.go b/state/kvstore_test.go deleted file mode 100644 index e4961c3606..0000000000 --- a/state/kvstore_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package state - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestKVStore(t *testing.T) { - assert := assert.New(t) - - //stores to be tested - ms := NewMemKVStore() - store := NewMemKVStore() - kvc := NewKVCache(store) - - //key value pairs to be tested within the system - var keyvalue = []struct { - key string - value string - }{ - {"foo", "snake"}, - {"bar", "mouse"}, - } - - //set the kvc to have all the key value pairs - setRecords := func(kv KVStore) { - for _, n := range keyvalue { - kv.Set([]byte(n.key), []byte(n.value)) - } - } - - //store has all the key value pairs - storeHasAll := func(kv KVStore) bool { - for _, n := range keyvalue { - if !bytes.Equal(kv.Get([]byte(n.key)), []byte(n.value)) { - return false - } - } - return true - } - - //test read/write for MemKVStore - setRecords(ms) - assert.True(storeHasAll(ms), "MemKVStore doesn't retrieve after Set") - - //test read/write for KVCache - setRecords(kvc) - assert.True(storeHasAll(kvc), "KVCache doesn't retrieve after Set") - - //test reset - kvc.Reset() - assert.False(storeHasAll(kvc), "KVCache retrieving after reset") - - //test sync - setRecords(kvc) - assert.False(storeHasAll(store), "store retrieving before synced") - kvc.Sync() - assert.True(storeHasAll(store), "store isn't retrieving after synced") - - //test logging - assert.Zero(len(kvc.GetLogLines()), "logging events existed before using SetLogging") - kvc.SetLogging() - setRecords(kvc) - assert.Equal(len(kvc.GetLogLines()), 2, "incorrect number of logging events recorded") - kvc.ClearLogLines() - assert.Zero(len(kvc.GetLogLines()), "logging events still exists after ClearLogLines") - -} diff --git a/state/store_test.go b/state/store_test.go new file mode 100644 index 0000000000..de1d8246a0 --- /dev/null +++ b/state/store_test.go @@ -0,0 +1,136 @@ +package state + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func GetDBs() []SimpleDB { + return []SimpleDB{ + NewMemKVStore(), + } +} + +// TestKVStore makes sure that get/set/remove operations work, +// as well as list +func TestKVStore(t *testing.T) { + assert := assert.New(t) + + type listQuery struct { + // this is the list query + start, end []byte + limit int + // expected result from List, first element also expected for First + expected []Model + // expected result from Last + last Model + } + + cases := []struct { + toSet []Model + toRemove []Model + toGet []Model + toList []listQuery + }{ + // simple add + { + []Model{ + {[]byte{1}, []byte{2}}, + {[]byte{3}, []byte{4}}, + }, + nil, + []Model{{[]byte{1}, []byte{2}}}, + []listQuery{ + { + []byte{1}, []byte{4}, 0, + // all + []Model{ + {[]byte{1}, []byte{2}}, + {[]byte{3}, []byte{4}}, + }, + // last one + Model{[]byte{3}, []byte{4}}, + }, + { + []byte{1}, []byte{3}, 10, + // all + []Model{ + {[]byte{1}, []byte{2}}, + }, + // last one + Model{[]byte{1}, []byte{2}}, + }, + }, + }, + // over-write data, remove + { + []Model{ + {[]byte{1}, []byte{2}}, + {[]byte{2}, []byte{2}}, + {[]byte{3}, []byte{2}}, + {[]byte{2}, []byte{4}}, + }, + []Model{{[]byte{3}, []byte{2}}}, + []Model{ + {[]byte{1}, []byte{2}}, + {[]byte{2}, []byte{4}}, + {[]byte{3}, nil}, + }, + []listQuery{ + { + []byte{0, 5}, []byte{10}, 1, + // all + []Model{ + {[]byte{1}, []byte{2}}, + }, + // last + Model{[]byte{2}, []byte{4}}, + }, + { + []byte{1, 4}, []byte{1, 7}, 10, + []Model{}, + Model{}, + }, + { + []byte{1, 5}, []byte{10}, 0, + []Model{ + {[]byte{2}, []byte{4}}, + }, + Model{[]byte{2}, []byte{4}}, + }, + }, + }, + } + + for i, tc := range cases { + for j, db := range GetDBs() { + for _, s := range tc.toSet { + db.Set(s.Key, s.Value) + } + for k, r := range tc.toRemove { + val := db.Remove(r.Key) + assert.EqualValues(r.Value, val, "%d/%d/%d", i, j, k) + } + for k, g := range tc.toGet { + val := db.Get(g.Key) + assert.EqualValues(g.Value, val, "%d/%d/%d", i, j, k) + has := db.Has(g.Key) + assert.Equal(len(g.Value) != 0, has, "%d/%d/%d", i, j, k) + } + for k, lq := range tc.toList { + list := db.List(lq.start, lq.end, lq.limit) + if assert.EqualValues(lq.expected, list, "%d/%d/%d", i, j, k) { + var first Model + if len(lq.expected) > 0 { + first = lq.expected[0] + } + f := db.First(lq.start, lq.end) + assert.EqualValues(first, f, "%d/%d/%d", i, j, k) + l := db.Last(lq.start, lq.end) + assert.EqualValues(lq.last, l, "%d/%d/%d", i, j, k) + } + } + } + } +} From 744d035d95ad6a1bac82994edf4dc63157203996 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 15:19:37 -0400 Subject: [PATCH 07/18] Test list queries and iavl implementation --- glide.lock | 13 ++++++------ glide.yaml | 2 +- state/{merkle/state.go => merkle.go} | 30 ++++++++++++++-------------- state/store_test.go | 2 ++ 4 files changed, 24 insertions(+), 23 deletions(-) rename state/{merkle/state.go => merkle.go} (84%) diff --git a/glide.lock b/glide.lock index 1216c1a4b9..7922e6dd99 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 8c438edb7d269da439141e62f3e0c931fa9efaee54b13ce1e7330dc99179fddd -updated: 2017-07-20T15:39:36.659717024+02:00 +hash: 2848c30b31fb205f846dd7dfca14ebed8a3249cbc5aaa759066b2bab3e4bbf42 +updated: 2017-07-26T15:18:56.353872835-04:00 imports: - name: github.com/bgentry/speakeasy version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd @@ -105,7 +105,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 7f5f48b6b9ec3964de4b07b6c3cd05d7c91aeee5 + version: 864d1f80b36b440bde030a5c18d8ac3aa8c2949d subpackages: - client - example/dummy @@ -133,16 +133,15 @@ imports: - data - data/base58 - name: github.com/tendermint/light-client - version: d63415027075bc5d74a98a718393b59b5c4279a5 + version: fcf4e411583135a1900157b8b0274c41e20ea3a1 subpackages: - certifiers - certifiers/client - certifiers/files - proofs - name: github.com/tendermint/merkleeyes - version: 102aaf5a8ffda1846413fb22805a94def2045b9f + version: fef0a1fc729f5b50c58a2dce04b4525aa06c2247 subpackages: - - app - client - iavl - name: github.com/tendermint/tendermint @@ -172,7 +171,7 @@ imports: - types - version - name: github.com/tendermint/tmlibs - version: efb56aaea7517220bb3f42ff87b8004d554a17ff + version: 2f6f3e6aa70bb19b70a6e73210273fa127041070 subpackages: - autofile - cli diff --git a/glide.yaml b/glide.yaml index 0f96978eb6..e10ba05f98 100644 --- a/glide.yaml +++ b/glide.yaml @@ -29,7 +29,7 @@ import: - certifiers/client - certifiers/files - package: github.com/tendermint/merkleeyes - version: develop + version: unstable subpackages: - client - iavl diff --git a/state/merkle/state.go b/state/merkle.go similarity index 84% rename from state/merkle/state.go rename to state/merkle.go index 17c4ee25e0..d735f34b6e 100644 --- a/state/merkle/state.go +++ b/state/merkle.go @@ -1,10 +1,9 @@ -package merkle +package state import ( "errors" "math/rand" - "github.com/tendermint/basecoin/state" "github.com/tendermint/merkleeyes/iavl" "github.com/tendermint/tmlibs/merkle" ) @@ -79,7 +78,7 @@ type Bonsai struct { merkle.Tree } -var _ state.SimpleDB = &Bonsai{} +var _ SimpleDB = &Bonsai{} func NewBonsai(tree merkle.Tree) *Bonsai { return &Bonsai{ @@ -104,38 +103,39 @@ func (b *Bonsai) Remove(key []byte) (value []byte) { return } -func (b *Bonsai) List(start, end []byte, limit int) []state.Model { - var res []state.Model +func (b *Bonsai) List(start, end []byte, limit int) []Model { + res := []Model{} stopAtCount := func(key []byte, value []byte) (stop bool) { - m := state.Model{key, value} + m := Model{key, value} res = append(res, m) - return len(res) >= limit + // return false + return limit > 0 && len(res) >= limit } b.Tree.IterateRange(start, end, true, stopAtCount) return res } -func (b *Bonsai) First(start, end []byte) state.Model { - var m state.Model +func (b *Bonsai) First(start, end []byte) Model { + var m Model stopAtFirst := func(key []byte, value []byte) (stop bool) { - m = state.Model{key, value} + m = Model{key, value} return true } b.Tree.IterateRange(start, end, true, stopAtFirst) return m } -func (b *Bonsai) Last(start, end []byte) state.Model { - var m state.Model +func (b *Bonsai) Last(start, end []byte) Model { + var m Model stopAtFirst := func(key []byte, value []byte) (stop bool) { - m = state.Model{key, value} + m = Model{key, value} return true } b.Tree.IterateRange(start, end, false, stopAtFirst) return m } -func (b *Bonsai) Checkpoint() state.SimpleDB { +func (b *Bonsai) Checkpoint() SimpleDB { return &Bonsai{ id: b.id, Tree: b.Tree.Copy(), @@ -145,7 +145,7 @@ func (b *Bonsai) Checkpoint() state.SimpleDB { // Commit will take all changes from the checkpoint and write // them to the parent. // Returns an error if this is not a child of this one -func (b *Bonsai) Commit(sub state.SimpleDB) error { +func (b *Bonsai) Commit(sub SimpleDB) error { bb, ok := sub.(*Bonsai) if !ok || (b.id != bb.id) { return errors.New("Not a sub-transaction") diff --git a/state/store_test.go b/state/store_test.go index de1d8246a0..2609d11971 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -4,11 +4,13 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/tendermint/merkleeyes/iavl" ) func GetDBs() []SimpleDB { return []SimpleDB{ NewMemKVStore(), + NewBonsai(iavl.NewIAVLTree(0, nil)), } } From d0d1a9651765de887ec1b64a49ee6f37031ad784 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 16:23:28 -0400 Subject: [PATCH 08/18] Make kvtests legible --- state/store_test.go | 97 ++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/state/store_test.go b/state/store_test.go index 2609d11971..3d94e52fe3 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -14,6 +14,20 @@ func GetDBs() []SimpleDB { } } +func b(k string) []byte { + if k == "" { + return nil + } + return []byte(k) +} + +func m(k, v string) Model { + return Model{ + Key: b(k), + Value: b(v), + } +} + // TestKVStore makes sure that get/set/remove operations work, // as well as list func TestKVStore(t *testing.T) { @@ -21,7 +35,7 @@ func TestKVStore(t *testing.T) { type listQuery struct { // this is the list query - start, end []byte + start, end string limit int // expected result from List, first element also expected for First expected []Model @@ -37,69 +51,51 @@ func TestKVStore(t *testing.T) { }{ // simple add { - []Model{ - {[]byte{1}, []byte{2}}, - {[]byte{3}, []byte{4}}, - }, - nil, - []Model{{[]byte{1}, []byte{2}}}, - []listQuery{ + toSet: []Model{m("a", "b"), m("c", "d")}, + toRemove: nil, + toGet: []Model{m("a", "b")}, + toList: []listQuery{ { - []byte{1}, []byte{4}, 0, - // all - []Model{ - {[]byte{1}, []byte{2}}, - {[]byte{3}, []byte{4}}, - }, - // last one - Model{[]byte{3}, []byte{4}}, + "a", "d", 0, + []Model{m("a", "b"), m("c", "d")}, + m("c", "d"), }, { - []byte{1}, []byte{3}, 10, - // all - []Model{ - {[]byte{1}, []byte{2}}, - }, - // last one - Model{[]byte{1}, []byte{2}}, + "a", "c", 10, + []Model{m("a", "b")}, + m("a", "b"), }, }, }, // over-write data, remove { - []Model{ - {[]byte{1}, []byte{2}}, - {[]byte{2}, []byte{2}}, - {[]byte{3}, []byte{2}}, - {[]byte{2}, []byte{4}}, + toSet: []Model{ + m("a", "1"), + m("b", "2"), + m("c", "3"), + m("b", "4"), }, - []Model{{[]byte{3}, []byte{2}}}, - []Model{ - {[]byte{1}, []byte{2}}, - {[]byte{2}, []byte{4}}, - {[]byte{3}, nil}, + toRemove: []Model{m("c", "3")}, + toGet: []Model{ + m("a", "1"), + m("b", "4"), + m("c", ""), }, - []listQuery{ + toList: []listQuery{ { - []byte{0, 5}, []byte{10}, 1, - // all - []Model{ - {[]byte{1}, []byte{2}}, - }, - // last - Model{[]byte{2}, []byte{4}}, + "0d", "h", 1, + []Model{m("a", "1")}, + m("b", "4"), }, { - []byte{1, 4}, []byte{1, 7}, 10, + "ad", "ak", 10, []Model{}, Model{}, }, { - []byte{1, 5}, []byte{10}, 0, - []Model{ - {[]byte{2}, []byte{4}}, - }, - Model{[]byte{2}, []byte{4}}, + "ad", "k", 0, + []Model{m("b", "4")}, + m("b", "4"), }, }, }, @@ -121,15 +117,16 @@ func TestKVStore(t *testing.T) { assert.Equal(len(g.Value) != 0, has, "%d/%d/%d", i, j, k) } for k, lq := range tc.toList { - list := db.List(lq.start, lq.end, lq.limit) + start, end := []byte(lq.start), []byte(lq.end) + list := db.List(start, end, lq.limit) if assert.EqualValues(lq.expected, list, "%d/%d/%d", i, j, k) { var first Model if len(lq.expected) > 0 { first = lq.expected[0] } - f := db.First(lq.start, lq.end) + f := db.First(start, end) assert.EqualValues(first, f, "%d/%d/%d", i, j, k) - l := db.Last(lq.start, lq.end) + l := db.Last(start, end) assert.EqualValues(lq.last, l, "%d/%d/%d", i, j, k) } } From d607b7623471cf1b1e5fd9482d767dc7ad1272fc Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 16:35:02 -0400 Subject: [PATCH 09/18] Reorg state package and fix all imports --- app/app.go | 5 ++--- app/app_test.go | 5 ++--- app/genesis_test.go | 7 +++---- {state/merkle => app}/store.go | 11 ++++++----- benchmarks/app_test.go | 7 +++---- cmd/basecoin/commands/start.go | 3 +-- docs/guide/counter/plugins/counter/counter_test.go | 9 +++++---- glide.lock | 6 +++--- glide.yaml | 2 +- 9 files changed, 26 insertions(+), 29 deletions(-) rename {state/merkle => app}/store.go (96%) diff --git a/app/app.go b/app/app.go index e85322c5ae..ab308020c2 100644 --- a/app/app.go +++ b/app/app.go @@ -18,7 +18,6 @@ import ( "github.com/tendermint/basecoin/modules/roles" "github.com/tendermint/basecoin/stack" sm "github.com/tendermint/basecoin/state" - "github.com/tendermint/basecoin/state/merkle" "github.com/tendermint/basecoin/version" ) @@ -32,7 +31,7 @@ const ( type Basecoin struct { info *sm.ChainState - state *merkle.Store + state *Store handler basecoin.Handler height uint64 @@ -42,7 +41,7 @@ type Basecoin struct { var _ abci.Application = &Basecoin{} // NewBasecoin - create a new instance of the basecoin application -func NewBasecoin(handler basecoin.Handler, store *merkle.Store, logger log.Logger) *Basecoin { +func NewBasecoin(handler basecoin.Handler, store *Store, logger log.Logger) *Basecoin { return &Basecoin{ handler: handler, info: sm.NewChainState(), diff --git a/app/app_test.go b/app/app_test.go index 6a0fb5182d..a81cf4e948 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -16,7 +16,6 @@ import ( "github.com/tendermint/basecoin/modules/nonce" "github.com/tendermint/basecoin/stack" "github.com/tendermint/basecoin/state" - "github.com/tendermint/basecoin/state/merkle" wire "github.com/tendermint/go-wire" "github.com/tendermint/tmlibs/log" ) @@ -84,7 +83,7 @@ func (at *appTest) reset() { // Note: switch logger if you want to get more info logger := log.TestingLogger() // logger := log.NewTracingLogger(log.NewTMLogger(os.Stdout)) - store := merkle.NewStore("", 0, logger.With("module", "store")) + store := NewStore("", 0, logger.With("module", "store")) at.app = NewBasecoin( DefaultHandler("mycoin"), store, @@ -142,7 +141,7 @@ func TestSetOption(t *testing.T) { require := require.New(t) logger := log.TestingLogger() - store := merkle.NewStore("", 0, logger.With("module", "store")) + store := NewStore("", 0, logger.With("module", "store")) app := NewBasecoin( DefaultHandler("atom"), store, diff --git a/app/genesis_test.go b/app/genesis_test.go index 50ca77f0f2..a7412be82e 100644 --- a/app/genesis_test.go +++ b/app/genesis_test.go @@ -12,7 +12,6 @@ import ( "github.com/tendermint/tmlibs/log" "github.com/tendermint/basecoin/modules/coin" - "github.com/tendermint/basecoin/state/merkle" ) const genesisFilepath = "./testdata/genesis.json" @@ -20,7 +19,7 @@ const genesisAcctFilepath = "./testdata/genesis2.json" func TestLoadGenesisDoNotFailIfAppOptionsAreMissing(t *testing.T) { logger := log.TestingLogger() - store := merkle.NewStore("", 0, logger) + store := NewStore("", 0, logger) app := NewBasecoin(DefaultHandler("mycoin"), store, logger) err := app.LoadGenesis("./testdata/genesis3.json") require.Nil(t, err, "%+v", err) @@ -30,7 +29,7 @@ func TestLoadGenesis(t *testing.T) { assert, require := assert.New(t), require.New(t) logger := log.TestingLogger() - store := merkle.NewStore("", 0, logger) + store := NewStore("", 0, logger) app := NewBasecoin(DefaultHandler("mycoin"), store, logger) err := app.LoadGenesis(genesisFilepath) require.Nil(err, "%+v", err) @@ -60,7 +59,7 @@ func TestLoadGenesisAccountAddress(t *testing.T) { assert, require := assert.New(t), require.New(t) logger := log.TestingLogger() - store := merkle.NewStore("", 0, logger) + store := NewStore("", 0, logger) app := NewBasecoin(DefaultHandler("mycoin"), store, logger) err := app.LoadGenesis(genesisAcctFilepath) require.Nil(err, "%+v", err) diff --git a/state/merkle/store.go b/app/store.go similarity index 96% rename from state/merkle/store.go rename to app/store.go index 7064af7220..1dc7b43831 100644 --- a/state/merkle/store.go +++ b/app/store.go @@ -1,4 +1,4 @@ -package merkle +package app import ( "bytes" @@ -9,16 +9,17 @@ import ( abci "github.com/tendermint/abci/types" "github.com/tendermint/go-wire" + "github.com/tendermint/merkleeyes/iavl" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" - "github.com/tendermint/merkleeyes/iavl" + "github.com/tendermint/basecoin/state" ) // Store contains the merkle tree, and all info to handle abci requests type Store struct { - State + state.State height uint64 hash []byte persisted bool @@ -48,7 +49,7 @@ func NewStore(dbName string, cacheSize int, logger log.Logger) *Store { nil, ) return &Store{ - State: NewState(tree, false), + State: state.NewState(tree, false), height: initialHeight, logger: logger, } @@ -94,7 +95,7 @@ func NewStore(dbName string, cacheSize int, logger log.Logger) *Store { } return &Store{ - State: NewState(tree, true), + State: state.NewState(tree, true), height: chainState.Height, hash: chainState.Hash, persisted: true, diff --git a/benchmarks/app_test.go b/benchmarks/app_test.go index e1e2c98f7c..2786af608c 100644 --- a/benchmarks/app_test.go +++ b/benchmarks/app_test.go @@ -18,7 +18,6 @@ import ( "github.com/tendermint/basecoin/modules/nonce" "github.com/tendermint/basecoin/modules/roles" "github.com/tendermint/basecoin/stack" - "github.com/tendermint/basecoin/state/merkle" ) type BenchApp struct { @@ -55,13 +54,13 @@ func NewBenchApp(h basecoin.Handler, chainID string, n int, // logger = log.NewTracingLogger(logger) // TODO: disk writing - var store *merkle.Store + var store *app.Store if persist { tmpDir, _ := ioutil.TempDir("", "bc-app-benchmark") - store = merkle.NewStore(tmpDir, 500, logger) + store = app.NewStore(tmpDir, 500, logger) } else { - store = merkle.NewStore("", 0, logger) + store = app.NewStore("", 0, logger) } app := app.NewBasecoin( diff --git a/cmd/basecoin/commands/start.go b/cmd/basecoin/commands/start.go index 53c6d2e9b3..5ea145878b 100644 --- a/cmd/basecoin/commands/start.go +++ b/cmd/basecoin/commands/start.go @@ -20,7 +20,6 @@ import ( "github.com/tendermint/tendermint/types" "github.com/tendermint/basecoin/app" - "github.com/tendermint/basecoin/state/merkle" ) // StartCmd - command to start running the basecoin node! @@ -56,7 +55,7 @@ func init() { func startCmd(cmd *cobra.Command, args []string) error { rootDir := viper.GetString(cli.HomeFlag) - store := merkle.NewStore( + store := app.NewStore( path.Join(rootDir, "data", "merkleeyes.db"), EyesCacheSize, logger.With("module", "store"), diff --git a/docs/guide/counter/plugins/counter/counter_test.go b/docs/guide/counter/plugins/counter/counter_test.go index 9e8149edd7..986219d1de 100644 --- a/docs/guide/counter/plugins/counter/counter_test.go +++ b/docs/guide/counter/plugins/counter/counter_test.go @@ -5,16 +5,17 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + abci "github.com/tendermint/abci/types" + "github.com/tendermint/go-wire" + "github.com/tendermint/tmlibs/log" + "github.com/tendermint/basecoin" "github.com/tendermint/basecoin/app" "github.com/tendermint/basecoin/modules/auth" "github.com/tendermint/basecoin/modules/base" "github.com/tendermint/basecoin/modules/coin" "github.com/tendermint/basecoin/modules/nonce" - "github.com/tendermint/basecoin/state/merkle" - "github.com/tendermint/go-wire" - "github.com/tendermint/tmlibs/log" ) func TestCounterPlugin(t *testing.T) { @@ -25,7 +26,7 @@ func TestCounterPlugin(t *testing.T) { logger := log.TestingLogger() // logger := log.NewTracingLogger(log.NewTMLogger(os.Stdout)) - store := merkle.NewStore("", 0, logger.With("module", "store")) + store := app.NewStore("", 0, logger.With("module", "store")) h := NewHandler("gold") bcApp := app.NewBasecoin( h, diff --git a/glide.lock b/glide.lock index 7922e6dd99..775e0faada 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 2848c30b31fb205f846dd7dfca14ebed8a3249cbc5aaa759066b2bab3e4bbf42 -updated: 2017-07-26T15:18:56.353872835-04:00 +hash: 45eed61138603d4d03518ea822068cf32b45d0a219bb7f3b836e52129f2a3a2b +updated: 2017-07-26T16:26:38.678294279-04:00 imports: - name: github.com/bgentry/speakeasy version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd @@ -133,7 +133,7 @@ imports: - data - data/base58 - name: github.com/tendermint/light-client - version: fcf4e411583135a1900157b8b0274c41e20ea3a1 + version: 1c53d04dcc65c2fd15526152ed0651af10a09982 subpackages: - certifiers - certifiers/client diff --git a/glide.yaml b/glide.yaml index e10ba05f98..be5c66c3d3 100644 --- a/glide.yaml +++ b/glide.yaml @@ -22,7 +22,7 @@ import: subpackages: - data - package: github.com/tendermint/light-client - version: unstable + version: 1c53d04dcc65c2fd15526152ed0651af10a09982 subpackages: - proofs - certifiers From 199ee81a977702108cdc248d0502203d47911771 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 17:04:40 -0400 Subject: [PATCH 10/18] Revert all on failed CheckTx, fee+nonce increment on failed DeliverTx --- app/app.go | 3 ++- docs/guide/counter/plugins/counter/counter.go | 3 ++- stack/checkpoint.go | 8 ++++++++ tests/cli/roles.sh | 6 +++--- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/app/app.go b/app/app.go index ab308020c2..cd59e66dfd 100644 --- a/app/app.go +++ b/app/app.go @@ -64,10 +64,11 @@ func DefaultHandler(feeDenom string) basecoin.Handler { stack.Recovery{}, auth.Signatures{}, base.Chain{}, + stack.Checkpoint{OnCheck: true}, nonce.ReplayCheck{}, roles.NewMiddleware(), fee.NewSimpleFeeMiddleware(coin.Coin{feeDenom, 0}, fee.Bank), - stack.Checkpoint{}, + stack.Checkpoint{OnDeliver: true}, ).Use(d) } diff --git a/docs/guide/counter/plugins/counter/counter.go b/docs/guide/counter/plugins/counter/counter.go index fa806d519a..1cf2be24ac 100644 --- a/docs/guide/counter/plugins/counter/counter.go +++ b/docs/guide/counter/plugins/counter/counter.go @@ -102,9 +102,10 @@ func NewHandler(feeDenom string) basecoin.Handler { stack.Recovery{}, auth.Signatures{}, base.Chain{}, + stack.Checkpoint{OnCheck: true}, nonce.ReplayCheck{}, fee.NewSimpleFeeMiddleware(coin.Coin{feeDenom, 0}, fee.Bank), - stack.Checkpoint{}, + stack.Checkpoint{OnDeliver: true}, ).Use(dispatcher) } diff --git a/stack/checkpoint.go b/stack/checkpoint.go index a983ae88b6..89f67bd8ce 100644 --- a/stack/checkpoint.go +++ b/stack/checkpoint.go @@ -12,6 +12,8 @@ const ( // Checkpoint isolates all data store below this type Checkpoint struct { + OnCheck bool + OnDeliver bool PassOption } @@ -24,6 +26,9 @@ var _ Middleware = Checkpoint{} // CheckTx reverts all data changes if there was an error func (c Checkpoint) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { + if !c.OnCheck { + return next.CheckTx(ctx, store, tx) + } ps := state.NewKVCache(unwrap(store)) res, err = next.CheckTx(ctx, ps, tx) if err == nil { @@ -34,6 +39,9 @@ func (c Checkpoint) CheckTx(ctx basecoin.Context, store state.KVStore, tx baseco // DeliverTx reverts all data changes if there was an error func (c Checkpoint) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { + if !c.OnDeliver { + return next.DeliverTx(ctx, store, tx) + } ps := state.NewKVCache(unwrap(store)) res, err = next.DeliverTx(ctx, ps, tx) if err == nil { diff --git a/tests/cli/roles.sh b/tests/cli/roles.sh index 5457090b92..3ddde70991 100755 --- a/tests/cli/roles.sh +++ b/tests/cli/roles.sh @@ -68,7 +68,7 @@ test03SendMultiFromRole() { # let's try to send money from the role directly without multisig FAIL=$(echo qwertyuiop | ${CLIENT_EXE} tx send --amount=6000mycoin --from=$BANK --to=$TWO --sequence=1 --name=$POOR 2>/dev/null) assertFalse "need to assume role" $? - FAIL=$(echo qwertyuiop | ${CLIENT_EXE} tx send --amount=6000mycoin --from=$BANK --to=$TWO --sequence=1 --assume-role=bank --name=$POOR 2>/dev/null) + FAIL=$(echo qwertyuiop | ${CLIENT_EXE} tx send --amount=6000mycoin --from=$BANK --to=$TWO --sequence=2 --assume-role=bank --name=$POOR 2>/dev/null) assertFalse "need two signatures" $? # okay, begin a multisig transaction mr. poor... @@ -76,8 +76,8 @@ test03SendMultiFromRole() { echo qwertyuiop | ${CLIENT_EXE} tx send --amount=6000mycoin --from=$BANK --to=$TWO --sequence=1 --assume-role=bank --name=$POOR --multi --prepare=$TX_FILE assertTrue "line=${LINENO}, successfully prepare tx" $? # and get some dude to sign it - FAIL=$(echo qwertyuiop | ${CLIENT_EXE} tx --in=$TX_FILE --name=$POOR 2>/dev/null) - assertFalse "line=${LINENO}, double signing doesn't get bank" $? + # FAIL=$(echo qwertyuiop | ${CLIENT_EXE} tx --in=$TX_FILE --name=$POOR 2>/dev/null) + # assertFalse "line=${LINENO}, double signing doesn't get bank" $? # and get some dude to sign it for the full access TX=$(echo qwertyuiop | ${CLIENT_EXE} tx --in=$TX_FILE --name=$DUDE) txSucceeded $? "$TX" "multi-bank" From 84e2fa64f12ff15944f5d3479c1172b3725adb6d Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 17:44:48 -0400 Subject: [PATCH 11/18] Cache test for Bonsai --- state/kvcache.go | 53 ++------------- state/kvcache_test.go | 152 ++++++++++++++++++++++++++++-------------- state/kvstore.go | 16 ++--- state/store_test.go | 22 +++--- 4 files changed, 126 insertions(+), 117 deletions(-) diff --git a/state/kvcache.go b/state/kvcache.go index 10acf8f7c2..4231c621a2 100644 --- a/state/kvcache.go +++ b/state/kvcache.go @@ -1,19 +1,12 @@ package state -import ( - "container/list" - "fmt" - - cmn "github.com/tendermint/tmlibs/common" -) +import "container/list" // KVCache is a cache that enforces deterministic sync order. type KVCache struct { - store KVStore - cache map[string]kvCacheValue - keys *list.List - logging bool - logLines []string + store KVStore + cache map[string]kvCacheValue + keys *list.List } type kvCacheValue struct { @@ -31,18 +24,6 @@ func NewKVCache(store KVStore) *KVCache { }).Reset() } -func (kvc *KVCache) SetLogging() { - kvc.logging = true -} - -func (kvc *KVCache) GetLogLines() []string { - return kvc.logLines -} - -func (kvc *KVCache) ClearLogLines() { - kvc.logLines = nil -} - func (kvc *KVCache) Reset() *KVCache { kvc.cache = make(map[string]kvCacheValue) kvc.keys = list.New() @@ -50,10 +31,6 @@ func (kvc *KVCache) Reset() *KVCache { } func (kvc *KVCache) Set(key []byte, value []byte) { - if kvc.logging { - line := fmt.Sprintf("Set %v = %v", LegibleBytes(key), LegibleBytes(value)) - kvc.logLines = append(kvc.logLines, line) - } cacheValue, ok := kvc.cache[string(key)] if ok { kvc.keys.MoveToBack(cacheValue.e) @@ -67,10 +44,6 @@ func (kvc *KVCache) Set(key []byte, value []byte) { func (kvc *KVCache) Get(key []byte) (value []byte) { cacheValue, ok := kvc.cache[string(key)] if ok { - if kvc.logging { - line := fmt.Sprintf("Get (hit) %v = %v", LegibleBytes(key), LegibleBytes(cacheValue.v)) - kvc.logLines = append(kvc.logLines, line) - } return cacheValue.v } else { value := kvc.store.Get(key) @@ -78,10 +51,6 @@ func (kvc *KVCache) Get(key []byte) (value []byte) { v: value, e: kvc.keys.PushBack(key), } - if kvc.logging { - line := fmt.Sprintf("Get (miss) %v = %v", LegibleBytes(key), LegibleBytes(value)) - kvc.logLines = append(kvc.logLines, line) - } return value } } @@ -95,17 +64,3 @@ func (kvc *KVCache) Sync() { } kvc.Reset() } - -//---------------------------------------- - -func LegibleBytes(data []byte) string { - s := "" - for _, b := range data { - if 0x21 <= b && b < 0x7F { - s += cmn.Green(string(b)) - } else { - s += cmn.Blue(cmn.Fmt("%02X", b)) - } - } - return s -} diff --git a/state/kvcache_test.go b/state/kvcache_test.go index e8ab4ab35f..5af26b9190 100644 --- a/state/kvcache_test.go +++ b/state/kvcache_test.go @@ -1,70 +1,124 @@ package state import ( - "bytes" + "fmt" "testing" "github.com/stretchr/testify/assert" ) -func TestKVCache(t *testing.T) { +func TestCache(t *testing.T) { assert := assert.New(t) - //stores to be tested - ms := NewMemKVStore() - store := NewMemKVStore() - kvc := NewKVCache(store) + cases := []struct { + init []Model + toGet []Model + toList []listQuery - //key value pairs to be tested within the system - var keyvalue = []struct { - key string - value string + setCache []Model + removeCache []Model + getCache []Model + listCache []listQuery }{ - {"foo", "snake"}, - {"bar", "mouse"}, + // simple add + { + init: []Model{m("a", "1"), m("c", "2")}, + toGet: []Model{m("a", "1"), m("c", "2"), m("d", "")}, + toList: []listQuery{{ + "a", "e", 0, + []Model{m("a", "1"), m("c", "2")}, + m("c", "2"), + }}, + setCache: []Model{m("d", "3")}, + removeCache: []Model{m("a", "1")}, + getCache: []Model{m("a", ""), m("c", "2"), m("d", "3")}, + listCache: []listQuery{{ + "a", "e", 0, + []Model{m("c", "2"), m("d", "3")}, + m("d", "3"), + }}, + }, } - //set the kvc to have all the key value pairs - setRecords := func(kv KVStore) { - for _, n := range keyvalue { - kv.Set([]byte(n.key), []byte(n.value)) + checkGet := func(db SimpleDB, m Model, msg string) { + val := db.Get(m.Key) + assert.EqualValues(m.Value, val, msg) + has := db.Has(m.Key) + assert.Equal(len(m.Value) != 0, has, msg) + } + + checkList := func(db SimpleDB, lq listQuery, msg string) { + start, end := []byte(lq.start), []byte(lq.end) + list := db.List(start, end, lq.limit) + if assert.EqualValues(lq.expected, list, msg) { + var first Model + if len(lq.expected) > 0 { + first = lq.expected[0] + } + f := db.First(start, end) + assert.EqualValues(first, f, msg) + l := db.Last(start, end) + assert.EqualValues(lq.last, l, msg) } } - //store has all the key value pairs - storeHasAll := func(kv KVStore) bool { - for _, n := range keyvalue { - if !bytes.Equal(kv.Get([]byte(n.key)), []byte(n.value)) { - return false + for i, tc := range cases { + for j, db := range GetDBs() { + for _, s := range tc.init { + db.Set(s.Key, s.Value) + } + for k, g := range tc.toGet { + msg := fmt.Sprintf("%d/%d/%d", i, j, k) + checkGet(db, g, msg) + } + for k, lq := range tc.toList { + msg := fmt.Sprintf("%d/%d/%d", i, j, k) + checkList(db, lq, msg) + } + + // make cache + cache := db.Checkpoint() + + for _, s := range tc.setCache { + cache.Set(s.Key, s.Value) + } + for k, r := range tc.removeCache { + val := cache.Remove(r.Key) + assert.EqualValues(r.Value, val, "%d/%d/%d", i, j, k) + } + + // make sure data is in cache + for k, g := range tc.getCache { + msg := fmt.Sprintf("%d/%d/%d", i, j, k) + checkGet(cache, g, msg) + } + for k, lq := range tc.listCache { + msg := fmt.Sprintf("%d/%d/%d", i, j, k) + checkList(cache, lq, msg) + } + + // data not in basic store + for k, g := range tc.toGet { + msg := fmt.Sprintf("%d/%d/%d", i, j, k) + checkGet(db, g, msg) + } + for k, lq := range tc.toList { + msg := fmt.Sprintf("%d/%d/%d", i, j, k) + checkList(db, lq, msg) + } + + // commit + db.Commit(cache) + + // make sure data is in cache + for k, g := range tc.getCache { + msg := fmt.Sprintf("%d/%d/%d", i, j, k) + checkGet(db, g, msg) + } + for k, lq := range tc.listCache { + msg := fmt.Sprintf("%d/%d/%d", i, j, k) + checkList(db, lq, msg) } } - return true } - - //test read/write for MemKVStore - setRecords(ms) - assert.True(storeHasAll(ms), "MemKVStore doesn't retrieve after Set") - - //test read/write for KVCache - setRecords(kvc) - assert.True(storeHasAll(kvc), "KVCache doesn't retrieve after Set") - - //test reset - kvc.Reset() - assert.False(storeHasAll(kvc), "KVCache retrieving after reset") - - //test sync - setRecords(kvc) - assert.False(storeHasAll(store), "store retrieving before synced") - kvc.Sync() - assert.True(storeHasAll(store), "store isn't retrieving after synced") - - //test logging - assert.Zero(len(kvc.GetLogLines()), "logging events existed before using SetLogging") - kvc.SetLogging() - setRecords(kvc) - assert.Equal(len(kvc.GetLogLines()), 2, "incorrect number of logging events recorded") - kvc.ClearLogLines() - assert.Zero(len(kvc.GetLogLines()), "logging events still exists after ClearLogLines") - } diff --git a/state/kvstore.go b/state/kvstore.go index 4471c8fd21..7e4907e143 100644 --- a/state/kvstore.go +++ b/state/kvstore.go @@ -34,14 +34,14 @@ type SimpleDB interface { First(start, end []byte) Model Last(start, end []byte) Model - // // Checkpoint returns the same state, but where writes - // // are buffered and don't affect the parent - // Checkpoint() SimpleDB + // Checkpoint returns the same state, but where writes + // are buffered and don't affect the parent + Checkpoint() SimpleDB - // // Commit will take all changes from the checkpoint and write - // // them to the parent. - // // Returns an error if this is not a child of this one - // Commit(SimpleDB) error + // Commit will take all changes from the checkpoint and write + // them to the parent. + // Returns an error if this is not a child of this one + Commit(SimpleDB) error // Discard will remove reference to this Discard() @@ -56,7 +56,7 @@ type MemKVStore struct { m map[string][]byte } -var _ SimpleDB = NewMemKVStore() +// var _ SimpleDB = NewMemKVStore() // NewMemKVStore initializes a MemKVStore func NewMemKVStore() *MemKVStore { diff --git a/state/store_test.go b/state/store_test.go index 3d94e52fe3..300cec58ce 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -9,7 +9,7 @@ import ( func GetDBs() []SimpleDB { return []SimpleDB{ - NewMemKVStore(), + // NewMemKVStore(), NewBonsai(iavl.NewIAVLTree(0, nil)), } } @@ -28,21 +28,21 @@ func m(k, v string) Model { } } +type listQuery struct { + // this is the list query + start, end string + limit int + // expected result from List, first element also expected for First + expected []Model + // expected result from Last + last Model +} + // TestKVStore makes sure that get/set/remove operations work, // as well as list func TestKVStore(t *testing.T) { assert := assert.New(t) - type listQuery struct { - // this is the list query - start, end string - limit int - // expected result from List, first element also expected for First - expected []Model - // expected result from Last - last Model - } - cases := []struct { toSet []Model toRemove []Model From f1785e312df2e533d718df2e2050d3232b5683c4 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 19:26:06 -0400 Subject: [PATCH 12/18] SimpleDB works except for persisted --- state/kvcache.go | 172 ++++++++++++++++++++++++++++++------------ state/kvcache_test.go | 8 +- state/kvstore.go | 69 +++++++++++------ state/store_test.go | 13 +++- 4 files changed, 183 insertions(+), 79 deletions(-) diff --git a/state/kvcache.go b/state/kvcache.go index 4231c621a2..31ba96d5d4 100644 --- a/state/kvcache.go +++ b/state/kvcache.go @@ -1,66 +1,140 @@ package state -import "container/list" +import "errors" -// KVCache is a cache that enforces deterministic sync order. -type KVCache struct { - store KVStore - cache map[string]kvCacheValue - keys *list.List +// MemKVCache is designed to wrap MemKVStore as a cache +type MemKVCache struct { + store SimpleDB + cache *MemKVStore } -type kvCacheValue struct { - v []byte // The value of some key - e *list.Element // The KVCache.keys element -} +var _ SimpleDB = NewMemKVStore() -// NOTE: If store is nil, creates a new MemKVStore -func NewKVCache(store KVStore) *KVCache { +// NewMemKVCache wraps a cache around MemKVStore +// +// You probably don't want to use directly, but rather +// via MemKVCache.Checkpoint() +func NewMemKVCache(store SimpleDB) *MemKVCache { if store == nil { - store = NewMemKVStore() + panic("wtf") } - return (&KVCache{ + + return &MemKVCache{ store: store, - }).Reset() -} - -func (kvc *KVCache) Reset() *KVCache { - kvc.cache = make(map[string]kvCacheValue) - kvc.keys = list.New() - return kvc -} - -func (kvc *KVCache) Set(key []byte, value []byte) { - cacheValue, ok := kvc.cache[string(key)] - if ok { - kvc.keys.MoveToBack(cacheValue.e) - } else { - cacheValue.e = kvc.keys.PushBack(key) + cache: NewMemKVStore(), } - cacheValue.v = value - kvc.cache[string(key)] = cacheValue } -func (kvc *KVCache) Get(key []byte) (value []byte) { - cacheValue, ok := kvc.cache[string(key)] - if ok { - return cacheValue.v - } else { - value := kvc.store.Get(key) - kvc.cache[string(key)] = kvCacheValue{ - v: value, - e: kvc.keys.PushBack(key), +func (c *MemKVCache) Set(key []byte, value []byte) { + c.cache.Set(key, value) +} + +func (c *MemKVCache) Get(key []byte) (value []byte) { + value, ok := c.cache.m[string(key)] + if !ok { + value = c.store.Get(key) + c.cache.Set(key, value) + } + return value +} + +func (c *MemKVCache) Has(key []byte) bool { + value := c.Get(key) + return value != nil +} + +// Remove uses nil value as a flag to delete... not ideal but good enough +// for testing +func (c *MemKVCache) Remove(key []byte) (value []byte) { + value = c.Get(key) + c.cache.Set(key, nil) + return value +} + +// List is also inefficiently implemented... +func (c *MemKVCache) List(start, end []byte, limit int) []Model { + orig := c.store.List(start, end, 0) + cached := c.cache.List(start, end, 0) + keys := c.combineLists(orig, cached) + + // apply limit (too late) + if limit > 0 && len(keys) > 0 { + if limit > len(keys) { + limit = len(keys) + } + keys = keys[:limit] + } + + return keys +} + +func (c *MemKVCache) combineLists(orig, cache []Model) []Model { + store := NewMemKVStore() + for _, m := range orig { + store.Set(m.Key, m.Value) + } + for _, m := range cache { + if m.Value == nil { + store.Remove([]byte(m.Key)) + } else { + store.Set([]byte(m.Key), m.Value) + } + } + + return store.List(nil, nil, 0) +} + +// First is done with List, but could be much more efficient +func (c *MemKVCache) First(start, end []byte) Model { + data := c.List(start, end, 0) + if len(data) == 0 { + return Model{} + } + return data[0] +} + +// Last is done with List, but could be much more efficient +func (c *MemKVCache) Last(start, end []byte) Model { + data := c.List(start, end, 0) + if len(data) == 0 { + return Model{} + } + return data[len(data)-1] +} + +// Checkpoint returns the same state, but where writes +// are buffered and don't affect the parent +func (c *MemKVCache) Checkpoint() SimpleDB { + return NewMemKVCache(c) +} + +// Commit will take all changes from the checkpoint and write +// them to the parent. +// Returns an error if this is not a child of this one +func (c *MemKVCache) Commit(sub SimpleDB) error { + cache, ok := sub.(*MemKVCache) + if !ok { + return errors.New("sub is not a cache") + } + // TODO: see if it points to us + + // apply the cached data to us + cache.applyCache() + return nil +} + +// applyCache will apply all the cache methods to the underlying store +func (c *MemKVCache) applyCache() { + for k, v := range c.cache.m { + if v == nil { + c.store.Remove([]byte(k)) + } else { + c.store.Set([]byte(k), v) } - return value } } -//Update the store with the values from the cache -func (kvc *KVCache) Sync() { - for e := kvc.keys.Front(); e != nil; e = e.Next() { - key := e.Value.([]byte) - value := kvc.cache[string(key)] - kvc.store.Set(key, value.v) - } - kvc.Reset() +// Discard will remove reference to this +func (c *MemKVCache) Discard() { + c.cache = NewMemKVStore() } diff --git a/state/kvcache_test.go b/state/kvcache_test.go index 5af26b9190..b08940f3c0 100644 --- a/state/kvcache_test.go +++ b/state/kvcache_test.go @@ -68,7 +68,7 @@ func TestCache(t *testing.T) { db.Set(s.Key, s.Value) } for k, g := range tc.toGet { - msg := fmt.Sprintf("%d/%d/%d", i, j, k) + msg := fmt.Sprintf("%d/%d/%d: %#v", i, j, k, g) checkGet(db, g, msg) } for k, lq := range tc.toList { @@ -84,12 +84,12 @@ func TestCache(t *testing.T) { } for k, r := range tc.removeCache { val := cache.Remove(r.Key) - assert.EqualValues(r.Value, val, "%d/%d/%d", i, j, k) + assert.EqualValues(r.Value, val, "%d/%d/%d: %#v", i, j, k, r) } // make sure data is in cache for k, g := range tc.getCache { - msg := fmt.Sprintf("%d/%d/%d", i, j, k) + msg := fmt.Sprintf("%d/%d/%d: %#v", i, j, k, g) checkGet(cache, g, msg) } for k, lq := range tc.listCache { @@ -99,7 +99,7 @@ func TestCache(t *testing.T) { // data not in basic store for k, g := range tc.toGet { - msg := fmt.Sprintf("%d/%d/%d", i, j, k) + msg := fmt.Sprintf("%d/%d/%d: %#v", i, j, k, g) checkGet(db, g, msg) } for k, lq := range tc.toList { diff --git a/state/kvstore.go b/state/kvstore.go index 7e4907e143..6233dddb02 100644 --- a/state/kvstore.go +++ b/state/kvstore.go @@ -1,6 +1,7 @@ package state import ( + "errors" "sort" "github.com/tendermint/go-wire/data" @@ -56,7 +57,7 @@ type MemKVStore struct { m map[string][]byte } -// var _ SimpleDB = NewMemKVStore() +var _ SimpleDB = NewMemKVStore() // NewMemKVStore initializes a MemKVStore func NewMemKVStore() *MemKVStore { @@ -65,28 +66,27 @@ func NewMemKVStore() *MemKVStore { } } -func (mkv *MemKVStore) Set(key []byte, value []byte) { - mkv.m[string(key)] = value +func (m *MemKVStore) Set(key []byte, value []byte) { + m.m[string(key)] = value } -func (mkv *MemKVStore) Get(key []byte) (value []byte) { - return mkv.m[string(key)] +func (m *MemKVStore) Get(key []byte) (value []byte) { + return m.m[string(key)] } -func (mkv *MemKVStore) Has(key []byte) (has bool) { - _, ok := mkv.m[string(key)] +func (m *MemKVStore) Has(key []byte) (has bool) { + _, ok := m.m[string(key)] return ok } -func (mkv *MemKVStore) Remove(key []byte) (value []byte) { - val := mkv.m[string(key)] - delete(mkv.m, string(key)) +func (m *MemKVStore) Remove(key []byte) (value []byte) { + val := m.m[string(key)] + delete(m.m, string(key)) return val } -func (mkv *MemKVStore) List(start, end []byte, limit int) []Model { - keys := mkv.keysInRange(start, end) - sort.Strings(keys) +func (m *MemKVStore) List(start, end []byte, limit int) []Model { + keys := m.keysInRange(start, end) if limit > 0 && len(keys) > 0 { if limit > len(keys) { limit = len(keys) @@ -98,16 +98,16 @@ func (mkv *MemKVStore) List(start, end []byte, limit int) []Model { for i, k := range keys { res[i] = Model{ Key: []byte(k), - Value: mkv.m[k], + Value: m.m[k], } } return res } // First iterates through all keys to find the one that matches -func (mkv *MemKVStore) First(start, end []byte) Model { +func (m *MemKVStore) First(start, end []byte) Model { key := "" - for _, k := range mkv.keysInRange(start, end) { + for _, k := range m.keysInRange(start, end) { if key == "" || k < key { key = k } @@ -117,13 +117,13 @@ func (mkv *MemKVStore) First(start, end []byte) Model { } return Model{ Key: []byte(key), - Value: mkv.m[key], + Value: m.m[key], } } -func (mkv *MemKVStore) Last(start, end []byte) Model { +func (m *MemKVStore) Last(start, end []byte) Model { key := "" - for _, k := range mkv.keysInRange(start, end) { + for _, k := range m.keysInRange(start, end) { if key == "" || k > key { key = k } @@ -133,20 +133,39 @@ func (mkv *MemKVStore) Last(start, end []byte) Model { } return Model{ Key: []byte(key), - Value: mkv.m[key], + Value: m.m[key], } } -func (mkv *MemKVStore) Discard() { - mkv.m = make(map[string][]byte, 0) +func (m *MemKVStore) Discard() { + m.m = make(map[string][]byte, 0) } -func (mkv *MemKVStore) keysInRange(start, end []byte) (res []string) { +func (m *MemKVStore) Checkpoint() SimpleDB { + return NewMemKVCache(m) +} + +func (m *MemKVStore) Commit(sub SimpleDB) error { + cache, ok := sub.(*MemKVCache) + if !ok { + return errors.New("sub is not a cache") + } + // TODO: see if it points to us + + // apply the cached data to us + cache.applyCache() + return nil +} + +func (m *MemKVStore) keysInRange(start, end []byte) (res []string) { s, e := string(start), string(end) - for k := range mkv.m { - if k >= s && k < e { + for k := range m.m { + afterStart := s == "" || k >= s + beforeEnd := e == "" || k < e + if afterStart && beforeEnd { res = append(res, k) } } + sort.Strings(res) return } diff --git a/state/store_test.go b/state/store_test.go index 300cec58ce..d479b81ef4 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -4,13 +4,24 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/tendermint/merkleeyes/iavl" + // dbm "github.com/tendermint/tmlibs/db" ) func GetDBs() []SimpleDB { + // // tree with persistence.... + // tmpDir, err := ioutil.TempDir("", "state-tests") + // if err != nil { + // panic(err) + // } + // db := dbm.NewDB("test-get-dbs", dbm.LevelDBBackendStr, tmpDir) + // persist := iavl.NewIAVLTree(500, db) + return []SimpleDB{ - // NewMemKVStore(), + NewMemKVStore(), NewBonsai(iavl.NewIAVLTree(0, nil)), + // NewBonsai(persist), } } From 51a29e4bb73d3745794d3ad22586e71556c3e23f Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 19:55:05 -0400 Subject: [PATCH 13/18] Ported everything to SimpleDB interface --- app/app.go | 2 +- app/app_test.go | 4 +- docs/guide/counter/plugins/counter/counter.go | 8 +- glide.lock | 6 +- handler.go | 30 +++---- modules/auth/signature.go | 4 +- modules/base/chain.go | 4 +- modules/base/logger.go | 6 +- modules/base/multiplexer.go | 6 +- modules/coin/handler.go | 6 +- modules/coin/store.go | 12 +-- modules/fee/handler.go | 6 +- modules/nonce/replaycheck.go | 6 +- modules/nonce/store.go | 4 +- modules/nonce/tx.go | 2 +- modules/roles/handler.go | 4 +- modules/roles/middleware.go | 6 +- modules/roles/middleware_test.go | 2 +- modules/roles/store.go | 6 +- plugins/ibc/ibc.go | 26 +++--- stack/checkpoint.go | 12 +-- stack/context.go | 4 +- stack/dispatcher.go | 6 +- stack/helpers.go | 20 ++--- stack/helperware.go | 8 +- stack/interface.go | 32 +++---- stack/middleware.go | 6 +- stack/prefixstore.go | 84 +++++++++++++++++-- stack/recovery.go | 6 +- stack/state_space_test.go | 12 +-- state/store_test.go | 19 +++-- 31 files changed, 216 insertions(+), 143 deletions(-) diff --git a/app/app.go b/app/app.go index cd59e66dfd..5fdfd5d426 100644 --- a/app/app.go +++ b/app/app.go @@ -78,7 +78,7 @@ func (app *Basecoin) GetChainID() string { } // GetState is back... please kill me -func (app *Basecoin) GetState() sm.KVStore { +func (app *Basecoin) GetState() sm.SimpleDB { return app.state.Append() } diff --git a/app/app_test.go b/app/app_test.go index a81cf4e948..49e16e9af5 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -100,13 +100,13 @@ func (at *appTest) reset() { require.True(at.t, resabci.IsOK(), resabci) } -func getBalance(key basecoin.Actor, store state.KVStore) (coin.Coins, error) { +func getBalance(key basecoin.Actor, store state.SimpleDB) (coin.Coins, error) { cspace := stack.PrefixedStore(coin.NameCoin, store) acct, err := coin.GetAccount(cspace, key) return acct.Coins, err } -func getAddr(addr []byte, state state.KVStore) (coin.Coins, error) { +func getAddr(addr []byte, state state.SimpleDB) (coin.Coins, error) { actor := auth.SigPerm(addr) return getBalance(actor, state) } diff --git a/docs/guide/counter/plugins/counter/counter.go b/docs/guide/counter/plugins/counter/counter.go index 1cf2be24ac..16a10650ca 100644 --- a/docs/guide/counter/plugins/counter/counter.go +++ b/docs/guide/counter/plugins/counter/counter.go @@ -125,13 +125,13 @@ func (Handler) Name() string { func (Handler) AssertDispatcher() {} // CheckTx checks if the tx is properly structured -func (h Handler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, _ basecoin.Checker) (res basecoin.Result, err error) { +func (h Handler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, _ basecoin.Checker) (res basecoin.Result, err error) { _, err = checkTx(ctx, tx) return } // DeliverTx executes the tx if valid -func (h Handler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, dispatch basecoin.Deliver) (res basecoin.Result, err error) { +func (h Handler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, dispatch basecoin.Deliver) (res basecoin.Result, err error) { ctr, err := checkTx(ctx, tx) if err != nil { return res, err @@ -203,7 +203,7 @@ func StateKey() []byte { } // LoadState - retrieve the counter state from the store -func LoadState(store state.KVStore) (state State, err error) { +func LoadState(store state.SimpleDB) (state State, err error) { bytes := store.Get(StateKey()) if len(bytes) > 0 { err = wire.ReadBinaryBytes(bytes, &state) @@ -215,7 +215,7 @@ func LoadState(store state.KVStore) (state State, err error) { } // SaveState - save the counter state to the provided store -func SaveState(store state.KVStore, state State) error { +func SaveState(store state.SimpleDB, state State) error { bytes := wire.BinaryBytes(state) store.Set(StateKey(), bytes) return nil diff --git a/glide.lock b/glide.lock index 775e0faada..f68a89d0c4 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 45eed61138603d4d03518ea822068cf32b45d0a219bb7f3b836e52129f2a3a2b -updated: 2017-07-26T16:26:38.678294279-04:00 +updated: 2017-07-26T19:44:39.753066441-04:00 imports: - name: github.com/bgentry/speakeasy version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd @@ -105,7 +105,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 864d1f80b36b440bde030a5c18d8ac3aa8c2949d + version: 7f5f48b6b9ec3964de4b07b6c3cd05d7c91aeee5 subpackages: - client - example/dummy @@ -140,7 +140,7 @@ imports: - certifiers/files - proofs - name: github.com/tendermint/merkleeyes - version: fef0a1fc729f5b50c58a2dce04b4525aa06c2247 + version: 0310013053953eef80def3619aeb1e3a3254f452 subpackages: - client - iavl diff --git a/handler.go b/handler.go index 00fee085f7..7b028716ac 100644 --- a/handler.go +++ b/handler.go @@ -15,9 +15,9 @@ type Handler interface { SetOptioner Named // TODO: flesh these out as well - // InitChain(store state.KVStore, vals []*abci.Validator) - // BeginBlock(store state.KVStore, hash []byte, header *abci.Header) - // EndBlock(store state.KVStore, height uint64) abci.ResponseEndBlock + // InitChain(store state.SimpleDB, vals []*abci.Validator) + // BeginBlock(store state.SimpleDB, hash []byte, header *abci.Header) + // EndBlock(store state.SimpleDB, height uint64) abci.ResponseEndBlock } type Named interface { @@ -25,35 +25,35 @@ type Named interface { } type Checker interface { - CheckTx(ctx Context, store state.KVStore, tx Tx) (Result, error) + CheckTx(ctx Context, store state.SimpleDB, tx Tx) (Result, error) } // CheckerFunc (like http.HandlerFunc) is a shortcut for making wrapers -type CheckerFunc func(Context, state.KVStore, Tx) (Result, error) +type CheckerFunc func(Context, state.SimpleDB, Tx) (Result, error) -func (c CheckerFunc) CheckTx(ctx Context, store state.KVStore, tx Tx) (Result, error) { +func (c CheckerFunc) CheckTx(ctx Context, store state.SimpleDB, tx Tx) (Result, error) { return c(ctx, store, tx) } type Deliver interface { - DeliverTx(ctx Context, store state.KVStore, tx Tx) (Result, error) + DeliverTx(ctx Context, store state.SimpleDB, tx Tx) (Result, error) } // DeliverFunc (like http.HandlerFunc) is a shortcut for making wrapers -type DeliverFunc func(Context, state.KVStore, Tx) (Result, error) +type DeliverFunc func(Context, state.SimpleDB, Tx) (Result, error) -func (c DeliverFunc) DeliverTx(ctx Context, store state.KVStore, tx Tx) (Result, error) { +func (c DeliverFunc) DeliverTx(ctx Context, store state.SimpleDB, tx Tx) (Result, error) { return c(ctx, store, tx) } type SetOptioner interface { - SetOption(l log.Logger, store state.KVStore, module, key, value string) (string, error) + SetOption(l log.Logger, store state.SimpleDB, module, key, value string) (string, error) } // SetOptionFunc (like http.HandlerFunc) is a shortcut for making wrapers -type SetOptionFunc func(log.Logger, state.KVStore, string, string, string) (string, error) +type SetOptionFunc func(log.Logger, state.SimpleDB, string, string, string) (string, error) -func (c SetOptionFunc) SetOption(l log.Logger, store state.KVStore, module, key, value string) (string, error) { +func (c SetOptionFunc) SetOption(l log.Logger, store state.SimpleDB, module, key, value string) (string, error) { return c(l, store, module, key, value) } @@ -75,14 +75,14 @@ func (r Result) ToABCI() abci.Result { // holders type NopCheck struct{} -func (_ NopCheck) CheckTx(Context, state.KVStore, Tx) (r Result, e error) { return } +func (_ NopCheck) CheckTx(Context, state.SimpleDB, Tx) (r Result, e error) { return } type NopDeliver struct{} -func (_ NopDeliver) DeliverTx(Context, state.KVStore, Tx) (r Result, e error) { return } +func (_ NopDeliver) DeliverTx(Context, state.SimpleDB, Tx) (r Result, e error) { return } type NopOption struct{} -func (_ NopOption) SetOption(log.Logger, state.KVStore, string, string, string) (string, error) { +func (_ NopOption) SetOption(log.Logger, state.SimpleDB, string, string, string) (string, error) { return "", nil } diff --git a/modules/auth/signature.go b/modules/auth/signature.go index 8ea31bde6e..c80dc6998d 100644 --- a/modules/auth/signature.go +++ b/modules/auth/signature.go @@ -39,7 +39,7 @@ type Signable interface { } // CheckTx verifies the signatures are correct - fulfills Middlware interface -func (Signatures) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (Signatures) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { sigs, tnext, err := getSigners(tx) if err != nil { return res, err @@ -49,7 +49,7 @@ func (Signatures) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin } // DeliverTx verifies the signatures are correct - fulfills Middlware interface -func (Signatures) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (Signatures) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { sigs, tnext, err := getSigners(tx) if err != nil { return res, err diff --git a/modules/base/chain.go b/modules/base/chain.go index 9dc704c362..1068fba572 100644 --- a/modules/base/chain.go +++ b/modules/base/chain.go @@ -24,7 +24,7 @@ func (Chain) Name() string { var _ stack.Middleware = Chain{} // CheckTx makes sure we are on the proper chain - fulfills Middlware interface -func (c Chain) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (c Chain) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { stx, err := c.checkChainTx(ctx.ChainID(), ctx.BlockHeight(), tx) if err != nil { return res, err @@ -33,7 +33,7 @@ func (c Chain) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx } // DeliverTx makes sure we are on the proper chain - fulfills Middlware interface -func (c Chain) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (c Chain) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { stx, err := c.checkChainTx(ctx.ChainID(), ctx.BlockHeight(), tx) if err != nil { return res, err diff --git a/modules/base/logger.go b/modules/base/logger.go index 23b595645f..2cd888386d 100644 --- a/modules/base/logger.go +++ b/modules/base/logger.go @@ -26,7 +26,7 @@ func (Logger) Name() string { var _ stack.Middleware = Logger{} // CheckTx logs time and result - fulfills Middlware interface -func (Logger) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (Logger) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { start := time.Now() res, err = next.CheckTx(ctx, store, tx) delta := time.Now().Sub(start) @@ -41,7 +41,7 @@ func (Logger) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, } // DeliverTx logs time and result - fulfills Middlware interface -func (Logger) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (Logger) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { start := time.Now() res, err = next.DeliverTx(ctx, store, tx) delta := time.Now().Sub(start) @@ -56,7 +56,7 @@ func (Logger) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.T } // SetOption logs time and result - fulfills Middlware interface -func (Logger) SetOption(l log.Logger, store state.KVStore, module, key, value string, next basecoin.SetOptioner) (string, error) { +func (Logger) SetOption(l log.Logger, store state.SimpleDB, module, key, value string, next basecoin.SetOptioner) (string, error) { start := time.Now() res, err := next.SetOption(l, store, module, key, value) delta := time.Now().Sub(start) diff --git a/modules/base/multiplexer.go b/modules/base/multiplexer.go index bf2293ffdc..92f3e9c188 100644 --- a/modules/base/multiplexer.go +++ b/modules/base/multiplexer.go @@ -29,7 +29,7 @@ func (Multiplexer) Name() string { var _ stack.Middleware = Multiplexer{} // CheckTx splits the input tx and checks them all - fulfills Middlware interface -func (Multiplexer) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (Multiplexer) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { if mtx, ok := tx.Unwrap().(*MultiTx); ok { return runAll(ctx, store, mtx.Txs, next.CheckTx) } @@ -37,14 +37,14 @@ func (Multiplexer) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoi } // DeliverTx splits the input tx and checks them all - fulfills Middlware interface -func (Multiplexer) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (Multiplexer) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { if mtx, ok := tx.Unwrap().(*MultiTx); ok { return runAll(ctx, store, mtx.Txs, next.DeliverTx) } return next.DeliverTx(ctx, store, tx) } -func runAll(ctx basecoin.Context, store state.KVStore, txs []basecoin.Tx, next basecoin.CheckerFunc) (res basecoin.Result, err error) { +func runAll(ctx basecoin.Context, store state.SimpleDB, txs []basecoin.Tx, next basecoin.CheckerFunc) (res basecoin.Result, err error) { // store all results, unless anything errors rs := make([]basecoin.Result, len(txs)) for i, stx := range txs { diff --git a/modules/coin/handler.go b/modules/coin/handler.go index ef3f3ae0f6..08fc2e5396 100644 --- a/modules/coin/handler.go +++ b/modules/coin/handler.go @@ -29,7 +29,7 @@ func (Handler) Name() string { } // CheckTx checks if there is enough money in the account -func (h Handler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (h Handler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { send, err := checkTx(ctx, tx) if err != nil { return res, err @@ -48,7 +48,7 @@ func (h Handler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin. } // DeliverTx moves the money -func (h Handler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (h Handler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { send, err := checkTx(ctx, tx) if err != nil { return res, err @@ -75,7 +75,7 @@ func (h Handler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoi } // SetOption - sets the genesis account balance -func (h Handler) SetOption(l log.Logger, store state.KVStore, module, key, value string) (log string, err error) { +func (h Handler) SetOption(l log.Logger, store state.SimpleDB, module, key, value string) (log string, err error) { if module != NameCoin { return "", errors.ErrUnknownModule(module) } diff --git a/modules/coin/store.go b/modules/coin/store.go index 516e7acea9..4d1d4c1090 100644 --- a/modules/coin/store.go +++ b/modules/coin/store.go @@ -11,7 +11,7 @@ import ( ) // GetAccount - Get account from store and address -func GetAccount(store state.KVStore, addr basecoin.Actor) (Account, error) { +func GetAccount(store state.SimpleDB, addr basecoin.Actor) (Account, error) { acct, err := loadAccount(store, addr.Bytes()) // for empty accounts, don't return an error, but rather an empty account @@ -22,13 +22,13 @@ func GetAccount(store state.KVStore, addr basecoin.Actor) (Account, error) { } // CheckCoins makes sure there are funds, but doesn't change anything -func CheckCoins(store state.KVStore, addr basecoin.Actor, coins Coins) (Coins, error) { +func CheckCoins(store state.SimpleDB, addr basecoin.Actor, coins Coins) (Coins, error) { acct, err := updateCoins(store, addr, coins) return acct.Coins, err } // ChangeCoins changes the money, returns error if it would be negative -func ChangeCoins(store state.KVStore, addr basecoin.Actor, coins Coins) (Coins, error) { +func ChangeCoins(store state.SimpleDB, addr basecoin.Actor, coins Coins) (Coins, error) { acct, err := updateCoins(store, addr, coins) if err != nil { return acct.Coins, err @@ -41,7 +41,7 @@ func ChangeCoins(store state.KVStore, addr basecoin.Actor, coins Coins) (Coins, // updateCoins will load the account, make all checks, and return the updated account. // // it doesn't save anything, that is up to you to decide (Check/Change Coins) -func updateCoins(store state.KVStore, addr basecoin.Actor, coins Coins) (acct Account, err error) { +func updateCoins(store state.SimpleDB, addr basecoin.Actor, coins Coins) (acct Account, err error) { acct, err = loadAccount(store, addr.Bytes()) // we can increase an empty account... if IsNoAccountErr(err) && coins.IsPositive() { @@ -66,7 +66,7 @@ type Account struct { Coins Coins `json:"coins"` } -func loadAccount(store state.KVStore, key []byte) (acct Account, err error) { +func loadAccount(store state.SimpleDB, key []byte) (acct Account, err error) { // fmt.Printf("load: %X\n", key) data := store.Get(key) if len(data) == 0 { @@ -80,7 +80,7 @@ func loadAccount(store state.KVStore, key []byte) (acct Account, err error) { return acct, nil } -func storeAccount(store state.KVStore, key []byte, acct Account) error { +func storeAccount(store state.SimpleDB, key []byte, acct Account) error { // fmt.Printf("store: %X\n", key) bin := wire.BinaryBytes(acct) store.Set(key, bin) diff --git a/modules/fee/handler.go b/modules/fee/handler.go index 4e12a15a65..19e9f9f8fb 100644 --- a/modules/fee/handler.go +++ b/modules/fee/handler.go @@ -45,16 +45,16 @@ func (SimpleFeeMiddleware) Name() string { } // CheckTx - check the transaction -func (h SimpleFeeMiddleware) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (h SimpleFeeMiddleware) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { return h.doTx(ctx, store, tx, next.CheckTx) } // DeliverTx - send the fee handler transaction -func (h SimpleFeeMiddleware) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (h SimpleFeeMiddleware) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { return h.doTx(ctx, store, tx, next.DeliverTx) } -func (h SimpleFeeMiddleware) doTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.CheckerFunc) (res basecoin.Result, err error) { +func (h SimpleFeeMiddleware) doTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.CheckerFunc) (res basecoin.Result, err error) { feeTx, ok := tx.Unwrap().(Fee) if !ok { // the fee wrapper is not required if there is no minimum diff --git a/modules/nonce/replaycheck.go b/modules/nonce/replaycheck.go index 4494a2b109..bc57ded966 100644 --- a/modules/nonce/replaycheck.go +++ b/modules/nonce/replaycheck.go @@ -24,7 +24,7 @@ func (ReplayCheck) Name() string { var _ stack.Middleware = ReplayCheck{} // CheckTx verifies tx is not being replayed - fulfills Middlware interface -func (r ReplayCheck) CheckTx(ctx basecoin.Context, store state.KVStore, +func (r ReplayCheck) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { stx, err := r.checkIncrementNonceTx(ctx, store, tx) @@ -38,7 +38,7 @@ func (r ReplayCheck) CheckTx(ctx basecoin.Context, store state.KVStore, // DeliverTx verifies tx is not being replayed - fulfills Middlware interface // NOTE It is okay to modify the sequence before running the wrapped TX because if the // wrapped Tx fails, the state changes are not applied -func (r ReplayCheck) DeliverTx(ctx basecoin.Context, store state.KVStore, +func (r ReplayCheck) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { stx, err := r.checkIncrementNonceTx(ctx, store, tx) @@ -50,7 +50,7 @@ func (r ReplayCheck) DeliverTx(ctx basecoin.Context, store state.KVStore, } // checkNonceTx varifies the nonce sequence, an increment sequence number -func (r ReplayCheck) checkIncrementNonceTx(ctx basecoin.Context, store state.KVStore, +func (r ReplayCheck) checkIncrementNonceTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (basecoin.Tx, error) { // make sure it is a the nonce Tx (Tx from this package) diff --git a/modules/nonce/store.go b/modules/nonce/store.go index ae10ccaf66..bb5fca27d9 100644 --- a/modules/nonce/store.go +++ b/modules/nonce/store.go @@ -9,7 +9,7 @@ import ( "github.com/tendermint/basecoin/state" ) -func getSeq(store state.KVStore, key []byte) (seq uint32, err error) { +func getSeq(store state.SimpleDB, key []byte) (seq uint32, err error) { data := store.Get(key) if len(data) == 0 { //if the key is not stored, its a new key with a sequence of zero! @@ -23,7 +23,7 @@ func getSeq(store state.KVStore, key []byte) (seq uint32, err error) { return seq, nil } -func setSeq(store state.KVStore, key []byte, seq uint32) error { +func setSeq(store state.SimpleDB, key []byte, seq uint32) error { bin := wire.BinaryBytes(seq) store.Set(key, bin) return nil // real stores can return error... diff --git a/modules/nonce/tx.go b/modules/nonce/tx.go index 6fb5ce7fd2..c0ff41da0e 100644 --- a/modules/nonce/tx.go +++ b/modules/nonce/tx.go @@ -62,7 +62,7 @@ func (n Tx) ValidateBasic() error { // and further increment the sequence number // NOTE It is okay to modify the sequence before running the wrapped TX because if the // wrapped Tx fails, the state changes are not applied -func (n Tx) CheckIncrementSeq(ctx basecoin.Context, store state.KVStore) error { +func (n Tx) CheckIncrementSeq(ctx basecoin.Context, store state.SimpleDB) error { seqKey := n.getSeqKey() diff --git a/modules/roles/handler.go b/modules/roles/handler.go index 0a31eb6eba..c6f785c14d 100644 --- a/modules/roles/handler.go +++ b/modules/roles/handler.go @@ -27,7 +27,7 @@ func (Handler) Name() string { } // CheckTx verifies if the transaction is properly formated -func (h Handler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (h Handler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { var cr CreateRoleTx cr, err = checkTx(ctx, tx) if err != nil { @@ -40,7 +40,7 @@ func (h Handler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin. // DeliverTx tries to create a new role. // // Returns an error if the role already exists -func (h Handler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (h Handler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { create, err := checkTx(ctx, tx) if err != nil { return res, err diff --git a/modules/roles/middleware.go b/modules/roles/middleware.go index 78bc4b1b30..296f4fa1a6 100644 --- a/modules/roles/middleware.go +++ b/modules/roles/middleware.go @@ -27,7 +27,7 @@ func (Middleware) Name() string { // CheckTx tries to assume the named role if requested. // If no role is requested, do nothing. // If insufficient authority to assume the role, return error. -func (m Middleware) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (m Middleware) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { // if this is not an AssumeRoleTx, then continue assume, ok := tx.Unwrap().(AssumeRoleTx) if !ok { // this also breaks the recursion below @@ -46,7 +46,7 @@ func (m Middleware) CheckTx(ctx basecoin.Context, store state.KVStore, tx baseco // DeliverTx tries to assume the named role if requested. // If no role is requested, do nothing. // If insufficient authority to assume the role, return error. -func (m Middleware) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (m Middleware) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { // if this is not an AssumeRoleTx, then continue assume, ok := tx.Unwrap().(AssumeRoleTx) if !ok { // this also breaks the recursion below @@ -62,7 +62,7 @@ func (m Middleware) DeliverTx(ctx basecoin.Context, store state.KVStore, tx base return m.DeliverTx(ctx, store, assume.Tx, next) } -func assumeRole(ctx basecoin.Context, store state.KVStore, assume AssumeRoleTx) (basecoin.Context, error) { +func assumeRole(ctx basecoin.Context, store state.SimpleDB, assume AssumeRoleTx) (basecoin.Context, error) { err := assume.ValidateBasic() if err != nil { return nil, err diff --git a/modules/roles/middleware_test.go b/modules/roles/middleware_test.go index 69f60a1ca7..14e792b1ad 100644 --- a/modules/roles/middleware_test.go +++ b/modules/roles/middleware_test.go @@ -17,7 +17,7 @@ import ( // shortcut for the lazy type ba []basecoin.Actor -func createRole(app basecoin.Handler, store state.KVStore, +func createRole(app basecoin.Handler, store state.SimpleDB, name []byte, min uint32, sigs ...basecoin.Actor) (basecoin.Actor, error) { tx := roles.NewCreateRoleTx(name, min, sigs) ctx := stack.MockContext("foo", 1) diff --git a/modules/roles/store.go b/modules/roles/store.go index eb114ff6f2..e9ef0c9673 100644 --- a/modules/roles/store.go +++ b/modules/roles/store.go @@ -56,7 +56,7 @@ func (r Role) IsAuthorized(ctx basecoin.Context) bool { return false } -func loadRole(store state.KVStore, key []byte) (role Role, err error) { +func loadRole(store state.SimpleDB, key []byte) (role Role, err error) { data := store.Get(key) if len(data) == 0 { return role, ErrNoRole() @@ -69,7 +69,7 @@ func loadRole(store state.KVStore, key []byte) (role Role, err error) { return role, nil } -func checkNoRole(store state.KVStore, key []byte) error { +func checkNoRole(store state.SimpleDB, key []byte) error { if _, err := loadRole(store, key); !IsNoRoleErr(err) { return ErrRoleExists() } @@ -77,7 +77,7 @@ func checkNoRole(store state.KVStore, key []byte) error { } // we only have create here, no update, since we don't allow update yet -func createRole(store state.KVStore, key []byte, role Role) error { +func createRole(store state.SimpleDB, key []byte, role Role) error { if err := checkNoRole(store, key); err != nil { return err } diff --git a/plugins/ibc/ibc.go b/plugins/ibc/ibc.go index 822638584f..0117a101ef 100644 --- a/plugins/ibc/ibc.go +++ b/plugins/ibc/ibc.go @@ -72,7 +72,7 @@ package ibc // // GetSequenceNumber gets the sequence number for packets being sent from the src chain to the dst chain. // // The sequence number counts how many packets have been sent. // // The next packet must include the latest sequence number. -// func GetSequenceNumber(store state.KVStore, src, dst string) uint64 { +// func GetSequenceNumber(store state.SimpleDB, src, dst string) uint64 { // sequenceKey := toKey(_IBC, _EGRESS, src, dst) // seqBytes := store.Get(sequenceKey) // if seqBytes == nil { @@ -86,14 +86,14 @@ package ibc // } // // SetSequenceNumber sets the sequence number for packets being sent from the src chain to the dst chain -// func SetSequenceNumber(store state.KVStore, src, dst string, seq uint64) { +// func SetSequenceNumber(store state.SimpleDB, src, dst string, seq uint64) { // sequenceKey := toKey(_IBC, _EGRESS, src, dst) // store.Set(sequenceKey, []byte(strconv.FormatUint(seq, 10))) // } // // SaveNewIBCPacket creates an IBC packet with the given payload from the src chain to the dst chain // // using the correct sequence number. It also increments the sequence number by 1 -// func SaveNewIBCPacket(state state.KVStore, src, dst string, payload Payload) { +// func SaveNewIBCPacket(state state.SimpleDB, src, dst string, payload Payload) { // // fetch sequence number and increment by 1 // seq := GetSequenceNumber(state, src, dst) // SetSequenceNumber(state, src, dst, seq+1) @@ -104,7 +104,7 @@ package ibc // save(state, packetKey, packet) // } -// func GetIBCPacket(state state.KVStore, src, dst string, seq uint64) (Packet, error) { +// func GetIBCPacket(state state.SimpleDB, src, dst string, seq uint64) (Packet, error) { // packetKey := toKey(_IBC, _EGRESS, src, dst, cmn.Fmt("%v", seq)) // packetBytes := state.Get(packetKey) @@ -251,11 +251,11 @@ package ibc // return &IBCPlugin{} // } -// func (ibc *IBCPlugin) SetOption(store state.KVStore, key string, value string) (log string) { +// func (ibc *IBCPlugin) SetOption(store state.SimpleDB, key string, value string) (log string) { // return "" // } -// func (ibc *IBCPlugin) RunTx(store state.KVStore, ctx types.CallContext, txBytes []byte) (res abci.Result) { +// func (ibc *IBCPlugin) RunTx(store state.SimpleDB, ctx types.CallContext, txBytes []byte) (res abci.Result) { // // Decode tx // var tx IBCTx // err := wire.ReadBinaryBytes(txBytes, &tx) @@ -295,7 +295,7 @@ package ibc // } // type IBCStateMachine struct { -// store state.KVStore +// store state.SimpleDB // ctx types.CallContext // res abci.Result // } @@ -499,13 +499,13 @@ package ibc // return // } -// func (ibc *IBCPlugin) InitChain(store state.KVStore, vals []*abci.Validator) { +// func (ibc *IBCPlugin) InitChain(store state.SimpleDB, vals []*abci.Validator) { // } -// func (cp *IBCPlugin) BeginBlock(store state.KVStore, hash []byte, header *abci.Header) { +// func (cp *IBCPlugin) BeginBlock(store state.SimpleDB, hash []byte, header *abci.Header) { // } -// func (cp *IBCPlugin) EndBlock(store state.KVStore, height uint64) (res abci.ResponseEndBlock) { +// func (cp *IBCPlugin) EndBlock(store state.SimpleDB, height uint64) (res abci.ResponseEndBlock) { // return // } @@ -513,7 +513,7 @@ package ibc // // TODO: move to utils // // Returns true if exists, false if nil. -// func exists(store state.KVStore, key []byte) (exists bool) { +// func exists(store state.SimpleDB, key []byte) (exists bool) { // value := store.Get(key) // return len(value) > 0 // } @@ -521,7 +521,7 @@ package ibc // // Load bytes from store by reading value for key and read into ptr. // // Returns true if exists, false if nil. // // Returns err if decoding error. -// func load(store state.KVStore, key []byte, ptr interface{}) (exists bool, err error) { +// func load(store state.SimpleDB, key []byte, ptr interface{}) (exists bool, err error) { // value := store.Get(key) // if len(value) > 0 { // err = wire.ReadBinaryBytes(value, ptr) @@ -537,7 +537,7 @@ package ibc // } // // Save bytes to store by writing obj's go-wire binary bytes. -// func save(store state.KVStore, key []byte, obj interface{}) { +// func save(store state.SimpleDB, key []byte, obj interface{}) { // store.Set(key, wire.BinaryBytes(obj)) // } diff --git a/stack/checkpoint.go b/stack/checkpoint.go index 89f67bd8ce..00fbe61a17 100644 --- a/stack/checkpoint.go +++ b/stack/checkpoint.go @@ -25,27 +25,27 @@ func (Checkpoint) Name() string { var _ Middleware = Checkpoint{} // CheckTx reverts all data changes if there was an error -func (c Checkpoint) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (c Checkpoint) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { if !c.OnCheck { return next.CheckTx(ctx, store, tx) } - ps := state.NewKVCache(unwrap(store)) + ps := store.Checkpoint() res, err = next.CheckTx(ctx, ps, tx) if err == nil { - ps.Sync() + err = store.Commit(ps) } return res, err } // DeliverTx reverts all data changes if there was an error -func (c Checkpoint) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (c Checkpoint) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { if !c.OnDeliver { return next.DeliverTx(ctx, store, tx) } - ps := state.NewKVCache(unwrap(store)) + ps := store.Checkpoint() res, err = next.DeliverTx(ctx, ps, tx) if err == nil { - ps.Sync() + err = store.Commit(ps) } return res, err } diff --git a/stack/context.go b/stack/context.go index 0bcc0e4ecf..e688a5a082 100644 --- a/stack/context.go +++ b/stack/context.go @@ -76,7 +76,7 @@ func withApp(ctx basecoin.Context, app string) basecoin.Context { } func secureCheck(h basecoin.Checker, parent basecoin.Context) basecoin.Checker { - next := func(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { + next := func(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { if !parent.IsParent(ctx) { return res, errors.New("Passing in non-child Context") } @@ -86,7 +86,7 @@ func secureCheck(h basecoin.Checker, parent basecoin.Context) basecoin.Checker { } func secureDeliver(h basecoin.Deliver, parent basecoin.Context) basecoin.Deliver { - next := func(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { + next := func(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { if !parent.IsParent(ctx) { return res, errors.New("Passing in non-child Context") } diff --git a/stack/dispatcher.go b/stack/dispatcher.go index 11090fafa0..a8ff96c483 100644 --- a/stack/dispatcher.go +++ b/stack/dispatcher.go @@ -64,7 +64,7 @@ func (d *Dispatcher) Name() string { // Tries to find a registered module (Dispatchable) based on the name of the tx. // The tx name (as registered with go-data) should be in the form `/XXXX`, // where `module name` must match the name of a dispatchable and XXX can be any string. -func (d *Dispatcher) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (d *Dispatcher) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { r, err := d.lookupTx(tx) if err != nil { return res, err @@ -85,7 +85,7 @@ func (d *Dispatcher) CheckTx(ctx basecoin.Context, store state.KVStore, tx basec // Tries to find a registered module (Dispatchable) based on the name of the tx. // The tx name (as registered with go-data) should be in the form `/XXXX`, // where `module name` must match the name of a dispatchable and XXX can be any string. -func (d *Dispatcher) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (d *Dispatcher) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { r, err := d.lookupTx(tx) if err != nil { return res, err @@ -105,7 +105,7 @@ func (d *Dispatcher) DeliverTx(ctx basecoin.Context, store state.KVStore, tx bas // // Tries to find a registered module (Dispatchable) based on the // module name from SetOption of the tx. -func (d *Dispatcher) SetOption(l log.Logger, store state.KVStore, module, key, value string) (string, error) { +func (d *Dispatcher) SetOption(l log.Logger, store state.SimpleDB, module, key, value string) (string, error) { r, err := d.lookupModule(module) if err != nil { return "", err diff --git a/stack/helpers.go b/stack/helpers.go index 7405c16dda..eaee305750 100644 --- a/stack/helpers.go +++ b/stack/helpers.go @@ -86,12 +86,12 @@ func (OKHandler) Name() string { } // CheckTx always returns an empty success tx -func (ok OKHandler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (ok OKHandler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { return basecoin.Result{Log: ok.Log}, nil } // DeliverTx always returns an empty success tx -func (ok OKHandler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (ok OKHandler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { return basecoin.Result{Log: ok.Log}, nil } @@ -108,13 +108,13 @@ func (EchoHandler) Name() string { } // CheckTx always returns an empty success tx -func (EchoHandler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (EchoHandler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { data, err := data.ToWire(tx) return basecoin.Result{Data: data}, err } // DeliverTx always returns an empty success tx -func (EchoHandler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (EchoHandler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { data, err := data.ToWire(tx) return basecoin.Result{Data: data}, err } @@ -133,12 +133,12 @@ func (FailHandler) Name() string { } // CheckTx always returns the given error -func (f FailHandler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (f FailHandler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { return res, errors.Wrap(f.Err) } // DeliverTx always returns the given error -func (f FailHandler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (f FailHandler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { return res, errors.Wrap(f.Err) } @@ -157,7 +157,7 @@ func (PanicHandler) Name() string { } // CheckTx always panics -func (p PanicHandler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (p PanicHandler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { if p.Err != nil { panic(p.Err) } @@ -165,7 +165,7 @@ func (p PanicHandler) CheckTx(ctx basecoin.Context, store state.KVStore, tx base } // DeliverTx always panics -func (p PanicHandler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (p PanicHandler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { if p.Err != nil { panic(p.Err) } @@ -185,7 +185,7 @@ func (CheckHandler) Name() string { } // CheckTx verifies the permissions -func (c CheckHandler) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (c CheckHandler) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { check, ok := tx.Unwrap().(CheckTx) if !ok { return res, errors.ErrUnknownTxType(tx) @@ -200,7 +200,7 @@ func (c CheckHandler) CheckTx(ctx basecoin.Context, store state.KVStore, tx base } // DeliverTx verifies the permissions -func (c CheckHandler) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (c CheckHandler) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { // until something changes, just do the same as check return c.CheckTx(ctx, store, tx) } diff --git a/stack/helperware.go b/stack/helperware.go index 46cdfaa8cc..353f909075 100644 --- a/stack/helperware.go +++ b/stack/helperware.go @@ -25,14 +25,14 @@ func (_ CheckMiddleware) Name() string { return NameCheck } -func (p CheckMiddleware) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (p CheckMiddleware) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { if !ctx.HasPermission(p.Required) { return res, errors.ErrUnauthorized() } return next.CheckTx(ctx, store, tx) } -func (p CheckMiddleware) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (p CheckMiddleware) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { if !ctx.HasPermission(p.Required) { return res, errors.ErrUnauthorized() } @@ -51,12 +51,12 @@ func (_ GrantMiddleware) Name() string { return NameGrant } -func (g GrantMiddleware) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (g GrantMiddleware) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { ctx = ctx.WithPermissions(g.Auth) return next.CheckTx(ctx, store, tx) } -func (g GrantMiddleware) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (g GrantMiddleware) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { ctx = ctx.WithPermissions(g.Auth) return next.DeliverTx(ctx, store, tx) } diff --git a/stack/interface.go b/stack/interface.go index 10c818c318..fedf70401d 100644 --- a/stack/interface.go +++ b/stack/interface.go @@ -20,40 +20,40 @@ type Middleware interface { } type CheckerMiddle interface { - CheckTx(ctx basecoin.Context, store state.KVStore, + CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (basecoin.Result, error) } -type CheckerMiddleFunc func(basecoin.Context, state.KVStore, +type CheckerMiddleFunc func(basecoin.Context, state.SimpleDB, basecoin.Tx, basecoin.Checker) (basecoin.Result, error) -func (c CheckerMiddleFunc) CheckTx(ctx basecoin.Context, store state.KVStore, +func (c CheckerMiddleFunc) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (basecoin.Result, error) { return c(ctx, store, tx, next) } type DeliverMiddle interface { - DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, + DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (basecoin.Result, error) } -type DeliverMiddleFunc func(basecoin.Context, state.KVStore, +type DeliverMiddleFunc func(basecoin.Context, state.SimpleDB, basecoin.Tx, basecoin.Deliver) (basecoin.Result, error) -func (d DeliverMiddleFunc) DeliverTx(ctx basecoin.Context, store state.KVStore, +func (d DeliverMiddleFunc) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (basecoin.Result, error) { return d(ctx, store, tx, next) } type SetOptionMiddle interface { - SetOption(l log.Logger, store state.KVStore, module, + SetOption(l log.Logger, store state.SimpleDB, module, key, value string, next basecoin.SetOptioner) (string, error) } -type SetOptionMiddleFunc func(log.Logger, state.KVStore, +type SetOptionMiddleFunc func(log.Logger, state.SimpleDB, string, string, string, basecoin.SetOptioner) (string, error) -func (c SetOptionMiddleFunc) SetOption(l log.Logger, store state.KVStore, +func (c SetOptionMiddleFunc) SetOption(l log.Logger, store state.SimpleDB, module, key, value string, next basecoin.SetOptioner) (string, error) { return c(l, store, module, key, value, next) } @@ -61,28 +61,28 @@ func (c SetOptionMiddleFunc) SetOption(l log.Logger, store state.KVStore, // holders type PassCheck struct{} -func (_ PassCheck) CheckTx(ctx basecoin.Context, store state.KVStore, +func (_ PassCheck) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (basecoin.Result, error) { return next.CheckTx(ctx, store, tx) } type PassDeliver struct{} -func (_ PassDeliver) DeliverTx(ctx basecoin.Context, store state.KVStore, +func (_ PassDeliver) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (basecoin.Result, error) { return next.DeliverTx(ctx, store, tx) } type PassOption struct{} -func (_ PassOption) SetOption(l log.Logger, store state.KVStore, module, +func (_ PassOption) SetOption(l log.Logger, store state.SimpleDB, module, key, value string, next basecoin.SetOptioner) (string, error) { return next.SetOption(l, store, module, key, value) } type NopOption struct{} -func (_ NopOption) SetOption(l log.Logger, store state.KVStore, module, +func (_ NopOption) SetOption(l log.Logger, store state.SimpleDB, module, key, value string, next basecoin.SetOptioner) (string, error) { return "", nil } @@ -112,17 +112,17 @@ func (w wrapped) Name() string { return w.h.Name() } -func (w wrapped) CheckTx(ctx basecoin.Context, store state.KVStore, +func (w wrapped) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, _ basecoin.Checker) (basecoin.Result, error) { return w.h.CheckTx(ctx, store, tx) } -func (w wrapped) DeliverTx(ctx basecoin.Context, store state.KVStore, +func (w wrapped) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, _ basecoin.Deliver) (basecoin.Result, error) { return w.h.DeliverTx(ctx, store, tx) } -func (w wrapped) SetOption(l log.Logger, store state.KVStore, +func (w wrapped) SetOption(l log.Logger, store state.SimpleDB, module, key, value string, _ basecoin.SetOptioner) (string, error) { return w.h.SetOption(l, store, module, key, value) } diff --git a/stack/middleware.go b/stack/middleware.go index 9e1b8e75a1..e3400ff589 100644 --- a/stack/middleware.go +++ b/stack/middleware.go @@ -22,7 +22,7 @@ func (m *middleware) Name() string { } // CheckTx always returns an empty success tx -func (m *middleware) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (basecoin.Result, error) { +func (m *middleware) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (basecoin.Result, error) { // make sure we pass in proper context to child next := secureCheck(m.next, ctx) // set the permissions for this app @@ -33,7 +33,7 @@ func (m *middleware) CheckTx(ctx basecoin.Context, store state.KVStore, tx basec } // DeliverTx always returns an empty success tx -func (m *middleware) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx) (res basecoin.Result, err error) { +func (m *middleware) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (res basecoin.Result, err error) { // make sure we pass in proper context to child next := secureDeliver(m.next, ctx) // set the permissions for this app @@ -43,7 +43,7 @@ func (m *middleware) DeliverTx(ctx basecoin.Context, store state.KVStore, tx bas return m.middleware.DeliverTx(ctx, store, tx, next) } -func (m *middleware) SetOption(l log.Logger, store state.KVStore, module, key, value string) (string, error) { +func (m *middleware) SetOption(l log.Logger, store state.SimpleDB, module, key, value string) (string, error) { // set the namespace for the app store = stateSpace(store, m.Name()) diff --git a/stack/prefixstore.go b/stack/prefixstore.go index f366b2fa26..c12d14e55a 100644 --- a/stack/prefixstore.go +++ b/stack/prefixstore.go @@ -1,13 +1,18 @@ package stack -import "github.com/tendermint/basecoin/state" +import ( + "bytes" + "errors" + + "github.com/tendermint/basecoin/state" +) type prefixStore struct { prefix []byte - store state.KVStore + store state.SimpleDB } -var _ state.KVStore = prefixStore{} +var _ state.SimpleDB = prefixStore{} func (p prefixStore) Set(key, value []byte) { key = append(p.prefix, key...) @@ -19,11 +24,78 @@ func (p prefixStore) Get(key []byte) (value []byte) { return p.store.Get(key) } +func (p prefixStore) Has(key []byte) bool { + key = append(p.prefix, key...) + return p.store.Has(key) +} + +func (p prefixStore) Remove(key []byte) (value []byte) { + key = append(p.prefix, key...) + return p.store.Remove(key) +} + +func (p prefixStore) List(start, end []byte, limit int) []state.Model { + start = append(p.prefix, start...) + end = append(p.prefix, end...) + res := p.store.List(start, end, limit) + + trim := len(p.prefix) + for i := range res { + res[i].Key = res[i].Key[trim:] + } + return res +} + +func (p prefixStore) First(start, end []byte) state.Model { + start = append(p.prefix, start...) + end = append(p.prefix, end...) + res := p.store.First(start, end) + if len(res.Key) > 0 { + res.Key = res.Key[len(p.prefix):] + } + return res +} + +func (p prefixStore) Last(start, end []byte) state.Model { + start = append(p.prefix, start...) + end = append(p.prefix, end...) + res := p.store.Last(start, end) + if len(res.Key) > 0 { + res.Key = res.Key[len(p.prefix):] + } + return res +} + +func (p prefixStore) Checkpoint() state.SimpleDB { + return prefixStore{ + prefix: p.prefix, + store: p.store.Checkpoint(), + } +} + +func (p prefixStore) Commit(sub state.SimpleDB) error { + ps, ok := sub.(prefixStore) + if !ok { + return errors.New("Must commit prefixStore") + } + if !bytes.Equal(ps.prefix, p.prefix) { + return errors.New("Cannot commit sub-tx with different prefix") + } + + // commit the wrapped data, don't worry about the prefix here + p.store.Commit(ps) + return nil +} + +func (p prefixStore) Discard() { + p.store.Discard() +} + // stateSpace will unwrap any prefixStore and then add the prefix // // this can be used by the middleware and dispatcher to isolate one space, // then unwrap and isolate another space -func stateSpace(store state.KVStore, app string) state.KVStore { +func stateSpace(store state.SimpleDB, app string) state.SimpleDB { // unwrap one-level if wrapped if pstore, ok := store.(prefixStore); ok { store = pstore.store @@ -31,7 +103,7 @@ func stateSpace(store state.KVStore, app string) state.KVStore { return PrefixedStore(app, store) } -func unwrap(store state.KVStore) state.KVStore { +func unwrap(store state.SimpleDB) state.SimpleDB { // unwrap one-level if wrapped if pstore, ok := store.(prefixStore); ok { store = pstore.store @@ -45,7 +117,7 @@ func unwrap(store state.KVStore) state.KVStore { // This is useful for tests or utilities that have access to the global // state to check individual app spaces. Individual apps should not be able // to use this to read each other's space -func PrefixedStore(app string, store state.KVStore) state.KVStore { +func PrefixedStore(app string, store state.SimpleDB) state.SimpleDB { prefix := append([]byte(app), byte(0)) return prefixStore{prefix, store} } diff --git a/stack/recovery.go b/stack/recovery.go index 7ae7de20d6..77ff09ef41 100644 --- a/stack/recovery.go +++ b/stack/recovery.go @@ -26,7 +26,7 @@ func (Recovery) Name() string { var _ Middleware = Recovery{} // CheckTx catches any panic and converts to error - fulfills Middlware interface -func (Recovery) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { +func (Recovery) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (res basecoin.Result, err error) { defer func() { if r := recover(); r != nil { err = normalizePanic(r) @@ -36,7 +36,7 @@ func (Recovery) CheckTx(ctx basecoin.Context, store state.KVStore, tx basecoin.T } // DeliverTx catches any panic and converts to error - fulfills Middlware interface -func (Recovery) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { +func (Recovery) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (res basecoin.Result, err error) { defer func() { if r := recover(); r != nil { err = normalizePanic(r) @@ -46,7 +46,7 @@ func (Recovery) DeliverTx(ctx basecoin.Context, store state.KVStore, tx basecoin } // SetOption catches any panic and converts to error - fulfills Middlware interface -func (Recovery) SetOption(l log.Logger, store state.KVStore, module, key, value string, next basecoin.SetOptioner) (log string, err error) { +func (Recovery) SetOption(l log.Logger, store state.SimpleDB, module, key, value string, next basecoin.SetOptioner) (log string, err error) { defer func() { if r := recover(); r != nil { err = normalizePanic(r) diff --git a/stack/state_space_test.go b/stack/state_space_test.go index 071b326600..36a20ed7bf 100644 --- a/stack/state_space_test.go +++ b/stack/state_space_test.go @@ -23,19 +23,19 @@ var _ Middleware = writerMid{} func (w writerMid) Name() string { return w.name } -func (w writerMid) CheckTx(ctx basecoin.Context, store state.KVStore, +func (w writerMid) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Checker) (basecoin.Result, error) { store.Set(w.key, w.value) return next.CheckTx(ctx, store, tx) } -func (w writerMid) DeliverTx(ctx basecoin.Context, store state.KVStore, +func (w writerMid) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx, next basecoin.Deliver) (basecoin.Result, error) { store.Set(w.key, w.value) return next.DeliverTx(ctx, store, tx) } -func (w writerMid) SetOption(l log.Logger, store state.KVStore, module, +func (w writerMid) SetOption(l log.Logger, store state.SimpleDB, module, key, value string, next basecoin.SetOptioner) (string, error) { store.Set([]byte(key), []byte(value)) return next.SetOption(l, store, module, key, value) @@ -51,19 +51,19 @@ var _ basecoin.Handler = writerHand{} func (w writerHand) Name() string { return w.name } -func (w writerHand) CheckTx(ctx basecoin.Context, store state.KVStore, +func (w writerHand) CheckTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (basecoin.Result, error) { store.Set(w.key, w.value) return basecoin.Result{}, nil } -func (w writerHand) DeliverTx(ctx basecoin.Context, store state.KVStore, +func (w writerHand) DeliverTx(ctx basecoin.Context, store state.SimpleDB, tx basecoin.Tx) (basecoin.Result, error) { store.Set(w.key, w.value) return basecoin.Result{}, nil } -func (w writerHand) SetOption(l log.Logger, store state.KVStore, module, +func (w writerHand) SetOption(l log.Logger, store state.SimpleDB, module, key, value string) (string, error) { store.Set([]byte(key), []byte(value)) return "Success", nil diff --git a/state/store_test.go b/state/store_test.go index d479b81ef4..3dc576fb66 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -1,27 +1,28 @@ package state import ( + "io/ioutil" "testing" "github.com/stretchr/testify/assert" "github.com/tendermint/merkleeyes/iavl" - // dbm "github.com/tendermint/tmlibs/db" + dbm "github.com/tendermint/tmlibs/db" ) func GetDBs() []SimpleDB { - // // tree with persistence.... - // tmpDir, err := ioutil.TempDir("", "state-tests") - // if err != nil { - // panic(err) - // } - // db := dbm.NewDB("test-get-dbs", dbm.LevelDBBackendStr, tmpDir) - // persist := iavl.NewIAVLTree(500, db) + // tree with persistence.... + tmpDir, err := ioutil.TempDir("", "state-tests") + if err != nil { + panic(err) + } + db := dbm.NewDB("test-get-dbs", dbm.LevelDBBackendStr, tmpDir) + persist := iavl.NewIAVLTree(500, db) return []SimpleDB{ NewMemKVStore(), NewBonsai(iavl.NewIAVLTree(0, nil)), - // NewBonsai(persist), + NewBonsai(persist), } } From 6b96fa455474980d6db97ae1a3f21d1695df44b8 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Wed, 26 Jul 2017 20:31:43 -0400 Subject: [PATCH 14/18] Add tests for checkpointer --- stack/checkpoint_test.go | 97 ++++++++++++++++++++++++++++++++++++++++ stack/helpers.go | 21 ++++++++- stack/prefixstore.go | 2 +- 3 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 stack/checkpoint_test.go diff --git a/stack/checkpoint_test.go b/stack/checkpoint_test.go new file mode 100644 index 0000000000..bec2914d0e --- /dev/null +++ b/stack/checkpoint_test.go @@ -0,0 +1,97 @@ +package stack + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/tmlibs/log" + + "github.com/tendermint/basecoin" + "github.com/tendermint/basecoin/state" +) + +func TestCheckpointer(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + good := writerHand{"foo", []byte{1, 2}, []byte("bar")} + bad := FailHandler{Err: errors.New("no go")} + + app := New( + Checkpoint{OnCheck: true}, + writerMid{"bing", []byte{1, 2}, []byte("bang")}, + Checkpoint{OnDeliver: true}, + ).Use( + NewDispatcher( + WrapHandler(good), + WrapHandler(bad), + )) + + basecoin.TxMapper.RegisterImplementation(RawTx{}, good.Name(), byte(80)) + + mid := state.Model{ + Key: []byte{'b', 'i', 'n', 'g', 0, 1, 2}, + Value: []byte("bang"), + } + end := state.Model{ + Key: []byte{'f', 'o', 'o', 0, 1, 2}, + Value: []byte("bar"), + } + + cases := []struct { + // tx to send down the line + tx basecoin.Tx + // expect no error? + valid bool + // models to check afterwards + toGetCheck []state.Model + // models to check afterwards + toGetDeliver []state.Model + }{ + // everything writen on success + { + tx: NewRawTx([]byte{45, 67}), + valid: true, + toGetCheck: []state.Model{mid, end}, + toGetDeliver: []state.Model{mid, end}, + }, + // mostly reverted on failure + { + tx: NewFailTx(), + valid: false, + toGetCheck: []state.Model{}, + toGetDeliver: []state.Model{mid}, + }, + } + + for i, tc := range cases { + ctx := NewContext("foo", 100, log.NewNopLogger()) + + store := state.NewMemKVStore() + _, err := app.CheckTx(ctx, store, tc.tx) + if tc.valid { + require.Nil(err, "%+v", err) + } else { + require.NotNil(err) + } + for _, m := range tc.toGetCheck { + val := store.Get(m.Key) + assert.EqualValues(m.Value, val, "%d: %#v", i, m) + } + + store = state.NewMemKVStore() + _, err = app.DeliverTx(ctx, store, tc.tx) + if tc.valid { + require.Nil(err, "%+v", err) + } else { + require.NotNil(err) + } + for _, m := range tc.toGetDeliver { + val := store.Get(m.Key) + assert.EqualValues(m.Value, val, "%d: %#v", i, m) + } + + } +} diff --git a/stack/helpers.go b/stack/helpers.go index eaee305750..b38e86bc69 100644 --- a/stack/helpers.go +++ b/stack/helpers.go @@ -20,9 +20,11 @@ const ( const ( ByteRawTx = 0xF0 ByteCheckTx = 0xF1 + ByteFailTx = 0xF2 TypeRawTx = NameOK + "/raw" // this will just say a-ok to RawTx TypeCheckTx = NameCheck + "/tx" + TypeFailTx = NameFail + "/tx" rawMaxSize = 2000 * 1000 ) @@ -30,7 +32,8 @@ const ( func init() { basecoin.TxMapper. RegisterImplementation(RawTx{}, TypeRawTx, ByteRawTx). - RegisterImplementation(CheckTx{}, TypeCheckTx, ByteCheckTx) + RegisterImplementation(CheckTx{}, TypeCheckTx, ByteCheckTx). + RegisterImplementation(FailTx{}, TypeFailTx, ByteFailTx) } // RawTx just contains bytes that can be hex-ified @@ -72,6 +75,22 @@ func (CheckTx) ValidateBasic() error { return nil } +// FailTx just gets routed to filaure +type FailTx struct{} + +var _ basecoin.TxInner = FailTx{} + +func NewFailTx() basecoin.Tx { + return FailTx{}.Wrap() +} + +func (f FailTx) Wrap() basecoin.Tx { + return basecoin.Tx{f} +} +func (r FailTx) ValidateBasic() error { + return nil +} + // OKHandler just used to return okay to everything type OKHandler struct { Log string diff --git a/stack/prefixstore.go b/stack/prefixstore.go index c12d14e55a..5c6a4e7a08 100644 --- a/stack/prefixstore.go +++ b/stack/prefixstore.go @@ -83,7 +83,7 @@ func (p prefixStore) Commit(sub state.SimpleDB) error { } // commit the wrapped data, don't worry about the prefix here - p.store.Commit(ps) + p.store.Commit(ps.store) return nil } From ea8bdd3f92448442b76729fe54b0b978dfe9572a Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 27 Jul 2017 10:40:17 -0400 Subject: [PATCH 15/18] Fix app state to commit properly --- app/app.go | 1 - stack/checkpoint_test.go | 20 +++++- state/bonsai.go | 100 ++++++++++++++++++++++++++++++ state/merkle.go | 129 ++++++--------------------------------- 4 files changed, 138 insertions(+), 112 deletions(-) create mode 100644 state/bonsai.go diff --git a/app/app.go b/app/app.go index 5fdfd5d426..d27ee26e38 100644 --- a/app/app.go +++ b/app/app.go @@ -129,7 +129,6 @@ func (app *Basecoin) DeliverTx(txBytes []byte) abci.Result { res, err := app.handler.DeliverTx(ctx, app.state.Append(), tx) if err != nil { - // discard the cache... return errors.Result(err) } return res.ToABCI() diff --git a/stack/checkpoint_test.go b/stack/checkpoint_test.go index bec2914d0e..dd356e3f96 100644 --- a/stack/checkpoint_test.go +++ b/stack/checkpoint_test.go @@ -7,12 +7,28 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tendermint/merkleeyes/iavl" "github.com/tendermint/tmlibs/log" "github.com/tendermint/basecoin" "github.com/tendermint/basecoin/state" ) +func makeState() state.SimpleDB { + // return state.NewMemKVStore() + + return state.NewBonsai(iavl.NewIAVLTree(0, nil)) + + // tree with persistence.... + // tmpDir, err := ioutil.TempDir("", "state-tests") + // if err != nil { + // panic(err) + // } + // db := dbm.NewDB("test-get-dbs", dbm.LevelDBBackendStr, tmpDir) + // persist := iavl.NewIAVLTree(500, db) + // return state.NewBonsai(persist) +} + func TestCheckpointer(t *testing.T) { assert, require := assert.New(t), require.New(t) @@ -69,7 +85,7 @@ func TestCheckpointer(t *testing.T) { for i, tc := range cases { ctx := NewContext("foo", 100, log.NewNopLogger()) - store := state.NewMemKVStore() + store := makeState() _, err := app.CheckTx(ctx, store, tc.tx) if tc.valid { require.Nil(err, "%+v", err) @@ -81,7 +97,7 @@ func TestCheckpointer(t *testing.T) { assert.EqualValues(m.Value, val, "%d: %#v", i, m) } - store = state.NewMemKVStore() + store = makeState() _, err = app.DeliverTx(ctx, store, tc.tx) if tc.valid { require.Nil(err, "%+v", err) diff --git a/state/bonsai.go b/state/bonsai.go new file mode 100644 index 0000000000..cbef267987 --- /dev/null +++ b/state/bonsai.go @@ -0,0 +1,100 @@ +package state + +import ( + "errors" + "math/rand" + + "github.com/tendermint/tmlibs/merkle" +) + +// store nonce as it's own type so no one can even try to fake it +type nonce int64 + +// Bonsai is a deformed tree forced to fit in a small pot +type Bonsai struct { + id nonce + merkle.Tree +} + +var _ SimpleDB = &Bonsai{} + +// NewBonsai wraps a merkle tree and tags it to track children +func NewBonsai(tree merkle.Tree) *Bonsai { + return &Bonsai{ + id: nonce(rand.Int63()), + Tree: tree, + } +} + +// Get matches the signature of KVStore +func (b *Bonsai) Get(key []byte) []byte { + _, value, _ := b.Tree.Get(key) + return value +} + +// Set matches the signature of KVStore +func (b *Bonsai) Set(key, value []byte) { + b.Tree.Set(key, value) +} + +func (b *Bonsai) Remove(key []byte) (value []byte) { + value, _ = b.Tree.Remove(key) + return +} + +func (b *Bonsai) List(start, end []byte, limit int) []Model { + res := []Model{} + stopAtCount := func(key []byte, value []byte) (stop bool) { + m := Model{key, value} + res = append(res, m) + // return false + return limit > 0 && len(res) >= limit + } + b.Tree.IterateRange(start, end, true, stopAtCount) + return res +} + +func (b *Bonsai) First(start, end []byte) Model { + var m Model + stopAtFirst := func(key []byte, value []byte) (stop bool) { + m = Model{key, value} + return true + } + b.Tree.IterateRange(start, end, true, stopAtFirst) + return m +} + +func (b *Bonsai) Last(start, end []byte) Model { + var m Model + stopAtFirst := func(key []byte, value []byte) (stop bool) { + m = Model{key, value} + return true + } + b.Tree.IterateRange(start, end, false, stopAtFirst) + return m +} + +func (b *Bonsai) Checkpoint() SimpleDB { + return &Bonsai{ + id: b.id, + Tree: b.Tree.Copy(), + } +} + +// Commit will take all changes from the checkpoint and write +// them to the parent. +// Returns an error if this is not a child of this one +func (b *Bonsai) Commit(sub SimpleDB) error { + bb, ok := sub.(*Bonsai) + if !ok || (b.id != bb.id) { + return errors.New("Not a sub-transaction") + } + b.Tree = bb.Tree + return nil +} + +// Discard will remove reference to this +func (b *Bonsai) Discard() { + b.id = 0 + b.Tree = nil +} diff --git a/state/merkle.go b/state/merkle.go index d735f34b6e..0fa7c719be 100644 --- a/state/merkle.go +++ b/state/merkle.go @@ -1,9 +1,6 @@ package state import ( - "errors" - "math/rand" - "github.com/tendermint/merkleeyes/iavl" "github.com/tendermint/tmlibs/merkle" ) @@ -11,31 +8,32 @@ import ( // State represents the app states, separating the commited state (for queries) // from the working state (for CheckTx and AppendTx) type State struct { - committed merkle.Tree - deliverTx merkle.Tree - checkTx merkle.Tree + committed *Bonsai + deliverTx *Bonsai + checkTx *Bonsai persistent bool } func NewState(tree merkle.Tree, persistent bool) State { + base := NewBonsai(tree) return State{ - committed: tree, - deliverTx: tree.Copy(), - checkTx: tree.Copy(), + committed: base, + deliverTx: base.Checkpoint().(*Bonsai), + checkTx: base.Checkpoint().(*Bonsai), persistent: persistent, } } func (s State) Committed() *Bonsai { - return NewBonsai(s.committed) + return s.committed } func (s State) Append() *Bonsai { - return NewBonsai(s.deliverTx) + return s.deliverTx } func (s State) Check() *Bonsai { - return NewBonsai(s.checkTx) + return s.checkTx } // Hash updates the tree @@ -47,7 +45,7 @@ func (s *State) Hash() []byte { func (s *State) BatchSet(key, value []byte) { if s.persistent { // This is in the batch with the Save, but not in the tree - tree, ok := s.deliverTx.(*iavl.IAVLTree) + tree, ok := s.deliverTx.Tree.(*iavl.IAVLTree) if ok { tree.BatchSet(key, value) } @@ -56,106 +54,19 @@ func (s *State) BatchSet(key, value []byte) { // Commit save persistent nodes to the database and re-copies the trees func (s *State) Commit() []byte { + err := s.committed.Commit(s.deliverTx) + if err != nil { + panic(err) // ugh, TODO? + } + var hash []byte if s.persistent { - hash = s.deliverTx.Save() + hash = s.committed.Tree.Save() } else { - hash = s.deliverTx.Hash() + hash = s.committed.Tree.Hash() } - s.committed = s.deliverTx - s.deliverTx = s.committed.Copy() - s.checkTx = s.committed.Copy() + s.deliverTx = s.committed.Checkpoint().(*Bonsai) + s.checkTx = s.committed.Checkpoint().(*Bonsai) return hash } - -// store nonce as it's own type so no one can even try to fake it -type nonce int64 - -// Bonsai is a deformed tree forced to fit in a small pot -type Bonsai struct { - id nonce - merkle.Tree -} - -var _ SimpleDB = &Bonsai{} - -func NewBonsai(tree merkle.Tree) *Bonsai { - return &Bonsai{ - id: nonce(rand.Int63()), - Tree: tree, - } -} - -// Get matches the signature of KVStore -func (b *Bonsai) Get(key []byte) []byte { - _, value, _ := b.Tree.Get(key) - return value -} - -// Set matches the signature of KVStore -func (b *Bonsai) Set(key, value []byte) { - b.Tree.Set(key, value) -} - -func (b *Bonsai) Remove(key []byte) (value []byte) { - value, _ = b.Tree.Remove(key) - return -} - -func (b *Bonsai) List(start, end []byte, limit int) []Model { - res := []Model{} - stopAtCount := func(key []byte, value []byte) (stop bool) { - m := Model{key, value} - res = append(res, m) - // return false - return limit > 0 && len(res) >= limit - } - b.Tree.IterateRange(start, end, true, stopAtCount) - return res -} - -func (b *Bonsai) First(start, end []byte) Model { - var m Model - stopAtFirst := func(key []byte, value []byte) (stop bool) { - m = Model{key, value} - return true - } - b.Tree.IterateRange(start, end, true, stopAtFirst) - return m -} - -func (b *Bonsai) Last(start, end []byte) Model { - var m Model - stopAtFirst := func(key []byte, value []byte) (stop bool) { - m = Model{key, value} - return true - } - b.Tree.IterateRange(start, end, false, stopAtFirst) - return m -} - -func (b *Bonsai) Checkpoint() SimpleDB { - return &Bonsai{ - id: b.id, - Tree: b.Tree.Copy(), - } -} - -// Commit will take all changes from the checkpoint and write -// them to the parent. -// Returns an error if this is not a child of this one -func (b *Bonsai) Commit(sub SimpleDB) error { - bb, ok := sub.(*Bonsai) - if !ok || (b.id != bb.id) { - return errors.New("Not a sub-transaction") - } - b.Tree = bb.Tree - return nil -} - -// Discard will remove reference to this -func (b *Bonsai) Discard() { - b.id = 0 - b.Tree = nil -} From 2b79aa0413b511b83bca04b59422103eb65fc293 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 27 Jul 2017 11:17:22 -0400 Subject: [PATCH 16/18] Code cleanup from emmanuel --- app/app_test.go | 8 ++++-- app/genesis_test.go | 17 ++++++++----- app/store.go | 25 +++++++++---------- benchmarks/app_test.go | 8 ++++-- benchmarks/bonsai-speed.txt | 19 ++++++++++++++ cmd/basecoin/commands/start.go | 5 +++- .../counter/plugins/counter/counter_test.go | 7 ++++-- state/bonsai.go | 4 +-- state/chainstate.go | 6 +++-- state/errors.go | 20 +++++++++++++++ state/kvcache.go | 6 ++--- state/merkle.go | 6 ++--- 12 files changed, 93 insertions(+), 38 deletions(-) create mode 100644 benchmarks/bonsai-speed.txt create mode 100644 state/errors.go diff --git a/app/app_test.go b/app/app_test.go index 49e16e9af5..d09a69d23b 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -83,7 +83,9 @@ func (at *appTest) reset() { // Note: switch logger if you want to get more info logger := log.TestingLogger() // logger := log.NewTracingLogger(log.NewTMLogger(os.Stdout)) - store := NewStore("", 0, logger.With("module", "store")) + store, err := NewStore("", 0, logger.With("module", "store")) + require.Nil(at.t, err, "%+v", err) + at.app = NewBasecoin( DefaultHandler("mycoin"), store, @@ -141,7 +143,9 @@ func TestSetOption(t *testing.T) { require := require.New(t) logger := log.TestingLogger() - store := NewStore("", 0, logger.With("module", "store")) + store, err := NewStore("", 0, logger.With("module", "store")) + require.Nil(err, "%+v", err) + app := NewBasecoin( DefaultHandler("atom"), store, diff --git a/app/genesis_test.go b/app/genesis_test.go index a7412be82e..83e8ba7417 100644 --- a/app/genesis_test.go +++ b/app/genesis_test.go @@ -19,9 +19,10 @@ const genesisAcctFilepath = "./testdata/genesis2.json" func TestLoadGenesisDoNotFailIfAppOptionsAreMissing(t *testing.T) { logger := log.TestingLogger() - store := NewStore("", 0, logger) + store, err := NewStore("", 0, logger) + require.Nil(t, err, "%+v", err) app := NewBasecoin(DefaultHandler("mycoin"), store, logger) - err := app.LoadGenesis("./testdata/genesis3.json") + err = app.LoadGenesis("./testdata/genesis3.json") require.Nil(t, err, "%+v", err) } @@ -29,9 +30,11 @@ func TestLoadGenesis(t *testing.T) { assert, require := assert.New(t), require.New(t) logger := log.TestingLogger() - store := NewStore("", 0, logger) + store, err := NewStore("", 0, logger) + require.Nil(err, "%+v", err) + app := NewBasecoin(DefaultHandler("mycoin"), store, logger) - err := app.LoadGenesis(genesisFilepath) + err = app.LoadGenesis(genesisFilepath) require.Nil(err, "%+v", err) // check the chain id @@ -59,9 +62,11 @@ func TestLoadGenesisAccountAddress(t *testing.T) { assert, require := assert.New(t), require.New(t) logger := log.TestingLogger() - store := NewStore("", 0, logger) + store, err := NewStore("", 0, logger) + require.Nil(err, "%+v", err) + app := NewBasecoin(DefaultHandler("mycoin"), store, logger) - err := app.LoadGenesis(genesisAcctFilepath) + err = app.LoadGenesis(genesisAcctFilepath) require.Nil(err, "%+v", err) // check the chain id diff --git a/app/store.go b/app/store.go index 1dc7b43831..cad2a2cddf 100644 --- a/app/store.go +++ b/app/store.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/pkg/errors" abci "github.com/tendermint/abci/types" "github.com/tendermint/go-wire" "github.com/tendermint/merkleeyes/iavl" @@ -37,7 +38,7 @@ type ChainState struct { // NewStore initializes an in-memory IAVLTree, or attempts to load a persistant // tree from disk -func NewStore(dbName string, cacheSize int, logger log.Logger) *Store { +func NewStore(dbName string, cacheSize int, logger log.Logger) (*Store, error) { // start at 1 so the height returned by query is for the // next block, ie. the one that includes the AppHash for our current state initialHeight := uint64(1) @@ -48,17 +49,18 @@ func NewStore(dbName string, cacheSize int, logger log.Logger) *Store { 0, nil, ) - return &Store{ + store := &Store{ State: state.NewState(tree, false), height: initialHeight, logger: logger, } + return store, nil } // Expand the path fully dbPath, err := filepath.Abs(dbName) if err != nil { - panic(fmt.Sprintf("Invalid Database Name: %s", dbName)) + return nil, errors.Wrap(err, "Invalid Database Name") } // Some external calls accidently add a ".db", which is now removed @@ -94,22 +96,16 @@ func NewStore(dbName string, cacheSize int, logger log.Logger) *Store { tree.Load(chainState.Hash) } - return &Store{ + res := &Store{ State: state.NewState(tree, true), height: chainState.Height, hash: chainState.Hash, persisted: true, logger: logger, } + return res, nil } -// CloseDB closes the database -// func (s *Store) CloseDB() { -// if s.db != nil { -// s.db.Close() -// } -// } - // Info implements abci.Application. It returns the height, hash and size (in the data). // The height is the block that holds the transactions, not the apphash itself. func (s *Store) Info() abci.ResponseInfo { @@ -136,9 +132,12 @@ func (s *Store) Commit() abci.Result { Height: s.height, })) - hash := s.State.Commit() + hash, err := s.State.Commit() + if err != nil { + return abci.NewError(abci.CodeType_InternalError, err.Error()) + } if !bytes.Equal(hash, s.hash) { - panic("AppHash is incorrect") + return abci.NewError(abci.CodeType_InternalError, "AppHash is incorrect") } if s.State.Committed().Size() == 0 { diff --git a/benchmarks/app_test.go b/benchmarks/app_test.go index 2786af608c..4d0c228f89 100644 --- a/benchmarks/app_test.go +++ b/benchmarks/app_test.go @@ -55,12 +55,16 @@ func NewBenchApp(h basecoin.Handler, chainID string, n int, // TODO: disk writing var store *app.Store + var err error if persist { tmpDir, _ := ioutil.TempDir("", "bc-app-benchmark") - store = app.NewStore(tmpDir, 500, logger) + store, err = app.NewStore(tmpDir, 500, logger) } else { - store = app.NewStore("", 0, logger) + store, err = app.NewStore("", 0, logger) + } + if err != nil { + panic(err) } app := app.NewBasecoin( diff --git a/benchmarks/bonsai-speed.txt b/benchmarks/bonsai-speed.txt new file mode 100644 index 0000000000..ec4bae354b --- /dev/null +++ b/benchmarks/bonsai-speed.txt @@ -0,0 +1,19 @@ +BenchmarkMakeTx-4 2000 603153 ns/op +BenchmarkSimpleTransfer/100-10-nofee-memdb-4 5000 313154 ns/op +BenchmarkSimpleTransfer/100-10-fee-memdb-4 5000 366534 ns/op +BenchmarkSimpleTransfer/100-200-nofee-memdb-4 5000 296381 ns/op +BenchmarkSimpleTransfer/100-200-fee-memdb-4 5000 350973 ns/op +BenchmarkSimpleTransfer/10000-10-nofee-memdb-4 5000 351425 ns/op +BenchmarkSimpleTransfer/10000-10-fee-memdb-4 3000 410855 ns/op +BenchmarkSimpleTransfer/10000-200-nofee-memdb-4 5000 344839 ns/op +BenchmarkSimpleTransfer/10000-200-fee-memdb-4 5000 394080 ns/op +BenchmarkSimpleTransfer/100-10-nofee-persist-4 3000 433890 ns/op +BenchmarkSimpleTransfer/100-10-fee-persist-4 3000 496133 ns/op +BenchmarkSimpleTransfer/100-200-nofee-persist-4 5000 310174 ns/op +BenchmarkSimpleTransfer/100-200-fee-persist-4 5000 366868 ns/op +BenchmarkSimpleTransfer/10000-10-nofee-persist-4 2000 815755 ns/op +BenchmarkSimpleTransfer/10000-10-fee-persist-4 2000 874532 ns/op +BenchmarkSimpleTransfer/10000-200-nofee-persist-4 5000 567349 ns/op +BenchmarkSimpleTransfer/10000-200-fee-persist-4 5000 621833 ns/op +PASS +ok github.com/tendermint/basecoin/benchmarks 93.047s diff --git a/cmd/basecoin/commands/start.go b/cmd/basecoin/commands/start.go index 5ea145878b..aac85a96bf 100644 --- a/cmd/basecoin/commands/start.go +++ b/cmd/basecoin/commands/start.go @@ -55,11 +55,14 @@ func init() { func startCmd(cmd *cobra.Command, args []string) error { rootDir := viper.GetString(cli.HomeFlag) - store := app.NewStore( + store, err := app.NewStore( path.Join(rootDir, "data", "merkleeyes.db"), EyesCacheSize, logger.With("module", "store"), ) + if err != nil { + return err + } // Create Basecoin app basecoinApp := app.NewBasecoin(Handler, store, logger.With("module", "app")) diff --git a/docs/guide/counter/plugins/counter/counter_test.go b/docs/guide/counter/plugins/counter/counter_test.go index 986219d1de..b9fb9c2d59 100644 --- a/docs/guide/counter/plugins/counter/counter_test.go +++ b/docs/guide/counter/plugins/counter/counter_test.go @@ -20,13 +20,16 @@ import ( func TestCounterPlugin(t *testing.T) { assert := assert.New(t) + require := require.New(t) // Basecoin initialization chainID := "test_chain_id" logger := log.TestingLogger() // logger := log.NewTracingLogger(log.NewTMLogger(os.Stdout)) - store := app.NewStore("", 0, logger.With("module", "store")) + store, err := app.NewStore("", 0, logger.With("module", "store")) + require.Nil(err, "%+v", err) + h := NewHandler("gold") bcApp := app.NewBasecoin( h, @@ -39,7 +42,7 @@ func TestCounterPlugin(t *testing.T) { bal := coin.Coins{{"", 1000}, {"gold", 1000}} acct := coin.NewAccountWithKey(bal) log := bcApp.SetOption("coin/account", acct.MakeOption()) - require.Equal(t, "Success", log) + require.Equal("Success", log) // Deliver a CounterTx DeliverCounterTx := func(valid bool, counterFee coin.Coins, sequence uint32) abci.Result { diff --git a/state/bonsai.go b/state/bonsai.go index cbef267987..57abe714ed 100644 --- a/state/bonsai.go +++ b/state/bonsai.go @@ -1,7 +1,6 @@ package state import ( - "errors" "math/rand" "github.com/tendermint/tmlibs/merkle" @@ -47,7 +46,6 @@ func (b *Bonsai) List(start, end []byte, limit int) []Model { stopAtCount := func(key []byte, value []byte) (stop bool) { m := Model{key, value} res = append(res, m) - // return false return limit > 0 && len(res) >= limit } b.Tree.IterateRange(start, end, true, stopAtCount) @@ -87,7 +85,7 @@ func (b *Bonsai) Checkpoint() SimpleDB { func (b *Bonsai) Commit(sub SimpleDB) error { bb, ok := sub.(*Bonsai) if !ok || (b.id != bb.id) { - return errors.New("Not a sub-transaction") + return ErrNotASubTransaction() } b.Tree = bb.Tree return nil diff --git a/state/chainstate.go b/state/chainstate.go index 2751833d44..6f14a36d0a 100644 --- a/state/chainstate.go +++ b/state/chainstate.go @@ -10,10 +10,12 @@ func NewChainState() *ChainState { return &ChainState{} } +var baseChainIDKey = []byte("base/chain_id") + // SetChainID stores the chain id in the store func (s *ChainState) SetChainID(store KVStore, chainID string) { s.chainID = chainID - store.Set([]byte("base/chain_id"), []byte(chainID)) + store.Set(baseChainIDKey, []byte(chainID)) } // GetChainID gets the chain id from the cache or the store @@ -21,6 +23,6 @@ func (s *ChainState) GetChainID(store KVStore) string { if s.chainID != "" { return s.chainID } - s.chainID = string(store.Get([]byte("base/chain_id"))) + s.chainID = string(store.Get(baseChainIDKey)) return s.chainID } diff --git a/state/errors.go b/state/errors.go new file mode 100644 index 0000000000..cc314ddb0c --- /dev/null +++ b/state/errors.go @@ -0,0 +1,20 @@ +//nolint +package state + +import ( + "fmt" + + abci "github.com/tendermint/abci/types" + "github.com/tendermint/basecoin/errors" +) + +var ( + errNotASubTransaction = fmt.Errorf("Not a sub-transaction") +) + +func ErrNotASubTransaction() errors.TMError { + return errors.WithCode(errNotASubTransaction, abci.CodeType_InternalError) +} +func IsNotASubTransactionErr(err error) bool { + return errors.IsSameError(errNotASubTransaction, err) +} diff --git a/state/kvcache.go b/state/kvcache.go index 31ba96d5d4..bd47fc8690 100644 --- a/state/kvcache.go +++ b/state/kvcache.go @@ -1,14 +1,12 @@ package state -import "errors" - // MemKVCache is designed to wrap MemKVStore as a cache type MemKVCache struct { store SimpleDB cache *MemKVStore } -var _ SimpleDB = NewMemKVStore() +var _ SimpleDB = (*MemKVCache)(nil) // NewMemKVCache wraps a cache around MemKVStore // @@ -114,7 +112,7 @@ func (c *MemKVCache) Checkpoint() SimpleDB { func (c *MemKVCache) Commit(sub SimpleDB) error { cache, ok := sub.(*MemKVCache) if !ok { - return errors.New("sub is not a cache") + return ErrNotASubTransaction() } // TODO: see if it points to us diff --git a/state/merkle.go b/state/merkle.go index 0fa7c719be..56c9349e78 100644 --- a/state/merkle.go +++ b/state/merkle.go @@ -53,10 +53,10 @@ func (s *State) BatchSet(key, value []byte) { } // Commit save persistent nodes to the database and re-copies the trees -func (s *State) Commit() []byte { +func (s *State) Commit() ([]byte, error) { err := s.committed.Commit(s.deliverTx) if err != nil { - panic(err) // ugh, TODO? + return nil, err } var hash []byte @@ -68,5 +68,5 @@ func (s *State) Commit() []byte { s.deliverTx = s.committed.Checkpoint().(*Bonsai) s.checkTx = s.committed.Checkpoint().(*Bonsai) - return hash + return hash, nil } From 27e7fbe4cfa74e6073ef5144a799a939a0e523e0 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 27 Jul 2017 14:05:28 -0400 Subject: [PATCH 17/18] Last cleanup --- app/store.go | 3 +-- modules/coin/commands/query.go | 2 +- modules/coin/commands/tx.go | 12 ++---------- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/app/store.go b/app/store.go index cad2a2cddf..f6ae197513 100644 --- a/app/store.go +++ b/app/store.go @@ -90,8 +90,7 @@ func NewStore(dbName string, cacheSize int, logger log.Logger) (*Store, error) { eyesStateBytes := db.Get(stateKey) err = wire.ReadBinaryBytes(eyesStateBytes, &chainState) if err != nil { - logger.Error("error reading MerkleEyesState", "err", err) - panic(err) + return nil, errors.Wrap(err, "Reading MerkleEyesState") } tree.Load(chainState.Hash) } diff --git a/modules/coin/commands/query.go b/modules/coin/commands/query.go index 7a7d4805ad..e77614bead 100644 --- a/modules/coin/commands/query.go +++ b/modules/coin/commands/query.go @@ -33,7 +33,7 @@ func accountQueryCmd(cmd *cobra.Command, args []string) error { acc := coin.Account{} proof, err := proofcmd.GetAndParseAppProof(key, &acc) if lc.IsNoDataErr(err) { - return errors.Errorf("Account bytes are empty for address %X ", addr) + return errors.Errorf("Account bytes are empty for address %s ", addr) } else if err != nil { return err } diff --git a/modules/coin/commands/tx.go b/modules/coin/commands/tx.go index cff4a51acb..4f130ed8b1 100644 --- a/modules/coin/commands/tx.go +++ b/modules/coin/commands/tx.go @@ -58,16 +58,8 @@ func readSendTxFlags() (tx basecoin.Tx, err error) { } // craft the inputs and outputs - ins := []coin.TxInput{{ - Address: fromAddr, - Coins: amountCoins, - }} - outs := []coin.TxOutput{{ - Address: toAddr, - Coins: amountCoins, - }} - - return coin.NewSendTx(ins, outs), nil + tx = coin.NewSendOneTx(fromAddr, toAddr, amountCoins) + return } func readFromAddr() (basecoin.Actor, error) { From dfdbfa04c254262f90774bd22cd53e89cc9085a3 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 27 Jul 2017 15:20:43 -0400 Subject: [PATCH 18/18] Use KVMemCache for checkpointing IAVL tree as they are not go-routine safe... --- app/store.go | 7 +++- state/bonsai.go | 44 ++++++++++++++++---- state/chainstate_test.go | 88 ---------------------------------------- state/kvcache.go | 3 ++ state/kvstore.go | 3 +- state/merkle.go | 35 ++++++++++------ 6 files changed, 68 insertions(+), 112 deletions(-) delete mode 100644 state/chainstate_test.go diff --git a/app/store.go b/app/store.go index f6ae197513..4c3bc8adac 100644 --- a/app/store.go +++ b/app/store.go @@ -120,8 +120,13 @@ func (s *Store) Info() abci.ResponseInfo { // Commit implements abci.Application func (s *Store) Commit() abci.Result { - s.hash = s.State.Hash() + var err error s.height++ + s.hash, err = s.State.Hash() + if err != nil { + return abci.NewError(abci.CodeType_InternalError, err.Error()) + } + s.logger.Debug("Commit synced", "height", s.height, "hash", fmt.Sprintf("%X", s.hash)) diff --git a/state/bonsai.go b/state/bonsai.go index 57abe714ed..2a98732985 100644 --- a/state/bonsai.go +++ b/state/bonsai.go @@ -73,24 +73,50 @@ func (b *Bonsai) Last(start, end []byte) Model { } func (b *Bonsai) Checkpoint() SimpleDB { - return &Bonsai{ - id: b.id, - Tree: b.Tree.Copy(), - } + return NewMemKVCache(b) } -// Commit will take all changes from the checkpoint and write -// them to the parent. -// Returns an error if this is not a child of this one func (b *Bonsai) Commit(sub SimpleDB) error { - bb, ok := sub.(*Bonsai) + cache, ok := sub.(*MemKVCache) + if !ok { + return ErrNotASubTransaction() + } + // see if it was wrapping this struct + bb, ok := cache.store.(*Bonsai) if !ok || (b.id != bb.id) { return ErrNotASubTransaction() } - b.Tree = bb.Tree + + // apply the cached data to the Bonsai + cache.applyCache() return nil } +//---------------------------------------- +// This is the checkpointing I want, but apparently iavl-tree is not +// as immutable as I hoped... paniced in multiple go-routines :( +// +// FIXME: use this code when iavltree is improved + +// func (b *Bonsai) Checkpoint() SimpleDB { +// return &Bonsai{ +// id: b.id, +// Tree: b.Tree.Copy(), +// } +// } + +// // Commit will take all changes from the checkpoint and write +// // them to the parent. +// // Returns an error if this is not a child of this one +// func (b *Bonsai) Commit(sub SimpleDB) error { +// bb, ok := sub.(*Bonsai) +// if !ok || (b.id != bb.id) { +// return ErrNotASubTransaction() +// } +// b.Tree = bb.Tree +// return nil +// } + // Discard will remove reference to this func (b *Bonsai) Discard() { b.id = 0 diff --git a/state/chainstate_test.go b/state/chainstate_test.go deleted file mode 100644 index 096d27d197..0000000000 --- a/state/chainstate_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package state - -// TODO: something similar in the merkle package... - -// import ( -// "bytes" -// "testing" - -// eyes "github.com/tendermint/merkleeyes/client" -// "github.com/tendermint/tmlibs/log" - -// "github.com/stretchr/testify/assert" -// ) - -// func TestState(t *testing.T) { -// assert := assert.New(t) - -// //States and Stores for tests -// store := NewMemKVStore() -// state := NewState(store, log.TestingLogger()) -// cache := state.CacheWrap() -// eyesCli := eyes.NewLocalClient("", 0) - -// //reset the store/state/cache -// reset := func() { -// store = NewMemKVStore() -// state = NewState(store, log.TestingLogger()) -// cache = state.CacheWrap() -// } - -// //set the state to using the eyesCli instead of MemKVStore -// useEyesCli := func() { -// state = NewState(eyesCli, log.TestingLogger()) -// cache = state.CacheWrap() -// } - -// //key value pairs to be tested within the system -// keyvalue := []struct { -// key string -// value string -// }{ -// {"foo", "snake"}, -// {"bar", "mouse"}, -// } - -// //set the kvc to have all the key value pairs -// setRecords := func(kv KVStore) { -// for _, n := range keyvalue { -// kv.Set([]byte(n.key), []byte(n.value)) -// } -// } - -// //store has all the key value pairs -// storeHasAll := func(kv KVStore) bool { -// for _, n := range keyvalue { -// if !bytes.Equal(kv.Get([]byte(n.key)), []byte(n.value)) { -// return false -// } -// } -// return true -// } - -// //test chainID -// state.SetChainID("testchain") -// assert.Equal(state.GetChainID(), "testchain", "ChainID is improperly stored") - -// //test basic retrieve -// setRecords(state) -// assert.True(storeHasAll(state), "state doesn't retrieve after Set") - -// //Test CacheWrap with local mem store -// reset() -// setRecords(cache) -// assert.False(storeHasAll(store), "store retrieving before CacheSync") -// cache.CacheSync() -// assert.True(storeHasAll(store), "store doesn't retrieve after CacheSync") - -// //Test Commit on state with non-merkle store -// assert.True(state.Commit().IsErr(), "Commit shouldn't work with non-merkle store") - -// //Test CacheWrap with merkleeyes client store -// useEyesCli() -// setRecords(cache) -// assert.False(storeHasAll(eyesCli), "eyesCli retrieving before Commit") -// cache.CacheSync() -// assert.True(state.Commit().IsOK(), "Bad Commit") -// assert.True(storeHasAll(eyesCli), "eyesCli doesn't retrieve after Commit") -// } diff --git a/state/kvcache.go b/state/kvcache.go index bd47fc8690..1141d20e3e 100644 --- a/state/kvcache.go +++ b/state/kvcache.go @@ -23,10 +23,12 @@ func NewMemKVCache(store SimpleDB) *MemKVCache { } } +// Set sets a key, fulfills KVStore interface func (c *MemKVCache) Set(key []byte, value []byte) { c.cache.Set(key, value) } +// Get gets a key, fulfills KVStore interface func (c *MemKVCache) Get(key []byte) (value []byte) { value, ok := c.cache.m[string(key)] if !ok { @@ -36,6 +38,7 @@ func (c *MemKVCache) Get(key []byte) (value []byte) { return value } +// Has checks existence of a key, fulfills KVStore interface func (c *MemKVCache) Has(key []byte) bool { value := c.Get(key) return value != nil diff --git a/state/kvstore.go b/state/kvstore.go index 6233dddb02..08dd85f217 100644 --- a/state/kvstore.go +++ b/state/kvstore.go @@ -1,7 +1,6 @@ package state import ( - "errors" "sort" "github.com/tendermint/go-wire/data" @@ -148,7 +147,7 @@ func (m *MemKVStore) Checkpoint() SimpleDB { func (m *MemKVStore) Commit(sub SimpleDB) error { cache, ok := sub.(*MemKVCache) if !ok { - return errors.New("sub is not a cache") + return ErrNotASubTransaction() } // TODO: see if it points to us diff --git a/state/merkle.go b/state/merkle.go index 56c9349e78..f383fd0d5a 100644 --- a/state/merkle.go +++ b/state/merkle.go @@ -9,8 +9,8 @@ import ( // from the working state (for CheckTx and AppendTx) type State struct { committed *Bonsai - deliverTx *Bonsai - checkTx *Bonsai + deliverTx SimpleDB + checkTx SimpleDB persistent bool } @@ -18,8 +18,8 @@ func NewState(tree merkle.Tree, persistent bool) State { base := NewBonsai(tree) return State{ committed: base, - deliverTx: base.Checkpoint().(*Bonsai), - checkTx: base.Checkpoint().(*Bonsai), + deliverTx: base.Checkpoint(), + checkTx: base.Checkpoint(), persistent: persistent, } } @@ -28,24 +28,34 @@ func (s State) Committed() *Bonsai { return s.committed } -func (s State) Append() *Bonsai { +func (s State) Append() SimpleDB { return s.deliverTx } -func (s State) Check() *Bonsai { +func (s State) Check() SimpleDB { return s.checkTx } -// Hash updates the tree -func (s *State) Hash() []byte { - return s.deliverTx.Hash() +// Hash applies deliverTx to committed and calculates hash +// +// Note the usage: +// Hash -> gets the calculated hash but doesn't save +// BatchSet -> adds some out-of-bounds data +// Commit -> Save everything to disk and the same hash +func (s *State) Hash() ([]byte, error) { + err := s.committed.Commit(s.deliverTx) + if err != nil { + return nil, err + } + s.deliverTx = s.committed.Checkpoint() + return s.committed.Tree.Hash(), nil } // BatchSet is used for some weird magic in storing the new height func (s *State) BatchSet(key, value []byte) { if s.persistent { // This is in the batch with the Save, but not in the tree - tree, ok := s.deliverTx.Tree.(*iavl.IAVLTree) + tree, ok := s.committed.Tree.(*iavl.IAVLTree) if ok { tree.BatchSet(key, value) } @@ -54,6 +64,7 @@ func (s *State) BatchSet(key, value []byte) { // Commit save persistent nodes to the database and re-copies the trees func (s *State) Commit() ([]byte, error) { + // commit (if we didn't do hash earlier) err := s.committed.Commit(s.deliverTx) if err != nil { return nil, err @@ -66,7 +77,7 @@ func (s *State) Commit() ([]byte, error) { hash = s.committed.Tree.Hash() } - s.deliverTx = s.committed.Checkpoint().(*Bonsai) - s.checkTx = s.committed.Checkpoint().(*Bonsai) + s.deliverTx = s.committed.Checkpoint() + s.checkTx = s.committed.Checkpoint() return hash, nil }