From 61cbd129766918492771e213791ed603791e07c7 Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Tue, 10 Jul 2018 02:38:09 -0400 Subject: [PATCH 01/17] cleanup logic in updateValidator --- x/stake/keeper/validator.go | 120 +++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 51 deletions(-) diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index 6962ca28a1..b1a0ebe866 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -199,78 +199,96 @@ func (k Keeper) ClearTendermintUpdates(ctx sdk.Context) { func (k Keeper) UpdateValidator(ctx sdk.Context, validator types.Validator) types.Validator { store := ctx.KVStore(k.storeKey) pool := k.GetPool(ctx) - ownerAddr := validator.Owner + oldValidator, oldFound := k.GetValidator(ctx, validator.Owner) - // always update the main list ordered by owner address before exiting - defer func() { - k.SetValidator(ctx, validator) - }() + validator = k.updateForRevoking(ctx, oldFound, oldValidator, validator) + powerIncreasing := k.getPowerIncreasing(ctx, oldFound, oldValidator, validator) + validator.BondHeight, validator.BondIntraTxCounter = k.getBondDetails(ctx, oldFound, oldValidator, validator) + valPower := k.updateValidatorPower(ctx, oldFound, oldValidator, validator, pool) + cliffPower := k.GetCliffValidatorPower(ctx) - // retrieve the old validator record - oldValidator, oldFound := k.GetValidator(ctx, ownerAddr) + switch { + // if already bonded and power increasing only need to update tendermint + case powerIncreasing && !validator.Revoked && + (oldFound && oldValidator.Status() == sdk.Bonded): - if validator.Revoked && oldValidator.Status() == sdk.Bonded { - validator = k.unbondValidator(ctx, validator) + bz := k.cdc.MustMarshalBinary(validator.ABCIValidator()) + store.Set(GetTendermintUpdatesKey(validator.Owner), bz) + + // if was unbonded/or is a new validator - and the new power is less than the cliff validator + case cliffPower != nil && + (!oldFound || (oldFound && oldValidator.Status() == sdk.Unbonded)) && + bytes.Compare(valPower, cliffPower) == -1: //(valPower < cliffPower + // skip to completion + + default: + // update the validator set for this validator + updatedVal := k.UpdateBondedValidators(ctx, validator) + if updatedVal.Owner != nil { // updates to validator occurred to be updated + validator = updatedVal + } else { + + // if decreased in power but still bonded, update Tendermint validator + // (if updatedVal is set, the validator has changed bonding status) + stillBonded := oldFound && oldValidator.Status() == sdk.Bonded + if stillBonded && oldValidator.PoolShares.Bonded().GT(validator.PoolShares.Bonded()) { + bz := k.cdc.MustMarshalBinary(validator.ABCIValidator()) + store.Set(GetTendermintUpdatesKey(validator.Owner), bz) + } + } + } + + k.SetValidator(ctx, validator) + return validator +} + +func (k Keeper) updateForRevoking(ctx sdk.Context, oldFound bool, oldValidator, newValidator types.Validator) types.Validator { + if newValidator.Revoked && oldFound && oldValidator.Status() == sdk.Bonded { + newValidator = k.unbondValidator(ctx, newValidator) // need to also clear the cliff validator spot because the revoke has // opened up a new spot which will be filled when // updateValidatorsBonded is called k.clearCliffValidator(ctx) } + return newValidator +} - powerIncreasing := false - if oldFound && oldValidator.PoolShares.Bonded().LT(validator.PoolShares.Bonded()) { - powerIncreasing = true +func (k Keeper) getPowerIncreasing(ctx sdk.Context, oldFound bool, oldValidator, newValidator types.Validator) bool { + if oldFound && oldValidator.PoolShares.Bonded().LT(newValidator.PoolShares.Bonded()) { + return true } + return false +} + +func (k Keeper) getBondDetails(ctx sdk.Context, oldFound bool, oldValidator, + newValidator types.Validator) (height int64, intraTxCounter int16) { // if already a validator, copy the old block height and counter, else set them if oldFound && oldValidator.Status() == sdk.Bonded { - validator.BondHeight = oldValidator.BondHeight - validator.BondIntraTxCounter = oldValidator.BondIntraTxCounter - } else { - validator.BondHeight = ctx.BlockHeight() - counter := k.GetIntraTxCounter(ctx) - validator.BondIntraTxCounter = counter - k.SetIntraTxCounter(ctx, counter+1) + height = oldValidator.BondHeight + intraTxCounter = oldValidator.BondIntraTxCounter + return } + height = ctx.BlockHeight() + counter := k.GetIntraTxCounter(ctx) + intraTxCounter = counter + k.SetIntraTxCounter(ctx, counter+1) + return +} + +func (k Keeper) updateValidatorPower(ctx sdk.Context, oldFound bool, oldValidator, + newValidator types.Validator, pool types.Pool) (valPower []byte) { + store := ctx.KVStore(k.storeKey) // update the list ordered by voting power if oldFound { store.Delete(GetValidatorsByPowerIndexKey(oldValidator, pool)) } - valPower := GetValidatorsByPowerIndexKey(validator, pool) - store.Set(valPower, validator.Owner) + valPower = GetValidatorsByPowerIndexKey(newValidator, pool) + store.Set(valPower, newValidator.Owner) - // efficiency case: - // if already bonded and power increasing only need to update tendermint - if powerIncreasing && !validator.Revoked && oldValidator.Status() == sdk.Bonded { - bz := k.cdc.MustMarshalBinary(validator.ABCIValidator()) - store.Set(GetTendermintUpdatesKey(ownerAddr), bz) - return validator - } - - // efficiency case: - // if was unbonded/or is a new validator - and the new power is less than the cliff validator - cliffPower := k.GetCliffValidatorPower(ctx) - if cliffPower != nil && - (!oldFound || (oldFound && oldValidator.Status() == sdk.Unbonded)) && - bytes.Compare(valPower, cliffPower) == -1 { //(valPower < cliffPower - return validator - } - - // update the validator set for this validator - updatedVal := k.UpdateBondedValidators(ctx, validator) - if updatedVal.Owner != nil { // updates to validator occurred to be updated - validator = updatedVal - } - // if decreased in power but still bonded, update Tendermint validator - // (if updatedVal is set, the validator has changed bonding status) - stillBonded := oldFound && oldValidator.Status() == sdk.Bonded && updatedVal.Owner == nil - if stillBonded && oldValidator.PoolShares.Bonded().GT(validator.PoolShares.Bonded()) { - bz := k.cdc.MustMarshalBinary(validator.ABCIValidator()) - store.Set(GetTendermintUpdatesKey(ownerAddr), bz) - } - return validator + return valPower } // Update the validator group and kick out any old validators. In addition this From 12932de40f89844923f38b6b3f85bbc6fa99994e Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Tue, 10 Jul 2018 10:23:31 -0400 Subject: [PATCH 02/17] further cleanup, also reorg updateBondedValidators --- x/stake/keeper/validator.go | 107 ++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index b1a0ebe866..2cc1c2d452 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -215,9 +215,14 @@ func (k Keeper) UpdateValidator(ctx sdk.Context, validator types.Validator) type bz := k.cdc.MustMarshalBinary(validator.ABCIValidator()) store.Set(GetTendermintUpdatesKey(validator.Owner), bz) - // if was unbonded/or is a new validator - and the new power is less than the cliff validator + // if is a new validator and the new power is less than the cliff validator + case cliffPower != nil && !oldFound && + bytes.Compare(valPower, cliffPower) == -1: //(valPower < cliffPower + // skip to completion + + // if was unbonded and the new power is less than the cliff validator case cliffPower != nil && - (!oldFound || (oldFound && oldValidator.Status() == sdk.Unbonded)) && + (oldFound && oldValidator.Status() == sdk.Unbonded) && bytes.Compare(valPower, cliffPower) == -1: //(valPower < cliffPower // skip to completion @@ -299,11 +304,13 @@ func (k Keeper) updateValidatorPower(ctx sdk.Context, oldFound bool, oldValidato // The correct subset is retrieved by iterating through an index of the // validators sorted by power, stored using the ValidatorsByPowerIndexKey. // Simultaneously the current validator records are updated in store with the -// ValidatorsBondedIndexKey. This store is used to determine if a validator is a -// validator without needing to iterate over the subspace as we do in +// ValidatorsBondedIndexKey. This store is used to determine if a validator is +// a validator without needing to iterate over the subspace as we do in // GetValidators. // -// Optionally also return the validator from a retrieve address if the validator has been bonded +// Optionally also return the validator from a retrieve address if the +// validator has been bonded +// // nolint: gocyclo // TODO: Remove the above golint func (k Keeper) UpdateBondedValidators(ctx sdk.Context, @@ -311,30 +318,20 @@ func (k Keeper) UpdateBondedValidators(ctx sdk.Context, store := ctx.KVStore(k.storeKey) - kickCliffValidator := false oldCliffValidatorAddr := k.GetCliffValidator(ctx) - - // add the actual validator power sorted store maxValidators := k.GetParams(ctx).MaxValidators - iterator := sdk.KVStoreReversePrefixIterator(store, ValidatorsByPowerIndexKey) // largest to smallest bondedValidatorsCount := 0 - var validator types.Validator + var validator, validatorToBond types.Validator + + iterator := sdk.KVStoreReversePrefixIterator(store, ValidatorsByPowerIndexKey) // largest to smallest for { if !iterator.Valid() || bondedValidatorsCount > int(maxValidators-1) { - - // TODO benchmark if we should read the current power and not write if it's the same - if bondedValidatorsCount == int(maxValidators) { // is cliff validator - k.setCliffValidator(ctx, validator, k.GetPool(ctx)) - } else if len(oldCliffValidatorAddr) > 0 { - k.clearCliffValidator(ctx) - } break } // either retrieve the original validator from the store, or under the - // situation that this is the "new validator" just use the validator - // provided because it has not yet been updated in the main validator - // store + // situation that this is the "affected validator" just use the + // validator provided because it has not yet been updated in the store ownerAddr := iterator.Value() if bytes.Equal(ownerAddr, affectedValidator.Owner) { validator = affectedValidator @@ -346,38 +343,48 @@ func (k Keeper) UpdateBondedValidators(ctx sdk.Context, } } - // if not previously a validator (and unrevoked), - // kick the cliff validator / bond this new validator - if validator.Status() != sdk.Bonded && !validator.Revoked { - kickCliffValidator = true - - validator = k.bondValidator(ctx, validator) - if bytes.Equal(ownerAddr, affectedValidator.Owner) { - updatedVal = validator - } - } - + // increment bondedValidatorsCount / get the validator to bond if !validator.Revoked { - bondedValidatorsCount++ - } else { - if validator.Status() == sdk.Bonded { - panic(fmt.Sprintf("revoked validator cannot be bonded, address: %v\n", ownerAddr)) + if validator.Status() != sdk.Bonded { + validatorToBond = validator } + bondedValidatorsCount++ + + // sanity check + } else if validator.Status() == sdk.Bonded { + panic(fmt.Sprintf("revoked validator cannot be bonded, address: %v\n", ownerAddr)) } iterator.Next() } iterator.Close() - // perform the actual kicks - if oldCliffValidatorAddr != nil && kickCliffValidator { - validator, found := k.GetValidator(ctx, oldCliffValidatorAddr) - if !found { - panic(fmt.Sprintf("validator record not found for address: %v\n", oldCliffValidatorAddr)) - } - k.unbondValidator(ctx, validator) + // clear or set the cliff validator + if bondedValidatorsCount == int(maxValidators) { + k.setCliffValidator(ctx, validator, k.GetPool(ctx)) + } else if len(oldCliffValidatorAddr) > 0 { + k.clearCliffValidator(ctx) } + // perform the validator swap for any new validators to add + if len(validatorToBond.Owner) > 0 { + + // unbond the cliff validator + if oldCliffValidatorAddr != nil { + cliffVal, found := k.GetValidator(ctx, oldCliffValidatorAddr) + if !found { + panic(fmt.Sprintf("validator record not found for address: %v\n", oldCliffValidatorAddr)) + } + k.unbondValidator(ctx, cliffVal) + + } + + // bond the new validator + validator = k.bondValidator(ctx, validatorToBond) + if bytes.Equal(validator.Owner, affectedValidator.Owner) { + updatedVal = validator + } + } return } @@ -395,17 +402,14 @@ func (k Keeper) UpdateBondedValidatorsFull(ctx sdk.Context) { } iterator.Close() - // add the actual validator power sorted store + oldCliffValidatorAddr := k.GetCliffValidator(ctx) maxValidators := k.GetParams(ctx).MaxValidators - iterator = sdk.KVStoreReversePrefixIterator(store, ValidatorsByPowerIndexKey) // largest to smallest bondedValidatorsCount := 0 + + iterator = sdk.KVStoreReversePrefixIterator(store, ValidatorsByPowerIndexKey) // largest to smallest var validator types.Validator for { if !iterator.Valid() || bondedValidatorsCount > int(maxValidators-1) { - - if bondedValidatorsCount == int(maxValidators) { // is cliff validator - k.setCliffValidator(ctx, validator, k.GetPool(ctx)) - } break } @@ -443,6 +447,13 @@ func (k Keeper) UpdateBondedValidatorsFull(ctx sdk.Context) { } iterator.Close() + // clear or set the cliff validator + if bondedValidatorsCount == int(maxValidators) { + k.setCliffValidator(ctx, validator, k.GetPool(ctx)) + } else if len(oldCliffValidatorAddr) > 0 { + k.clearCliffValidator(ctx) + } + // perform the actual kicks kickOutValidators(k, ctx, toKickOut) return From 053f4e955b7819372eb5432bd2d16625a7ed6c4a Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Tue, 10 Jul 2018 14:25:56 -0400 Subject: [PATCH 03/17] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 222f52e4ab..dabd90a514 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ IMPROVEMENTS * [stake] keeper always loads the store (instead passing around which doesn't really boost efficiency) * [stake] edit-validator changes now can use the keyword [do-not-modify] to not modify unspecified `--flag` (aka won't set them to `""` value) * [stake] offload more generic functionality from the handler into the keeper +* [stake] clearer staking logic * [types] added common tag constants * [keys] improve error message when deleting non-existent key * [gaiacli] improve error messages on `send` and `account` commands From e8e5e8c6007e83bf6d01f906018d0e2176ea482b Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Tue, 10 Jul 2018 18:36:57 -0400 Subject: [PATCH 04/17] bez updates --- x/stake/keeper/validator.go | 52 +++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index 2cc1c2d452..60177aa6e0 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -191,9 +191,11 @@ func (k Keeper) ClearTendermintUpdates(ctx sdk.Context) { //___________________________________________________________________________ -// perfom all the nessisary steps for when a validator changes its power -// updates all validator stores as well as tendermint update store -// may kick out validators if new validator is entering the bonded validator group +// Perfom all the nessisary steps for when a validator changes its power. This +// function updates all validator stores as well as tendermint update store. +// It may kick out validators if new validator is entering the bonded validator +// group. +// // nolint: gocyclo // TODO: Remove above nolint, function needs to be simplified func (k Keeper) UpdateValidator(ctx sdk.Context, validator types.Validator) types.Validator { @@ -203,7 +205,7 @@ func (k Keeper) UpdateValidator(ctx sdk.Context, validator types.Validator) type validator = k.updateForRevoking(ctx, oldFound, oldValidator, validator) powerIncreasing := k.getPowerIncreasing(ctx, oldFound, oldValidator, validator) - validator.BondHeight, validator.BondIntraTxCounter = k.getBondDetails(ctx, oldFound, oldValidator, validator) + validator.BondHeight, validator.BondIntraTxCounter = k.bondIncrement(ctx, oldFound, oldValidator, validator) valPower := k.updateValidatorPower(ctx, oldFound, oldValidator, validator, pool) cliffPower := k.GetCliffValidatorPower(ctx) @@ -228,8 +230,8 @@ func (k Keeper) UpdateValidator(ctx sdk.Context, validator types.Validator) type default: // update the validator set for this validator - updatedVal := k.UpdateBondedValidators(ctx, validator) - if updatedVal.Owner != nil { // updates to validator occurred to be updated + updatedVal, updated := k.UpdateBondedValidators(ctx, validator) + if updated { // updates to validator occurred to be updated validator = updatedVal } else { @@ -266,7 +268,8 @@ func (k Keeper) getPowerIncreasing(ctx sdk.Context, oldFound bool, oldValidator, return false } -func (k Keeper) getBondDetails(ctx sdk.Context, oldFound bool, oldValidator, +// get the bond height and incremented intra-tx counter +func (k Keeper) bondIncrement(ctx sdk.Context, oldFound bool, oldValidator, newValidator types.Validator) (height int64, intraTxCounter int16) { // if already a validator, copy the old block height and counter, else set them @@ -296,25 +299,22 @@ func (k Keeper) updateValidatorPower(ctx sdk.Context, oldFound bool, oldValidato return valPower } -// Update the validator group and kick out any old validators. In addition this -// function adds (or doesn't add) a validator which has updated its bonded -// tokens to the validator group. -> this validator is specified through the -// updatedValidatorAddr term. +// Update the bonded validator group based on a change to the validator +// affectedValidator. This function potentially adds the affectedValidator to +// the bonded validator group which kicks out the cliff validator. Under this +// situation this function returns the updated affectedValidator. // -// The correct subset is retrieved by iterating through an index of the -// validators sorted by power, stored using the ValidatorsByPowerIndexKey. -// Simultaneously the current validator records are updated in store with the -// ValidatorsBondedIndexKey. This store is used to determine if a validator is -// a validator without needing to iterate over the subspace as we do in -// GetValidators. -// -// Optionally also return the validator from a retrieve address if the -// validator has been bonded +// The correct bonded subset of validators is retrieved by iterating through an +// index of the validators sorted by power, stored using the +// ValidatorsByPowerIndexKey. Simultaneously the current validator records are +// updated in store with the ValidatorsBondedIndexKey. This store is used to +// determine if a validator is a validator without needing to iterate over all +// validators. // // nolint: gocyclo // TODO: Remove the above golint func (k Keeper) UpdateBondedValidators(ctx sdk.Context, - affectedValidator types.Validator) (updatedVal types.Validator) { + affectedValidator types.Validator) (updatedVal types.Validator, updated bool) { store := ctx.KVStore(k.storeKey) @@ -322,6 +322,7 @@ func (k Keeper) UpdateBondedValidators(ctx sdk.Context, maxValidators := k.GetParams(ctx).MaxValidators bondedValidatorsCount := 0 var validator, validatorToBond types.Validator + newValidatorBonded := false iterator := sdk.KVStoreReversePrefixIterator(store, ValidatorsByPowerIndexKey) // largest to smallest for { @@ -347,6 +348,7 @@ func (k Keeper) UpdateBondedValidators(ctx sdk.Context, if !validator.Revoked { if validator.Status() != sdk.Bonded { validatorToBond = validator + newValidatorBonded = true } bondedValidatorsCount++ @@ -366,8 +368,8 @@ func (k Keeper) UpdateBondedValidators(ctx sdk.Context, k.clearCliffValidator(ctx) } - // perform the validator swap for any new validators to add - if len(validatorToBond.Owner) > 0 { + // swap the cliff validator for a new validator if the affected validator was bonded + if newValidatorBonded { // unbond the cliff validator if oldCliffValidatorAddr != nil { @@ -382,10 +384,10 @@ func (k Keeper) UpdateBondedValidators(ctx sdk.Context, // bond the new validator validator = k.bondValidator(ctx, validatorToBond) if bytes.Equal(validator.Owner, affectedValidator.Owner) { - updatedVal = validator + return validator, true } } - return + return Validator{}, false } // full update of the bonded validator set, many can be added/kicked From 561eda3fc337b6cbeeaae80f435d7c0190c0d19b Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Tue, 10 Jul 2018 18:39:52 -0400 Subject: [PATCH 05/17] typo --- x/stake/keeper/validator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index 60177aa6e0..db10ebd362 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -387,7 +387,7 @@ func (k Keeper) UpdateBondedValidators(ctx sdk.Context, return validator, true } } - return Validator{}, false + return types.Validator{}, false } // full update of the bonded validator set, many can be added/kicked From 9eeb47517d02129dc5b1c5955a933579ec5acb5c Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Tue, 10 Jul 2018 19:27:39 -0400 Subject: [PATCH 06/17] cwgoes comments --- x/stake/keeper/validator.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index db10ebd362..e96a1330b5 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -228,7 +228,11 @@ func (k Keeper) UpdateValidator(ctx sdk.Context, validator types.Validator) type bytes.Compare(valPower, cliffPower) == -1: //(valPower < cliffPower // skip to completion + // default case - validator was either: + // a) not-bonded and now has power-rank greater than cliff validator + // b) bonded and now has decreased in power default: + // update the validator set for this validator updatedVal, updated := k.UpdateBondedValidators(ctx, validator) if updated { // updates to validator occurred to be updated From cba7fd7ee831ab1b690d2414fec91d6674e7c09d Mon Sep 17 00:00:00 2001 From: Sunny Aggarwal Date: Tue, 10 Jul 2018 17:59:07 -0700 Subject: [PATCH 07/17] gov enums marshal to String --- CHANGELOG.md | 2 + client/lcd/lcd_test.go | 60 ++++---- cmd/gaia/cli_test/cli_test.go | 18 +-- x/gov/client/cli/tx.go | 12 +- x/gov/client/rest/rest.go | 75 +++------- x/gov/depositsvotes.go | 156 +++++++++++---------- x/gov/errors.go | 9 +- x/gov/keeper.go | 6 +- x/gov/msgs.go | 44 ++---- x/gov/msgs_test.go | 2 +- x/gov/proposals.go | 252 ++++++++++++++++++++++------------ 11 files changed, 335 insertions(+), 301 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a4c6acf7b..f7a12d87fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,8 @@ BREAKING CHANGES * [lcd] Switch key creation output to return bech32 * [lcd] Removed shorthand CLI flags (`a`, `c`, `n`, `o`) * [gaiad] genesis transactions now use bech32 addresses / pubkeys +* [gov] VoteStatus renamed to ProposalStatus +* [gov] VoteOption, ProposalType, and ProposalStatus all marshal to string form in JSON DEPRECATED * [cli] Deprecated `--name` flag in commands that send txs, in favor of `--from` diff --git a/client/lcd/lcd_test.go b/client/lcd/lcd_test.go index b5800ca292..d4ca9d0d38 100644 --- a/client/lcd/lcd_test.go +++ b/client/lcd/lcd_test.go @@ -443,7 +443,7 @@ func TestSubmitProposal(t *testing.T) { // query proposal proposal := getProposal(t, port, proposalID) - require.Equal(t, "Test", proposal.Title) + require.Equal(t, "Test", proposal.GetTitle()) } func TestDeposit(t *testing.T) { @@ -465,7 +465,7 @@ func TestDeposit(t *testing.T) { // query proposal proposal := getProposal(t, port, proposalID) - require.Equal(t, "Test", proposal.Title) + require.Equal(t, "Test", proposal.GetTitle()) // create SubmitProposal TX resultTx = doDeposit(t, port, seed, name, password, addr, proposalID) @@ -473,7 +473,7 @@ func TestDeposit(t *testing.T) { // query proposal proposal = getProposal(t, port, proposalID) - require.True(t, proposal.TotalDeposit.IsEqual(sdk.Coins{sdk.NewCoin("steak", 10)})) + require.True(t, proposal.GetTotalDeposit().IsEqual(sdk.Coins{sdk.NewCoin("steak", 10)})) // query deposit deposit := getDeposit(t, port, proposalID, addr) @@ -499,7 +499,7 @@ func TestVote(t *testing.T) { // query proposal proposal := getProposal(t, port, proposalID) - require.Equal(t, "Test", proposal.Title) + require.Equal(t, "Test", proposal.GetTitle()) // create SubmitProposal TX resultTx = doDeposit(t, port, seed, name, password, addr, proposalID) @@ -507,7 +507,7 @@ func TestVote(t *testing.T) { // query proposal proposal = getProposal(t, port, proposalID) - require.Equal(t, gov.StatusToString(gov.StatusVotingPeriod), proposal.Status) + require.Equal(t, gov.StatusVotingPeriod, proposal.GetStatus()) // create SubmitProposal TX resultTx = doVote(t, port, seed, name, password, addr, proposalID) @@ -515,7 +515,7 @@ func TestVote(t *testing.T) { vote := getVote(t, port, proposalID, addr) require.Equal(t, proposalID, vote.ProposalID) - require.Equal(t, gov.VoteOptionToString(gov.OptionYes), vote.Option) + require.Equal(t, gov.OptionYes, vote.Option) } func TestUnrevoke(t *testing.T) { @@ -576,31 +576,31 @@ func TestProposalsQuery(t *testing.T) { // Test query all proposals proposals := getProposalsAll(t, port) - require.Equal(t, proposalID1, (proposals[0]).ProposalID) - require.Equal(t, proposalID2, (proposals[1]).ProposalID) - require.Equal(t, proposalID3, (proposals[2]).ProposalID) + require.Equal(t, proposalID1, (proposals[0]).GetProposalID()) + require.Equal(t, proposalID2, (proposals[1]).GetProposalID()) + require.Equal(t, proposalID3, (proposals[2]).GetProposalID()) // Test query deposited by addr1 proposals = getProposalsFilterDepositer(t, port, addr) - require.Equal(t, proposalID1, (proposals[0]).ProposalID) + require.Equal(t, proposalID1, (proposals[0]).GetProposalID()) // Test query deposited by addr2 proposals = getProposalsFilterDepositer(t, port, addr2) - require.Equal(t, proposalID2, (proposals[0]).ProposalID) - require.Equal(t, proposalID3, (proposals[1]).ProposalID) + require.Equal(t, proposalID2, (proposals[0]).GetProposalID()) + require.Equal(t, proposalID3, (proposals[1]).GetProposalID()) // Test query voted by addr1 proposals = getProposalsFilterVoter(t, port, addr) - require.Equal(t, proposalID2, (proposals[0]).ProposalID) - require.Equal(t, proposalID3, (proposals[1]).ProposalID) + require.Equal(t, proposalID2, (proposals[0]).GetProposalID()) + require.Equal(t, proposalID3, (proposals[1]).GetProposalID()) // Test query voted by addr2 proposals = getProposalsFilterVoter(t, port, addr2) - require.Equal(t, proposalID3, (proposals[0]).ProposalID) + require.Equal(t, proposalID3, (proposals[0]).GetProposalID()) // Test query voted and deposited by addr1 proposals = getProposalsFilterVoterDepositer(t, port, addr, addr) - require.Equal(t, proposalID2, (proposals[0]).ProposalID) + require.Equal(t, proposalID2, (proposals[0]).GetProposalID()) } //_____________________________________________________________________________ @@ -838,68 +838,68 @@ func getValidators(t *testing.T, port string) []stakerest.StakeValidatorOutput { return validators } -func getProposal(t *testing.T, port string, proposalID int64) gov.ProposalRest { +func getProposal(t *testing.T, port string, proposalID int64) gov.Proposal { res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals/%d", proposalID), nil) require.Equal(t, http.StatusOK, res.StatusCode, body) - var proposal gov.ProposalRest + var proposal gov.Proposal err := cdc.UnmarshalJSON([]byte(body), &proposal) require.Nil(t, err) return proposal } -func getDeposit(t *testing.T, port string, proposalID int64, depositerAddr sdk.AccAddress) gov.DepositRest { +func getDeposit(t *testing.T, port string, proposalID int64, depositerAddr sdk.AccAddress) gov.Deposit { res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals/%d/deposits/%s", proposalID, depositerAddr), nil) require.Equal(t, http.StatusOK, res.StatusCode, body) - var deposit gov.DepositRest + var deposit gov.Deposit err := cdc.UnmarshalJSON([]byte(body), &deposit) require.Nil(t, err) return deposit } -func getVote(t *testing.T, port string, proposalID int64, voterAddr sdk.AccAddress) gov.VoteRest { +func getVote(t *testing.T, port string, proposalID int64, voterAddr sdk.AccAddress) gov.Vote { res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals/%d/votes/%s", proposalID, voterAddr), nil) require.Equal(t, http.StatusOK, res.StatusCode, body) - var vote gov.VoteRest + var vote gov.Vote err := cdc.UnmarshalJSON([]byte(body), &vote) require.Nil(t, err) return vote } -func getProposalsAll(t *testing.T, port string) []gov.ProposalRest { +func getProposalsAll(t *testing.T, port string) []gov.Proposal { res, body := Request(t, port, "GET", "/gov/proposals", nil) require.Equal(t, http.StatusOK, res.StatusCode, body) - var proposals []gov.ProposalRest + var proposals []gov.Proposal err := cdc.UnmarshalJSON([]byte(body), &proposals) require.Nil(t, err) return proposals } -func getProposalsFilterDepositer(t *testing.T, port string, depositerAddr sdk.AccAddress) []gov.ProposalRest { +func getProposalsFilterDepositer(t *testing.T, port string, depositerAddr sdk.AccAddress) []gov.Proposal { res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals?depositer=%s", depositerAddr), nil) require.Equal(t, http.StatusOK, res.StatusCode, body) - var proposals []gov.ProposalRest + var proposals []gov.Proposal err := cdc.UnmarshalJSON([]byte(body), &proposals) require.Nil(t, err) return proposals } -func getProposalsFilterVoter(t *testing.T, port string, voterAddr sdk.AccAddress) []gov.ProposalRest { +func getProposalsFilterVoter(t *testing.T, port string, voterAddr sdk.AccAddress) []gov.Proposal { res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals?voter=%s", voterAddr), nil) require.Equal(t, http.StatusOK, res.StatusCode, body) - var proposals []gov.ProposalRest + var proposals []gov.Proposal err := cdc.UnmarshalJSON([]byte(body), &proposals) require.Nil(t, err) return proposals } -func getProposalsFilterVoterDepositer(t *testing.T, port string, voterAddr, depositerAddr sdk.AccAddress) []gov.ProposalRest { +func getProposalsFilterVoterDepositer(t *testing.T, port string, voterAddr, depositerAddr sdk.AccAddress) []gov.Proposal { res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals?depositer=%s&voter=%s", depositerAddr, voterAddr), nil) require.Equal(t, http.StatusOK, res.StatusCode, body) - var proposals []gov.ProposalRest + var proposals []gov.Proposal err := cdc.UnmarshalJSON([]byte(body), &proposals) require.Nil(t, err) return proposals diff --git a/cmd/gaia/cli_test/cli_test.go b/cmd/gaia/cli_test/cli_test.go index 38a347b241..645a6c276a 100644 --- a/cmd/gaia/cli_test/cli_test.go +++ b/cmd/gaia/cli_test/cli_test.go @@ -183,8 +183,8 @@ func TestGaiaCLISubmitProposal(t *testing.T) { require.Equal(t, int64(45), fooAcc.GetCoins().AmountOf("steak").Int64()) proposal1 := executeGetProposal(t, fmt.Sprintf("gaiacli gov query-proposal --proposalID=1 --output=json %v", flags)) - require.Equal(t, int64(1), proposal1.ProposalID) - require.Equal(t, gov.StatusToString(gov.StatusDepositPeriod), proposal1.Status) + require.Equal(t, int64(1), proposal1.GetProposalID()) + require.Equal(t, gov.StatusDepositPeriod, proposal1.GetStatus()) executeWrite(t, fmt.Sprintf("gaiacli gov deposit %v --depositer=%s --deposit=10steak --proposalID=1 --from=foo", flags, fooAddr), pass) tests.WaitForNextHeightTM(port) @@ -192,15 +192,15 @@ func TestGaiaCLISubmitProposal(t *testing.T) { fooAcc = executeGetAccount(t, fmt.Sprintf("gaiacli account %s %v", fooAddr, flags)) require.Equal(t, int64(35), fooAcc.GetCoins().AmountOf("steak").Int64()) proposal1 = executeGetProposal(t, fmt.Sprintf("gaiacli gov query-proposal --proposalID=1 --output=json %v", flags)) - require.Equal(t, int64(1), proposal1.ProposalID) - require.Equal(t, gov.StatusToString(gov.StatusVotingPeriod), proposal1.Status) + require.Equal(t, int64(1), proposal1.GetProposalID()) + require.Equal(t, gov.StatusVotingPeriod, proposal1.GetStatus()) executeWrite(t, fmt.Sprintf("gaiacli gov vote %v --proposalID=1 --voter=%s --option=Yes --from=foo", flags, fooAddr), pass) tests.WaitForNextHeightTM(port) vote := executeGetVote(t, fmt.Sprintf("gaiacli gov query-vote --proposalID=1 --voter=%s --output=json %v", fooAddr, flags)) require.Equal(t, int64(1), vote.ProposalID) - require.Equal(t, gov.VoteOptionToString(gov.OptionYes), vote.Option) + require.Equal(t, gov.OptionYes, vote.Option) } //___________________________________________________________________________________ @@ -288,18 +288,18 @@ func executeGetValidator(t *testing.T, cmdStr string) stake.Validator { return validator } -func executeGetProposal(t *testing.T, cmdStr string) gov.ProposalRest { +func executeGetProposal(t *testing.T, cmdStr string) gov.Proposal { out := tests.ExecuteT(t, cmdStr) - var proposal gov.ProposalRest + var proposal gov.Proposal cdc := app.MakeCodec() err := cdc.UnmarshalJSON([]byte(out), &proposal) require.NoError(t, err, "out %v\n, err %v", out, err) return proposal } -func executeGetVote(t *testing.T, cmdStr string) gov.VoteRest { +func executeGetVote(t *testing.T, cmdStr string) gov.Vote { out := tests.ExecuteT(t, cmdStr) - var vote gov.VoteRest + var vote gov.Vote cdc := app.MakeCodec() err := cdc.UnmarshalJSON([]byte(out), &vote) require.NoError(t, err, "out %v\n, err %v", out, err) diff --git a/x/gov/client/cli/tx.go b/x/gov/client/cli/tx.go index 1c5c11b71e..8369c99273 100644 --- a/x/gov/client/cli/tx.go +++ b/x/gov/client/cli/tx.go @@ -48,7 +48,7 @@ func GetCmdSubmitProposal(cdc *wire.Codec) *cobra.Command { return err } - proposalType, err := gov.StringToProposalType(strProposalType) + proposalType, err := gov.ProposalTypeFromString(strProposalType) if err != nil { return err } @@ -145,7 +145,7 @@ func GetCmdVote(cdc *wire.Codec) *cobra.Command { option := viper.GetString(flagOption) - byteVoteOption, err := gov.StringToVoteOption(option) + byteVoteOption, err := gov.VoteOptionFromString(option) if err != nil { return err } @@ -158,7 +158,7 @@ func GetCmdVote(cdc *wire.Codec) *cobra.Command { return err } - fmt.Printf("Vote[Voter:%s,ProposalID:%d,Option:%s]", bechVoter, msg.ProposalID, gov.VoteOptionToString(msg.Option)) + fmt.Printf("Vote[Voter:%s,ProposalID:%d,Option:%s]", bechVoter, msg.ProposalID, msg.Option) // build and sign the transaction, then broadcast to Tendermint ctx := context.NewCoreContextFromViper().WithDecoder(authcmd.GetAccountDecoder(cdc)) @@ -195,8 +195,7 @@ func GetCmdQueryProposal(storeName string, cdc *wire.Codec) *cobra.Command { var proposal gov.Proposal cdc.MustUnmarshalBinary(res, &proposal) - proposalRest := gov.ProposalToRest(proposal) - output, err := wire.MarshalJSONIndent(cdc, proposalRest) + output, err := wire.MarshalJSONIndent(cdc, proposal) if err != nil { return err } @@ -232,8 +231,7 @@ func GetCmdQueryVote(storeName string, cdc *wire.Codec) *cobra.Command { var vote gov.Vote cdc.MustUnmarshalBinary(res, &vote) - voteRest := gov.VoteToRest(vote) - output, err := wire.MarshalJSONIndent(cdc, voteRest) + output, err := wire.MarshalJSONIndent(cdc, vote) if err != nil { return err } diff --git a/x/gov/client/rest/rest.go b/x/gov/client/rest/rest.go index 3e121d4d41..1473045871 100644 --- a/x/gov/client/rest/rest.go +++ b/x/gov/client/rest/rest.go @@ -36,24 +36,24 @@ func RegisterRoutes(ctx context.CoreContext, r *mux.Router, cdc *wire.Codec) { } type postProposalReq struct { - BaseReq baseReq `json:"base_req"` - Title string `json:"title"` // Title of the proposal - Description string `json:"description"` // Description of the proposal - ProposalType string `json:"proposal_type"` // Type of proposal. Initial set {PlainTextProposal, SoftwareUpgradeProposal} - Proposer string `json:"proposer"` // Address of the proposer - InitialDeposit sdk.Coins `json:"initial_deposit"` // Coins to add to the proposal's deposit + BaseReq baseReq `json:"base_req"` + Title string `json:"title"` // Title of the proposal + Description string `json:"description"` // Description of the proposal + ProposalType gov.ProposalKind `json:"proposal_type"` // Type of proposal. Initial set {PlainTextProposal, SoftwareUpgradeProposal} + Proposer sdk.AccAddress `json:"proposer"` // Address of the proposer + InitialDeposit sdk.Coins `json:"initial_deposit"` // Coins to add to the proposal's deposit } type depositReq struct { - BaseReq baseReq `json:"base_req"` - Depositer string `json:"depositer"` // Address of the depositer - Amount sdk.Coins `json:"amount"` // Coins to add to the proposal's deposit + BaseReq baseReq `json:"base_req"` + Depositer sdk.AccAddress `json:"depositer"` // Address of the depositer + Amount sdk.Coins `json:"amount"` // Coins to add to the proposal's deposit } type voteReq struct { - BaseReq baseReq `json:"base_req"` - Voter string `json:"voter"` // address of the voter - Option string `json:"option"` // option from OptionSet chosen by the voter + BaseReq baseReq `json:"base_req"` + Voter sdk.AccAddress `json:"voter"` // address of the voter + Option gov.VoteOption `json:"option"` // option from OptionSet chosen by the voter } func postProposalHandlerFn(cdc *wire.Codec, ctx context.CoreContext) http.HandlerFunc { @@ -68,20 +68,8 @@ func postProposalHandlerFn(cdc *wire.Codec, ctx context.CoreContext) http.Handle return } - proposer, err := sdk.AccAddressFromBech32(req.Proposer) - if err != nil { - writeErr(&w, http.StatusBadRequest, err.Error()) - return - } - - proposalTypeByte, err := gov.StringToProposalType(req.ProposalType) - if err != nil { - writeErr(&w, http.StatusBadRequest, err.Error()) - return - } - // create the message - msg := gov.NewMsgSubmitProposal(req.Title, req.Description, proposalTypeByte, proposer, req.InitialDeposit) + msg := gov.NewMsgSubmitProposal(req.Title, req.Description, req.ProposalType, req.Proposer, req.InitialDeposit) err = msg.ValidateBasic() if err != nil { writeErr(&w, http.StatusBadRequest, err.Error()) @@ -117,19 +105,12 @@ func depositHandlerFn(cdc *wire.Codec, ctx context.CoreContext) http.HandlerFunc if err != nil { return } - if !req.BaseReq.baseReqValidate(w) { return } - depositer, err := sdk.AccAddressFromBech32(req.Depositer) - if err != nil { - writeErr(&w, http.StatusBadRequest, err.Error()) - return - } - // create the message - msg := gov.NewMsgDeposit(depositer, proposalID, req.Amount) + msg := gov.NewMsgDeposit(req.Depositer, proposalID, req.Amount) err = msg.ValidateBasic() if err != nil { writeErr(&w, http.StatusBadRequest, err.Error()) @@ -165,25 +146,12 @@ func voteHandlerFn(cdc *wire.Codec, ctx context.CoreContext) http.HandlerFunc { if err != nil { return } - if !req.BaseReq.baseReqValidate(w) { return } - voter, err := sdk.AccAddressFromBech32(req.Voter) - if err != nil { - writeErr(&w, http.StatusBadRequest, err.Error()) - return - } - - voteOptionByte, err := gov.StringToVoteOption(req.Option) - if err != nil { - writeErr(&w, http.StatusBadRequest, err.Error()) - return - } - // create the message - msg := gov.NewMsgVote(voter, proposalID, voteOptionByte) + msg := gov.NewMsgVote(req.Voter, proposalID, req.Option) err = msg.ValidateBasic() if err != nil { writeErr(&w, http.StatusBadRequest, err.Error()) @@ -225,8 +193,7 @@ func queryProposalHandlerFn(cdc *wire.Codec) http.HandlerFunc { var proposal gov.Proposal cdc.MustUnmarshalBinary(res, &proposal) - proposalRest := gov.ProposalToRest(proposal) - output, err := wire.MarshalJSONIndent(cdc, proposalRest) + output, err := wire.MarshalJSONIndent(cdc, proposal) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) @@ -291,8 +258,7 @@ func queryDepositHandlerFn(cdc *wire.Codec) http.HandlerFunc { var deposit gov.Deposit cdc.MustUnmarshalBinary(res, &deposit) - depositRest := gov.DepositToRest(deposit) - output, err := wire.MarshalJSONIndent(cdc, depositRest) + output, err := wire.MarshalJSONIndent(cdc, deposit) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) @@ -358,8 +324,7 @@ func queryVoteHandlerFn(cdc *wire.Codec) http.HandlerFunc { var vote gov.Vote cdc.MustUnmarshalBinary(res, &vote) - voteRest := gov.VoteToRest(vote) - output, err := wire.MarshalJSONIndent(cdc, voteRest) + output, err := wire.MarshalJSONIndent(cdc, vote) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) @@ -411,7 +376,7 @@ func queryProposalsWithParameterFn(cdc *wire.Codec) http.HandlerFunc { var maxProposalID int64 cdc.MustUnmarshalBinary(res, &maxProposalID) - matchingProposals := []gov.ProposalRest{} + matchingProposals := []gov.Proposal{} for proposalID := int64(0); proposalID < maxProposalID; proposalID++ { if voterAddr != nil { @@ -435,7 +400,7 @@ func queryProposalsWithParameterFn(cdc *wire.Codec) http.HandlerFunc { var proposal gov.Proposal cdc.MustUnmarshalBinary(res, &proposal) - matchingProposals = append(matchingProposals, gov.ProposalToRest(proposal)) + matchingProposals = append(matchingProposals, proposal) } output, err := wire.MarshalJSONIndent(cdc, matchingProposals) diff --git a/x/gov/depositsvotes.go b/x/gov/depositsvotes.go index 5ec5651897..19ed97f69d 100644 --- a/x/gov/depositsvotes.go +++ b/x/gov/depositsvotes.go @@ -1,19 +1,11 @@ package gov import ( + "encoding/json" + "fmt" + sdk "github.com/cosmos/cosmos-sdk/types" -) - -// Type that represents VoteOption as a byte -type VoteOption = byte - -//nolint -const ( - OptionEmpty VoteOption = 0x00 - OptionYes VoteOption = 0x01 - OptionAbstain VoteOption = 0x02 - OptionNo VoteOption = 0x03 - OptionNoWithVeto VoteOption = 0x04 + "github.com/pkg/errors" ) // Vote @@ -30,9 +22,80 @@ type Deposit struct { Amount sdk.Coins `json:"amount"` // Deposit amount } -// ProposalTypeToString for pretty prints of ProposalType -func VoteOptionToString(option VoteOption) string { - switch option { +// Type that represents VoteOption as a byte +type VoteOption byte + +//nolint +const ( + OptionEmpty VoteOption = 0x00 + OptionYes VoteOption = 0x01 + OptionAbstain VoteOption = 0x02 + OptionNo VoteOption = 0x03 + OptionNoWithVeto VoteOption = 0x04 +) + +// String to proposalType byte. Returns ff if invalid. +func VoteOptionFromString(str string) (VoteOption, error) { + switch str { + case "Yes": + return OptionYes, nil + case "Abstain": + return OptionAbstain, nil + case "No": + return OptionNo, nil + case "NoWithVeto": + return OptionNoWithVeto, nil + default: + return VoteOption(0xff), errors.Errorf("'%s' is not a valid vote option", str) + } +} + +// Is defined VoteOption +func validVoteOption(option VoteOption) bool { + if option == OptionYes || + option == OptionAbstain || + option == OptionNo || + option == OptionNoWithVeto { + return true + } + return false +} + +// Marshal needed for protobuf compatibility +func (vo VoteOption) Marshal() ([]byte, error) { + return []byte{byte(vo)}, nil +} + +// Unmarshal needed for protobuf compatibility +func (vo *VoteOption) Unmarshal(data []byte) error { + *vo = VoteOption(data[0]) + return nil +} + +// Marshals to JSON using string +func (vo VoteOption) MarshalJSON() ([]byte, error) { + return json.Marshal(vo.String()) +} + +// Unmarshals from JSON assuming Bech32 encoding +func (vo *VoteOption) UnmarshalJSON(data []byte) error { + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return nil + } + + bz2, err := VoteOptionFromString(s) + if err != nil { + return err + } + *vo = bz2 + return nil +} + +// Turns VoteOption byte to String +func (vo VoteOption) String() string { + switch vo { case OptionYes: return "Yes" case OptionAbstain: @@ -46,63 +109,12 @@ func VoteOptionToString(option VoteOption) string { } } -func validVoteOption(option VoteOption) bool { - if option == OptionYes || - option == OptionAbstain || - option == OptionNo || - option == OptionNoWithVeto { - return true - } - return false -} - -// String to proposalType byte. Returns ff if invalid. -func StringToVoteOption(str string) (VoteOption, sdk.Error) { - switch str { - case "Yes": - return OptionYes, nil - case "Abstain": - return OptionAbstain, nil - case "No": - return OptionNo, nil - case "NoWithVeto": - return OptionNoWithVeto, nil +// For Printf / Sprintf, returns bech32 when using %s +func (vo VoteOption) Format(s fmt.State, verb rune) { + switch verb { + case 's': + s.Write([]byte(fmt.Sprintf("%s", vo.String()))) default: - return VoteOption(0xff), ErrInvalidVote(DefaultCodespace, str) - } -} - -//----------------------------------------------------------- -// REST - -// Rest Deposits -type DepositRest struct { - Depositer sdk.AccAddress `json:"depositer"` // address of the depositer - ProposalID int64 `json:"proposal_id"` // proposalID of the proposal - Amount sdk.Coins `json:"option"` -} - -// Turn any Deposit to a DepositRest -func DepositToRest(deposit Deposit) DepositRest { - return DepositRest{ - Depositer: deposit.Depositer, - ProposalID: deposit.ProposalID, - Amount: deposit.Amount, - } -} - -// Rest Votes -type VoteRest struct { - Voter sdk.AccAddress `json:"voter"` // address of the voter - ProposalID int64 `json:"proposal_id"` // proposalID of the proposal - Option string `json:"option"` -} - -// Turn any Vote to a VoteRest -func VoteToRest(vote Vote) VoteRest { - return VoteRest{ - Voter: vote.Voter, - ProposalID: vote.ProposalID, - Option: VoteOptionToString(vote.Option), + s.Write([]byte(fmt.Sprintf("%v", byte(vo)))) } } diff --git a/x/gov/errors.go b/x/gov/errors.go index e1f2f27937..0825e00f8c 100644 --- a/x/gov/errors.go +++ b/x/gov/errors.go @@ -20,6 +20,7 @@ const ( CodeInvalidProposalType sdk.CodeType = 8 CodeInvalidVote sdk.CodeType = 9 CodeInvalidGenesis sdk.CodeType = 10 + CodeInvalidProposalStatus sdk.CodeType = 11 ) //---------------------------------------- @@ -53,12 +54,12 @@ func ErrInvalidDescription(codespace sdk.CodespaceType, description string) sdk. return sdk.NewError(codespace, CodeInvalidDescription, fmt.Sprintf("Proposal Desciption '%s' is not valid", description)) } -func ErrInvalidProposalType(codespace sdk.CodespaceType, strProposalType string) sdk.Error { - return sdk.NewError(codespace, CodeInvalidProposalType, fmt.Sprintf("Proposal Type '%s' is not valid", strProposalType)) +func ErrInvalidProposalType(codespace sdk.CodespaceType, proposalType ProposalKind) sdk.Error { + return sdk.NewError(codespace, CodeInvalidProposalType, fmt.Sprintf("Proposal Type '%s' is not valid", proposalType)) } -func ErrInvalidVote(codespace sdk.CodespaceType, strOption string) sdk.Error { - return sdk.NewError(codespace, CodeInvalidVote, fmt.Sprintf("'%s' is not a valid voting option", strOption)) +func ErrInvalidVote(codespace sdk.CodespaceType, voteOption VoteOption) sdk.Error { + return sdk.NewError(codespace, CodeInvalidVote, fmt.Sprintf("'%v' is not a valid voting option", voteOption)) } func ErrInvalidGenesis(codespace sdk.CodespaceType, msg string) sdk.Error { diff --git a/x/gov/keeper.go b/x/gov/keeper.go index d966a78a2f..b60404b8ce 100644 --- a/x/gov/keeper.go +++ b/x/gov/keeper.go @@ -48,7 +48,7 @@ func (keeper Keeper) WireCodec() *wire.Codec { // Proposals // Creates a NewProposal -func (keeper Keeper) NewTextProposal(ctx sdk.Context, title string, description string, proposalType byte) Proposal { +func (keeper Keeper) NewTextProposal(ctx sdk.Context, title string, description string, proposalType ProposalKind) Proposal { proposalID, err := keeper.getNewProposalID(ctx) if err != nil { return nil @@ -165,8 +165,8 @@ func (keeper Keeper) AddVote(ctx sdk.Context, proposalID int64, voterAddr sdk.Ac return ErrInactiveProposal(keeper.codespace, proposalID) } - if option != OptionYes && option != OptionAbstain && option != OptionNo && option != OptionNoWithVeto { - return ErrInvalidVote(keeper.codespace, VoteOptionToString(option)) + if !validVoteOption(option) { + return ErrInvalidVote(keeper.codespace, option) } vote := Vote{ diff --git a/x/gov/msgs.go b/x/gov/msgs.go index 5b38861eef..5d85f689e5 100644 --- a/x/gov/msgs.go +++ b/x/gov/msgs.go @@ -41,7 +41,7 @@ func (msg MsgSubmitProposal) ValidateBasic() sdk.Error { return ErrInvalidDescription(DefaultCodespace, msg.Description) // TODO: Proper Error } if !validProposalType(msg.ProposalType) { - return ErrInvalidProposalType(DefaultCodespace, ProposalTypeToString(msg.ProposalType)) + return ErrInvalidProposalType(DefaultCodespace, msg.ProposalType) } if len(msg.Proposer) == 0 { return sdk.ErrInvalidAddress(msg.Proposer.String()) @@ -56,7 +56,7 @@ func (msg MsgSubmitProposal) ValidateBasic() sdk.Error { } func (msg MsgSubmitProposal) String() string { - return fmt.Sprintf("MsgSubmitProposal{%v, %v, %v, %v}", msg.Title, msg.Description, ProposalTypeToString(msg.ProposalType), msg.InitialDeposit) + return fmt.Sprintf("MsgSubmitProposal{%s, %s, %s, %v}", msg.Title, msg.Description, msg.ProposalType, msg.InitialDeposit) } // Implements Msg. @@ -66,19 +66,7 @@ func (msg MsgSubmitProposal) Get(key interface{}) (value interface{}) { // Implements Msg. func (msg MsgSubmitProposal) GetSignBytes() []byte { - b, err := msgCdc.MarshalJSON(struct { - Title string `json:"title"` - Description string `json:"description"` - ProposalType string `json:"proposal_type"` - Proposer sdk.AccAddress `json:"proposer"` - InitialDeposit sdk.Coins `json:"deposit"` - }{ - Title: msg.Title, - Description: msg.Description, - ProposalType: ProposalTypeToString(msg.ProposalType), - Proposer: msg.Proposer, - InitialDeposit: msg.InitialDeposit, - }) + b, err := msgCdc.MarshalJSON(msg) if err != nil { panic(err) } @@ -127,7 +115,7 @@ func (msg MsgDeposit) ValidateBasic() sdk.Error { } func (msg MsgDeposit) String() string { - return fmt.Sprintf("MsgDeposit{%v=>%v: %v}", msg.Depositer, msg.ProposalID, msg.Amount) + return fmt.Sprintf("MsgDeposit{%s=>%v: %v}", msg.Depositer, msg.ProposalID, msg.Amount) } // Implements Msg. @@ -137,15 +125,7 @@ func (msg MsgDeposit) Get(key interface{}) (value interface{}) { // Implements Msg. func (msg MsgDeposit) GetSignBytes() []byte { - b, err := msgCdc.MarshalJSON(struct { - ProposalID int64 `json:"proposalID"` - Depositer sdk.AccAddress `json:"proposer"` - Amount sdk.Coins `json:"deposit"` - }{ - ProposalID: msg.ProposalID, - Depositer: msg.Depositer, - Amount: msg.Amount, - }) + b, err := msgCdc.MarshalJSON(msg) if err != nil { panic(err) } @@ -185,13 +165,13 @@ func (msg MsgVote) ValidateBasic() sdk.Error { return ErrUnknownProposal(DefaultCodespace, msg.ProposalID) } if !validVoteOption(msg.Option) { - return ErrInvalidVote(DefaultCodespace, VoteOptionToString(msg.Option)) + return ErrInvalidVote(DefaultCodespace, msg.Option) } return nil } func (msg MsgVote) String() string { - return fmt.Sprintf("MsgVote{%v - %v}", msg.ProposalID, msg.Option) + return fmt.Sprintf("MsgVote{%v - %s}", msg.ProposalID, msg.Option) } // Implements Msg. @@ -201,15 +181,7 @@ func (msg MsgVote) Get(key interface{}) (value interface{}) { // Implements Msg. func (msg MsgVote) GetSignBytes() []byte { - b, err := msgCdc.MarshalJSON(struct { - ProposalID int64 `json:"proposalID"` - Voter sdk.AccAddress `json:"voter"` - Option string `json:"option"` - }{ - ProposalID: msg.ProposalID, - Voter: msg.Voter, - Option: VoteOptionToString(msg.Option), - }) + b, err := msgCdc.MarshalJSON(msg) if err != nil { panic(err) } diff --git a/x/gov/msgs_test.go b/x/gov/msgs_test.go index fd526c7fa7..186b8365ba 100644 --- a/x/gov/msgs_test.go +++ b/x/gov/msgs_test.go @@ -22,7 +22,7 @@ func TestMsgSubmitProposal(t *testing.T) { _, addrs, _, _ := mock.CreateGenAccounts(1, sdk.Coins{}) tests := []struct { title, description string - proposalType byte + proposalType ProposalKind proposerAddr sdk.AccAddress initialDeposit sdk.Coins expectPass bool diff --git a/x/gov/proposals.go b/x/gov/proposals.go index c81d61e21d..64ffe6bfa2 100644 --- a/x/gov/proposals.go +++ b/x/gov/proposals.go @@ -1,27 +1,14 @@ package gov import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + sdk "github.com/cosmos/cosmos-sdk/types" ) -// Type that represents Status as a byte -type VoteStatus = byte - -// Type that represents Proposal Type as a byte -type ProposalKind = byte - -//nolint -const ( - StatusDepositPeriod VoteStatus = 0x01 - StatusVotingPeriod VoteStatus = 0x02 - StatusPassed VoteStatus = 0x03 - StatusRejected VoteStatus = 0x04 - - ProposalTypeText ProposalKind = 0x01 - ProposalTypeParameterChange ProposalKind = 0x02 - ProposalTypeSoftwareUpgrade ProposalKind = 0x03 -) - //----------------------------------------------------------- // Proposal interface type Proposal interface { @@ -37,8 +24,8 @@ type Proposal interface { GetProposalType() ProposalKind SetProposalType(ProposalKind) - GetStatus() VoteStatus - SetStatus(VoteStatus) + GetStatus() ProposalStatus + SetStatus(ProposalStatus) GetSubmitBlock() int64 SetSubmitBlock(int64) @@ -73,7 +60,7 @@ type TextProposal struct { Description string `json:"description"` // Description of the proposal ProposalType ProposalKind `json:"proposal_type"` // Type of proposal. Initial set {PlainTextProposal, SoftwareUpgradeProposal} - Status VoteStatus `json:"string"` // Status of the Proposal {Pending, Active, Passed, Rejected} + Status ProposalStatus `json:"string"` // Status of the Proposal {Pending, Active, Passed, Rejected} SubmitBlock int64 `json:"submit_block"` // Height of the block where TxGovSubmitProposal was included TotalDeposit sdk.Coins `json:"total_deposit"` // Current deposit on this proposal. Initial value is set at InitialDeposit @@ -93,8 +80,8 @@ func (tp TextProposal) GetDescription() string { return tp.D func (tp *TextProposal) SetDescription(description string) { tp.Description = description } func (tp TextProposal) GetProposalType() ProposalKind { return tp.ProposalType } func (tp *TextProposal) SetProposalType(proposalType ProposalKind) { tp.ProposalType = proposalType } -func (tp TextProposal) GetStatus() VoteStatus { return tp.Status } -func (tp *TextProposal) SetStatus(status VoteStatus) { tp.Status = status } +func (tp TextProposal) GetStatus() ProposalStatus { return tp.Status } +func (tp *TextProposal) SetStatus(status ProposalStatus) { tp.Status = status } func (tp TextProposal) GetSubmitBlock() int64 { return tp.SubmitBlock } func (tp *TextProposal) SetSubmitBlock(submitBlock int64) { tp.SubmitBlock = submitBlock } func (tp TextProposal) GetTotalDeposit() sdk.Coins { return tp.TotalDeposit } @@ -104,12 +91,82 @@ func (tp *TextProposal) SetVotingStartBlock(votingStartBlock int64) { tp.VotingStartBlock = votingStartBlock } -// Current Active Proposals +//----------------------------------------------------------- +// ProposalQueue type ProposalQueue []int64 -// ProposalTypeToString for pretty prints of ProposalType -func ProposalTypeToString(proposalType ProposalKind) string { - switch proposalType { +//----------------------------------------------------------- +// ProposalKind + +// Type that represents Proposal Type as a byte +type ProposalKind byte + +//nolint +const ( + ProposalTypeText ProposalKind = 0x01 + ProposalTypeParameterChange ProposalKind = 0x02 + ProposalTypeSoftwareUpgrade ProposalKind = 0x03 +) + +// String to proposalType byte. Returns ff if invalid. +func ProposalTypeFromString(str string) (ProposalKind, error) { + switch str { + case "Text": + return ProposalTypeText, nil + case "ParameterChange": + return ProposalTypeParameterChange, nil + case "SoftwareUpgrade": + return ProposalTypeSoftwareUpgrade, nil + default: + return ProposalKind(0xff), errors.Errorf("'%s' is not a valid proposal type", str) + } +} + +// is defined ProposalType? +func validProposalType(pt ProposalKind) bool { + if pt == ProposalTypeText || + pt == ProposalTypeParameterChange || + pt == ProposalTypeSoftwareUpgrade { + return true + } + return false +} + +// Marshal needed for protobuf compatibility +func (pt ProposalKind) Marshal() ([]byte, error) { + return []byte{byte(pt)}, nil +} + +// Unmarshal needed for protobuf compatibility +func (pt *ProposalKind) Unmarshal(data []byte) error { + *pt = ProposalKind(data[0]) + return nil +} + +// Marshals to JSON using string +func (pt ProposalKind) MarshalJSON() ([]byte, error) { + return json.Marshal(pt.String()) +} + +// Unmarshals from JSON assuming Bech32 encoding +func (pt *ProposalKind) UnmarshalJSON(data []byte) error { + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return nil + } + + bz2, err := ProposalTypeFromString(s) + if err != nil { + return err + } + *pt = bz2 + return nil +} + +// Turns VoteOption byte to String +func (pt ProposalKind) String() string { + switch pt { case 0x00: return "Text" case 0x01: @@ -121,31 +178,91 @@ func ProposalTypeToString(proposalType ProposalKind) string { } } -func validProposalType(proposalType ProposalKind) bool { - if proposalType == ProposalTypeText || - proposalType == ProposalTypeParameterChange || - proposalType == ProposalTypeSoftwareUpgrade { +// For Printf / Sprintf, returns bech32 when using %s +func (pt ProposalKind) Format(s fmt.State, verb rune) { + switch verb { + case 's': + s.Write([]byte(fmt.Sprintf("%s", pt.String()))) + default: + s.Write([]byte(fmt.Sprintf("%v", pt))) + } +} + +//----------------------------------------------------------- +// ProposalStatus + +// Type that represents Proposal Status as a byte +type ProposalStatus byte + +//nolint +const ( + StatusDepositPeriod ProposalStatus = 0x01 + StatusVotingPeriod ProposalStatus = 0x02 + StatusPassed ProposalStatus = 0x03 + StatusRejected ProposalStatus = 0x04 +) + +// ProposalStatusToString turns a string into a ProposalStatus +func ProposalStatusFromString(str string) (ProposalStatus, error) { + switch str { + case "DepositPeriod": + return StatusDepositPeriod, nil + case "VotingPeriod": + return StatusVotingPeriod, nil + case "Passed": + return StatusPassed, nil + case "Rejected": + return StatusRejected, nil + default: + return ProposalStatus(0xff), errors.Errorf("'%s' is not a valid proposal status", str) + } +} + +// is defined ProposalType? +func validProposalStatus(status ProposalStatus) bool { + if status == StatusDepositPeriod || + status == StatusVotingPeriod || + status == StatusPassed || + status == StatusRejected { return true } return false } -// String to proposalType byte. Returns ff if invalid. -func StringToProposalType(str string) (ProposalKind, sdk.Error) { - switch str { - case "Text": - return ProposalTypeText, nil - case "ParameterChange": - return ProposalTypeParameterChange, nil - case "SoftwareUpgrade": - return ProposalTypeSoftwareUpgrade, nil - default: - return ProposalKind(0xff), ErrInvalidProposalType(DefaultCodespace, str) - } +// Marshal needed for protobuf compatibility +func (status ProposalStatus) Marshal() ([]byte, error) { + return []byte{byte(status)}, nil } -// StatusToString for pretty prints of Status -func StatusToString(status VoteStatus) string { +// Unmarshal needed for protobuf compatibility +func (status *ProposalStatus) Unmarshal(data []byte) error { + *status = ProposalStatus(data[0]) + return nil +} + +// Marshals to JSON using string +func (status ProposalStatus) MarshalJSON() ([]byte, error) { + return json.Marshal(status.String()) +} + +// Unmarshals from JSON assuming Bech32 encoding +func (status *ProposalStatus) UnmarshalJSON(data []byte) error { + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return nil + } + + bz2, err := ProposalStatusFromString(s) + if err != nil { + return err + } + *status = bz2 + return nil +} + +// Turns VoteStatus byte to String +func (status ProposalStatus) String() string { switch status { case StatusDepositPeriod: return "DepositPeriod" @@ -160,45 +277,12 @@ func StatusToString(status VoteStatus) string { } } -// StatusToString for pretty prints of Status -func StringToStatus(status string) VoteStatus { - switch status { - case "DepositPeriod": - return StatusDepositPeriod - case "VotingPeriod": - return StatusVotingPeriod - case "Passed": - return StatusPassed - case "Rejected": - return StatusRejected +// For Printf / Sprintf, returns bech32 when using %s +func (status ProposalStatus) Format(s fmt.State, verb rune) { + switch verb { + case 's': + s.Write([]byte(fmt.Sprintf("%s", status.String()))) default: - return VoteStatus(0xff) - } -} - -//----------------------------------------------------------- -// Rest Proposals -type ProposalRest struct { - ProposalID int64 `json:"proposal_id"` // ID of the proposal - Title string `json:"title"` // Title of the proposal - Description string `json:"description"` // Description of the proposal - ProposalType string `json:"proposal_type"` // Type of proposal. Initial set {PlainTextProposal, SoftwareUpgradeProposal} - Status string `json:"string"` // Status of the Proposal {Pending, Active, Passed, Rejected} - SubmitBlock int64 `json:"submit_block"` // Height of the block where TxGovSubmitProposal was included - TotalDeposit sdk.Coins `json:"total_deposit"` // Current deposit on this proposal. Initial value is set at InitialDeposit - VotingStartBlock int64 `json:"voting_start_block"` // Height of the block where MinDeposit was reached. -1 if MinDeposit is not reached -} - -// Turn any Proposal to a ProposalRest -func ProposalToRest(proposal Proposal) ProposalRest { - return ProposalRest{ - ProposalID: proposal.GetProposalID(), - Title: proposal.GetTitle(), - Description: proposal.GetDescription(), - ProposalType: ProposalTypeToString(proposal.GetProposalType()), - Status: StatusToString(proposal.GetStatus()), - SubmitBlock: proposal.GetSubmitBlock(), - TotalDeposit: proposal.GetTotalDeposit(), - VotingStartBlock: proposal.GetVotingStartBlock(), + s.Write([]byte(fmt.Sprintf("%v", status))) } } From fc4c563e292f7bbdd8ceaf2c2cc549503e0b2ee2 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Tue, 10 Jul 2018 19:56:09 -0700 Subject: [PATCH 08/17] keys: Keybase.Update no longer asks for newpass if oldpass is incorrect Achieved by refactoring the parameter newpass as follows: * (newpass string) -> (getNewpass func() (string, error)) Closes #1629 --- CHANGELOG.md | 8 ++++++++ client/keys/update.go | 24 +++++++++++++----------- crypto/keys/keybase.go | 9 +++++++-- crypto/keys/keybase_test.go | 10 ++++++---- crypto/keys/types.go | 2 +- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44fa7290ef..bfb5523d52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Pending + +BREAKING CHANGES +- [keys] Keybase.Update function now takes in a function to get the newpass, rather than the password itself + +BUG FIXES +- [keys] \#1629 - updating password no longer asks for a new password when the first entered password was incorrect + ## 0.20.0 *July 10th, 2018* diff --git a/client/keys/update.go b/client/keys/update.go index 0308b89bdb..78a81bf0e6 100644 --- a/client/keys/update.go +++ b/client/keys/update.go @@ -26,23 +26,23 @@ func runUpdateCmd(cmd *cobra.Command, args []string) error { name := args[0] buf := client.BufferStdin() + kb, err := GetKeyBase() + if err != nil { + return err + } oldpass, err := client.GetPassword( "Enter the current passphrase:", buf) if err != nil { return err } - newpass, err := client.GetCheckPassword( - "Enter the new passphrase:", - "Repeat the new passphrase:", buf) - if err != nil { - return err + + getNewpass := func() (string, error) { + return client.GetCheckPassword( + "Enter the new passphrase:", + "Repeat the new passphrase:", buf) } - kb, err := GetKeyBase() - if err != nil { - return err - } - err = kb.Update(name, oldpass, newpass) + err = kb.Update(name, oldpass, getNewpass) if err != nil { return err } @@ -81,8 +81,10 @@ func UpdateKeyRequestHandler(w http.ResponseWriter, r *http.Request) { return } + getNewpass := func() (string, error) { return m.NewPassword, nil } + // TODO check if account exists and if password is correct - err = kb.Update(name, m.OldPassword, m.NewPassword) + err = kb.Update(name, m.OldPassword, getNewpass) if err != nil { w.WriteHeader(401) w.Write([]byte(err.Error())) diff --git a/crypto/keys/keybase.go b/crypto/keys/keybase.go index 7cf43eea67..80de47d3df 100644 --- a/crypto/keys/keybase.go +++ b/crypto/keys/keybase.go @@ -330,8 +330,9 @@ func (kb dbKeybase) Delete(name, passphrase string) error { // encrypted. // // oldpass must be the current passphrase used for encryption, -// newpass will be the only valid passphrase from this time forward. -func (kb dbKeybase) Update(name, oldpass, newpass string) error { +// getNewpass is a function to get the passphrase to permanently replace +// the current passphrase +func (kb dbKeybase) Update(name, oldpass string, getNewpass func() (string, error)) error { info, err := kb.Get(name) if err != nil { return err @@ -343,6 +344,10 @@ func (kb dbKeybase) Update(name, oldpass, newpass string) error { if err != nil { return err } + newpass, err := getNewpass() + if err != nil { + return err + } kb.writeLocalKey(key, name, newpass) return nil default: diff --git a/crypto/keys/keybase_test.go b/crypto/keys/keybase_test.go index c910849865..1ca4317922 100644 --- a/crypto/keys/keybase_test.go +++ b/crypto/keys/keybase_test.go @@ -167,9 +167,10 @@ func TestSignVerify(t *testing.T) { } func assertPassword(t *testing.T, cstore Keybase, name, pass, badpass string) { - err := cstore.Update(name, badpass, pass) + getNewpass := func() (string, error) { return pass, nil } + err := cstore.Update(name, badpass, getNewpass) require.NotNil(t, err) - err = cstore.Update(name, pass, pass) + err = cstore.Update(name, pass, getNewpass) require.Nil(t, err, "%+v", err) } @@ -265,12 +266,13 @@ func TestAdvancedKeyManagement(t *testing.T) { assertPassword(t, cstore, n1, p1, p2) // update password requires the existing password - err = cstore.Update(n1, "jkkgkg", p2) + getNewpass := func() (string, error) { return p2, nil } + err = cstore.Update(n1, "jkkgkg", getNewpass) require.NotNil(t, err) assertPassword(t, cstore, n1, p1, p2) // then it changes the password when correct - err = cstore.Update(n1, p1, p2) + err = cstore.Update(n1, p1, getNewpass) require.NoError(t, err) // p2 is now the proper one! assertPassword(t, cstore, n1, p2, p1) diff --git a/crypto/keys/types.go b/crypto/keys/types.go index 0e77725c31..74cedf9199 100644 --- a/crypto/keys/types.go +++ b/crypto/keys/types.go @@ -34,7 +34,7 @@ type Keybase interface { CreateOffline(name string, pubkey crypto.PubKey) (info Info, err error) // The following operations will *only* work on locally-stored keys - Update(name, oldpass, newpass string) error + Update(name, oldpass string, getNewpass func() (string, error)) error Import(name string, armor string) (err error) ImportPubKey(name string, armor string) (err error) Export(name string) (armor string, err error) From e400e83c43d6ec0b1f938d0fd8218bdba380e0c9 Mon Sep 17 00:00:00 2001 From: Fabian Weber Date: Thu, 12 Jul 2018 15:28:28 +0200 Subject: [PATCH 09/17] fix key import creating random account --- client/keys/add.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/keys/add.go b/client/keys/add.go index 1bd13dd0a2..d462db1c06 100644 --- a/client/keys/add.go +++ b/client/keys/add.go @@ -161,6 +161,7 @@ func printCreate(info keys.Info, seed string) { type NewKeyBody struct { Name string `json:"name"` Password string `json:"password"` + Seed string `json:"seed"` } // add new key REST handler @@ -205,7 +206,11 @@ func AddNewKeyRequestHandler(w http.ResponseWriter, r *http.Request) { } // create account - info, mnemonic, err := kb.CreateMnemonic(m.Name, keys.English, m.Password, keys.Secp256k1) + seed := m.Seed + if seed == "" { + seed = getSeed(keys.Secp256k1) + } + info, err := kb.CreateKey(m.Name, seed, m.Password) if err != nil { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(err.Error())) @@ -219,7 +224,7 @@ func AddNewKeyRequestHandler(w http.ResponseWriter, r *http.Request) { return } - keyOutput.Seed = mnemonic + keyOutput.Seed = seed bz, err := json.Marshal(keyOutput) if err != nil { From f8f46b2b8c670d53649a5c160b992d0aa27e6c69 Mon Sep 17 00:00:00 2001 From: Fabian Weber Date: Thu, 12 Jul 2018 16:19:20 +0200 Subject: [PATCH 10/17] added test for account importing --- client/lcd/lcd_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/lcd/lcd_test.go b/client/lcd/lcd_test.go index 48c1529c57..8d1a030a05 100644 --- a/client/lcd/lcd_test.go +++ b/client/lcd/lcd_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/spf13/viper" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" cryptoKeys "github.com/cosmos/cosmos-sdk/crypto/keys" @@ -52,7 +53,7 @@ func TestKeys(t *testing.T) { newPassword := "0987654321" // add key - jsonStr := []byte(fmt.Sprintf(`{"name":"%s", "password":"%s"}`, newName, newPassword)) + jsonStr := []byte(fmt.Sprintf(`{"name":"%s", "password":"%s", "seed":"%s"}`, newName, newPassword, seed)) res, body = Request(t, port, "POST", "/keys", jsonStr) require.Equal(t, http.StatusOK, res.StatusCode, body) @@ -64,6 +65,11 @@ func TestKeys(t *testing.T) { _, err = sdk.AccAddressFromBech32(addr2Bech32) require.NoError(t, err, "Failed to return a correct bech32 address") + // test if created account is the correct account + expectedInfo, _ := GetKB(t).CreateKey(newName, seed, newPassword) + expectedAccount := sdk.AccAddress(expectedInfo.GetPubKey().Address().Bytes()) + assert.Equal(t, expectedAccount.String(), addr2Bech32) + // existing keys res, body = Request(t, port, "GET", "/keys", nil) require.Equal(t, http.StatusOK, res.StatusCode, body) From c3bbb37df3e37c8c3903a00a9587f98f18af9902 Mon Sep 17 00:00:00 2001 From: Fabian Weber Date: Thu, 12 Jul 2018 16:22:26 +0200 Subject: [PATCH 11/17] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4adaecaa78..6396fe5861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,7 @@ BUG FIXES * \#1287 - prevent zero power validators at genesis * [x/stake] fix bug when unbonding/redelegating using `--shares-percent` * \#1010 - two validators can't bond with the same pubkey anymore +* [lcd] importing an account would create a random account ## 0.19.0 From 8b9f5396de2831347dfcfea932e60e84286613d6 Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Thu, 12 Jul 2018 19:45:09 +0200 Subject: [PATCH 12/17] Move changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6396fe5861..7e57b8a324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ FEATURES BUG FIXES * [keys] \#1629 - updating password no longer asks for a new password when the first entered password was incorrect +* [lcd] importing an account would create a random account ## 0.20.0 @@ -151,7 +152,6 @@ BUG FIXES * \#1287 - prevent zero power validators at genesis * [x/stake] fix bug when unbonding/redelegating using `--shares-percent` * \#1010 - two validators can't bond with the same pubkey anymore -* [lcd] importing an account would create a random account ## 0.19.0 From 41dd01906ae46aa20ecdbfbebbcf8f76c4f87cf4 Mon Sep 17 00:00:00 2001 From: Jeremiah Andrews Date: Thu, 12 Jul 2018 13:03:18 -0700 Subject: [PATCH 13/17] Merge PR #1654: Wait for 2 blocks attempt --- cmd/gaia/cli_test/cli_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/gaia/cli_test/cli_test.go b/cmd/gaia/cli_test/cli_test.go index 645a6c276a..e37e4d5219 100644 --- a/cmd/gaia/cli_test/cli_test.go +++ b/cmd/gaia/cli_test/cli_test.go @@ -49,7 +49,7 @@ func TestGaiaCLISend(t *testing.T) { defer proc.Stop(false) tests.WaitForTMStart(port) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) fooAddr, _ := executeGetAddrPK(t, fmt.Sprintf("gaiacli keys show foo --output=json --home=%s", gaiacliHome)) barAddr, _ := executeGetAddrPK(t, fmt.Sprintf("gaiacli keys show bar --output=json --home=%s", gaiacliHome)) @@ -58,7 +58,7 @@ func TestGaiaCLISend(t *testing.T) { require.Equal(t, int64(50), fooAcc.GetCoins().AmountOf("steak").Int64()) executeWrite(t, fmt.Sprintf("gaiacli send %v --amount=10steak --to=%s --from=foo", flags, barAddr), pass) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) barAcc := executeGetAccount(t, fmt.Sprintf("gaiacli account %s %v", barAddr, flags)) require.Equal(t, int64(10), barAcc.GetCoins().AmountOf("steak").Int64()) @@ -67,7 +67,7 @@ func TestGaiaCLISend(t *testing.T) { // test autosequencing executeWrite(t, fmt.Sprintf("gaiacli send %v --amount=10steak --to=%s --from=foo", flags, barAddr), pass) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) barAcc = executeGetAccount(t, fmt.Sprintf("gaiacli account %s %v", barAddr, flags)) require.Equal(t, int64(20), barAcc.GetCoins().AmountOf("steak").Int64()) @@ -76,7 +76,7 @@ func TestGaiaCLISend(t *testing.T) { // test memo executeWrite(t, fmt.Sprintf("gaiacli send %v --amount=10steak --to=%s --from=foo --memo 'testmemo'", flags, barAddr), pass) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) barAcc = executeGetAccount(t, fmt.Sprintf("gaiacli account %s %v", barAddr, flags)) require.Equal(t, int64(30), barAcc.GetCoins().AmountOf("steak").Int64()) @@ -101,14 +101,14 @@ func TestGaiaCLICreateValidator(t *testing.T) { defer proc.Stop(false) tests.WaitForTMStart(port) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) fooAddr, _ := executeGetAddrPK(t, fmt.Sprintf("gaiacli keys show foo --output=json --home=%s", gaiacliHome)) barAddr, barPubKey := executeGetAddrPK(t, fmt.Sprintf("gaiacli keys show bar --output=json --home=%s", gaiacliHome)) barCeshPubKey := sdk.MustBech32ifyValPub(barPubKey) executeWrite(t, fmt.Sprintf("gaiacli send %v --amount=10steak --to=%s --from=foo", flags, barAddr), pass) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) barAcc := executeGetAccount(t, fmt.Sprintf("gaiacli account %s %v", barAddr, flags)) require.Equal(t, int64(10), barAcc.GetCoins().AmountOf("steak").Int64()) @@ -124,7 +124,7 @@ func TestGaiaCLICreateValidator(t *testing.T) { cvStr += fmt.Sprintf(" --moniker=%v", "bar-vally") executeWrite(t, cvStr, pass) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) barAcc = executeGetAccount(t, fmt.Sprintf("gaiacli account %s %v", barAddr, flags)) require.Equal(t, int64(8), barAcc.GetCoins().AmountOf("steak").Int64(), "%v", barAcc) @@ -142,7 +142,7 @@ func TestGaiaCLICreateValidator(t *testing.T) { success := executeWrite(t, unbondStr, pass) require.True(t, success) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) /* // this won't be what we expect because we've only started unbonding, haven't completed barAcc = executeGetAccount(t, fmt.Sprintf("gaiacli account %v %v", barCech, flags)) @@ -169,7 +169,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) { defer proc.Stop(false) tests.WaitForTMStart(port) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) fooAddr, _ := executeGetAddrPK(t, fmt.Sprintf("gaiacli keys show foo --output=json --home=%s", gaiacliHome)) @@ -177,7 +177,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) { require.Equal(t, int64(50), fooAcc.GetCoins().AmountOf("steak").Int64()) executeWrite(t, fmt.Sprintf("gaiacli gov submit-proposal %v --proposer=%s --deposit=5steak --type=Text --title=Test --description=test --from=foo", flags, fooAddr), pass) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) fooAcc = executeGetAccount(t, fmt.Sprintf("gaiacli account %s %v", fooAddr, flags)) require.Equal(t, int64(45), fooAcc.GetCoins().AmountOf("steak").Int64()) @@ -187,7 +187,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) { require.Equal(t, gov.StatusDepositPeriod, proposal1.GetStatus()) executeWrite(t, fmt.Sprintf("gaiacli gov deposit %v --depositer=%s --deposit=10steak --proposalID=1 --from=foo", flags, fooAddr), pass) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) fooAcc = executeGetAccount(t, fmt.Sprintf("gaiacli account %s %v", fooAddr, flags)) require.Equal(t, int64(35), fooAcc.GetCoins().AmountOf("steak").Int64()) @@ -196,7 +196,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) { require.Equal(t, gov.StatusVotingPeriod, proposal1.GetStatus()) executeWrite(t, fmt.Sprintf("gaiacli gov vote %v --proposalID=1 --voter=%s --option=Yes --from=foo", flags, fooAddr), pass) - tests.WaitForNextHeightTM(port) + tests.WaitForNextNBlocksTM(2, port) vote := executeGetVote(t, fmt.Sprintf("gaiacli gov query-vote --proposalID=1 --voter=%s --output=json %v", fooAddr, flags)) require.Equal(t, int64(1), vote.ProposalID) From 98bc419d5e3cbf01b1d0be7e99291ef8306b2314 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Thu, 12 Jul 2018 15:12:20 -0700 Subject: [PATCH 14/17] Merge PR #1659: crypto/keys: Allow one to export their private key * crypto/keys: Allow one to export their private key * Update changelog --- CHANGELOG.md | 1 + crypto/keys/keybase.go | 25 +++++++++++++++++++++++++ crypto/keys/types.go | 3 +++ 3 files changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e57b8a324..1cadfc352d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ FEATURES - Auth has its invariants checked within the framework * [tests] Add WaitForNextNBlocksTM helper method * [keys] New keys now have 24 word recovery keys, for heightened security +- [keys] Add a temporary method for exporting the private key IMPROVEMENTS * [x/bank] Now uses go-wire codec instead of 'encoding/json' diff --git a/crypto/keys/keybase.go b/crypto/keys/keybase.go index 80de47d3df..ef2c6243a6 100644 --- a/crypto/keys/keybase.go +++ b/crypto/keys/keybase.go @@ -240,6 +240,31 @@ func (kb dbKeybase) Sign(name, passphrase string, msg []byte) (sig tcrypto.Signa return sig, pub, nil } +func (kb dbKeybase) ExportPrivateKeyObject(name string, passphrase string) (tcrypto.PrivKey, error) { + info, err := kb.Get(name) + if err != nil { + return nil, err + } + var priv tcrypto.PrivKey + switch info.(type) { + case localInfo: + linfo := info.(localInfo) + if linfo.PrivKeyArmor == "" { + err = fmt.Errorf("private key not available") + return nil, err + } + priv, err = unarmorDecryptPrivKey(linfo.PrivKeyArmor, passphrase) + if err != nil { + return nil, err + } + case ledgerInfo: + return nil, errors.New("Only works on local private keys") + case offlineInfo: + return nil, errors.New("Only works on local private keys") + } + return priv, nil +} + func (kb dbKeybase) Export(name string) (armor string, err error) { bz := kb.db.Get(infoKey(name)) if bz == nil { diff --git a/crypto/keys/types.go b/crypto/keys/types.go index 74cedf9199..c3f5d7834f 100644 --- a/crypto/keys/types.go +++ b/crypto/keys/types.go @@ -39,6 +39,9 @@ type Keybase interface { ImportPubKey(name string, armor string) (err error) Export(name string) (armor string, err error) ExportPubKey(name string) (armor string, err error) + + // *only* works on locally-stored keys. Temporary method until we redo the exporting API + ExportPrivateKeyObject(name string, passphrase string) (crypto.PrivKey, error) } // Info is the publicly exposed information about a keypair From 2885ac586e96a48eccfd07bb6e7192d49912ee6b Mon Sep 17 00:00:00 2001 From: Rigel Date: Thu, 12 Jul 2018 19:38:35 -0400 Subject: [PATCH 15/17] Merge PR #1660: Redelegation doesn't subtract from liquid * fix redelegation subtracting source coins * changelog * Add testcases for balance subtraction * Move changelog entry --- CHANGELOG.md | 2 +- x/stake/handler.go | 4 ++-- x/stake/handler_test.go | 25 ++++++++++++++++++++++++- x/stake/keeper/delegation.go | 19 +++++++++++-------- 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cadfc352d..8ac7bd4372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ BREAKING CHANGES FEATURES * [baseapp] NewBaseApp now takes option functions as parameters - BUG FIXES +* \#1630 - redelegation nolonger removes tokens from the delegator liquid account * [keys] \#1629 - updating password no longer asks for a new password when the first entered password was incorrect * [lcd] importing an account would create a random account diff --git a/x/stake/handler.go b/x/stake/handler.go index c355179cf2..031edda43b 100644 --- a/x/stake/handler.go +++ b/x/stake/handler.go @@ -81,7 +81,7 @@ func handleMsgCreateValidator(ctx sdk.Context, msg types.MsgCreateValidator, k k // move coins from the msg.Address account to a (self-delegation) delegator account // the validator account and global shares are updated within here - _, err := k.Delegate(ctx, msg.DelegatorAddr, msg.Delegation, validator) + _, err := k.Delegate(ctx, msg.DelegatorAddr, msg.Delegation, validator, true) if err != nil { return err.Result() } @@ -136,7 +136,7 @@ func handleMsgDelegate(ctx sdk.Context, msg types.MsgDelegate, k keeper.Keeper) if validator.Revoked == true { return ErrValidatorRevoked(k.Codespace()).Result() } - _, err := k.Delegate(ctx, msg.DelegatorAddr, msg.Delegation, validator) + _, err := k.Delegate(ctx, msg.DelegatorAddr, msg.Delegation, validator, true) if err != nil { return err.Result() } diff --git a/x/stake/handler_test.go b/x/stake/handler_test.go index 95a0788022..dac938e6be 100644 --- a/x/stake/handler_test.go +++ b/x/stake/handler_test.go @@ -268,6 +268,7 @@ func TestIncrementsMsgUnbond(t *testing.T) { initBond := int64(1000) ctx, accMapper, keeper := keep.CreateTestInput(t, false, initBond) params := setInstantUnbondPeriod(keeper, ctx) + denom := params.BondDenom // create validator, delegate validatorAddr, delegatorAddr := keep.Addrs[0], keep.Addrs[1] @@ -276,10 +277,17 @@ func TestIncrementsMsgUnbond(t *testing.T) { got := handleMsgCreateValidator(ctx, msgCreateValidator, keeper) require.True(t, got.IsOK(), "expected create-validator to be ok, got %v", got) + // initial balance + amt1 := accMapper.GetAccount(ctx, delegatorAddr).GetCoins().AmountOf(denom) + msgDelegate := newTestMsgDelegate(delegatorAddr, validatorAddr, initBond) got = handleMsgDelegate(ctx, msgDelegate, keeper) require.True(t, got.IsOK(), "expected delegation to be ok, got %v", got) + // balance should have been subtracted after delegation + amt2 := accMapper.GetAccount(ctx, delegatorAddr).GetCoins().AmountOf(denom) + require.Equal(t, amt1.Sub(sdk.NewInt(initBond)).Int64(), amt2.Int64(), "expected coins to be subtracted") + validator, found := keeper.GetValidator(ctx, validatorAddr) require.True(t, found) require.Equal(t, initBond*2, validator.DelegatorShares.RoundInt64()) @@ -528,8 +536,9 @@ func TestUnbondingPeriod(t *testing.T) { } func TestRedelegationPeriod(t *testing.T) { - ctx, _, keeper := keep.CreateTestInput(t, false, 1000) + ctx, AccMapper, keeper := keep.CreateTestInput(t, false, 1000) validatorAddr, validatorAddr2 := keep.Addrs[0], keep.Addrs[1] + denom := keeper.GetParams(ctx).BondDenom // set the unbonding time params := keeper.GetParams(ctx) @@ -538,18 +547,32 @@ func TestRedelegationPeriod(t *testing.T) { // create the validators msgCreateValidator := newTestMsgCreateValidator(validatorAddr, keep.PKs[0], 10) + + // initial balance + amt1 := AccMapper.GetAccount(ctx, validatorAddr).GetCoins().AmountOf(denom) + got := handleMsgCreateValidator(ctx, msgCreateValidator, keeper) require.True(t, got.IsOK(), "expected no error on runMsgCreateValidator") + // balance should have been subtracted after creation + amt2 := AccMapper.GetAccount(ctx, validatorAddr).GetCoins().AmountOf(denom) + require.Equal(t, amt1.Sub(sdk.NewInt(10)).Int64(), amt2.Int64(), "expected coins to be subtracted") + msgCreateValidator = newTestMsgCreateValidator(validatorAddr2, keep.PKs[1], 10) got = handleMsgCreateValidator(ctx, msgCreateValidator, keeper) require.True(t, got.IsOK(), "expected no error on runMsgCreateValidator") + bal1 := AccMapper.GetAccount(ctx, validatorAddr).GetCoins() + // begin redelegate msgBeginRedelegate := NewMsgBeginRedelegate(validatorAddr, validatorAddr, validatorAddr2, sdk.NewRat(10)) got = handleMsgBeginRedelegate(ctx, msgBeginRedelegate, keeper) require.True(t, got.IsOK(), "expected no error, %v", got) + // origin account should not lose tokens as with a regular delegation + bal2 := AccMapper.GetAccount(ctx, validatorAddr).GetCoins() + require.Equal(t, bal1, bal2) + // cannot complete redelegation at same time msgCompleteRedelegate := NewMsgCompleteRedelegate(validatorAddr, validatorAddr, validatorAddr2) got = handleMsgCompleteRedelegate(ctx, msgCompleteRedelegate, keeper) diff --git a/x/stake/keeper/delegation.go b/x/stake/keeper/delegation.go index e2f96f3d9f..6bf357e79b 100644 --- a/x/stake/keeper/delegation.go +++ b/x/stake/keeper/delegation.go @@ -200,9 +200,9 @@ func (k Keeper) RemoveRedelegation(ctx sdk.Context, red types.Redelegation) { //_____________________________________________________________________________________ -// Perform a delegation, set/update everything necessary within the store +// Perform a delegation, set/update everything necessary within the store. func (k Keeper) Delegate(ctx sdk.Context, delegatorAddr sdk.AccAddress, bondAmt sdk.Coin, - validator types.Validator) (newShares sdk.Rat, err sdk.Error) { + validator types.Validator, subtractAccount bool) (newShares sdk.Rat, err sdk.Error) { // Get or create the delegator delegation delegation, found := k.GetDelegation(ctx, delegatorAddr, validator.Owner) @@ -214,12 +214,15 @@ func (k Keeper) Delegate(ctx sdk.Context, delegatorAddr sdk.AccAddress, bondAmt } } - // Account new shares, save - pool := k.GetPool(ctx) - _, _, err = k.coinKeeper.SubtractCoins(ctx, delegation.DelegatorAddr, sdk.Coins{bondAmt}) - if err != nil { - return + if subtractAccount { + // Account new shares, save + _, _, err = k.coinKeeper.SubtractCoins(ctx, delegation.DelegatorAddr, sdk.Coins{bondAmt}) + if err != nil { + return + } } + + pool := k.GetPool(ctx) validator, pool, newShares = validator.AddTokensFromDel(pool, bondAmt.Amount.Int64()) delegation.Shares = delegation.Shares.Add(newShares) @@ -358,7 +361,7 @@ func (k Keeper) BeginRedelegation(ctx sdk.Context, delegatorAddr, validatorSrcAd if !found { return types.ErrBadRedelegationDst(k.Codespace()) } - sharesCreated, err := k.Delegate(ctx, delegatorAddr, returnCoin, dstValidator) + sharesCreated, err := k.Delegate(ctx, delegatorAddr, returnCoin, dstValidator, false) if err != nil { return err } From 5983a07fb6d1c9a2cac8151901e076657d5288cc Mon Sep 17 00:00:00 2001 From: Alexander Bezobchuk Date: Thu, 12 Jul 2018 19:58:51 -0400 Subject: [PATCH 16/17] Merge PR #1599: Implementation of TraceKVStore --- CHANGELOG.md | 1 + baseapp/baseapp.go | 81 +++++-- client/lcd/test_helpers.go | 2 +- cmd/gaia/app/app.go | 14 +- cmd/gaia/cmd/gaiad/main.go | 13 +- cmd/gaia/cmd/gaiadebug/hack.go | 5 +- docs/clients/node.md | 37 +++- examples/basecoin/cmd/basecoind/main.go | 5 +- examples/democoin/cmd/democoind/main.go | 5 +- server/constructors.go | 67 ++++-- server/export.go | 10 +- server/mock/store.go | 26 +++ server/start.go | 45 ++-- store/cachekvstore.go | 24 +- store/cachemultistore.go | 74 ++++++- store/dbstoreadapter.go | 7 + store/gaskvstore.go | 9 +- store/iavlstore.go | 6 + store/prefixstore.go | 7 + store/rootmultistore.go | 62 +++++- store/tracekvstore.go | 198 +++++++++++++++++ store/tracekvstore_test.go | 283 ++++++++++++++++++++++++ store/types.go | 37 ++-- types/store.go | 40 +++- 24 files changed, 942 insertions(+), 116 deletions(-) create mode 100644 store/tracekvstore.go create mode 100644 store/tracekvstore_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ac7bd4372..59c444f9d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ BREAKING CHANGES FEATURES * [baseapp] NewBaseApp now takes option functions as parameters +* [store] Added support for tracing multi-store operations via `--trace-store` BUG FIXES * \#1630 - redelegation nolonger removes tokens from the delegator liquid account diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go index b17e797774..40b3c2bc17 100644 --- a/baseapp/baseapp.go +++ b/baseapp/baseapp.go @@ -2,12 +2,14 @@ package baseapp import ( "fmt" + "io" "runtime/debug" "strings" "github.com/pkg/errors" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" @@ -37,7 +39,7 @@ const ( runTxModeDeliver runTxMode = iota ) -// The ABCI application +// BaseApp reflects the ABCI application implementation. type BaseApp struct { // initialized on creation Logger log.Logger @@ -71,7 +73,12 @@ type BaseApp struct { var _ abci.Application = (*BaseApp)(nil) -// Create and name new BaseApp +// NewBaseApp returns a reference to an initialized BaseApp. +// +// TODO: Determine how to use a flexible and robust configuration paradigm that +// allows for sensible defaults while being highly configurable +// (e.g. functional options). +// // NOTE: The db is used to store the version number for now. // Accepts variable number of option functions, which act on the BaseApp to set configuration choices func NewBaseApp(name string, cdc *wire.Codec, logger log.Logger, db dbm.DB, options ...func(*BaseApp)) *BaseApp { @@ -85,7 +92,9 @@ func NewBaseApp(name string, cdc *wire.Codec, logger log.Logger, db dbm.DB, opti codespacer: sdk.NewCodespacer(), txDecoder: defaultTxDecoder(cdc), } - // Register the undefined & root codespaces, which should not be used by any modules + + // Register the undefined & root codespaces, which should not be used by + // any modules. app.codespacer.RegisterOrPanic(sdk.CodespaceRoot) for _, option := range options { option(app) @@ -98,6 +107,12 @@ func (app *BaseApp) Name() string { return app.name } +// SetCommitMultiStoreTracer sets the store tracer on the BaseApp's underlying +// CommitMultiStore. +func (app *BaseApp) SetCommitMultiStoreTracer(w io.Writer) { + app.cms.WithTracer(w) +} + // Register the next available codespace through the baseapp's codespacer, starting from a default func (app *BaseApp) RegisterCodespace(codespace sdk.CodespaceType) sdk.CodespaceType { return app.codespacer.RegisterNext(codespace) @@ -392,13 +407,18 @@ func handleQueryP2P(app *BaseApp, path []string, req abci.RequestQuery) (res abc return sdk.ErrUnknownRequest(msg).QueryResult() } -// Implements ABCI +// BeginBlock implements the ABCI application interface. func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeginBlock) { - // Initialize the DeliverTx state. - // If this is the first block, it should already - // be initialized in InitChain. - // Otherwise app.deliverState will be nil, since it - // is reset on Commit. + if app.cms.TracingEnabled() { + app.cms.ResetTraceContext() + app.cms.WithTracingContext(sdk.TraceContext( + map[string]interface{}{"blockHeight": req.Header.Height}, + )) + } + + // Initialize the DeliverTx state. If this is the first block, it should + // already be initialized in InitChain. Otherwise app.deliverState will be + // nil, since it is reset on Commit. if app.deliverState == nil { app.setDeliverState(req.Header) } else { @@ -406,9 +426,11 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg // by InitChain. Context is now updated with Header information. app.deliverState.ctx = app.deliverState.ctx.WithBlockHeader(req.Header) } + if app.beginBlocker != nil { res = app.beginBlocker(app.deliverState.ctx, req) } + // set the signed validators for addition to context in deliverTx app.signedValidators = req.Validators return @@ -548,25 +570,26 @@ func (app *BaseApp) runMsgs(ctx sdk.Context, msgs []sdk.Msg) (result sdk.Result) return result } -// Returns deliverState if app is in runTxModeDeliver, otherwhise returns checkstate +// Returns the applicantion's deliverState if app is in runTxModeDeliver, +// otherwise it returns the application's checkstate. func getState(app *BaseApp, mode runTxMode) *state { if mode == runTxModeCheck || mode == runTxModeSimulate { return app.checkState } + return app.deliverState } -// txBytes may be nil in some cases, eg. in tests. -// Also, in the future we may support "internal" transactions. +// runTx processes a transaction. The transactions is proccessed via an +// anteHandler. txBytes may be nil in some cases, eg. in tests. Also, in the +// future we may support "internal" transactions. func (app *BaseApp) runTx(mode runTxMode, txBytes []byte, tx sdk.Tx) (result sdk.Result) { - //NOTE: GasWanted should be returned by the AnteHandler. - // GasUsed is determined by the GasMeter. - // We need access to the context to get the gas meter so - // we initialize upfront + // NOTE: GasWanted should be returned by the AnteHandler. GasUsed is + // determined by the GasMeter. We need access to the context to get the gas + // meter so we initialize upfront. var gasWanted int64 ctx := app.getContextForAnte(mode, txBytes) - // Handle any panics. defer func() { if r := recover(); r != nil { switch rType := r.(type) { @@ -578,11 +601,11 @@ func (app *BaseApp) runTx(mode runTxMode, txBytes []byte, tx sdk.Tx) (result sdk result = sdk.ErrInternal(log).Result() } } + result.GasWanted = gasWanted result.GasUsed = ctx.GasMeter().GasConsumed() }() - // Get the Msg. var msgs = tx.GetMsgs() err := validateBasicTxMsgs(msgs) @@ -590,7 +613,7 @@ func (app *BaseApp) runTx(mode runTxMode, txBytes []byte, tx sdk.Tx) (result sdk return err.Result() } - // Run the ante handler. + // run the ante handler if app.anteHandler != nil { newCtx, anteResult, abort := app.anteHandler(ctx, tx) if abort { @@ -599,17 +622,24 @@ func (app *BaseApp) runTx(mode runTxMode, txBytes []byte, tx sdk.Tx) (result sdk if !newCtx.IsZero() { ctx = newCtx } + gasWanted = result.GasWanted } - // CacheWrap the state in case it fails. + // Keep the state in a transient CacheWrap in case processing the messages + // fails. msCache := getState(app, mode).CacheMultiStore() - ctx = ctx.WithMultiStore(msCache) + if msCache.TracingEnabled() { + msCache = msCache.WithTracingContext(sdk.TraceContext( + map[string]interface{}{"txHash": cmn.HexBytes(tmhash.Sum(txBytes)).String()}, + )).(sdk.CacheMultiStore) + } + ctx = ctx.WithMultiStore(msCache) result = app.runMsgs(ctx, msgs) result.GasWanted = gasWanted - // Only update state if all messages pass and we're not in a simulation. + // only update state if all messages pass and we're not in a simulation if result.IsOK() && mode != runTxModeSimulate { msCache.Write() } @@ -617,11 +647,16 @@ func (app *BaseApp) runTx(mode runTxMode, txBytes []byte, tx sdk.Tx) (result sdk return } -// Implements ABCI +// EndBlock implements the ABCI application interface. func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBlock) { + if app.deliverState.ms.TracingEnabled() { + app.deliverState.ms = app.deliverState.ms.ResetTraceContext().(sdk.CacheMultiStore) + } + if app.endBlocker != nil { res = app.endBlocker(app.deliverState.ctx, req) } + return } diff --git a/client/lcd/test_helpers.go b/client/lcd/test_helpers.go index 5967b2319f..4eeb7253b6 100644 --- a/client/lcd/test_helpers.go +++ b/client/lcd/test_helpers.go @@ -105,7 +105,7 @@ func InitializeTestLCD(t *testing.T, nValidators int, initAddrs []sdk.AccAddress privVal := pvm.LoadOrGenFilePV(privValidatorFile) privVal.Reset() db := dbm.NewMemDB() - app := gapp.NewGaiaApp(logger, db) + app := gapp.NewGaiaApp(logger, db, nil) cdc = gapp.MakeCodec() genesisFile := config.GenesisFile() diff --git a/cmd/gaia/app/app.go b/cmd/gaia/app/app.go index 40e8686694..62b3e3fc4f 100644 --- a/cmd/gaia/app/app.go +++ b/cmd/gaia/app/app.go @@ -2,6 +2,7 @@ package app import ( "encoding/json" + "io" "os" abci "github.com/tendermint/tendermint/abci/types" @@ -55,12 +56,19 @@ type GaiaApp struct { govKeeper gov.Keeper } -func NewGaiaApp(logger log.Logger, db dbm.DB) *GaiaApp { +// NewGaiaApp returns a reference to an initialized GaiaApp. +// +// TODO: Determine how to use a flexible and robust configuration paradigm that +// allows for sensible defaults while being highly configurable +// (e.g. functional options). +func NewGaiaApp(logger log.Logger, db dbm.DB, traceStore io.Writer) *GaiaApp { cdc := MakeCodec() - // create your application object + bApp := bam.NewBaseApp(appName, cdc, logger, db) + bApp.SetCommitMultiStoreTracer(traceStore) + var app = &GaiaApp{ - BaseApp: bam.NewBaseApp(appName, cdc, logger, db), + BaseApp: bApp, cdc: cdc, keyMain: sdk.NewKVStoreKey("main"), keyAccount: sdk.NewKVStoreKey("acc"), diff --git a/cmd/gaia/cmd/gaiad/main.go b/cmd/gaia/cmd/gaiad/main.go index 9c8434eb06..6f4a427142 100644 --- a/cmd/gaia/cmd/gaiad/main.go +++ b/cmd/gaia/cmd/gaiad/main.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "io" "github.com/spf13/cobra" @@ -38,11 +39,13 @@ func main() { } } -func newApp(logger log.Logger, db dbm.DB) abci.Application { - return app.NewGaiaApp(logger, db) +func newApp(logger log.Logger, db dbm.DB, traceStore io.Writer) abci.Application { + return app.NewGaiaApp(logger, db, traceStore) } -func exportAppStateAndTMValidators(logger log.Logger, db dbm.DB) (json.RawMessage, []tmtypes.GenesisValidator, error) { - gapp := app.NewGaiaApp(logger, db) - return gapp.ExportAppStateAndValidators() +func exportAppStateAndTMValidators( + logger log.Logger, db dbm.DB, traceStore io.Writer, +) (json.RawMessage, []tmtypes.GenesisValidator, error) { + gApp := app.NewGaiaApp(logger, db, traceStore) + return gApp.ExportAppStateAndValidators() } diff --git a/cmd/gaia/cmd/gaiadebug/hack.go b/cmd/gaia/cmd/gaiadebug/hack.go index 121f437a3c..38921edc35 100644 --- a/cmd/gaia/cmd/gaiadebug/hack.go +++ b/cmd/gaia/cmd/gaiadebug/hack.go @@ -143,9 +143,12 @@ type GaiaApp struct { func NewGaiaApp(logger log.Logger, db dbm.DB) *GaiaApp { cdc := MakeCodec() + bApp := bam.NewBaseApp(appName, cdc, logger, db) + bApp.SetCommitMultiStoreTracer(os.Stdout) + // create your application object var app = &GaiaApp{ - BaseApp: bam.NewBaseApp(appName, cdc, logger, db), + BaseApp: bApp, cdc: cdc, keyMain: sdk.NewKVStoreKey("main"), keyAccount: sdk.NewKVStoreKey("acc"), diff --git a/docs/clients/node.md b/docs/clients/node.md index 71d4846fd9..fb3ff07b15 100644 --- a/docs/clients/node.md +++ b/docs/clients/node.md @@ -1,10 +1,45 @@ # Running a Node -TODO: document `gaiad` +> TODO: Improve documentation of `gaiad` + + +## Basics + +To start a node: + +```shell +$ gaiad start +``` Options for running the `gaiad` binary are effectively the same as for `tendermint`. See `gaiad --help` and the [guide to using Tendermint](https://github.com/tendermint/tendermint/blob/master/docs/using-tendermint.md) for more details. +## Debugging +Optionally, you can run `gaiad` with `--trace-store` to trace all store operations +to a specified file. + +```shell +$ gaiad start --trace-store=/path/to/trace.out +``` + +Key/value pairs will be base64 encoded. Additionally, the block number and any +correlated transaction hash will be included as metadata. + +e.g. +```json +... +{"operation":"write","key":"ATW6Bu997eeuUeRBwv1EPGvXRfPR","value":"BggEEBYgFg==","metadata":{"blockHeight":12,"txHash":"5AAC197EC45E6C5DE0798C4A4E2F54BBB695CA9E"}} +{"operation":"write","key":"AjW6Bu997eeuUeRBwv1EPGvXRfPRCgAAAAAAAAA=","value":"AQE=","metadata":{"blockHeight":12,"txHash":"5AAC197EC45E6C5DE0798C4A4E2F54BBB695CA9E"}} +{"operation":"read","key":"ATW6Bu997eeuUeRBwv1EPGvXRfPR","value":"BggEEBYgFg==","metadata":{"blockHeight":13}} +{"operation":"read","key":"AjW6Bu997eeuUeRBwv1EPGvXRfPRCwAAAAAAAAA=","value":"","metadata":{"blockHeight":13}} +... +``` + +You can then query for the various traced operations using a tool like [jq](https://github.com/stedolan/jq). + +```shell +$ jq -s '.[] | select((.key=="ATW6Bu997eeuUeRBwv1EPGvXRfPR") and .metadata.blockHeight==14)' /path/to/trace.out +``` \ No newline at end of file diff --git a/examples/basecoin/cmd/basecoind/main.go b/examples/basecoin/cmd/basecoind/main.go index a4f233f0a7..49be70a535 100644 --- a/examples/basecoin/cmd/basecoind/main.go +++ b/examples/basecoin/cmd/basecoind/main.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "io" "os" "github.com/cosmos/cosmos-sdk/examples/basecoin/app" @@ -39,11 +40,11 @@ func main() { } } -func newApp(logger log.Logger, db dbm.DB) abci.Application { +func newApp(logger log.Logger, db dbm.DB, storeTracer io.Writer) abci.Application { return app.NewBasecoinApp(logger, db) } -func exportAppStateAndTMValidators(logger log.Logger, db dbm.DB) (json.RawMessage, []tmtypes.GenesisValidator, error) { +func exportAppStateAndTMValidators(logger log.Logger, db dbm.DB, storeTracer io.Writer) (json.RawMessage, []tmtypes.GenesisValidator, error) { bapp := app.NewBasecoinApp(logger, db) return bapp.ExportAppStateAndValidators() } diff --git a/examples/democoin/cmd/democoind/main.go b/examples/democoin/cmd/democoind/main.go index 0b6a9b8b10..528cafe1c7 100644 --- a/examples/democoin/cmd/democoind/main.go +++ b/examples/democoin/cmd/democoind/main.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "io" "os" "github.com/spf13/cobra" @@ -50,11 +51,11 @@ func CoolAppGenState(cdc *wire.Codec, appGenTxs []json.RawMessage) (appState jso return } -func newApp(logger log.Logger, db dbm.DB) abci.Application { +func newApp(logger log.Logger, db dbm.DB, _ io.Writer) abci.Application { return app.NewDemocoinApp(logger, db) } -func exportAppStateAndTMValidators(logger log.Logger, db dbm.DB) (json.RawMessage, []tmtypes.GenesisValidator, error) { +func exportAppStateAndTMValidators(logger log.Logger, db dbm.DB, _ io.Writer) (json.RawMessage, []tmtypes.GenesisValidator, error) { dapp := app.NewDemocoinApp(logger, db) return dapp.ExportAppStateAndValidators() } diff --git a/server/constructors.go b/server/constructors.go index ab7a949e6c..84be219bbb 100644 --- a/server/constructors.go +++ b/server/constructors.go @@ -2,6 +2,8 @@ package server import ( "encoding/json" + "io" + "os" "path/filepath" abci "github.com/tendermint/tendermint/abci/types" @@ -10,34 +12,73 @@ import ( tmtypes "github.com/tendermint/tendermint/types" ) -// AppCreator lets us lazily initialize app, using home dir -// and other flags (?) to start -type AppCreator func(string, log.Logger) (abci.Application, error) +type ( + // AppCreator reflects a function that allows us to lazily initialize an + // application using various configurations. + AppCreator func(home string, logger log.Logger, traceStore string) (abci.Application, error) -// AppExporter dumps all app state to JSON-serializable structure and returns the current validator set -type AppExporter func(home string, log log.Logger) (json.RawMessage, []tmtypes.GenesisValidator, error) + // AppExporter reflects a function that dumps all app state to + // JSON-serializable structure and returns the current validator set. + AppExporter func(home string, logger log.Logger, traceStore string) (json.RawMessage, []tmtypes.GenesisValidator, error) -// ConstructAppCreator returns an application generation function -func ConstructAppCreator(appFn func(log.Logger, dbm.DB) abci.Application, name string) AppCreator { - return func(rootDir string, logger log.Logger) (abci.Application, error) { + // AppCreatorInit reflects a function that performs initialization of an + // AppCreator. + AppCreatorInit func(log.Logger, dbm.DB, io.Writer) abci.Application + + // AppExporterInit reflects a function that performs initialization of an + // AppExporter. + AppExporterInit func(log.Logger, dbm.DB, io.Writer) (json.RawMessage, []tmtypes.GenesisValidator, error) +) + +// ConstructAppCreator returns an application generation function. +func ConstructAppCreator(appFn AppCreatorInit, name string) AppCreator { + return func(rootDir string, logger log.Logger, traceStore string) (abci.Application, error) { dataDir := filepath.Join(rootDir, "data") + db, err := dbm.NewGoLevelDB(name, dataDir) if err != nil { return nil, err } - app := appFn(logger, db) + + var traceStoreWriter io.Writer + if traceStore != "" { + traceStoreWriter, err = os.OpenFile( + traceStore, + os.O_WRONLY|os.O_APPEND|os.O_CREATE, + 0666, + ) + if err != nil { + return nil, err + } + } + + app := appFn(logger, db, traceStoreWriter) return app, nil } } -// ConstructAppExporter returns an application export function -func ConstructAppExporter(appFn func(log.Logger, dbm.DB) (json.RawMessage, []tmtypes.GenesisValidator, error), name string) AppExporter { - return func(rootDir string, logger log.Logger) (json.RawMessage, []tmtypes.GenesisValidator, error) { +// ConstructAppExporter returns an application export function. +func ConstructAppExporter(appFn AppExporterInit, name string) AppExporter { + return func(rootDir string, logger log.Logger, traceStore string) (json.RawMessage, []tmtypes.GenesisValidator, error) { dataDir := filepath.Join(rootDir, "data") + db, err := dbm.NewGoLevelDB(name, dataDir) if err != nil { return nil, nil, err } - return appFn(logger, db) + + var traceStoreWriter io.Writer + if traceStore != "" { + traceStoreWriter, err = os.OpenFile( + traceStore, + os.O_WRONLY|os.O_APPEND|os.O_CREATE, + 0666, + ) + if err != nil { + return nil, nil, err + } + } + + return appFn(logger, db, traceStoreWriter) } } diff --git a/server/export.go b/server/export.go index 0ff7456add..5982e7b473 100644 --- a/server/export.go +++ b/server/export.go @@ -11,27 +11,33 @@ import ( tmtypes "github.com/tendermint/tendermint/types" ) -// ExportCmd dumps app state to JSON +// ExportCmd dumps app state to JSON. func ExportCmd(ctx *Context, cdc *wire.Codec, appExporter AppExporter) *cobra.Command { return &cobra.Command{ Use: "export", Short: "Export state to JSON", RunE: func(cmd *cobra.Command, args []string) error { home := viper.GetString("home") - appState, validators, err := appExporter(home, ctx.Logger) + traceStore := viper.GetString(flagTraceStore) + + appState, validators, err := appExporter(home, ctx.Logger, traceStore) if err != nil { return errors.Errorf("error exporting state: %v\n", err) } + doc, err := tmtypes.GenesisDocFromFile(ctx.Config.GenesisFile()) if err != nil { return err } + doc.AppStateJSON = appState doc.Validators = validators + encoded, err := wire.MarshalJSONIndent(cdc, doc) if err != nil { return err } + fmt.Println(string(encoded)) return nil }, diff --git a/server/mock/store.go b/server/mock/store.go index d62b9ad23e..ed673f956a 100644 --- a/server/mock/store.go +++ b/server/mock/store.go @@ -1,6 +1,8 @@ package mock import ( + "io" + dbm "github.com/tendermint/tendermint/libs/db" sdk "github.com/cosmos/cosmos-sdk/types" @@ -18,6 +20,26 @@ func (ms multiStore) CacheWrap() sdk.CacheWrap { panic("not implemented") } +func (ms multiStore) CacheWrapWithTrace(_ io.Writer, _ sdk.TraceContext) sdk.CacheWrap { + panic("not implemented") +} + +func (ms multiStore) ResetTraceContext() sdk.MultiStore { + panic("not implemented") +} + +func (ms multiStore) TracingEnabled() bool { + panic("not implemented") +} + +func (ms multiStore) WithTracingContext(tc sdk.TraceContext) sdk.MultiStore { + panic("not implemented") +} + +func (ms multiStore) WithTracer(w io.Writer) sdk.MultiStore { + panic("not implemented") +} + func (ms multiStore) Commit() sdk.CommitID { panic("not implemented") } @@ -70,6 +92,10 @@ func (kv kvStore) CacheWrap() sdk.CacheWrap { panic("not implemented") } +func (kv kvStore) CacheWrapWithTrace(w io.Writer, tc sdk.TraceContext) sdk.CacheWrap { + panic("not implemented") +} + func (kv kvStore) GetStoreType() sdk.StoreType { panic("not implemented") } diff --git a/server/start.go b/server/start.go index 37e40c9992..a40d73b394 100644 --- a/server/start.go +++ b/server/start.go @@ -17,10 +17,11 @@ import ( const ( flagWithTendermint = "with-tendermint" flagAddress = "address" + flagTraceStore = "trace-store" ) -// StartCmd runs the service passed in, either -// stand-alone, or in-process with tendermint +// StartCmd runs the service passed in, either stand-alone or in-process with +// Tendermint. func StartCmd(ctx *Context, appCreator AppCreator) *cobra.Command { cmd := &cobra.Command{ Use: "start", @@ -30,26 +31,30 @@ func StartCmd(ctx *Context, appCreator AppCreator) *cobra.Command { ctx.Logger.Info("Starting ABCI without Tendermint") return startStandAlone(ctx, appCreator) } + ctx.Logger.Info("Starting ABCI with Tendermint") + _, err := startInProcess(ctx, appCreator) return err }, } - // basic flags for abci app - cmd.Flags().Bool(flagWithTendermint, true, "run abci app embedded in-process with tendermint") + // core flags for the ABCI application + cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint") cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address") + cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") - // AddNodeFlags adds support for all tendermint-specific command line options + // add support for all Tendermint-specific command line options tcmd.AddNodeFlags(cmd) return cmd } func startStandAlone(ctx *Context, appCreator AppCreator) error { - // Generate the app in the proper dir addr := viper.GetString(flagAddress) home := viper.GetString("home") - app, err := appCreator(home, ctx.Logger) + traceStore := viper.GetString(flagTraceStore) + + app, err := appCreator(home, ctx.Logger, traceStore) if err != nil { return err } @@ -58,15 +63,17 @@ func startStandAlone(ctx *Context, appCreator AppCreator) error { if err != nil { return errors.Errorf("error creating listener: %v\n", err) } + svr.SetLogger(ctx.Logger.With("module", "abci-server")) + err = svr.Start() if err != nil { cmn.Exit(err.Error()) } - // Wait forever + // wait forever cmn.TrapSignal(func() { - // Cleanup + // cleanup err = svr.Stop() if err != nil { cmn.Exit(err.Error()) @@ -78,29 +85,33 @@ func startStandAlone(ctx *Context, appCreator AppCreator) error { func startInProcess(ctx *Context, appCreator AppCreator) (*node.Node, error) { cfg := ctx.Config home := cfg.RootDir - app, err := appCreator(home, ctx.Logger) + traceStore := viper.GetString(flagTraceStore) + + app, err := appCreator(home, ctx.Logger, traceStore) if err != nil { return nil, err } - // Create & start tendermint node - n, err := node.NewNode(cfg, + // create & start tendermint node + tmNode, err := node.NewNode( + cfg, pvm.LoadOrGenFilePV(cfg.PrivValidatorFile()), proxy.NewLocalClientCreator(app), node.DefaultGenesisDocProviderFunc(cfg), node.DefaultDBProvider, node.DefaultMetricsProvider, - ctx.Logger.With("module", "node")) + ctx.Logger.With("module", "node"), + ) if err != nil { return nil, err } - err = n.Start() + err = tmNode.Start() if err != nil { return nil, err } - // Trap signal, run forever. - n.RunForever() - return n, nil + // trap signal (run forever) + tmNode.RunForever() + return tmNode, nil } diff --git a/store/cachekvstore.go b/store/cachekvstore.go index 1263cef5f3..efc4e89113 100644 --- a/store/cachekvstore.go +++ b/store/cachekvstore.go @@ -2,6 +2,7 @@ package store import ( "bytes" + "io" "sort" "sync" @@ -27,11 +28,10 @@ var _ CacheKVStore = (*cacheKVStore)(nil) // nolint func NewCacheKVStore(parent KVStore) *cacheKVStore { - ci := &cacheKVStore{ + return &cacheKVStore{ cache: make(map[string]cValue), parent: parent, } - return ci } // Implements Store. @@ -98,6 +98,7 @@ func (ci *cacheKVStore) Write() { keys = append(keys, key) } } + sort.Strings(keys) // TODO: Consider allowing usage of Batch, which would allow the write to @@ -125,6 +126,11 @@ func (ci *cacheKVStore) CacheWrap() CacheWrap { return NewCacheKVStore(ci) } +// CacheWrapWithTrace implements the CacheWrapper interface. +func (ci *cacheKVStore) CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap { + return NewCacheKVStore(NewTraceKVStore(ci, w, tc)) +} + //---------------------------------------- // Iteration @@ -140,32 +146,39 @@ func (ci *cacheKVStore) ReverseIterator(start, end []byte) Iterator { func (ci *cacheKVStore) iterator(start, end []byte, ascending bool) Iterator { var parent, cache Iterator + if ascending { parent = ci.parent.Iterator(start, end) } else { parent = ci.parent.ReverseIterator(start, end) } + items := ci.dirtyItems(ascending) cache = newMemIterator(start, end, items) + return newCacheMergeIterator(parent, cache, ascending) } // Constructs a slice of dirty items, to use w/ memIterator. func (ci *cacheKVStore) dirtyItems(ascending bool) []cmn.KVPair { items := make([]cmn.KVPair, 0, len(ci.cache)) + for key, cacheValue := range ci.cache { if !cacheValue.dirty { continue } - items = append(items, - cmn.KVPair{[]byte(key), cacheValue.value}) + + items = append(items, cmn.KVPair{Key: []byte(key), Value: cacheValue.value}) } + sort.Slice(items, func(i, j int) bool { if ascending { return bytes.Compare(items[i].Key, items[j].Key) < 0 } + return bytes.Compare(items[i].Key, items[j].Key) > 0 }) + return items } @@ -180,10 +193,9 @@ func (ci *cacheKVStore) assertValidKey(key []byte) { // Only entrypoint to mutate ci.cache. func (ci *cacheKVStore) setCacheValue(key, value []byte, deleted bool, dirty bool) { - cacheValue := cValue{ + ci.cache[string(key)] = cValue{ value: value, deleted: deleted, dirty: dirty, } - ci.cache[string(key)] = cacheValue } diff --git a/store/cachemultistore.go b/store/cachemultistore.go index 47878fb155..af89bde692 100644 --- a/store/cachemultistore.go +++ b/store/cachemultistore.go @@ -1,6 +1,8 @@ package store import ( + "io" + sdk "github.com/cosmos/cosmos-sdk/types" ) @@ -13,33 +15,86 @@ type cacheMultiStore struct { db CacheKVStore stores map[StoreKey]CacheWrap keysByName map[string]StoreKey + + traceWriter io.Writer + traceContext TraceContext } var _ CacheMultiStore = cacheMultiStore{} func newCacheMultiStoreFromRMS(rms *rootMultiStore) cacheMultiStore { cms := cacheMultiStore{ - db: NewCacheKVStore(dbStoreAdapter{rms.db}), - stores: make(map[StoreKey]CacheWrap, len(rms.stores)), - keysByName: rms.keysByName, + db: NewCacheKVStore(dbStoreAdapter{rms.db}), + stores: make(map[StoreKey]CacheWrap, len(rms.stores)), + keysByName: rms.keysByName, + traceWriter: rms.traceWriter, + traceContext: rms.traceContext, } + for key, store := range rms.stores { - cms.stores[key] = store.CacheWrap() + if cms.TracingEnabled() { + cms.stores[key] = store.CacheWrapWithTrace(cms.traceWriter, cms.traceContext) + } else { + cms.stores[key] = store.CacheWrap() + } } + return cms } func newCacheMultiStoreFromCMS(cms cacheMultiStore) cacheMultiStore { cms2 := cacheMultiStore{ - db: NewCacheKVStore(cms.db), - stores: make(map[StoreKey]CacheWrap, len(cms.stores)), + db: NewCacheKVStore(cms.db), + stores: make(map[StoreKey]CacheWrap, len(cms.stores)), + traceWriter: cms.traceWriter, + traceContext: cms.traceContext, } + for key, store := range cms.stores { - cms2.stores[key] = store.CacheWrap() + if cms2.TracingEnabled() { + cms2.stores[key] = store.CacheWrapWithTrace(cms2.traceWriter, cms2.traceContext) + } else { + cms2.stores[key] = store.CacheWrap() + } } + return cms2 } +// WithTracer sets the tracer for the MultiStore that the underlying +// stores will utilize to trace operations. A MultiStore is returned. +func (cms cacheMultiStore) WithTracer(w io.Writer) MultiStore { + cms.traceWriter = w + return cms +} + +// WithTracingContext updates the tracing context for the MultiStore by merging +// the given context with the existing context by key. Any existing keys will +// be overwritten. It is implied that the caller should update the context when +// necessary between tracing operations. It returns a modified MultiStore. +func (cms cacheMultiStore) WithTracingContext(tc TraceContext) MultiStore { + if cms.traceContext != nil { + for k, v := range tc { + cms.traceContext[k] = v + } + } else { + cms.traceContext = tc + } + + return cms +} + +// TracingEnabled returns if tracing is enabled for the MultiStore. +func (cms cacheMultiStore) TracingEnabled() bool { + return cms.traceWriter != nil +} + +// ResetTraceContext resets the current tracing context. +func (cms cacheMultiStore) ResetTraceContext() MultiStore { + cms.traceContext = nil + return cms +} + // Implements Store. func (cms cacheMultiStore) GetStoreType() StoreType { return sdk.StoreTypeMulti @@ -58,6 +113,11 @@ func (cms cacheMultiStore) CacheWrap() CacheWrap { return cms.CacheMultiStore().(CacheWrap) } +// CacheWrapWithTrace implements the CacheWrapper interface. +func (cms cacheMultiStore) CacheWrapWithTrace(_ io.Writer, _ TraceContext) CacheWrap { + return cms.CacheWrap() +} + // Implements MultiStore. func (cms cacheMultiStore) CacheMultiStore() CacheMultiStore { return newCacheMultiStoreFromCMS(cms) diff --git a/store/dbstoreadapter.go b/store/dbstoreadapter.go index 09d48cf9d9..cd384c892c 100644 --- a/store/dbstoreadapter.go +++ b/store/dbstoreadapter.go @@ -1,6 +1,8 @@ package store import ( + "io" + sdk "github.com/cosmos/cosmos-sdk/types" dbm "github.com/tendermint/tendermint/libs/db" ) @@ -19,6 +21,11 @@ func (dsa dbStoreAdapter) CacheWrap() CacheWrap { return NewCacheKVStore(dsa) } +// CacheWrapWithTrace implements the KVStore interface. +func (dsa dbStoreAdapter) CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap { + return NewCacheKVStore(NewTraceKVStore(dsa, w, tc)) +} + // Implements KVStore func (dsa dbStoreAdapter) Prefix(prefix []byte) KVStore { return prefixStore{dsa, prefix} diff --git a/store/gaskvstore.go b/store/gaskvstore.go index 6dc699dfb3..22f89e6315 100644 --- a/store/gaskvstore.go +++ b/store/gaskvstore.go @@ -1,6 +1,8 @@ package store import ( + "io" + sdk "github.com/cosmos/cosmos-sdk/types" ) @@ -82,7 +84,12 @@ func (gi *gasKVStore) ReverseIterator(start, end []byte) sdk.Iterator { // Implements KVStore. func (gi *gasKVStore) CacheWrap() sdk.CacheWrap { - panic("you cannot CacheWrap a GasKVStore") + panic("cannot CacheWrap a GasKVStore") +} + +// CacheWrapWithTrace implements the KVStore interface. +func (gi *gasKVStore) CacheWrapWithTrace(_ io.Writer, _ TraceContext) CacheWrap { + panic("cannot CacheWrapWithTrace a GasKVStore") } func (gi *gasKVStore) iterator(start, end []byte, ascending bool) sdk.Iterator { diff --git a/store/iavlstore.go b/store/iavlstore.go index d05c867c2e..e02ede1304 100644 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -2,6 +2,7 @@ package store import ( "fmt" + "io" "sync" "github.com/tendermint/go-amino" @@ -117,6 +118,11 @@ func (st *iavlStore) CacheWrap() CacheWrap { return NewCacheKVStore(st) } +// CacheWrapWithTrace implements the Store interface. +func (st *iavlStore) CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap { + return NewCacheKVStore(NewTraceKVStore(st, w, tc)) +} + // Implements KVStore. func (st *iavlStore) Set(key, value []byte) { st.tree.Set(key, value) diff --git a/store/prefixstore.go b/store/prefixstore.go index c9f124a889..7e7f8d4931 100644 --- a/store/prefixstore.go +++ b/store/prefixstore.go @@ -1,6 +1,8 @@ package store import ( + "io" + sdk "github.com/cosmos/cosmos-sdk/types" ) @@ -19,6 +21,11 @@ func (s prefixStore) CacheWrap() CacheWrap { return NewCacheKVStore(s) } +// CacheWrapWithTrace implements the KVStore interface. +func (s prefixStore) CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap { + return NewCacheKVStore(NewTraceKVStore(s, w, tc)) +} + // Implements KVStore func (s prefixStore) Get(key []byte) []byte { return s.store.Get(append(s.prefix, key...)) diff --git a/store/rootmultistore.go b/store/rootmultistore.go index 10926e4bcb..528de9a848 100644 --- a/store/rootmultistore.go +++ b/store/rootmultistore.go @@ -2,6 +2,7 @@ package store import ( "fmt" + "io" "strings" "golang.org/x/crypto/ripemd160" @@ -18,16 +19,18 @@ const ( commitInfoKeyFmt = "s/%d" // s/ ) -// rootMultiStore is composed of many CommitStores. -// Name contrasts with cacheMultiStore which is for cache-wrapping -// other MultiStores. -// Implements MultiStore. +// rootMultiStore is composed of many CommitStores. Name contrasts with +// cacheMultiStore which is for cache-wrapping other MultiStores. It implements +// the CommitMultiStore interface. type rootMultiStore struct { db dbm.DB lastCommitID CommitID storesParams map[StoreKey]storeParams stores map[StoreKey]CommitStore keysByName map[string]StoreKey + + traceWriter io.Writer + traceContext TraceContext } var _ CommitMultiStore = (*rootMultiStore)(nil) @@ -130,6 +133,40 @@ func (rs *rootMultiStore) LoadVersion(ver int64) error { return nil } +// WithTracer sets the tracer for the MultiStore that the underlying +// stores will utilize to trace operations. A MultiStore is returned. +func (rs *rootMultiStore) WithTracer(w io.Writer) MultiStore { + rs.traceWriter = w + return rs +} + +// WithTracingContext updates the tracing context for the MultiStore by merging +// the given context with the existing context by key. Any existing keys will +// be overwritten. It is implied that the caller should update the context when +// necessary between tracing operations. It returns a modified MultiStore. +func (rs *rootMultiStore) WithTracingContext(tc TraceContext) MultiStore { + if rs.traceContext != nil { + for k, v := range tc { + rs.traceContext[k] = v + } + } else { + rs.traceContext = tc + } + + return rs +} + +// TracingEnabled returns if tracing is enabled for the MultiStore. +func (rs *rootMultiStore) TracingEnabled() bool { + return rs.traceWriter != nil +} + +// ResetTraceContext resets the current tracing context. +func (rs *rootMultiStore) ResetTraceContext() MultiStore { + rs.traceContext = nil + return rs +} + //---------------------------------------- // +CommitStore @@ -165,6 +202,11 @@ func (rs *rootMultiStore) CacheWrap() CacheWrap { return rs.CacheMultiStore().(CacheWrap) } +// CacheWrapWithTrace implements the CacheWrapper interface. +func (rs *rootMultiStore) CacheWrapWithTrace(_ io.Writer, _ TraceContext) CacheWrap { + return rs.CacheWrap() +} + //---------------------------------------- // +MultiStore @@ -178,9 +220,17 @@ func (rs *rootMultiStore) GetStore(key StoreKey) Store { return rs.stores[key] } -// Implements MultiStore. +// GetKVStore implements the MultiStore interface. If tracing is enabled on the +// rootMultiStore, a wrapped TraceKVStore will be returned with the given +// tracer, otherwise, the original KVStore will be returned. func (rs *rootMultiStore) GetKVStore(key StoreKey) KVStore { - return rs.stores[key].(KVStore) + store := rs.stores[key].(KVStore) + + if rs.TracingEnabled() { + store = NewTraceKVStore(store, rs.traceWriter, rs.traceContext) + } + + return store } // Implements MultiStore. diff --git a/store/tracekvstore.go b/store/tracekvstore.go new file mode 100644 index 0000000000..f769c6690a --- /dev/null +++ b/store/tracekvstore.go @@ -0,0 +1,198 @@ +package store + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + + sdk "github.com/cosmos/cosmos-sdk/types" +) + +const ( + writeOp operation = "write" + readOp operation = "read" + deleteOp operation = "delete" + iterKeyOp operation = "iterKey" + iterValueOp operation = "iterValue" +) + +type ( + // TraceKVStore implements the KVStore interface with tracing enabled. + // Operations are traced on each core KVStore call and written to the + // underlying io.writer. + // + // TODO: Should we use a buffered writer and implement Commit on + // TraceKVStore? + TraceKVStore struct { + parent sdk.KVStore + writer io.Writer + context TraceContext + } + + // operation represents an IO operation + operation string + + // traceOperation implements a traced KVStore operation + traceOperation struct { + Operation operation `json:"operation"` + Key string `json:"key"` + Value string `json:"value"` + Metadata map[string]interface{} `json:"metadata"` + } +) + +// NewTraceKVStore returns a reference to a new traceKVStore given a parent +// KVStore implementation and a buffered writer. +func NewTraceKVStore(parent sdk.KVStore, writer io.Writer, tc TraceContext) *TraceKVStore { + return &TraceKVStore{parent: parent, writer: writer, context: tc} +} + +// Get implements the KVStore interface. It traces a read operation and +// delegates a Get call to the parent KVStore. +func (tkv *TraceKVStore) Get(key []byte) []byte { + value := tkv.parent.Get(key) + + writeOperation(tkv.writer, readOp, tkv.context, key, value) + return value +} + +// Set implements the KVStore interface. It traces a write operation and +// delegates the Set call to the parent KVStore. +func (tkv *TraceKVStore) Set(key []byte, value []byte) { + writeOperation(tkv.writer, writeOp, tkv.context, key, value) + tkv.parent.Set(key, value) +} + +// Delete implements the KVStore interface. It traces a write operation and +// delegates the Delete call to the parent KVStore. +func (tkv *TraceKVStore) Delete(key []byte) { + writeOperation(tkv.writer, deleteOp, tkv.context, key, nil) + tkv.parent.Delete(key) +} + +// Has implements the KVStore interface. It delegates the Has call to the +// parent KVStore. +func (tkv *TraceKVStore) Has(key []byte) bool { + return tkv.parent.Has(key) +} + +// Prefix implements the KVStore interface. +func (tkv *TraceKVStore) Prefix(prefix []byte) KVStore { + return prefixStore{tkv, prefix} +} + +// Iterator implements the KVStore interface. It delegates the Iterator call +// the to the parent KVStore. +func (tkv *TraceKVStore) Iterator(start, end []byte) sdk.Iterator { + return tkv.iterator(start, end, true) +} + +// ReverseIterator implements the KVStore interface. It delegates the +// ReverseIterator call the to the parent KVStore. +func (tkv *TraceKVStore) ReverseIterator(start, end []byte) sdk.Iterator { + return tkv.iterator(start, end, false) +} + +// iterator facilitates iteration over a KVStore. It delegates the necessary +// calls to it's parent KVStore. +func (tkv *TraceKVStore) iterator(start, end []byte, ascending bool) sdk.Iterator { + var parent sdk.Iterator + + if ascending { + parent = tkv.parent.Iterator(start, end) + } else { + parent = tkv.parent.ReverseIterator(start, end) + } + + return newTraceIterator(tkv.writer, parent, tkv.context) +} + +type traceIterator struct { + parent sdk.Iterator + writer io.Writer + context TraceContext +} + +func newTraceIterator(w io.Writer, parent sdk.Iterator, tc TraceContext) sdk.Iterator { + return &traceIterator{writer: w, parent: parent, context: tc} +} + +// Domain implements the Iterator interface. +func (ti *traceIterator) Domain() (start []byte, end []byte) { + return ti.parent.Domain() +} + +// Valid implements the Iterator interface. +func (ti *traceIterator) Valid() bool { + return ti.parent.Valid() +} + +// Next implements the Iterator interface. +func (ti *traceIterator) Next() { + ti.parent.Next() +} + +// Key implements the Iterator interface. +func (ti *traceIterator) Key() []byte { + key := ti.parent.Key() + + writeOperation(ti.writer, iterKeyOp, ti.context, key, nil) + return key +} + +// Value implements the Iterator interface. +func (ti *traceIterator) Value() []byte { + value := ti.parent.Value() + + writeOperation(ti.writer, iterValueOp, ti.context, nil, value) + return value +} + +// Close implements the Iterator interface. +func (ti *traceIterator) Close() { + ti.parent.Close() +} + +// GetStoreType implements the KVStore interface. It returns the underlying +// KVStore type. +func (tkv *TraceKVStore) GetStoreType() sdk.StoreType { + return tkv.parent.GetStoreType() +} + +// CacheWrap implements the KVStore interface. It panics as a TraceKVStore +// cannot be cache wrapped. +func (tkv *TraceKVStore) CacheWrap() sdk.CacheWrap { + panic("cannot CacheWrap a TraceKVStore") +} + +// CacheWrapWithTrace implements the KVStore interface. It panics as a +// TraceKVStore cannot be cache wrapped. +func (tkv *TraceKVStore) CacheWrapWithTrace(_ io.Writer, _ TraceContext) CacheWrap { + panic("cannot CacheWrapWithTrace a TraceKVStore") +} + +// writeOperation writes a KVStore operation to the underlying io.Writer as +// JSON-encoded data where the key/value pair is base64 encoded. +func writeOperation(w io.Writer, op operation, tc TraceContext, key, value []byte) { + traceOp := traceOperation{ + Operation: op, + Key: base64.StdEncoding.EncodeToString(key), + Value: base64.StdEncoding.EncodeToString(value), + } + + if tc != nil { + traceOp.Metadata = tc + } + + raw, err := json.Marshal(traceOp) + if err != nil { + panic(fmt.Sprintf("failed to serialize trace operation: %v", err)) + } + + if _, err := w.Write(raw); err != nil { + panic(fmt.Sprintf("failed to write trace operation: %v", err)) + } + + io.WriteString(w, "\n") +} diff --git a/store/tracekvstore_test.go b/store/tracekvstore_test.go new file mode 100644 index 0000000000..e6f1bf3633 --- /dev/null +++ b/store/tracekvstore_test.go @@ -0,0 +1,283 @@ +package store + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" + + dbm "github.com/tendermint/tendermint/libs/db" +) + +var kvPairs = []KVPair{ + KVPair{Key: keyFmt(1), Value: valFmt(1)}, + KVPair{Key: keyFmt(2), Value: valFmt(2)}, + KVPair{Key: keyFmt(3), Value: valFmt(3)}, +} + +func newTraceKVStore(w io.Writer) *TraceKVStore { + store := newEmptyTraceKVStore(w) + + for _, kvPair := range kvPairs { + store.Set(kvPair.Key, kvPair.Value) + } + + return store +} + +func newEmptyTraceKVStore(w io.Writer) *TraceKVStore { + memDB := dbStoreAdapter{dbm.NewMemDB()} + tc := TraceContext(map[string]interface{}{"blockHeight": 64}) + + return NewTraceKVStore(memDB, w, tc) +} + +func TestTraceKVStoreGet(t *testing.T) { + testCases := []struct { + key []byte + expectedValue []byte + expectedOut string + }{ + { + key: []byte{}, + expectedValue: nil, + expectedOut: "{\"operation\":\"read\",\"key\":\"\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + }, + { + key: kvPairs[0].Key, + expectedValue: kvPairs[0].Value, + expectedOut: "{\"operation\":\"read\",\"key\":\"a2V5MDAwMDAwMDE=\",\"value\":\"dmFsdWUwMDAwMDAwMQ==\",\"metadata\":{\"blockHeight\":64}}\n", + }, + { + key: []byte("does-not-exist"), + expectedValue: nil, + expectedOut: "{\"operation\":\"read\",\"key\":\"ZG9lcy1ub3QtZXhpc3Q=\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + + store := newTraceKVStore(&buf) + buf.Reset() + value := store.Get(tc.key) + + require.Equal(t, tc.expectedValue, value) + require.Equal(t, tc.expectedOut, buf.String()) + } +} + +func TestTraceKVStoreSet(t *testing.T) { + testCases := []struct { + key []byte + value []byte + expectedOut string + }{ + { + key: []byte{}, + value: nil, + expectedOut: "{\"operation\":\"write\",\"key\":\"\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + }, + { + key: kvPairs[0].Key, + value: kvPairs[0].Value, + expectedOut: "{\"operation\":\"write\",\"key\":\"a2V5MDAwMDAwMDE=\",\"value\":\"dmFsdWUwMDAwMDAwMQ==\",\"metadata\":{\"blockHeight\":64}}\n", + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + + store := newEmptyTraceKVStore(&buf) + buf.Reset() + store.Set(tc.key, tc.value) + + require.Equal(t, tc.expectedOut, buf.String()) + } +} + +func TestTraceKVStoreDelete(t *testing.T) { + testCases := []struct { + key []byte + expectedOut string + }{ + { + key: []byte{}, + expectedOut: "{\"operation\":\"delete\",\"key\":\"\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + }, + { + key: kvPairs[0].Key, + expectedOut: "{\"operation\":\"delete\",\"key\":\"a2V5MDAwMDAwMDE=\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + + store := newTraceKVStore(&buf) + buf.Reset() + store.Delete(tc.key) + + require.Equal(t, tc.expectedOut, buf.String()) + } +} + +func TestTraceKVStoreHas(t *testing.T) { + testCases := []struct { + key []byte + expected bool + }{ + { + key: []byte{}, + expected: false, + }, + { + key: kvPairs[0].Key, + expected: true, + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + + store := newTraceKVStore(&buf) + buf.Reset() + ok := store.Has(tc.key) + + require.Equal(t, tc.expected, ok) + } +} + +func TestTestTraceKVStoreIterator(t *testing.T) { + var buf bytes.Buffer + + store := newTraceKVStore(&buf) + iterator := store.Iterator(nil, nil) + + s, e := iterator.Domain() + require.Equal(t, []uint8([]byte(nil)), s) + require.Equal(t, []uint8([]byte(nil)), e) + + testCases := []struct { + expectedKey []byte + expectedValue []byte + expectedKeyOut string + expectedvalueOut string + }{ + { + expectedKey: kvPairs[0].Key, + expectedValue: kvPairs[0].Value, + expectedKeyOut: "{\"operation\":\"iterKey\",\"key\":\"a2V5MDAwMDAwMDE=\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + expectedvalueOut: "{\"operation\":\"iterValue\",\"key\":\"\",\"value\":\"dmFsdWUwMDAwMDAwMQ==\",\"metadata\":{\"blockHeight\":64}}\n", + }, + { + expectedKey: kvPairs[1].Key, + expectedValue: kvPairs[1].Value, + expectedKeyOut: "{\"operation\":\"iterKey\",\"key\":\"a2V5MDAwMDAwMDI=\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + expectedvalueOut: "{\"operation\":\"iterValue\",\"key\":\"\",\"value\":\"dmFsdWUwMDAwMDAwMg==\",\"metadata\":{\"blockHeight\":64}}\n", + }, + { + expectedKey: kvPairs[2].Key, + expectedValue: kvPairs[2].Value, + expectedKeyOut: "{\"operation\":\"iterKey\",\"key\":\"a2V5MDAwMDAwMDM=\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + expectedvalueOut: "{\"operation\":\"iterValue\",\"key\":\"\",\"value\":\"dmFsdWUwMDAwMDAwMw==\",\"metadata\":{\"blockHeight\":64}}\n", + }, + } + + for _, tc := range testCases { + buf.Reset() + ka := iterator.Key() + require.Equal(t, tc.expectedKeyOut, buf.String()) + + buf.Reset() + va := iterator.Value() + require.Equal(t, tc.expectedvalueOut, buf.String()) + + require.Equal(t, tc.expectedKey, ka) + require.Equal(t, tc.expectedValue, va) + + iterator.Next() + } + + require.False(t, iterator.Valid()) + require.Panics(t, iterator.Next) + require.NotPanics(t, iterator.Close) +} + +func TestTestTraceKVStoreReverseIterator(t *testing.T) { + var buf bytes.Buffer + + store := newTraceKVStore(&buf) + iterator := store.ReverseIterator(nil, nil) + + s, e := iterator.Domain() + require.Equal(t, []uint8([]byte(nil)), s) + require.Equal(t, []uint8([]byte(nil)), e) + + testCases := []struct { + expectedKey []byte + expectedValue []byte + expectedKeyOut string + expectedvalueOut string + }{ + { + expectedKey: kvPairs[2].Key, + expectedValue: kvPairs[2].Value, + expectedKeyOut: "{\"operation\":\"iterKey\",\"key\":\"a2V5MDAwMDAwMDM=\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + expectedvalueOut: "{\"operation\":\"iterValue\",\"key\":\"\",\"value\":\"dmFsdWUwMDAwMDAwMw==\",\"metadata\":{\"blockHeight\":64}}\n", + }, + { + expectedKey: kvPairs[1].Key, + expectedValue: kvPairs[1].Value, + expectedKeyOut: "{\"operation\":\"iterKey\",\"key\":\"a2V5MDAwMDAwMDI=\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + expectedvalueOut: "{\"operation\":\"iterValue\",\"key\":\"\",\"value\":\"dmFsdWUwMDAwMDAwMg==\",\"metadata\":{\"blockHeight\":64}}\n", + }, + { + expectedKey: kvPairs[0].Key, + expectedValue: kvPairs[0].Value, + expectedKeyOut: "{\"operation\":\"iterKey\",\"key\":\"a2V5MDAwMDAwMDE=\",\"value\":\"\",\"metadata\":{\"blockHeight\":64}}\n", + expectedvalueOut: "{\"operation\":\"iterValue\",\"key\":\"\",\"value\":\"dmFsdWUwMDAwMDAwMQ==\",\"metadata\":{\"blockHeight\":64}}\n", + }, + } + + for _, tc := range testCases { + buf.Reset() + ka := iterator.Key() + require.Equal(t, tc.expectedKeyOut, buf.String()) + + buf.Reset() + va := iterator.Value() + require.Equal(t, tc.expectedvalueOut, buf.String()) + + require.Equal(t, tc.expectedKey, ka) + require.Equal(t, tc.expectedValue, va) + + iterator.Next() + } + + require.False(t, iterator.Valid()) + require.Panics(t, iterator.Next) + require.NotPanics(t, iterator.Close) +} + +func TestTraceKVStorePrefix(t *testing.T) { + store := newEmptyTraceKVStore(nil) + pStore := store.Prefix([]byte("trace_prefix")) + require.IsType(t, prefixStore{}, pStore) +} + +func TestTraceKVStoreGetStoreType(t *testing.T) { + memDB := dbStoreAdapter{dbm.NewMemDB()} + store := newEmptyTraceKVStore(nil) + require.Equal(t, memDB.GetStoreType(), store.GetStoreType()) +} + +func TestTraceKVStoreCacheWrap(t *testing.T) { + store := newEmptyTraceKVStore(nil) + require.Panics(t, func() { store.CacheWrap() }) +} +func TestTraceKVStoreCacheWrapWithTrace(t *testing.T) { + store := newEmptyTraceKVStore(nil) + require.Panics(t, func() { store.CacheWrapWithTrace(nil, nil) }) +} diff --git a/store/types.go b/store/types.go index e232e6ec71..4c36a004b3 100644 --- a/store/types.go +++ b/store/types.go @@ -6,20 +6,23 @@ import ( // Import cosmos-sdk/types/store.go for convenience. // nolint -type Store = types.Store -type Committer = types.Committer -type CommitStore = types.CommitStore -type MultiStore = types.MultiStore -type CacheMultiStore = types.CacheMultiStore -type CommitMultiStore = types.CommitMultiStore -type KVStore = types.KVStore -type KVPair = types.KVPair -type Iterator = types.Iterator -type CacheKVStore = types.CacheKVStore -type CommitKVStore = types.CommitKVStore -type CacheWrapper = types.CacheWrapper -type CacheWrap = types.CacheWrap -type CommitID = types.CommitID -type StoreKey = types.StoreKey -type StoreType = types.StoreType -type Queryable = types.Queryable +type ( + Store = types.Store + Committer = types.Committer + CommitStore = types.CommitStore + MultiStore = types.MultiStore + CacheMultiStore = types.CacheMultiStore + CommitMultiStore = types.CommitMultiStore + KVStore = types.KVStore + KVPair = types.KVPair + Iterator = types.Iterator + CacheKVStore = types.CacheKVStore + CommitKVStore = types.CommitKVStore + CacheWrapper = types.CacheWrapper + CacheWrap = types.CacheWrap + CommitID = types.CommitID + StoreKey = types.StoreKey + StoreType = types.StoreType + Queryable = types.Queryable + TraceContext = types.TraceContext +) diff --git a/types/store.go b/types/store.go index 04de2c7c1a..aaa9d2f2fe 100644 --- a/types/store.go +++ b/types/store.go @@ -2,6 +2,7 @@ package types import ( "fmt" + "io" abci "github.com/tendermint/tendermint/abci/types" cmn "github.com/tendermint/tendermint/libs/common" @@ -50,6 +51,21 @@ type MultiStore interface { //nolint GetStore(StoreKey) Store GetKVStore(StoreKey) KVStore GetKVStoreWithGas(GasMeter, StoreKey) KVStore + + // TracingEnabled returns if tracing is enabled for the MultiStore. + TracingEnabled() bool + + // WithTracer sets the tracer for the MultiStore that the underlying + // stores will utilize to trace operations. A MultiStore is returned. + WithTracer(w io.Writer) MultiStore + + // WithTracingContext sets the tracing context for a MultiStore. It is + // implied that the caller should update the context when necessary between + // tracing operations. A MultiStore is returned. + WithTracingContext(TraceContext) MultiStore + + // ResetTraceContext resets the current tracing context. + ResetTraceContext() MultiStore } // From MultiStore.CacheMultiStore().... @@ -163,25 +179,27 @@ type KVStoreGetter interface { //---------------------------------------- // CacheWrap -/* - CacheWrap() makes the most appropriate cache-wrap. For example, - IAVLStore.CacheWrap() returns a CacheKVStore. - - CacheWrap() should not return a Committer, since Commit() on - cache-wraps make no sense. It can return KVStore, HeapStore, - SpaceStore, etc. -*/ +// CacheWrap makes the most appropriate cache-wrap. For example, +// IAVLStore.CacheWrap() returns a CacheKVStore. CacheWrap should not return +// a Committer, since Commit cache-wraps make no sense. It can return KVStore, +// HeapStore, SpaceStore, etc. type CacheWrap interface { - // Write syncs with the underlying store. Write() // CacheWrap recursively wraps again. CacheWrap() CacheWrap + + // CacheWrapWithTrace recursively wraps again with tracing enabled. + CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap } type CacheWrapper interface { //nolint + // CacheWrap cache wraps. CacheWrap() CacheWrap + + // CacheWrapWithTrace cache wraps with tracing enabled. + CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap } //---------------------------------------- @@ -298,3 +316,7 @@ func (getter PrefixStoreGetter) KVStore(ctx Context) KVStore { type KVPair cmn.KVPair //---------------------------------------- + +// TraceContext contains TraceKVStore context data. It will be written with +// every trace operation. +type TraceContext map[string]interface{} From 43b9cc6df0765335304d891a0bda746e0229a035 Mon Sep 17 00:00:00 2001 From: Jeremiah Andrews Date: Thu, 12 Jul 2018 18:20:26 -0700 Subject: [PATCH 17/17] Merge PR #1533: Pruning Cleanup --- CHANGELOG.md | 1 + baseapp/options.go | 28 +++++++++++++++ cmd/gaia/app/app.go | 8 ++--- cmd/gaia/cmd/gaiad/main.go | 5 ++- cmd/gaia/cmd/gaiadebug/hack.go | 9 +++-- examples/basecoin/app/app.go | 4 +-- examples/basecoin/cmd/basecoind/main.go | 5 ++- server/mock/store.go | 4 +++ server/start.go | 2 ++ store/iavlstore.go | 46 +++++++++++++++++-------- store/iavlstore_test.go | 41 +++++++++++++++++++--- store/rootmultistore.go | 11 +++++- types/store.go | 15 ++++++++ 13 files changed, 146 insertions(+), 33 deletions(-) create mode 100644 baseapp/options.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 59c444f9d6..97881d5f66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ BREAKING CHANGES FEATURES * [baseapp] NewBaseApp now takes option functions as parameters * [store] Added support for tracing multi-store operations via `--trace-store` +* [store] Pruning strategy configurable with pruning flag on gaiad start BUG FIXES * \#1630 - redelegation nolonger removes tokens from the delegator liquid account diff --git a/baseapp/options.go b/baseapp/options.go new file mode 100644 index 0000000000..0a404217ae --- /dev/null +++ b/baseapp/options.go @@ -0,0 +1,28 @@ +package baseapp + +import ( + "fmt" + + sdk "github.com/cosmos/cosmos-sdk/types" +) + +// File for storing in-package BaseApp optional functions, +// for options that need access to non-exported fields of the BaseApp + +// SetPruning sets a pruning option on the multistore associated with the app +func SetPruning(pruning string) func(*BaseApp) { + var pruningEnum sdk.PruningStrategy + switch pruning { + case "nothing": + pruningEnum = sdk.PruneNothing + case "everything": + pruningEnum = sdk.PruneEverything + case "syncable": + pruningEnum = sdk.PruneSyncable + default: + panic(fmt.Sprintf("Invalid pruning strategy: %s", pruning)) + } + return func(bap *BaseApp) { + bap.cms.SetPruning(pruningEnum) + } +} diff --git a/cmd/gaia/app/app.go b/cmd/gaia/app/app.go index 62b3e3fc4f..ac1d27d39d 100644 --- a/cmd/gaia/app/app.go +++ b/cmd/gaia/app/app.go @@ -57,14 +57,10 @@ type GaiaApp struct { } // NewGaiaApp returns a reference to an initialized GaiaApp. -// -// TODO: Determine how to use a flexible and robust configuration paradigm that -// allows for sensible defaults while being highly configurable -// (e.g. functional options). -func NewGaiaApp(logger log.Logger, db dbm.DB, traceStore io.Writer) *GaiaApp { +func NewGaiaApp(logger log.Logger, db dbm.DB, traceStore io.Writer, baseAppOptions ...func(*bam.BaseApp)) *GaiaApp { cdc := MakeCodec() - bApp := bam.NewBaseApp(appName, cdc, logger, db) + bApp := bam.NewBaseApp(appName, cdc, logger, db, baseAppOptions...) bApp.SetCommitMultiStoreTracer(traceStore) var app = &GaiaApp{ diff --git a/cmd/gaia/cmd/gaiad/main.go b/cmd/gaia/cmd/gaiad/main.go index 6f4a427142..aa5978407d 100644 --- a/cmd/gaia/cmd/gaiad/main.go +++ b/cmd/gaia/cmd/gaiad/main.go @@ -4,7 +4,10 @@ import ( "encoding/json" "io" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/spf13/cobra" + "github.com/spf13/viper" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/cli" @@ -40,7 +43,7 @@ func main() { } func newApp(logger log.Logger, db dbm.DB, traceStore io.Writer) abci.Application { - return app.NewGaiaApp(logger, db, traceStore) + return app.NewGaiaApp(logger, db, traceStore, baseapp.SetPruning(viper.GetString("pruning"))) } func exportAppStateAndTMValidators( diff --git a/cmd/gaia/cmd/gaiadebug/hack.go b/cmd/gaia/cmd/gaiadebug/hack.go index 38921edc35..0d8af92eee 100644 --- a/cmd/gaia/cmd/gaiadebug/hack.go +++ b/cmd/gaia/cmd/gaiadebug/hack.go @@ -7,7 +7,10 @@ import ( "os" "path" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/spf13/cobra" + "github.com/spf13/viper" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto" cmn "github.com/tendermint/tendermint/libs/common" @@ -44,7 +47,7 @@ func runHackCmd(cmd *cobra.Command, args []string) error { fmt.Println(err) os.Exit(1) } - app := NewGaiaApp(logger, db) + app := NewGaiaApp(logger, db, baseapp.SetPruning(viper.GetString("pruning"))) // print some info id := app.LastCommitID() @@ -140,10 +143,10 @@ type GaiaApp struct { slashingKeeper slashing.Keeper } -func NewGaiaApp(logger log.Logger, db dbm.DB) *GaiaApp { +func NewGaiaApp(logger log.Logger, db dbm.DB, baseAppOptions ...func(*bam.BaseApp)) *GaiaApp { cdc := MakeCodec() - bApp := bam.NewBaseApp(appName, cdc, logger, db) + bApp := bam.NewBaseApp(appName, cdc, logger, db, baseAppOptions...) bApp.SetCommitMultiStoreTracer(os.Stdout) // create your application object diff --git a/examples/basecoin/app/app.go b/examples/basecoin/app/app.go index 7b2c7af3a3..14d4550d32 100644 --- a/examples/basecoin/app/app.go +++ b/examples/basecoin/app/app.go @@ -46,14 +46,14 @@ type BasecoinApp struct { // In addition, all necessary mappers and keepers are created, routes // registered, and finally the stores being mounted along with any necessary // chain initialization. -func NewBasecoinApp(logger log.Logger, db dbm.DB) *BasecoinApp { +func NewBasecoinApp(logger log.Logger, db dbm.DB, baseAppOptions ...func(*bam.BaseApp)) *BasecoinApp { // create and register app-level codec for TXs and accounts cdc := MakeCodec() // create your application type var app = &BasecoinApp{ cdc: cdc, - BaseApp: bam.NewBaseApp(appName, cdc, logger, db), + BaseApp: bam.NewBaseApp(appName, cdc, logger, db, baseAppOptions...), keyMain: sdk.NewKVStoreKey("main"), keyAccount: sdk.NewKVStoreKey("acc"), keyIBC: sdk.NewKVStoreKey("ibc"), diff --git a/examples/basecoin/cmd/basecoind/main.go b/examples/basecoin/cmd/basecoind/main.go index 49be70a535..420508e723 100644 --- a/examples/basecoin/cmd/basecoind/main.go +++ b/examples/basecoin/cmd/basecoind/main.go @@ -5,9 +5,12 @@ import ( "io" "os" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/examples/basecoin/app" "github.com/cosmos/cosmos-sdk/server" "github.com/spf13/cobra" + "github.com/spf13/viper" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/cli" dbm "github.com/tendermint/tendermint/libs/db" @@ -41,7 +44,7 @@ func main() { } func newApp(logger log.Logger, db dbm.DB, storeTracer io.Writer) abci.Application { - return app.NewBasecoinApp(logger, db) + return app.NewBasecoinApp(logger, db, baseapp.SetPruning(viper.GetString("pruning"))) } func exportAppStateAndTMValidators(logger log.Logger, db dbm.DB, storeTracer io.Writer) (json.RawMessage, []tmtypes.GenesisValidator, error) { diff --git a/server/mock/store.go b/server/mock/store.go index ed673f956a..5f598621ff 100644 --- a/server/mock/store.go +++ b/server/mock/store.go @@ -48,6 +48,10 @@ func (ms multiStore) LastCommitID() sdk.CommitID { panic("not implemented") } +func (ms multiStore) SetPruning(s sdk.PruningStrategy) { + panic("not implemented") +} + func (ms multiStore) GetCommitKVStore(key sdk.StoreKey) sdk.CommitKVStore { panic("not implemented") } diff --git a/server/start.go b/server/start.go index a40d73b394..64bd9fd457 100644 --- a/server/start.go +++ b/server/start.go @@ -18,6 +18,7 @@ const ( flagWithTendermint = "with-tendermint" flagAddress = "address" flagTraceStore = "trace-store" + flagPruning = "pruning" ) // StartCmd runs the service passed in, either stand-alone or in-process with @@ -43,6 +44,7 @@ func StartCmd(ctx *Context, appCreator AppCreator) *cobra.Command { cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint") cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address") cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") + cmd.Flags().String(flagPruning, "syncable", "Pruning strategy: syncable, nothing, everything") // add support for all Tendermint-specific command line options tcmd.AddNodeFlags(cmd) diff --git a/store/iavlstore.go b/store/iavlstore.go index e02ede1304..80c693288c 100644 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -15,20 +15,19 @@ import ( ) const ( - defaultIAVLCacheSize = 10000 - defaultIAVLNumRecent = 100 - defaultIAVLStoreEvery = 1 + defaultIAVLCacheSize = 10000 ) // load the iavl store -func LoadIAVLStore(db dbm.DB, id CommitID) (CommitStore, error) { +func LoadIAVLStore(db dbm.DB, id CommitID, pruning sdk.PruningStrategy) (CommitStore, error) { tree := iavl.NewVersionedTree(db, defaultIAVLCacheSize) _, err := tree.LoadVersion(id.Version) if err != nil { return nil, err } - store := newIAVLStore(tree, defaultIAVLNumRecent, defaultIAVLStoreEvery) - return store, nil + iavl := newIAVLStore(tree, int64(0), int64(0)) + iavl.SetPruning(pruning) + return iavl, nil } //---------------------------------------- @@ -44,16 +43,15 @@ type iavlStore struct { tree *iavl.VersionedTree // How many old versions we hold onto. - // A value of 0 means keep no recent states + // A value of 0 means keep no recent states. numRecent int64 - // Distance between state-sync waypoint states to be stored + // This is the distance between state-sync waypoint states to be stored. // See https://github.com/tendermint/tendermint/issues/828 - // A value of 1 means store every state - // A value of 0 means store no waypoints (node cannot assist in state-sync) + // A value of 1 means store every state. + // A value of 0 means store no waypoints. (node cannot assist in state-sync) // By default this value should be set the same across all nodes, - // so that nodes can know the waypoints their peers store - // TODO if set to non-default, signal to peers that the node is not suitable as a state sync source + // so that nodes can know the waypoints their peers store. storeEvery int64 } @@ -77,13 +75,13 @@ func (st *iavlStore) Commit() CommitID { panic(err) } - // Release an old version of history, if not a sync waypoint + // Release an old version of history, if not a sync waypoint. previous := version - 1 if st.numRecent < previous { toRelease := previous - st.numRecent if st.storeEvery == 0 || toRelease%st.storeEvery != 0 { err := st.tree.DeleteVersion(toRelease) - if err != nil { + if err != nil && err.(cmn.Error).Data() != iavl.ErrVersionDoesNotExist { panic(err) } } @@ -103,7 +101,21 @@ func (st *iavlStore) LastCommitID() CommitID { } } -// VersionExists returns whether or not a given version is stored +// Implements Committer. +func (st *iavlStore) SetPruning(pruning sdk.PruningStrategy) { + switch pruning { + case sdk.PruneEverything: + st.numRecent = 0 + st.storeEvery = 0 + case sdk.PruneNothing: + st.storeEvery = 1 + case sdk.PruneSyncable: + st.numRecent = 100 + st.storeEvery = 10000 + } +} + +// VersionExists returns whether or not a given version is stored. func (st *iavlStore) VersionExists(version int64) bool { return st.tree.VersionExists(version) } @@ -196,6 +208,10 @@ func (st *iavlStore) Query(req abci.RequestQuery) (res abci.ResponseQuery) { case "/store", "/key": // Get by key key := req.Data // Data holds the key bytes res.Key = key + if !st.VersionExists(res.Height) { + res.Log = cmn.ErrorWrap(iavl.ErrVersionDoesNotExist, "").Error() + break + } if req.Prove { value, proof, err := tree.GetVersionedWithProof(key, res.Height) if err != nil { diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go index f6f236dd8e..ab117252fb 100644 --- a/store/iavlstore_test.go +++ b/store/iavlstore_test.go @@ -266,13 +266,11 @@ func nextVersion(iavl *iavlStore) { iavl.Set(key, value) iavl.Commit() } + func TestIAVLDefaultPruning(t *testing.T) { //Expected stored / deleted version numbers for: //numRecent = 5, storeEvery = 3 - var states = []struct { - stored []int64 - deleted []int64 - }{ + var states = []pruneState{ {[]int64{}, []int64{}}, {[]int64{1}, []int64{}}, {[]int64{1, 2}, []int64{}}, @@ -290,6 +288,39 @@ func TestIAVLDefaultPruning(t *testing.T) { {[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}}, {[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}}, } + testPruning(t, int64(5), int64(3), states) +} + +func TestIAVLAlternativePruning(t *testing.T) { + //Expected stored / deleted version numbers for: + //numRecent = 3, storeEvery = 5 + var states = []pruneState{ + {[]int64{}, []int64{}}, + {[]int64{1}, []int64{}}, + {[]int64{1, 2}, []int64{}}, + {[]int64{1, 2, 3}, []int64{}}, + {[]int64{1, 2, 3, 4}, []int64{}}, + {[]int64{2, 3, 4, 5}, []int64{1}}, + {[]int64{3, 4, 5, 6}, []int64{1, 2}}, + {[]int64{4, 5, 6, 7}, []int64{1, 2, 3}}, + {[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}}, + {[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}}, + {[]int64{5, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 6}}, + {[]int64{5, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7}}, + {[]int64{5, 9, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8}}, + {[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, + {[]int64{5, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, + {[]int64{5, 10, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}}, + } + testPruning(t, int64(3), int64(5), states) +} + +type pruneState struct { + stored []int64 + deleted []int64 +} + +func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []pruneState) { db := dbm.NewMemDB() tree := iavl.NewVersionedTree(db, cacheSize) iavlStore := newIAVLStore(tree, numRecent, storeEvery) @@ -307,6 +338,7 @@ func TestIAVLDefaultPruning(t *testing.T) { nextVersion(iavlStore) } } + func TestIAVLNoPrune(t *testing.T) { db := dbm.NewMemDB() tree := iavl.NewVersionedTree(db, cacheSize) @@ -321,6 +353,7 @@ func TestIAVLNoPrune(t *testing.T) { nextVersion(iavlStore) } } + func TestIAVLPruneEverything(t *testing.T) { db := dbm.NewMemDB() tree := iavl.NewVersionedTree(db, cacheSize) diff --git a/store/rootmultistore.go b/store/rootmultistore.go index 528de9a848..255afffbd5 100644 --- a/store/rootmultistore.go +++ b/store/rootmultistore.go @@ -25,6 +25,7 @@ const ( type rootMultiStore struct { db dbm.DB lastCommitID CommitID + pruning sdk.PruningStrategy storesParams map[StoreKey]storeParams stores map[StoreKey]CommitStore keysByName map[string]StoreKey @@ -46,6 +47,14 @@ func NewCommitMultiStore(db dbm.DB) *rootMultiStore { } } +// Implements CommitMultiStore +func (rs *rootMultiStore) SetPruning(pruning sdk.PruningStrategy) { + rs.pruning = pruning + for _, substore := range rs.stores { + substore.SetPruning(pruning) + } +} + // Implements Store. func (rs *rootMultiStore) GetStoreType() StoreType { return sdk.StoreTypeMulti @@ -313,7 +322,7 @@ func (rs *rootMultiStore) loadCommitStoreFromParams(id CommitID, params storePar // TODO: id? // return NewCommitMultiStore(db, id) case sdk.StoreTypeIAVL: - store, err = LoadIAVLStore(db, id) + store, err = LoadIAVLStore(db, id, rs.pruning) return case sdk.StoreTypeDB: panic("dbm.DB is not a CommitStore") diff --git a/types/store.go b/types/store.go index aaa9d2f2fe..e8fe9067a6 100644 --- a/types/store.go +++ b/types/store.go @@ -11,6 +11,20 @@ import ( // NOTE: These are implemented in cosmos-sdk/store. +// PruningStrategy specfies how old states will be deleted over time +type PruningStrategy uint8 + +const ( + // PruneSyncable means only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th) + PruneSyncable PruningStrategy = iota + + // PruneEverything means all saved states will be deleted, storing only the current state + PruneEverything PruningStrategy = iota + + // PruneNothing means all historic states will be saved, nothing will be deleted + PruneNothing PruningStrategy = iota +) + type Store interface { //nolint GetStoreType() StoreType CacheWrapper @@ -20,6 +34,7 @@ type Store interface { //nolint type Committer interface { Commit() CommitID LastCommitID() CommitID + SetPruning(PruningStrategy) } // Stores of MultiStore must implement CommitStore.