Merge branch 'master' into spoorthi/9706-refactor-validate-balance

CHANGELOG.md

@@ -64,6 +64,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * [\#9576](https://github.com/cosmos/cosmos-sdk/pull/9576) Add debug error message to `sdkerrors.QueryResult` when enabled
 * [\#9650](https://github.com/cosmos/cosmos-sdk/pull/9650) Removed deprecated message handler implementation from the SDK modules.
 * (x/capability) [\#9836](https://github.com/cosmos/cosmos-sdk/pull/9836) Removed `InitializeAndSeal(ctx sdk.Context)` and replaced with `Seal()`. App must add x/capability to begin blockers which will assure that the x/capability keeper is properly initialized. The x/capability begin blocker must be run before any other module which uses x/capability.
+* (x/bank) [\#9832] (https://github.com/cosmos/cosmos-sdk/pull/9832) `AddressFromBalancesStore` renamed to `AddressAndDenomFromBalancesStore`.
 
 ### Client Breaking Changes
 
@@ -103,6 +104,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 * (x/bank) [\#9611](https://github.com/cosmos/cosmos-sdk/pull/9611) Introduce a new index to act as a reverse index between a denomination and address allowing to query for
   token holders of a specific denomination. `DenomOwners` is updated to use the new reverse index.
+* (x/bank) [\#9832] (https://github.com/cosmos/cosmos-sdk/pull/9832) Account balance is stored as `sdk.Int` rather than `sdk.Coin`.
 
 ## [v0.43.0-rc0](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.43.0-rc0) - 2021-06-25
 

contrib/rosetta/configuration/bootstrap.json

@@ -1,7 +1,7 @@
 [
   {
     "account_identifier": {
-      "address":"cosmos1kezmr2chzy7w00nhh7qxhpqphdwum3j0mgdaw0"
+      "address":"cosmos1y3awd3vl7g29q44uvz0yrevcduf2exvkwxk3uq"
     },
     "currency":{
       "symbol":"stake",

cosmovisor/README.md

@@ -22,6 +22,7 @@ All arguments passed to `cosmovisor` will be passed to the application binary (a
 * `DAEMON_NAME` is the name of the binary itself (e.g. `gaiad`, `regend`, `simd`, etc.).
 * `DAEMON_ALLOW_DOWNLOAD_BINARIES` (*optional*), if set to `true`, will enable auto-downloading of new binaries (for security reasons, this is intended for full nodes rather than validators). By default, `cosmovisor` will not auto-download new binaries.
 * `DAEMON_RESTART_AFTER_UPGRADE` (*optional*), if set to `true`, will restart the subprocess with the same command-line arguments and flags (but with the new binary) after a successful upgrade. By default, `cosmovisor` stops running after an upgrade and requires the system administrator to manually restart it. Note that `cosmovisor` will not auto-restart the subprocess if there was an error.
+* `UNSAFE_SKIP_BACKUP` (defaults to `false`), if set to `false`, will backup the data before trying the upgrade. Otherwise it will upgrade directly without doing any backup. This is useful (and recommended) in case of failures and when needed to rollback. It is advised to use backup option, i.e., `UNSAFE_SKIP_BACKUP=false`
 
 ## Folder Layout
 

cosmovisor/args.go

@@ -24,6 +24,7 @@ type Config struct {
 	AllowDownloadBinaries bool
 	RestartAfterUpgrade   bool
 	LogBufferSize         int
+	UnsafeSkipBackup      bool
 }
 
 // Root returns the root directory where all info lives
@@ -113,6 +114,8 @@ func GetConfigFromEnv() (*Config, error) {
 		cfg.LogBufferSize = bufio.MaxScanTokenSize
 	}
 
+	cfg.UnsafeSkipBackup = os.Getenv("UNSAFE_SKIP_BACKUP") == "true"
+
 	if err := cfg.validate(); err != nil {
 		return nil, err
 	}

cosmovisor/cmd/cosmovisor/main.go

@@ -22,6 +22,7 @@ func Run(args []string) error {
 	}
 
 	doUpgrade, err := cosmovisor.LaunchProcess(cfg, args, os.Stdout, os.Stderr)
+
 	// if RestartAfterUpgrade, we launch after a successful upgrade (only condition LaunchProcess returns nil)
 	for cfg.RestartAfterUpgrade && err == nil && doUpgrade {
 		doUpgrade, err = cosmovisor.LaunchProcess(cfg, args, os.Stdout, os.Stderr)

cosmovisor/process.go

@@ -2,15 +2,21 @@ package cosmovisor
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"sync"
 	"syscall"
+	"time"
+
+	"github.com/otiai10/copy"
 )
 
 // LaunchProcess runs a subprocess and returns when the subprocess exits,
@@ -70,12 +76,59 @@ func LaunchProcess(cfg *Config, args []string, stdout, stderr io.Writer) (bool,
 	}
 
 	if upgradeInfo != nil {
+		if err := doBackup(cfg); err != nil {
+			return false, err
+		}
+
 		return true, DoUpgrade(cfg, upgradeInfo)
 	}
 
 	return false, nil
 }
 
+func doBackup(cfg *Config) error {
+	// take backup if `UNSAFE_SKIP_BACKUP` is not set.
+	if !cfg.UnsafeSkipBackup {
+		// check if upgrade-info.json is not empty.
+		var uInfo UpgradeInfo
+		upgradeInfoFile, err := ioutil.ReadFile(filepath.Join(cfg.Home, "data", "upgrade-info.json"))
+		if err != nil {
+			return fmt.Errorf("error while reading upgrade-info.json: %w", err)
+		}
+
+		err = json.Unmarshal(upgradeInfoFile, &uInfo)
+		if err != nil {
+			return err
+		}
+
+		if uInfo.Name == "" {
+			return fmt.Errorf("upgrade-info.json is empty")
+		}
+
+		// a destination directory, Format YYYY-MM-DD
+		st := time.Now()
+		stStr := fmt.Sprintf("%d-%d-%d", st.Year(), st.Month(), st.Day())
+		dst := filepath.Join(cfg.Home, fmt.Sprintf("data"+"-backup-%s", stStr))
+
+		fmt.Printf("starting to take backup of data directory at time %s", st)
+
+		// copy the $DAEMON_HOME/data to a backup dir
+		err = copy.Copy(filepath.Join(cfg.Home, "data"), dst)
+
+		if err != nil {
+			return fmt.Errorf("error while taking data backup: %w", err)
+		}
+
+		// backup is done, lets check endtime to calculate total time taken for backup process
+		et := time.Now()
+		timeTaken := et.Sub(st)
+		fmt.Printf("backup saved at location: %s, completed at time: %s\n"+
+			"time taken to complete the backup: %s", dst, et, timeTaken)
+	}
+
+	return nil
+}
+
 // WaitResult is used to wrap feedback on cmd state with some mutex logic.
 // This is needed as multiple go-routines can affect this - two read pipes that can trigger upgrade
 // As well as the command, which can fail

cosmovisor/process_test.go

@@ -23,7 +23,7 @@ func TestProcessTestSuite(t *testing.T) {
 // and args are passed through
 func (s *processTestSuite) TestLaunchProcess() {
 	home := copyTestData(s.T(), "validate")
-	cfg := &cosmovisor.Config{Home: home, Name: "dummyd"}
+	cfg := &cosmovisor.Config{Home: home, Name: "dummyd", UnsafeSkipBackup: true}
 
 	// should run the genesis binary and produce expected output
 	var stdout, stderr bytes.Buffer
@@ -65,7 +65,7 @@ func (s *processTestSuite) TestLaunchProcessWithDownloads() {
 	// zip_binary -> "chain3" = ref_zipped -> zip_directory
 	// zip_directory no upgrade
 	home := copyTestData(s.T(), "download")
-	cfg := &cosmovisor.Config{Home: home, Name: "autod", AllowDownloadBinaries: true}
+	cfg := &cosmovisor.Config{Home: home, Name: "autod", AllowDownloadBinaries: true, UnsafeSkipBackup: true}
 
 	// should run the genesis binary and produce expected output
 	var stdout, stderr bytes.Buffer

types/query/filtered_pagination_test.go

@@ -171,7 +171,7 @@ func (s *paginationTestSuite) TestReverseFilteredPaginations() {
 }
 
 func ExampleFilteredPaginate() {
-	app, ctx, appCodec := setupTest()
+	app, ctx, _ := setupTest()
 
 	var balances sdk.Coins
 	for i := 0; i < numBalances; i++ {
@@ -200,16 +200,16 @@ func ExampleFilteredPaginate() {
 
 	var balResult sdk.Coins
 	pageRes, err := query.FilteredPaginate(accountStore, pageReq, func(key []byte, value []byte, accumulate bool) (bool, error) {
-		var bal sdk.Coin
-		err := appCodec.Unmarshal(value, &bal)
+		var amount sdk.Int
+		err := amount.Unmarshal(value)
 		if err != nil {
 			return false, err
 		}
 
-		// filter balances with amount greater than 100
-		if bal.Amount.Int64() > int64(100) {
+		// filter amount greater than 100
+		if amount.Int64() > int64(100) {
 			if accumulate {
-				balResult = append(balResult, bal)
+				balResult = append(balResult, sdk.NewCoin(string(key), amount))
 			}
 
 			return true, nil
@@ -232,16 +232,16 @@ func execFilterPaginate(store sdk.KVStore, pageReq *query.PageRequest, appCodec
 
 	var balResult sdk.Coins
 	res, err = query.FilteredPaginate(accountStore, pageReq, func(key []byte, value []byte, accumulate bool) (bool, error) {
-		var bal sdk.Coin
-		err := appCodec.Unmarshal(value, &bal)
+		var amount sdk.Int
+		err := amount.Unmarshal(value)
 		if err != nil {
 			return false, err
 		}
 
-		// filter balances with amount greater than 100
-		if bal.Amount.Int64() > int64(100) {
+		// filter amount greater than 100
+		if amount.Int64() > int64(100) {
 			if accumulate {
-				balResult = append(balResult, bal)
+				balResult = append(balResult, sdk.NewCoin(string(key), amount))
 			}
 
 			return true, nil

types/query/pagination_test.go

@@ -319,12 +319,12 @@ func ExamplePaginate() {
 	balancesStore := prefix.NewStore(authStore, types.BalancesPrefix)
 	accountStore := prefix.NewStore(balancesStore, address.MustLengthPrefix(addr1))
 	pageRes, err := query.Paginate(accountStore, request.Pagination, func(key []byte, value []byte) error {
-		var tempRes sdk.Coin
-		err := app.AppCodec().Unmarshal(value, &tempRes)
+		var amount sdk.Int
+		err := amount.Unmarshal(value)
 		if err != nil {
 			return err
 		}
-		balResult = append(balResult, tempRes)
+		balResult = append(balResult, sdk.NewCoin(string(key), amount))
 		return nil
 	})
 	if err != nil { // should return no error

x/bank/keeper/grpc_query.go

@@ -59,13 +59,12 @@ func (k BaseKeeper) AllBalances(ctx context.Context, req *types.QueryAllBalances
 	balances := sdk.NewCoins()
 	accountStore := k.getAccountStore(sdkCtx, addr)
 
-	pageRes, err := query.Paginate(accountStore, req.Pagination, func(_, value []byte) error {
-		var result sdk.Coin
-		err := k.cdc.Unmarshal(value, &result)
-		if err != nil {
+	pageRes, err := query.Paginate(accountStore, req.Pagination, func(key, value []byte) error {
+		var amount sdk.Int
+		if err := amount.Unmarshal(value); err != nil {
 			return err
 		}
-		balances = append(balances, result)
+		balances = append(balances, sdk.NewCoin(string(key), amount))
 		return nil
 	})
 
@@ -187,7 +186,7 @@ func (k BaseKeeper) DenomOwners(
 		req.Pagination,
 		func(key []byte, value []byte, accumulate bool) (bool, error) {
 			if accumulate {
-				address, err := types.AddressFromBalancesStore(key)
+				address, _, err := types.AddressAndDenomFromBalancesStore(key)
 				if err != nil {
 					return false, err
 				}

x/bank/keeper/send.go

@@ -243,8 +243,11 @@ func (k BaseSendKeeper) initBalances(ctx sdk.Context, addr sdk.AccAddress, balan
 
 		// x/bank invariants prohibit persistence of zero balances
 		if !balance.IsZero() {
-			bz := k.cdc.MustMarshal(&balance)
-			accountStore.Set([]byte(balance.Denom), bz)
+			amount, err := balance.Amount.Marshal()
+			if err != nil {
+				return err
+			}
+			accountStore.Set([]byte(balance.Denom), amount)
 
 			denomPrefixStore, ok := denomPrefixStores[balance.Denom]
 			if !ok {
@@ -278,8 +281,11 @@ func (k BaseSendKeeper) setBalance(ctx sdk.Context, addr sdk.AccAddress, balance
 		accountStore.Delete([]byte(balance.Denom))
 		denomPrefixStore.Delete(address.MustLengthPrefix(addr))
 	} else {
-		bz := k.cdc.MustMarshal(&balance)
-		accountStore.Set([]byte(balance.Denom), bz)
+		amount, err := balance.Amount.Marshal()
+		if err != nil {
+			return err
+		}
+		accountStore.Set([]byte(balance.Denom), amount)
 
 		// Store a reverse index from denomination to account address with a
 		// sentinel value.

x/bank/keeper/view.go

@@ -98,16 +98,17 @@ func (k BaseViewKeeper) GetAccountsBalances(ctx sdk.Context) []types.Balance {
 // by address.
 func (k BaseViewKeeper) GetBalance(ctx sdk.Context, addr sdk.AccAddress, denom string) sdk.Coin {
 	accountStore := k.getAccountStore(ctx, addr)
-
+	amount := sdk.ZeroInt()
 	bz := accountStore.Get([]byte(denom))
 	if bz == nil {
-		return sdk.NewCoin(denom, sdk.ZeroInt())
+		return sdk.NewCoin(denom, amount)
 	}
 
-	var balance sdk.Coin
-	k.cdc.MustUnmarshal(bz, &balance)
+	if err := amount.Unmarshal(bz); err != nil {
+		panic(err)
+	}
 
-	return balance
+	return sdk.NewCoin(denom, amount)
 }
 
 // IterateAccountBalances iterates over the balances of a single account and
@@ -120,10 +121,12 @@ func (k BaseViewKeeper) IterateAccountBalances(ctx sdk.Context, addr sdk.AccAddr
 	defer iterator.Close()
 
 	for ; iterator.Valid(); iterator.Next() {
-		var balance sdk.Coin
-		k.cdc.MustUnmarshal(iterator.Value(), &balance)
+		var amount sdk.Int
+		if err := amount.Unmarshal(iterator.Value()); err != nil {
+			panic(err)
+		}
 
-		if cb(balance) {
+		if cb(sdk.NewCoin(string(iterator.Key()), amount)) {
 			break
 		}
 	}
@@ -140,7 +143,7 @@ func (k BaseViewKeeper) IterateAllBalances(ctx sdk.Context, cb func(sdk.AccAddre
 	defer iterator.Close()
 
 	for ; iterator.Valid(); iterator.Next() {
-		address, err := types.AddressFromBalancesStore(iterator.Key())
+		address, denom, err := types.AddressAndDenomFromBalancesStore(iterator.Key())
 		if err != nil {
 			k.Logger(ctx).With("key", iterator.Key(), "err", err).Error("failed to get address from balances store")
 			// TODO: revisit, for now, panic here to keep same behavior as in 0.42
@@ -148,10 +151,12 @@ func (k BaseViewKeeper) IterateAllBalances(ctx sdk.Context, cb func(sdk.AccAddre
 			panic(err)
 		}
 
-		var balance sdk.Coin
-		k.cdc.MustUnmarshal(iterator.Value(), &balance)
+		var amount sdk.Int
+		if err := amount.Unmarshal(iterator.Value()); err != nil {
+			panic(err)
+		}
 
-		if cb(address, balance) {
+		if cb(address, sdk.NewCoin(denom, amount)) {
 			break
 		}
 	}

x/bank/migrations/v044/store.go

@@ -6,11 +6,13 @@ import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/types/address"
 	v043 "github.com/cosmos/cosmos-sdk/x/bank/migrations/v043"
+	"github.com/cosmos/cosmos-sdk/x/bank/types"
 )
 
 // MigrateStore performs in-place store migrations from v0.43 to v0.44. The
 // migration includes:
 //
+// - Migrate coin storage to save only amount.
 // - Add an additional reverse index from denomination to address.
 func MigrateStore(ctx sdk.Context, storeKey sdk.StoreKey, cdc codec.BinaryCodec) error {
 	store := ctx.KVStore(storeKey)
@@ -36,6 +38,19 @@ func addDenomReverseIndex(store sdk.KVStore, cdc codec.BinaryCodec) error {
 			return err
 		}
 
+		var coin sdk.DecCoin
+		if err := cdc.Unmarshal(oldBalancesIter.Value(), &coin); err != nil {
+			return err
+		}
+
+		bz, err := coin.Amount.Marshal()
+		if err != nil {
+			return err
+		}
+
+		newStore := prefix.NewStore(store, types.CreateAccountBalancesPrefix(addr))
+		newStore.Set([]byte(coin.Denom), bz)
+
 		denomPrefixStore, ok := denomPrefixStores[balance.Denom]
 		if !ok {
 			denomPrefixStore = prefix.NewStore(store, CreateAddressDenomPrefix(balance.Denom))

x/bank/migrations/v044/store_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/types/address"
 	v043 "github.com/cosmos/cosmos-sdk/x/bank/migrations/v043"
 	v044 "github.com/cosmos/cosmos-sdk/x/bank/migrations/v044"
+	"github.com/cosmos/cosmos-sdk/x/bank/types"
 )
 
 func TestMigrateStore(t *testing.T) {
@@ -37,6 +38,14 @@ func TestMigrateStore(t *testing.T) {
 
 	require.NoError(t, v044.MigrateStore(ctx, bankKey, encCfg.Codec))
 
+	for _, b := range balances {
+		addrPrefixStore := prefix.NewStore(store, types.CreateAccountBalancesPrefix(addr))
+		bz := addrPrefixStore.Get([]byte(b.Denom))
+		var expected sdk.Int
+		require.NoError(t, expected.Unmarshal(bz))
+		require.Equal(t, expected, b.Amount)
+	}
+
 	for _, b := range balances {
 		denomPrefixStore := prefix.NewStore(store, v044.CreateAddressDenomPrefix(b.Denom))
 		bz := denomPrefixStore.Get(address.MustLengthPrefix(addr))

x/bank/types/key.go

@@ -37,26 +37,25 @@ func DenomMetadataKey(denom string) []byte {
 	return append(DenomMetadataPrefix, d...)
 }
 
-// AddressFromBalancesStore returns an account address from a balances prefix
+// AddressAndDenomFromBalancesStore returns an account address and denom from a balances prefix
 // store. The key must not contain the prefix BalancesPrefix as the prefix store
 // iterator discards the actual prefix.
 //
-// If invalid key is passed, AddressFromBalancesStore returns ErrInvalidKey.
-func AddressFromBalancesStore(key []byte) (sdk.AccAddress, error) {
+// If invalid key is passed, AddressAndDenomFromBalancesStore returns ErrInvalidKey.
+func AddressAndDenomFromBalancesStore(key []byte) (sdk.AccAddress, string, error) {
 	if len(key) == 0 {
-		return nil, ErrInvalidKey
+		return nil, "", ErrInvalidKey
 	}
 
 	kv.AssertKeyAtLeastLength(key, 1)
 
-	addrLen := key[0]
-	bound := int(addrLen)
+	addrBound := int(key[0])
 
-	if len(key)-1 < bound {
-		return nil, ErrInvalidKey
+	if len(key)-1 < addrBound {
+		return nil, "", ErrInvalidKey
 	}
 
-	return key[1 : bound+1], nil
+	return key[1 : addrBound+1], string(key[addrBound+1:]), nil
 }
 
 // CreateAccountBalancesPrefix creates the prefix for an account's balances.

x/bank/types/key_test.go

@@ -42,7 +42,7 @@ func TestAddressFromBalancesStore(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-			addr, err := types.AddressFromBalancesStore(tc.key)
+			addr, denom, err := types.AddressAndDenomFromBalancesStore(tc.key)
 			if tc.wantErr {
 				assert.Error(t, err)
 				assert.True(t, errors.Is(types.ErrInvalidKey, err))
@@ -51,6 +51,7 @@ func TestAddressFromBalancesStore(t *testing.T) {
 			}
 			if len(tc.expectedKey) > 0 {
 				assert.Equal(t, tc.expectedKey, addr)
+				assert.Equal(t, "stake", denom)
 			}
 		})
 	}