feat: Add a cli cmd to prune old states according to current settings (#12742)

* add PruningCmd and change PruneStores signature

* the mimimum default pruning interval is 10

Co-authored-by: Marko <marbar3778@yahoo.com>

CHANGELOG.md

@@ -44,6 +44,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (testutil) [#12973](https://github.com/cosmos/cosmos-sdk/pull/12973) Add generic `testutil.RandSliceElem` function which selects a random element from the list.
 * (client) [#12936](https://github.com/cosmos/cosmos-sdk/pull/12936) Add capability to preprocess transactions before broadcasting from a higher level chain.
 * (x/authz) [#13047](https://github.com/cosmos/cosmos-sdk/pull/13047) Add a GetAuthorization function to the keeper.
+* (cli) [#12742](https://github.com/cosmos/cosmos-sdk/pull/12742) Add the `prune` CLI cmd to manually prune app store history versions based on the pruning options.
 
 ### Improvements
 

client/pruning/main.go

@@ -0,0 +1,117 @@
+package pruning
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/cosmos/cosmos-sdk/client/flags"
+	pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types"
+	"github.com/cosmos/cosmos-sdk/server"
+	servertypes "github.com/cosmos/cosmos-sdk/server/types"
+	"github.com/cosmos/cosmos-sdk/store/rootmulti"
+	"github.com/tendermint/tendermint/libs/log"
+	dbm "github.com/tendermint/tm-db"
+)
+
+const FlagAppDBBackend = "app-db-backend"
+
+// PruningCmd prunes the sdk root multi store history versions based on the pruning options
+// specified by command flags.
+func PruningCmd(appCreator servertypes.AppCreator) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "prune",
+		Short: "Prune app history states by keeping the recent heights and deleting old heights",
+		Long: `Prune app history states by keeping the recent heights and deleting old heights.
+		The pruning option is provided via the '--pruning' flag or alternatively with '--pruning-keep-recent'
+		
+		For '--pruning' the options are as follows:
+		
+		default: the last 362880 states are kept
+		nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node)
+		everything: 2 latest states will be kept
+		custom: allow pruning options to be manually specified through 'pruning-keep-recent'.
+		besides pruning options, database home directory and database backend type should also be specified via flags
+		'--home' and '--app-db-backend'.
+		valid app-db-backend type includes 'goleveldb', 'cleveldb', 'rocksdb', 'boltdb', and 'badgerdb'.
+		`,
+		Example: "prune --home './' --app-db-backend 'goleveldb' --pruning 'custom' --pruning-keep-recent 100",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			vp := viper.New()
+
+			// Bind flags to the Context's Viper so we can get pruning options.
+			if err := vp.BindPFlags(cmd.Flags()); err != nil {
+				return err
+			}
+			pruningOptions, err := server.GetPruningOptionsFromFlags(vp)
+			if err != nil {
+				return err
+			}
+			fmt.Printf("get pruning options from command flags, strategy: %v, keep-recent: %v\n",
+				pruningOptions.Strategy,
+				pruningOptions.KeepRecent,
+			)
+
+			home := vp.GetString(flags.FlagHome)
+			db, err := openDB(home, server.GetAppDBBackend(vp))
+			if err != nil {
+				return err
+			}
+
+			logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+			app := appCreator(logger, db, nil, vp)
+			cms := app.CommitMultiStore()
+
+			rootMultiStore, ok := cms.(*rootmulti.Store)
+			if !ok {
+				return fmt.Errorf("currently only support the pruning of rootmulti.Store type")
+			}
+			latestHeight := rootmulti.GetLatestVersion(db)
+			// valid heights should be greater than 0.
+			if latestHeight <= 0 {
+				return fmt.Errorf("the database has no valid heights to prune, the latest height: %v", latestHeight)
+			}
+
+			var pruningHeights []int64
+			for height := int64(1); height < latestHeight; height++ {
+				if height < latestHeight-int64(pruningOptions.KeepRecent) {
+					pruningHeights = append(pruningHeights, height)
+				}
+			}
+			if len(pruningHeights) == 0 {
+				fmt.Printf("no heights to prune\n")
+				return nil
+			}
+			fmt.Printf(
+				"pruning heights start from %v, end at %v\n",
+				pruningHeights[0],
+				pruningHeights[len(pruningHeights)-1],
+			)
+
+			err = rootMultiStore.PruneStores(false, pruningHeights)
+			if err != nil {
+				return err
+			}
+			fmt.Printf("successfully pruned the application root multi stores\n")
+			return nil
+		},
+	}
+
+	cmd.Flags().String(flags.FlagHome, "", "The database home directory")
+	cmd.Flags().String(FlagAppDBBackend, "", "The type of database for application and snapshots databases")
+	cmd.Flags().String(server.FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)")
+	cmd.Flags().Uint64(server.FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')")
+	cmd.Flags().Uint64(server.FlagPruningInterval, 10,
+		`Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom'), 
+		this is not used by this command but kept for compatibility with the complete pruning options`)
+
+	return cmd
+}
+
+func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) {
+	dataDir := filepath.Join(rootDir, "data")
+	return dbm.NewDB("application", backendType, dataDir)
+}

simapp/simd/cmd/root.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/client/debug"
 	"github.com/cosmos/cosmos-sdk/client/flags"
 	"github.com/cosmos/cosmos-sdk/client/keys"
+	"github.com/cosmos/cosmos-sdk/client/pruning"
 	"github.com/cosmos/cosmos-sdk/client/rpc"
 	"github.com/cosmos/cosmos-sdk/server"
 	serverconfig "github.com/cosmos/cosmos-sdk/server/config"
@@ -182,6 +183,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
 		NewTestnetCmd(simapp.ModuleBasics, banktypes.GenesisBalancesIterator{}),
 		debug.Cmd(),
 		config.Cmd(),
+		pruning.PruningCmd(newApp),
 	)
 
 	server.AddCommands(rootCmd, simapp.DefaultNodeHome, newApp, appExport, addModuleInitFlags)

store/rootmulti/store.go

@@ -162,7 +162,7 @@ func (rs *Store) StoreKeysByName() map[string]types.StoreKey {
 
 // LoadLatestVersionAndUpgrade implements CommitMultiStore
 func (rs *Store) LoadLatestVersionAndUpgrade(upgrades *types.StoreUpgrades) error {
-	ver := getLatestVersion(rs.db)
+	ver := GetLatestVersion(rs.db)
 	return rs.loadVersion(ver, upgrades)
 }
 
@@ -173,7 +173,7 @@ func (rs *Store) LoadVersionAndUpgrade(ver int64, upgrades *types.StoreUpgrades)
 
 // LoadLatestVersion implements CommitMultiStore.
 func (rs *Store) LoadLatestVersion() error {
-	ver := getLatestVersion(rs.db)
+	ver := GetLatestVersion(rs.db)
 	return rs.loadVersion(ver, nil)
 }
 
@@ -391,7 +391,7 @@ func (rs *Store) ListeningEnabled(key types.StoreKey) bool {
 func (rs *Store) LastCommitID() types.CommitID {
 	if rs.lastCommitInfo == nil {
 		return types.CommitID{
-			Version: getLatestVersion(rs.db),
+			Version: GetLatestVersion(rs.db),
 		}
 	}
 
@@ -539,17 +539,28 @@ func (rs *Store) handlePruning(version int64) error {
 	}
 	rs.logger.Info("prune start", "height", version)
 	defer rs.logger.Info("prune end", "height", version)
-	return rs.pruneStores()
+	return rs.PruneStores(true, nil)
 }
 
-func (rs *Store) pruneStores() error {
-	pruningHeights, err := rs.pruningManager.GetFlushAndResetPruningHeights()
-	if err != nil {
-		return err
+// PruneStores prunes the specific heights of the multi store.
+// If clearPruningManager is true, the pruning manager will return the pruning heights,
+// and they are appended to the pruningHeights to be pruned.
+func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) (err error) {
+	if clearPruningManager {
+		heights, err := rs.pruningManager.GetFlushAndResetPruningHeights()
+		if err != nil {
+			return err
+		}
+
+		if len(heights) == 0 {
+			rs.logger.Debug("no heights to be pruned from pruning manager")
+		}
+
+		pruningHeights = append(pruningHeights, heights...)
 	}
 
 	if len(pruningHeights) == 0 {
-		rs.logger.Debug("pruning skipped; no heights to prune")
+		rs.logger.Debug("no heights need to be pruned")
 		return nil
 	}
 
@@ -689,7 +700,7 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
 	if height == 0 {
 		return sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot snapshot height 0")
 	}
-	if height > uint64(getLatestVersion(rs.db)) {
+	if height > uint64(GetLatestVersion(rs.db)) {
 		return sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot snapshot future height %v", height)
 	}
 
@@ -981,7 +992,7 @@ type storeParams struct {
 	initialVersion uint64
 }
 
-func getLatestVersion(db dbm.DB) int64 {
+func GetLatestVersion(db dbm.DB) int64 {
 	bz, err := db.Get([]byte(latestVersionKey))
 	if err != nil {
 		panic(err)