fix: removed potential sources of non-determinism in upgrades (#10189)

forced deterministic iteration order in upgrade migrations, x/upgrade and store during upgrades

Co-authored-by: Robert Zaremba <robert@zaremba.ch>

store/rootmulti/store.go

@@ -192,7 +192,22 @@ func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error {
 	// load each Store (note this doesn't panic on unmounted keys now)
 	var newStores = make(map[types.StoreKey]types.CommitKVStore)
 
-	for key, storeParams := range rs.storesParams {
+	storesKeys := make([]types.StoreKey, 0, len(rs.storesParams))
+
+	for key := range rs.storesParams {
+		storesKeys = append(storesKeys, key)
+	}
+	if upgrades != nil {
+		// deterministic iteration order for upgrades
+		// (as the underlying store may change and
+		// upgrades make store changes where the execution order may matter)
+		sort.Slice(storesKeys, func(i, j int) bool {
+			return storesKeys[i].Name() < storesKeys[j].Name()
+		})
+	}
+
+	for _, key := range storesKeys {
+		storeParams := rs.storesParams[key]
 		commitID := rs.getCommitID(infos, key.Name())
 
 		// If it has been added, set the initial version