diff --git a/CHANGELOG.md b/CHANGELOG.md
index 012c65d293..4a7bbb4909 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -69,6 +69,10 @@ longer panics if the store to load contains substores that we didn't explicitly
 
 ### Bug Fixes
 
+* (baseapp) \#4903 Various height query fixes:
+  * Move height with proof check from `CLIContext` to `BaseApp` as the height
+  can automatically be injected there.
+  * Update `handleQueryStore` to resemble `handleQueryCustom`
 * (cli) [\#4763](https://github.com/cosmos/cosmos-sdk/issues/4763) Fix flag `--min-self-delegation` for staking `EditValidator`
 * (keys) Fix ledger custom coin type support bug
 
diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go
index 43a4ab6387..d336e79fca 100644
--- a/baseapp/baseapp.go
+++ b/baseapp/baseapp.go
@@ -581,6 +581,15 @@ func handleQueryStore(app *BaseApp, path []string, req abci.RequestQuery) abci.R
 
 	req.Path = "/" + strings.Join(path[1:], "/")
 
+	// when a client did not provide a query height, manually inject the latest
+	if req.Height == 0 {
+		req.Height = app.LastBlockHeight()
+	}
+
+	if req.Height <= 1 && req.Prove {
+		return sdk.ErrInternal("cannot query with proof when height <= 1; please provide a valid height").QueryResult()
+	}
+
 	resp := queryable.Query(req)
 	resp.Height = req.Height
 
@@ -631,6 +640,10 @@ func handleQueryCustom(app *BaseApp, path []string, req abci.RequestQuery) (res
 		req.Height = app.LastBlockHeight()
 	}
 
+	if req.Height <= 1 && req.Prove {
+		return sdk.ErrInternal("cannot query with proof when height <= 1; please provide a valid height").QueryResult()
+	}
+
 	cacheMS, err := app.cms.CacheMultiStoreWithVersion(req.Height)
 	if err != nil {
 		return sdk.ErrInternal(
diff --git a/client/context/query.go b/client/context/query.go
index 9e8ec0ae57..49ec0c59f7 100644
--- a/client/context/query.go
+++ b/client/context/query.go
@@ -83,10 +83,6 @@ func (ctx CLIContext) query(path string, key cmn.HexBytes) (res []byte, height i
 		return res, height, err
 	}
 
-	if ctx.Height <= 1 && !ctx.TrustNode {
-		return res, height, errors.New("cannot query with proof when height <= 1; please provide a valid height")
-	}
-
 	opts := rpcclient.ABCIQueryOptions{
 		Height: ctx.Height,
 		Prove:  !ctx.TrustNode,