chore: prepare systemtests release (#22658)

This commit is contained in:
Julien Robert 2024-11-27 07:32:59 +01:00 committed by GitHub
parent 48f2b36854
commit f296a5005c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 44 additions and 263 deletions

View File

@ -36,6 +36,8 @@ Ref: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.j
## [Unreleased]
## [v1.0.0-rc.1] - 2024-11-26
### Features
* [#22578](https://github.com/cosmos/cosmos-sdk/pull/22578) Extract system test framework

View File

@ -2,34 +2,16 @@ module cosmossdk.io/systemtests
go 1.23
require (
github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect
github.com/cosmos/cosmos-sdk v0.50.6
github.com/cosmos/gogogateway v1.2.0 // indirect
github.com/cosmos/gogoproto v1.7.0 // indirect
github.com/cosmos/iavl v1.1.4 // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.20.5 // indirect
github.com/spf13/cast v1.7.0 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.9.0
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/grpc v1.68.0
)
require (
cosmossdk.io/math v1.4.0
github.com/cometbft/cometbft v0.38.15
github.com/cometbft/cometbft/api v1.0.0-rc.1
github.com/cosmos/cosmos-sdk v0.50.6
github.com/creachadair/tomledit v0.0.26
github.com/stretchr/testify v1.9.0
github.com/tidwall/gjson v1.14.2
github.com/tidwall/sjson v1.2.5
google.golang.org/grpc v1.68.0
)
require (
@ -63,7 +45,11 @@ require (
github.com/cometbft/cometbft-db v0.14.1 // indirect
github.com/cosmos/btcutil v1.0.5 // indirect
github.com/cosmos/cosmos-db v1.0.3-0.20240829004618-717cba019b33 // indirect
github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect
github.com/cosmos/go-bip39 v1.0.0 // indirect
github.com/cosmos/gogogateway v1.2.0 // indirect
github.com/cosmos/gogoproto v1.7.0 // indirect
github.com/cosmos/iavl v1.1.4 // indirect
github.com/cosmos/ics23/go v0.11.0 // indirect
github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
@ -73,6 +59,7 @@ require (
github.com/dgraph-io/badger/v4 v4.2.0 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/emicklei/dot v1.6.2 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
@ -86,14 +73,17 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.2.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/orderedcode v0.0.1 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@ -126,7 +116,9 @@ require (
github.com/oklog/run v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.20.5 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.60.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
@ -139,8 +131,12 @@ require (
github.com/sasha-s/go-deadlock v0.3.5 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.0 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
github.com/tendermint/go-amino v0.16.0 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
@ -159,6 +155,7 @@ require (
golang.org/x/sys v0.27.0 // indirect
golang.org/x/term v0.26.0 // indirect
golang.org/x/text v0.20.0 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/protobuf v1.35.2 // indirect

View File

@ -1,215 +0,0 @@
# Getting started with a new system test
## Preparation
Build a new binary from current branch and copy it to the `tests/systemtests/binaries` folder by running system tests.
In project root:
```shell
make test-system
```
Or via manual steps
```shell
make build
mkdir -p ./tests/systemtests/binaries
cp ./build/simd ./tests/systemtests/binaries/
```
## Part 1: Writing the first system test
Switch to the `tests/systemtests` folder to work from here.
If there is no test file matching your use case, start a new test file here.
for example `bank_test.go` to begin with:
```go
//go:build system_test
package systemtests
import (
"testing"
)
func TestQueryTotalSupply(t *testing.T) {
sut.ResetChain(t)
sut.StartChain(t)
cli := NewCLIWrapper(t, sut, verbose)
raw := cli.CustomQuery("q", "bank", "total-supply")
t.Log("### got: " + raw)
}
```
The file begins with a Go build tag to exclude it from regular go test runs.
All tests in the `systemtests` folder build upon the *test runner* initialized in `main_test.go`.
This gives you a multi node chain started on your box.
It is a good practice to reset state in the beginning so that you have a stable base.
The system tests framework comes with a CLI wrapper that makes it easier to interact or parse results.
In this example we want to execute `simd q bank total-supply --output json --node tcp://localhost:26657` which queries
the bank module.
Then print the result to for the next steps
### Run the test
```shell
go test -mod=readonly -tags='system_test' -v ./... --run TestQueryTotalSupply --verbose
```
This give very verbose output. You would see all simd CLI commands used for starting the server or by the client to interact.
In the example code, we just log the output. Watch out for
```shell
bank_test.go:15: ### got: {
"supply": [
{
"denom": "stake",
"amount": "2000000190"
},
{
"denom": "testtoken",
"amount": "4000000000"
}
],
"pagination": {
"total": "2"
}
}
```
At the end is a tail from the server log printed. This can sometimes be handy when debugging issues.
### Tips
* Passing `--nodes-count=1` overwrites the default node count and can speed up your test for local runs
## Part 2: Working with json
When we have a json response, the [gjson](https://github.com/tidwall/gjson) lib can shine. It comes with jquery like
syntax that makes it easy to navigation within the document.
For example `gjson.Get(raw, "supply").Array()` gives us all the childs to `supply` as an array.
Or `gjson.Get("supply.#(denom==stake).amount").Int()` for the amount of the stake token as int64 type.
In order to test our assumptions in the system test, we modify the code to use `gjson` to fetch the data:
```go
raw := cli.CustomQuery("q", "bank", "total-supply")
exp := map[string]int64{
"stake": int64(500000000 * sut.nodesCount),
"testtoken": int64(1000000000 * sut.nodesCount),
}
require.Len(t, gjson.Get(raw, "supply").Array(), len(exp), raw)
for k, v := range exp {
got := gjson.Get(raw, fmt.Sprintf("supply.#(denom==%q).amount", k)).Int()
assert.Equal(t, v, got, raw)
}
```
The assumption on the staking token usually fails due to inflation minted on the staking token. Let's fix this in the next step
### Run the test
```shell
go test -mod=readonly -tags='system_test' -v ./... --run TestQueryTotalSupply --verbose
```
### Tips
* Putting the `raw` json response to the assert/require statements helps with debugging on failures. You are usually lacking
context when you look at the values only.
## Part 3: Setting state via genesis
First step is to disable inflation. This can be done via the `ModifyGenesisJSON` helper. But to add some complexity,
we also introduce a new token and update the balance of the account for key `node0`.
The setup code looks quite big and unreadable now. Usually a good time to think about extracting helper functions for
common operations. The `genesis_io.go` file contains some examples already. I would skip this and take this to showcase the mix
of `gjson`, `sjson` and stdlib json operations.
```go
sut.ResetChain(t)
cli := NewCLIWrapper(t, sut, verbose)
sut.ModifyGenesisJSON(t, func(genesis []byte) []byte {
// disable inflation
genesis, err := sjson.SetRawBytes(genesis, "app_state.mint.minter.inflation", []byte(`"0.000000000000000000"`))
require.NoError(t, err)
// add new token to supply
var supply []json.RawMessage
rawSupply := gjson.Get(string(genesis), "app_state.bank.supply").String()
require.NoError(t, json.Unmarshal([]byte(rawSupply), &supply))
supply = append(supply, json.RawMessage(`{"denom": "mytoken","amount": "1000000"}`))
newSupply, err := json.Marshal(supply)
require.NoError(t, err)
genesis, err = sjson.SetRawBytes(genesis, "app_state.bank.supply", newSupply)
require.NoError(t, err)
// add amount to any balance
anyAddr := cli.GetKeyAddr("node0")
newBalances := GetGenesisBalance(genesis, anyAddr).Add(sdk.NewInt64Coin("mytoken", 1000000))
newBalancesBz, err := newBalances.MarshalJSON()
require.NoError(t, err)
newState, err := sjson.SetRawBytes(genesis, fmt.Sprintf("app_state.bank.balances.#[address==%q]#.coins", anyAddr), newBalancesBz)
require.NoError(t, err)
return newState
})
sut.StartChain(t)
```
Next step is to add the new token to the assert map. But we can also make it more resilient to different node counts.
```go
exp := map[string]int64{
"stake": int64(500000000 * sut.nodesCount),
"testtoken": int64(1000000000 * sut.nodesCount),
"mytoken": 1000000,
}
```
```shell
go test -mod=readonly -tags='system_test' -v ./... --run TestQueryTotalSupply --verbose --nodes-count=1
```
## Part 4: Set state via TX
Complexer workflows and tests require modifying state on a running chain. This works only with builtin logic and operations.
If we want to burn some of our new tokens, we need to submit a bank burn message to do this.
The CLI wrapper works similar to the query. Just pass the parameters. It uses the `node0` key as *default*:
```go
// and when
txHash := cli.Run("tx", "bank", "burn", "node0", "400000mytoken")
RequireTxSuccess(t, txHash)
```
`RequireTxSuccess` or `RequireTxFailure` can be used to ensure the expected result of the operation.
Next, check that the changes are applied.
```go
exp["mytoken"] = 600_000 // update expected state
raw = cli.CustomQuery("q", "bank", "total-supply")
for k, v := range exp {
got := gjson.Get(raw, fmt.Sprintf("supply.#(denom==%q).amount", k)).Int()
assert.Equal(t, v, got, raw)
}
assert.Equal(t, int64(600_000), cli.QueryBalance(cli.GetKeyAddr("node0"), "mytoken"))
```
While tests are still more or less readable, it can gets harder the longer they are. I found it helpful to add
some comments at the beginning to describe what the intention is. For example:
```go
// scenario:
// given a chain with a custom token on genesis
// when an amount is burned
// then this is reflected in the total supply
```

View File

@ -1,33 +1,11 @@
module cosmossdk.io/tests/systemtests
module github.com/cosmos/cosmos-sdk/tests/systemtests
go 1.23
require (
github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect
github.com/cosmos/cosmos-sdk v0.50.6
github.com/cosmos/gogogateway v1.2.0 // indirect
github.com/cosmos/gogoproto v1.7.0 // indirect
github.com/cosmos/iavl v1.1.4 // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.20.5 // indirect
github.com/spf13/cast v1.7.0 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.10.0
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/grpc v1.68.0 // indirect
)
require (
cosmossdk.io/math v1.4.0
cosmossdk.io/systemtests v0.0.0-00010101000000-000000000000
github.com/tidwall/gjson v1.14.2
github.com/tidwall/sjson v1.2.5
cosmossdk.io/systemtests v0.0.0-20241126144654-14d98d277124
github.com/cosmos/cosmos-sdk v0.50.6
)
require (
@ -63,7 +41,11 @@ require (
github.com/cometbft/cometbft/api v1.0.0-rc.1 // indirect
github.com/cosmos/btcutil v1.0.5 // indirect
github.com/cosmos/cosmos-db v1.0.3-0.20240829004618-717cba019b33 // indirect
github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect
github.com/cosmos/go-bip39 v1.0.0 // indirect
github.com/cosmos/gogogateway v1.2.0 // indirect
github.com/cosmos/gogoproto v1.7.0 // indirect
github.com/cosmos/iavl v1.1.4 // indirect
github.com/cosmos/ics23/go v0.11.0 // indirect
github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect
github.com/creachadair/tomledit v0.0.26 // indirect
@ -74,6 +56,7 @@ require (
github.com/dgraph-io/badger/v4 v4.2.0 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/emicklei/dot v1.6.2 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
@ -87,14 +70,17 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.2.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/orderedcode v0.0.1 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@ -127,7 +113,9 @@ require (
github.com/oklog/run v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.20.5 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.60.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
@ -140,12 +128,19 @@ require (
github.com/sasha-s/go-deadlock v0.3.5 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.0 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/stretchr/testify v1.10.0
github.com/subosito/gotenv v1.6.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
github.com/tendermint/go-amino v0.16.0 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/tidwall/gjson v1.14.2
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/sjson v1.2.5
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/zondax/hid v0.9.2 // indirect
github.com/zondax/ledger-go v0.14.3 // indirect
@ -160,8 +155,10 @@ require (
golang.org/x/sys v0.27.0 // indirect
golang.org/x/term v0.26.0 // indirect
golang.org/x/text v0.20.0 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/grpc v1.68.0 // indirect
google.golang.org/protobuf v1.35.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
@ -170,5 +167,3 @@ require (
pgregory.net/rapid v1.1.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
replace cosmossdk.io/systemtests => ../../systemtests

View File

@ -16,6 +16,8 @@ cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ=
cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk=
cosmossdk.io/store v1.1.0 h1:LnKwgYMc9BInn9PhpTFEQVbL9UK475G2H911CGGnWHk=
cosmossdk.io/store v1.1.0/go.mod h1:oZfW/4Fc/zYqu3JmQcQdUJ3fqu5vnYTn3LZFFy8P8ng=
cosmossdk.io/systemtests v0.0.0-20241126144654-14d98d277124 h1:pzhysmpO6+HA/vnJKVTfmtZAtCJJzLqsBZAwzOCFa7Q=
cosmossdk.io/systemtests v0.0.0-20241126144654-14d98d277124/go.mod h1:IQrj3oVhIBn1X8183/IL1z3l7HgjePVSHLeZtUUjGjI=
cosmossdk.io/x/tx v0.13.3-0.20240419091757-db5906b1e894 h1:kHEvzVqpNv/9pnaEPBsgE/FMc+cVmWjSsInRufkZkpQ=
cosmossdk.io/x/tx v0.13.3-0.20240419091757-db5906b1e894/go.mod h1:Tb6/tpONmtL5qFdOMdv1pdvrtJNxcazZBoz04HB71ss=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=