From 946c62410f30ebf03aae199e4d15baf9213bf595 Mon Sep 17 00:00:00 2001 From: Marko Date: Fri, 10 May 2024 17:44:17 +0200 Subject: [PATCH] chore: upstream app manager (#20315) Co-authored-by: testinginprod <98415576+testinginprod@users.noreply.github.com> --- .github/workflows/v2-test.yml | 24 ++++ core/transaction/transaction.go | 2 + server/v2/appmanager/appmanager.go | 152 +++++++++++++++++++++ server/v2/appmanager/appmanager_builder.go | 41 ++++++ server/v2/appmanager/config.go | 9 ++ server/v2/appmanager/genesis.go | 14 ++ server/v2/appmanager/go.mod | 23 ++++ server/v2/appmanager/go.sum | 38 ++++++ server/v2/appmanager/store/types.go | 17 +++ server/v2/appmanager/types.go | 50 +++++++ server/v2/stf/stf.go | 1 - 11 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 server/v2/appmanager/appmanager.go create mode 100644 server/v2/appmanager/appmanager_builder.go create mode 100644 server/v2/appmanager/config.go create mode 100644 server/v2/appmanager/genesis.go create mode 100644 server/v2/appmanager/go.mod create mode 100644 server/v2/appmanager/go.sum create mode 100644 server/v2/appmanager/store/types.go create mode 100644 server/v2/appmanager/types.go diff --git a/.github/workflows/v2-test.yml b/.github/workflows/v2-test.yml index 3371751287..2cced36e47 100644 --- a/.github/workflows/v2-test.yml +++ b/.github/workflows/v2-test.yml @@ -37,3 +37,27 @@ jobs: if: env.GIT_DIFF run: | cd server/v2/stf && go test -mod=readonly -race -timeout 30m -covermode=atomic -tags='ledger test_ledger_mock' + + appamanger: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.22" + check-latest: true + cache: true + cache-dependency-path: go.sum + - uses: technote-space/get-diff-action@v6.1.2 + id: git_diff + with: + PATTERNS: | + server/v2/appmanager/**/*.go + server/v2/appmanager/go.mod + server/v2/appmanager/go.sum + - name: test & coverage report creation + if: env.GIT_DIFF + run: | + cd server/v2/appmanager && go test -mod=readonly -race -timeout 30m -covermode=atomic -tags='ledger test_ledger_mock' \ No newline at end of file diff --git a/core/transaction/transaction.go b/core/transaction/transaction.go index c6cca1865e..e2f9fc9c35 100644 --- a/core/transaction/transaction.go +++ b/core/transaction/transaction.go @@ -14,6 +14,8 @@ type Codec[T Tx] interface { // Decode decodes the tx bytes into a DecodedTx, containing // both concrete and bytes representation of the tx. Decode([]byte) (T, error) + // DecodeJSON decodes the tx JSON bytes into a DecodedTx + DecodeJSON([]byte) (T, error) } type Tx interface { diff --git a/server/v2/appmanager/appmanager.go b/server/v2/appmanager/appmanager.go new file mode 100644 index 0000000000..f20abad556 --- /dev/null +++ b/server/v2/appmanager/appmanager.go @@ -0,0 +1,152 @@ +package appmanager + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + appmanager "cosmossdk.io/core/app" + corestore "cosmossdk.io/core/store" + "cosmossdk.io/core/transaction" + "cosmossdk.io/server/v2/appmanager/store" +) + +// AppManager is a coordinator for all things related to an application +// TODO: add exportGenesis function +type AppManager[T transaction.Tx] struct { + config Config + + db store.Store + + initGenesis InitGenesis + exportGenesis ExportGenesis + + stf StateTransitionFunction[T] +} + +func (a AppManager[T]) InitGenesis( + ctx context.Context, + blockRequest *appmanager.BlockRequest[T], + initGenesisJSON []byte, + txDecoder transaction.Codec[T], +) (*appmanager.BlockResponse, corestore.WriterMap, error) { + v, zeroState, err := a.db.StateLatest() + if err != nil { + return nil, nil, fmt.Errorf("unable to get latest state: %w", err) + } + if v != 0 { // TODO: genesis state may be > 0, we need to set version on store + return nil, nil, fmt.Errorf("cannot init genesis on non-zero state") + } + + var genTxs []T + zeroState, err = a.stf.RunWithCtx(ctx, zeroState, func(ctx context.Context) error { + return a.initGenesis(ctx, bytes.NewBuffer(initGenesisJSON), func(jsonTx json.RawMessage) error { + genTx, err := txDecoder.DecodeJSON(jsonTx) + if err != nil { + return fmt.Errorf("failed to decode genesis transaction: %w", err) + } + genTxs = append(genTxs, genTx) + return nil + }) + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to import genesis state: %w", err) + } + // run block + // TODO: in an ideal world, genesis state is simply an initial state being applied + // unaware of what that state means in relation to every other, so here we can + // chain genesis + blockRequest.Txs = genTxs + + blockresponse, genesisState, err := a.stf.DeliverBlock(ctx, blockRequest, zeroState) + if err != nil { + return blockresponse, nil, fmt.Errorf("failed to deliver block %d: %w", blockRequest.Height, err) + } + + return blockresponse, genesisState, err + // consensus server will need to set the version of the store +} + +// ExportGenesis exports the genesis state of the application. +func (a AppManager[T]) ExportGenesis(ctx context.Context, version uint64) ([]byte, error) { + bz, err := a.exportGenesis(ctx, version) + if err != nil { + return nil, fmt.Errorf("failed to export genesis state: %w", err) + } + + return bz, nil +} + +func (a AppManager[T]) DeliverBlock( + ctx context.Context, + block *appmanager.BlockRequest[T], +) (*appmanager.BlockResponse, corestore.WriterMap, error) { + latestVersion, currentState, err := a.db.StateLatest() + if err != nil { + return nil, nil, fmt.Errorf("unable to create new state for height %d: %w", block.Height, err) + } + + if latestVersion+1 != block.Height { + return nil, nil, fmt.Errorf("invalid DeliverBlock height wanted %d, got %d", latestVersion+1, block.Height) + } + + blockResponse, newState, err := a.stf.DeliverBlock(ctx, block, currentState) + if err != nil { + return nil, nil, fmt.Errorf("block delivery failed: %w", err) + } + + return blockResponse, newState, nil +} + +// ValidateTx will validate the tx against the latest storage state. This means that +// only the stateful validation will be run, not the execution portion of the tx. +// If full execution is needed, Simulate must be used. +func (a AppManager[T]) ValidateTx(ctx context.Context, tx T) (appmanager.TxResult, error) { + _, latestState, err := a.db.StateLatest() + if err != nil { + return appmanager.TxResult{}, err + } + return a.stf.ValidateTx(ctx, latestState, a.config.ValidateTxGasLimit, tx), nil +} + +// Simulate runs validation and execution flow of a Tx. +func (a AppManager[T]) Simulate(ctx context.Context, tx T) (appmanager.TxResult, corestore.WriterMap, error) { + _, state, err := a.db.StateLatest() + if err != nil { + return appmanager.TxResult{}, nil, err + } + result, cs := a.stf.Simulate(ctx, state, a.config.SimulationGasLimit, tx) // TODO: check if this is done in the antehandler + return result, cs, nil +} + +// Query queries the application at the provided version. +// CONTRACT: Version must always be provided, if 0, get latest +func (a AppManager[T]) Query(ctx context.Context, version uint64, request transaction.Msg) (transaction.Msg, error) { + // if version is provided attempt to do a height query. + if version != 0 { + queryState, err := a.db.StateAt(version) + if err != nil { + return nil, err + } + return a.stf.Query(ctx, queryState, a.config.QueryGasLimit, request) + } + + // otherwise rely on latest available state. + _, queryState, err := a.db.StateLatest() + if err != nil { + return nil, err + } + return a.stf.Query(ctx, queryState, a.config.QueryGasLimit, request) +} + +// QueryWithState executes a query with the provided state. This allows to process a query +// independently of the db state. For example, it can be used to process a query with temporary +// and uncommitted state +func (a AppManager[T]) QueryWithState( + ctx context.Context, + state corestore.ReaderMap, + request transaction.Msg, +) (transaction.Msg, error) { + return a.stf.Query(ctx, state, a.config.QueryGasLimit, request) +} diff --git a/server/v2/appmanager/appmanager_builder.go b/server/v2/appmanager/appmanager_builder.go new file mode 100644 index 0000000000..7b952c92e9 --- /dev/null +++ b/server/v2/appmanager/appmanager_builder.go @@ -0,0 +1,41 @@ +package appmanager + +import ( + "cosmossdk.io/core/transaction" + "cosmossdk.io/server/v2/appmanager/store" +) + +// Builder is a struct that represents the application builder for managing transactions. +// It contains various fields and methods for initializing the application and handling transactions. +type Builder[T transaction.Tx] struct { + STF StateTransitionFunction[T] // The state transition function for processing transactions. + DB store.Store // The database for storing application data. + + // Gas limits for validating, querying, and simulating transactions. + ValidateTxGasLimit uint64 + QueryGasLimit uint64 + SimulationGasLimit uint64 + + // InitGenesis is a function that initializes the application state from a genesis file. + // It takes a context, a source reader for the genesis file, and a transaction handler function. + InitGenesis InitGenesis + // ExportGenesis is a function that exports the application state to a genesis file. + // It takes a context and a version number for the genesis file. + ExportGenesis ExportGenesis +} + +// Build creates a new instance of AppManager with the provided configuration and returns it. +// It initializes the AppManager with the given database, export state, import state, initGenesis function, and state transition function. +func (b Builder[T]) Build() (*AppManager[T], error) { + return &AppManager[T]{ + config: Config{ + ValidateTxGasLimit: b.ValidateTxGasLimit, + QueryGasLimit: b.QueryGasLimit, + SimulationGasLimit: b.SimulationGasLimit, + }, + db: b.DB, + initGenesis: b.InitGenesis, + exportGenesis: b.ExportGenesis, + stf: b.STF, + }, nil +} diff --git a/server/v2/appmanager/config.go b/server/v2/appmanager/config.go new file mode 100644 index 0000000000..8101a47ae8 --- /dev/null +++ b/server/v2/appmanager/config.go @@ -0,0 +1,9 @@ +package appmanager + +// Config represents the configuration options for the app manager. +// TODO: implement comments for toml +type Config struct { + ValidateTxGasLimit uint64 `mapstructure:"validate-tx-gas-limit"` //TODO: check how this works on app mempool + QueryGasLimit uint64 `mapstructure:"query-gas-limit"` + SimulationGasLimit uint64 `mapstructure:"simulation-gas-limit"` +} diff --git a/server/v2/appmanager/genesis.go b/server/v2/appmanager/genesis.go new file mode 100644 index 0000000000..2d058fa238 --- /dev/null +++ b/server/v2/appmanager/genesis.go @@ -0,0 +1,14 @@ +package appmanager + +import ( + "context" + "encoding/json" + "io" +) + +type ( + // exportGenesis is a function type that represents the export of the genesis state. + ExportGenesis func(ctx context.Context, version uint64) ([]byte, error) + // InitGenesis is a function type that represents the initialization of the genesis state. + InitGenesis func(ctx context.Context, src io.Reader, txHandler func(json.RawMessage) error) error +) diff --git a/server/v2/appmanager/go.mod b/server/v2/appmanager/go.mod new file mode 100644 index 0000000000..f70b766e95 --- /dev/null +++ b/server/v2/appmanager/go.mod @@ -0,0 +1,23 @@ +module cosmossdk.io/server/v2/appmanager + +go 1.21 + +// TODO: remove prior to final release +replace cosmossdk.io/core => ../../../core + +require cosmossdk.io/core v0.12.0 + +require ( + cosmossdk.io/log v1.3.1 // indirect + github.com/cosmos/gogoproto v1.4.12 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rs/zerolog v1.32.0 // indirect + golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f // indirect + golang.org/x/sys v0.20.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect +) diff --git a/server/v2/appmanager/go.sum b/server/v2/appmanager/go.sum new file mode 100644 index 0000000000..ea7ed79061 --- /dev/null +++ b/server/v2/appmanager/go.sum @@ -0,0 +1,38 @@ +cosmossdk.io/log v1.3.1 h1:UZx8nWIkfbbNEWusZqzAx3ZGvu54TZacWib3EzUYmGI= +cosmossdk.io/log v1.3.1/go.mod h1:2/dIomt8mKdk6vl3OWJcPk2be3pGOS8OQaLUM/3/tCM= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cosmos/gogoproto v1.4.12 h1:vB6Lbe/rtnYGjQuFxkPiPYiCybqFT8QvLipDZP8JpFE= +github.com/cosmos/gogoproto v1.4.12/go.mod h1:LnZob1bXRdUoqMMtwYlcR3wjiElmlC+FkjaZRv1/eLY= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f h1:3CW0unweImhOzd5FmYuRsD4Y4oQFKZIjAnKbjV4WIrw= +golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/server/v2/appmanager/store/types.go b/server/v2/appmanager/store/types.go new file mode 100644 index 0000000000..1993df5174 --- /dev/null +++ b/server/v2/appmanager/store/types.go @@ -0,0 +1,17 @@ +package store + +import ( + "cosmossdk.io/core/store" +) + +// Store defines the underlying storage engine of an app. +type Store interface { + // StateLatest returns a readonly view over the latest + // committed state of the store. Alongside the version + // associated with it. + StateLatest() (uint64, store.ReaderMap, error) + + // StateAt returns a readonly view over the provided + // state. Must error when the version does not exist. + StateAt(version uint64) (store.ReaderMap, error) +} diff --git a/server/v2/appmanager/types.go b/server/v2/appmanager/types.go new file mode 100644 index 0000000000..a63044e9dd --- /dev/null +++ b/server/v2/appmanager/types.go @@ -0,0 +1,50 @@ +package appmanager + +import ( + "context" + + appmanager "cosmossdk.io/core/app" + "cosmossdk.io/core/store" + "cosmossdk.io/core/transaction" +) + +// StateTransitionFunction is an interface for processing transactions and blocks. +type StateTransitionFunction[T transaction.Tx] interface { + // DeliverBlock executes a block of transactions. + DeliverBlock( + ctx context.Context, + block *appmanager.BlockRequest[T], + state store.ReaderMap, + ) (blockResult *appmanager.BlockResponse, newState store.WriterMap, err error) + + // ValidateTx validates a transaction. + ValidateTx( + ctx context.Context, + state store.ReaderMap, + gasLimit uint64, + tx T, + ) appmanager.TxResult + + // Simulate executes a transaction in simulation mode. + Simulate( + ctx context.Context, + state store.ReaderMap, + gasLimit uint64, + tx T, + ) (appmanager.TxResult, store.WriterMap) + + // Query executes a query on the application. + Query( + ctx context.Context, + state store.ReaderMap, + gasLimit uint64, + req transaction.Msg, + ) (transaction.Msg, error) + + // TODO: remove + RunWithCtx( + ctx context.Context, + state store.ReaderMap, + closure func(ctx context.Context) error, + ) (store.WriterMap, error) +} diff --git a/server/v2/stf/stf.go b/server/v2/stf/stf.go index 6601bf9fbf..205d161e69 100644 --- a/server/v2/stf/stf.go +++ b/server/v2/stf/stf.go @@ -503,7 +503,6 @@ func (s STF[T]) RunWithCtx( closure func(ctx context.Context) error, ) (store.WriterMap, error) { branchedState := s.branchFn(state) - // TODO do we need headerinfo for genesis? stfCtx := s.makeContext(ctx, nil, branchedState, corecontext.ExecModeFinalize) return branchedState, closure(stfCtx) }