cosmos-sdk/server/v2/cometbft/abci.go

package cometbft

import (
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"strings"
	"sync/atomic"

	abci "github.com/cometbft/cometbft/abci/types"
	abciproto "github.com/cometbft/cometbft/api/cometbft/abci/v1"
	gogoproto "github.com/cosmos/gogoproto/proto"
	"google.golang.org/protobuf/reflect/protoreflect"
	"google.golang.org/protobuf/reflect/protoregistry"

	"cosmossdk.io/collections"
	appmodulev2 "cosmossdk.io/core/appmodule/v2"
	"cosmossdk.io/core/comet"
	corecontext "cosmossdk.io/core/context"
	"cosmossdk.io/core/server"
	"cosmossdk.io/core/store"
	"cosmossdk.io/core/transaction"
	errorsmod "cosmossdk.io/errors/v2"
	"cosmossdk.io/log"
	"cosmossdk.io/schema/appdata"
	"cosmossdk.io/server/v2/appmanager"
	"cosmossdk.io/server/v2/cometbft/handlers"
	"cosmossdk.io/server/v2/cometbft/mempool"
	"cosmossdk.io/server/v2/cometbft/oe"
	"cosmossdk.io/server/v2/cometbft/types"
	cometerrors "cosmossdk.io/server/v2/cometbft/types/errors"
	"cosmossdk.io/server/v2/streaming"
	"cosmossdk.io/store/v2/snapshots"
	consensustypes "cosmossdk.io/x/consensus/types"
)

const (
	QueryPathApp   = "app"
	QueryPathP2P   = "p2p"
	QueryPathStore = "store"
)

var _ abci.Application = (*consensus[transaction.Tx])(nil)

// consensus contains the implementation of the ABCI interface for CometBFT.
type consensus[T transaction.Tx] struct {
	logger           log.Logger
	appName, version string
	app              appmanager.AppManager[T]
	store            types.Store
	listener         *appdata.Listener
	snapshotManager  *snapshots.Manager
	streamingManager streaming.Manager
	mempool          mempool.Mempool[T]
	appCodecs        AppCodecs[T]

	cfg               Config
	chainID           string
	indexedABCIEvents map[string]struct{}

	initialHeight uint64
	// this is only available after this node has committed a block (in FinalizeBlock),
	// otherwise it will be empty and we will need to query the app for the last
	// committed block.
	lastCommittedHeight atomic.Int64

	prepareProposalHandler handlers.PrepareHandler[T]
	processProposalHandler handlers.ProcessHandler[T]
	verifyVoteExt          handlers.VerifyVoteExtensionHandler
	extendVote             handlers.ExtendVoteHandler
	checkTxHandler         handlers.CheckTxHandler[T]

	// optimisticExec contains the context required for Optimistic Execution,
	// including the goroutine handling.This is experimental and must be enabled
	// by developers.
	optimisticExec *oe.OptimisticExecution[T]

	addrPeerFilter types.PeerFilter // filter peers by address and port
	idPeerFilter   types.PeerFilter // filter peers by node ID

	queryHandlersMap map[string]appmodulev2.Handler
	getProtoRegistry func() (*protoregistry.Files, error)
	cfgMap           server.ConfigMap
}

// CheckTx implements types.Application.
// It is called by cometbft to verify transaction validity
func (c *consensus[T]) CheckTx(ctx context.Context, req *abciproto.CheckTxRequest) (*abciproto.CheckTxResponse, error) {
	decodedTx, err := c.appCodecs.TxCodec.Decode(req.Tx)
	if err != nil {
		return nil, err
	}

	if c.checkTxHandler == nil {
		resp, err := c.app.ValidateTx(ctx, decodedTx)
		// we do not want to return a cometbft error, but a check tx response with the error
		if err != nil && !errors.Is(err, resp.Error) {
			return nil, err
		}

		events := make([]abci.Event, 0)
		if !c.cfg.AppTomlConfig.DisableABCIEvents {
			events, err = intoABCIEvents(
				resp.Events,
				c.indexedABCIEvents,
				c.cfg.AppTomlConfig.DisableIndexABCIEvents,
			)
			if err != nil {
				return nil, err
			}
		}

		cometResp := &abciproto.CheckTxResponse{
			Code:      0,
			GasWanted: uint64ToInt64(resp.GasWanted),
			GasUsed:   uint64ToInt64(resp.GasUsed),
			Events:    events,
		}

		if resp.Error != nil {
			space, code, log := errorsmod.ABCIInfo(resp.Error, c.cfg.AppTomlConfig.Trace)
			cometResp.Code = code
			cometResp.Codespace = space
			cometResp.Log = log
		}

		return cometResp, nil
	}

	return c.checkTxHandler(c.app.ValidateTx)
}

// Info implements types.Application.
func (c *consensus[T]) Info(ctx context.Context, _ *abciproto.InfoRequest) (*abciproto.InfoResponse, error) {
	version, _, err := c.store.StateLatest()
	if err != nil {
		return nil, err
	}

	// if height is 0, we dont know the consensus params
	var appVersion uint64 = 0
	if version > 0 {
		cp, err := GetConsensusParams(ctx, c.app)
		// if the consensus params are not found, we set the app version to 0
		// in the case that the start version is > 0
		if cp == nil || errors.Is(err, collections.ErrNotFound) {
			appVersion = 0
		} else if err != nil {
			return nil, err
		} else {
			appVersion = cp.Version.GetApp()
		}
		if err != nil {
			return nil, err
		}
	}

	cid, err := c.store.LastCommitID()
	if err != nil {
		return nil, err
	}

	return &abciproto.InfoResponse{
		Data:             c.appName,
		Version:          c.version,
		AppVersion:       appVersion,
		LastBlockHeight:  int64(version),
		LastBlockAppHash: cid.Hash,
	}, nil
}

// Query implements types.Application.
// It is called by cometbft to query application state.
func (c *consensus[T]) Query(ctx context.Context, req *abciproto.QueryRequest) (resp *abciproto.QueryResponse, err error) {
	// when a client did not provide a query height, manually inject the latest
	// for modules queries, AppManager does it automatically
	if req.Height == 0 {
		latestVersion, err := c.store.GetLatestVersion()
		if err != nil {
			return nil, err
		}
		req.Height = int64(latestVersion)
	}

	resp, isGRPC, err := c.maybeRunGRPCQuery(ctx, req)
	if isGRPC {
		return resp, err
	}

	// this error most probably means that we can't handle it with a proto message, so
	// it must be an app/p2p/store query
	path := splitABCIQueryPath(req.Path)
	if len(path) == 0 {
		return queryResult(errorsmod.Wrap(cometerrors.ErrUnknownRequest, "no query path provided"), c.cfg.AppTomlConfig.Trace), nil
	}

	switch path[0] {
	case QueryPathApp:
		resp, err = c.handleQueryApp(ctx, path, req)

	case QueryPathStore:
		resp, err = c.handleQueryStore(path, req)

	case QueryPathP2P:
		resp, err = c.handleQueryP2P(path)

	default:
		resp = queryResult(errorsmod.Wrapf(cometerrors.ErrUnknownRequest, "unknown query path %s", req.Path), c.cfg.AppTomlConfig.Trace)
	}

	if err != nil {
		return queryResult(err, c.cfg.AppTomlConfig.Trace), nil
	}

	return resp, nil
}

func (c *consensus[T]) maybeRunGRPCQuery(ctx context.Context, req *abci.QueryRequest) (resp *abciproto.QueryResponse, isGRPC bool, err error) {
	// if this fails then we cannot serve queries anymore
	registry, err := c.getProtoRegistry()
	if err != nil {
		return nil, false, err
	}

	path := strings.TrimPrefix(req.Path, "/")
	pathFullName := protoreflect.FullName(strings.ReplaceAll(path, "/", "."))

	// in order to check if it's a gRPC query we ensure that there's a descriptor
	// for the path, if such descriptor exists, and it is a method descriptor
	// then we assume this is a gRPC query.
	desc, err := registry.FindDescriptorByName(pathFullName)
	if err != nil {
		return nil, false, err
	}

	var handlerFullName string
	md, isGRPC := desc.(protoreflect.MethodDescriptor)
	if !isGRPC {
		handlerFullName = string(desc.FullName())
	} else {
		handlerFullName = string(md.Input().FullName())
	}

	// special case for non-module services as they are external gRPC registered on the grpc server component
	// and not on the app itself, so it won't pass the router afterwards.

	externalResp, err := c.maybeHandleExternalServices(ctx, req)
	if err != nil {
		return nil, true, err
	} else if externalResp != nil {
		resp, err = queryResponse(externalResp, req.Height)
		return resp, true, err
	}

	handler, found := c.queryHandlersMap[handlerFullName]
	if !found {
		return nil, true, fmt.Errorf("no query handler found for %s", req.Path)
	}
	protoRequest := handler.MakeMsg()
	err = gogoproto.Unmarshal(req.Data, protoRequest) // TODO: use codec
	if err != nil {
		return nil, true, fmt.Errorf("unable to decode gRPC request with path %s from ABCI.Query: %w", req.Path, err)
	}

	res, err := c.app.Query(ctx, uint64(req.Height), protoRequest)
	if err != nil {
		resp := gRPCErrorToSDKError(err)
		resp.Height = req.Height
		return resp, true, nil
	}

	resp, err = queryResponse(res, req.Height)
	return resp, true, err
}

// InitChain implements types.Application.
func (c *consensus[T]) InitChain(ctx context.Context, req *abciproto.InitChainRequest) (*abciproto.InitChainResponse, error) {
	c.logger.Info("InitChain", "initialHeight", req.InitialHeight, "chainID", req.ChainId)

	// store chainID to be used later on in execution
	c.chainID = req.ChainId

	// TODO: check if we need to load the config from genesis.json or config.toml
	c.initialHeight = uint64(req.InitialHeight)
	if c.initialHeight == 0 { // If initial height is 0, set it to 1
		c.initialHeight = 1
	}

	if req.ConsensusParams != nil {
		ctx = context.WithValue(ctx, corecontext.CometParamsInitInfoKey, &consensustypes.MsgUpdateParams{
			Block:     req.ConsensusParams.Block,
			Evidence:  req.ConsensusParams.Evidence,
			Validator: req.ConsensusParams.Validator,
			Abci:      req.ConsensusParams.Abci,
			Synchrony: req.ConsensusParams.Synchrony,
			Feature:   req.ConsensusParams.Feature,
		})
	}

	ci, err := c.store.LastCommitID()
	if err != nil {
		return nil, err
	}

	// populate hash with empty byte slice instead of nil
	bz := sha256.Sum256([]byte{})

	br := &server.BlockRequest[T]{
		Height:    uint64(req.InitialHeight - 1),
		Time:      req.Time,
		Hash:      bz[:],
		AppHash:   ci.Hash,
		ChainId:   req.ChainId,
		IsGenesis: true,
	}

	blockResponse, genesisState, err := c.app.InitGenesis(
		ctx,
		br,
		req.AppStateBytes,
		c.appCodecs.TxCodec)
	if err != nil {
		return nil, fmt.Errorf("genesis state init failure: %w", err)
	}

	for _, txRes := range blockResponse.TxResults {
		if err := txRes.Error; err != nil {
			space, code, txLog := errorsmod.ABCIInfo(err, c.cfg.AppTomlConfig.Trace)
			c.logger.Warn("genesis tx failed", "codespace", space, "code", code, "log", txLog)
		}
	}

	validatorUpdates := intoABCIValidatorUpdates(blockResponse.ValidatorUpdates)

	if err := c.store.SetInitialVersion(uint64(req.InitialHeight - 1)); err != nil {
		return nil, fmt.Errorf("failed to set initial version: %w", err)
	}

	stateChanges, err := genesisState.GetStateChanges()
	if err != nil {
		return nil, err
	}
	cs := &store.Changeset{
		Version: uint64(req.InitialHeight - 1),
		Changes: stateChanges,
	}
	stateRoot, err := c.store.Commit(cs)
	if err != nil {
		return nil, fmt.Errorf("unable to write the changeset: %w", err)
	}

	return &abciproto.InitChainResponse{
		ConsensusParams: req.ConsensusParams,
		Validators:      validatorUpdates,
		AppHash:         stateRoot,
	}, nil
}

// PrepareProposal implements types.Application.
// It is called by cometbft to prepare a proposal block.
func (c *consensus[T]) PrepareProposal(
	ctx context.Context,
	req *abciproto.PrepareProposalRequest,
) (resp *abciproto.PrepareProposalResponse, err error) {
	if req.Height < 1 {
		return nil, errors.New("PrepareProposal called with invalid height")
	}

	if c.prepareProposalHandler == nil {
		return nil, errors.New("no prepare proposal function was set")
	}

	// Abort any running OE so it cannot overlap with `PrepareProposal`. This could happen if optimistic
	// `internalFinalizeBlock` from previous round takes a long time, but consensus has moved on to next round.
	// Overlap is undesirable, since `internalFinalizeBlock` and `PrepareProposal` could share access to
	// in-memory structs depending on application implementation.
	// No-op if OE is not enabled.
	// Similar call to Abort() is done in `ProcessProposal`.
	c.optimisticExec.Abort()

	ciCtx := contextWithCometInfo(ctx, comet.Info{
		Evidence:        toCoreEvidence(req.Misbehavior),
		ValidatorsHash:  req.NextValidatorsHash,
		ProposerAddress: req.ProposerAddress,
		LastCommit:      toCoreExtendedCommitInfo(req.LocalLastCommit),
	})

	txs, err := c.prepareProposalHandler(ciCtx, c.app, c.appCodecs.TxCodec, req, c.chainID)
	if err != nil {
		return nil, err
	}

	encodedTxs := make([][]byte, len(txs))
	for i, tx := range txs {
		encodedTxs[i] = tx.Bytes()
	}

	return &abciproto.PrepareProposalResponse{
		Txs: encodedTxs,
	}, nil
}

// ProcessProposal implements types.Application.
// It is called by cometbft to process/verify a proposal block.
func (c *consensus[T]) ProcessProposal(
	ctx context.Context,
	req *abciproto.ProcessProposalRequest,
) (*abciproto.ProcessProposalResponse, error) {
	if req.Height < 1 {
		return nil, errors.New("ProcessProposal called with invalid height")
	}

	if c.processProposalHandler == nil {
		return nil, errors.New("no process proposal function was set")
	}

	// Since the application can get access to FinalizeBlock state and write to it,
	// we must be sure to reset it in case ProcessProposal timeouts and is called
	// again in a subsequent round. However, we only want to do this after we've
	// processed the first block, as we want to avoid overwriting the finalizeState
	// after state changes during InitChain.
	if req.Height > int64(c.initialHeight) {
		// abort any running OE
		c.optimisticExec.Abort()
	}

	ciCtx := contextWithCometInfo(ctx, comet.Info{
		Evidence:        toCoreEvidence(req.Misbehavior),
		ValidatorsHash:  req.NextValidatorsHash,
		ProposerAddress: req.ProposerAddress,
		LastCommit:      toCoreCommitInfo(req.ProposedLastCommit),
	})

	err := c.processProposalHandler(ciCtx, c.app, c.appCodecs.TxCodec, req, c.chainID)
	if err != nil {
		c.logger.Error("failed to process proposal", "height", req.Height, "time", req.Time, "hash", fmt.Sprintf("%X", req.Hash), "err", err)
		return &abciproto.ProcessProposalResponse{
			Status: abciproto.PROCESS_PROPOSAL_STATUS_REJECT,
		}, nil
	}

	// Only execute optimistic execution if the proposal is accepted, OE is
	// enabled and the block height is greater than the initial height. During
	// the first block we'll be carrying state from InitChain, so it would be
	// impossible for us to easily revert.
	// After the first block has been processed, the next blocks will get executed
	// optimistically, so that when the ABCI client calls `FinalizeBlock` the app
	// can have a response ready.
	if req.Height > int64(c.initialHeight) {
		c.optimisticExec.Execute(req)
	}

	return &abciproto.ProcessProposalResponse{
		Status: abciproto.PROCESS_PROPOSAL_STATUS_ACCEPT,
	}, nil
}

// FinalizeBlock implements types.Application.
// It is called by cometbft to finalize a block.
func (c *consensus[T]) FinalizeBlock(
	ctx context.Context,
	req *abciproto.FinalizeBlockRequest,
) (*abciproto.FinalizeBlockResponse, error) {
	var (
		resp       *server.BlockResponse
		newState   store.WriterMap
		decodedTxs []T
		err        error
	)

	if c.optimisticExec.Initialized() {
		// check if the hash we got is the same as the one we are executing
		aborted := c.optimisticExec.AbortIfNeeded(req.Hash)

		// Wait for the OE to finish, regardless of whether it was aborted or not
		res, optimistErr := c.optimisticExec.WaitResult()

		if !aborted {
			if res != nil {
				resp = res.Resp
				newState = res.StateChanges
				decodedTxs = res.DecodedTxs
			}

			if optimistErr != nil {
				return nil, optimistErr
			}
		}

		c.optimisticExec.Reset()
	}

	if resp == nil { // if we didn't run OE, run the normal finalize block
		resp, newState, decodedTxs, err = c.internalFinalizeBlock(ctx, req)
		if err != nil {
			return nil, err
		}
	}

	// after we get the changeset we can produce the commit hash,
	// from the store.
	stateChanges, err := newState.GetStateChanges()
	if err != nil {
		return nil, err
	}
	appHash, err := c.store.Commit(&store.Changeset{Version: uint64(req.Height), Changes: stateChanges})
	if err != nil {
		return nil, fmt.Errorf("unable to commit the changeset: %w", err)
	}

	// listen to state streaming changes in accordance with the block
	err = c.streamDeliverBlockChanges(ctx, req.Height, req.Txs, decodedTxs, *resp, stateChanges)
	if err != nil {
		return nil, err
	}

	// remove txs from the mempool
	for _, tx := range decodedTxs {
		if err = c.mempool.Remove(tx); err != nil {
			return nil, fmt.Errorf("unable to remove tx: %w", err)
		}
	}

	c.lastCommittedHeight.Store(req.Height)

	cp, err := GetConsensusParams(ctx, c.app) // we get the consensus params from the latest state because we committed state above
	if err != nil {
		return nil, err
	}

	return finalizeBlockResponse(
		resp,
		cp,
		appHash,
		c.indexedABCIEvents,
		c.cfg.AppTomlConfig,
	)
}

func (c *consensus[T]) internalFinalizeBlock(
	ctx context.Context,
	req *abciproto.FinalizeBlockRequest,
) (*server.BlockResponse, store.WriterMap, []T, error) {
	if err := c.validateFinalizeBlockHeight(req); err != nil {
		return nil, nil, nil, err
	}

	if err := c.checkHalt(req.Height, req.Time); err != nil {
		return nil, nil, nil, err
	}

	// TODO(tip): can we expect some txs to not decode? if so, what we do in this case? this does not seem to be the case,
	// considering that prepare and process always decode txs, assuming they're the ones providing txs we should never
	// have a tx that fails decoding.
	decodedTxs, err := decodeTxs(c.logger, req.Txs, c.appCodecs.TxCodec)
	if err != nil {
		return nil, nil, nil, err
	}

	cid, err := c.store.LastCommitID()
	if err != nil {
		return nil, nil, nil, err
	}

	blockReq := &server.BlockRequest[T]{
		Height:  uint64(req.Height),
		Time:    req.Time,
		Hash:    req.Hash,
		AppHash: cid.Hash,
		ChainId: c.chainID,
		Txs:     decodedTxs,
	}

	ciCtx := contextWithCometInfo(ctx, comet.Info{
		Evidence:        toCoreEvidence(req.Misbehavior),
		ValidatorsHash:  req.NextValidatorsHash,
		ProposerAddress: req.ProposerAddress,
		LastCommit:      toCoreCommitInfo(req.DecidedLastCommit),
	})

	resp, stateChanges, err := c.app.DeliverBlock(ciCtx, blockReq)

	return resp, stateChanges, decodedTxs, err
}

// Commit implements types.Application.
// It is called by cometbft to notify the application that a block was committed.
func (c *consensus[T]) Commit(ctx context.Context, _ *abciproto.CommitRequest) (*abciproto.CommitResponse, error) {
	lastCommittedHeight := c.lastCommittedHeight.Load()

	c.snapshotManager.SnapshotIfApplicable(lastCommittedHeight)

	cp, err := GetConsensusParams(ctx, c.app)
	if err != nil {
		return nil, err
	}

	return &abci.CommitResponse{
		RetainHeight: c.GetBlockRetentionHeight(cp, lastCommittedHeight),
	}, nil
}

// Vote extensions

// VerifyVoteExtension implements types.Application.
func (c *consensus[T]) VerifyVoteExtension(
	ctx context.Context,
	req *abciproto.VerifyVoteExtensionRequest,
) (*abciproto.VerifyVoteExtensionResponse, error) {
	// If vote extensions are not enabled, as a safety precaution, we return an
	// error.
	cp, err := GetConsensusParams(ctx, c.app)
	if err != nil {
		return nil, err
	}

	// Note: we verify votes extensions on VoteExtensionsEnableHeight+1. Check
	// comment in ExtendVote and ValidateVoteExtensions for more details.
	// Since Abci was deprecated, should check both Feature & Abci
	extsEnabled := cp.Feature.VoteExtensionsEnableHeight != nil && req.Height >= cp.Feature.VoteExtensionsEnableHeight.Value && cp.Feature.VoteExtensionsEnableHeight.Value != 0
	if !extsEnabled {
		// check abci params
		extsEnabled = cp.Abci != nil && req.Height >= cp.Abci.VoteExtensionsEnableHeight && cp.Abci.VoteExtensionsEnableHeight != 0
		if !extsEnabled {
			return nil, fmt.Errorf("vote extensions are not enabled; unexpected call to VerifyVoteExtension at height %d", req.Height)
		}
	}

	if c.verifyVoteExt == nil {
		return nil, errors.New("vote extensions are enabled but no verify function was set")
	}

	_, latestStore, err := c.store.StateLatest()
	if err != nil {
		return nil, err
	}

	resp, err := c.verifyVoteExt(ctx, latestStore, req)
	if err != nil {
		c.logger.Error("failed to verify vote extension", "height", req.Height, "err", err)
		return &abciproto.VerifyVoteExtensionResponse{Status: abciproto.VERIFY_VOTE_EXTENSION_STATUS_REJECT}, nil
	}

	return resp, err
}

// ExtendVote implements types.Application.
func (c *consensus[T]) ExtendVote(ctx context.Context, req *abciproto.ExtendVoteRequest) (*abciproto.ExtendVoteResponse, error) {
	// If vote extensions are not enabled, as a safety precaution, we return an
	// error.
	cp, err := GetConsensusParams(ctx, c.app)
	if err != nil {
		return nil, err
	}

	// Note: In this case, we do want to extend vote if the height is equal or
	// greater than VoteExtensionsEnableHeight. This defers from the check done
	// in ValidateVoteExtensions and PrepareProposal in which we'll check for
	// vote extensions on VoteExtensionsEnableHeight+1.
	// Since Abci was deprecated, should check both Feature & Abci
	extsEnabled := cp.Feature.VoteExtensionsEnableHeight != nil && req.Height >= cp.Feature.VoteExtensionsEnableHeight.Value && cp.Feature.VoteExtensionsEnableHeight.Value != 0
	if !extsEnabled {
		// check abci params
		extsEnabled = cp.Abci != nil && req.Height >= cp.Abci.VoteExtensionsEnableHeight && cp.Abci.VoteExtensionsEnableHeight != 0
		if !extsEnabled {
			return nil, fmt.Errorf("vote extensions are not enabled; unexpected call to ExtendVote at height %d", req.Height)
		}
	}

	if c.extendVote == nil {
		return nil, errors.New("vote extensions are enabled but no extend function was set")
	}

	_, latestStore, err := c.store.StateLatest()
	if err != nil {
		return nil, err
	}

	resp, err := c.extendVote(ctx, latestStore, req)
	if err != nil {
		c.logger.Error("failed to extend vote", "height", req.Height, "err", err)
		return &abciproto.ExtendVoteResponse{}, nil
	}

	return resp, err
}

func decodeTxs[T transaction.Tx](logger log.Logger, rawTxs [][]byte, codec transaction.Codec[T]) ([]T, error) {
	txs := make([]T, len(rawTxs))
	for i, rawTx := range rawTxs {
		tx, err := codec.Decode(rawTx)
		if err != nil {
			// do not return an error here, as we want to deliver the block even if some txs are invalid
			logger.Debug("failed to decode tx", "err", err)
			txs[i] = RawTx(rawTx).(T) // allows getting the raw bytes down the line
			continue
		}
		txs[i] = tx
	}
	return txs, nil
}