tests: add testutils for integration testing (#132)

* tests: add testutils for integration testing

* update util

* fix config

* more updates

* rand chain-id

* add rpc client integration test example

* makefile

* rename

* updates to makefile

testutil/network/doc.go

@@ -0,0 +1,65 @@
+/*
+Package network implements and exposes a fully operational in-process Tendermint
+test network that consists of at least one or potentially many validators. This
+test network can be used primarily for integration tests or unit test suites.
+
+The test network utilizes SimApp as the ABCI application and uses all the modules
+defined in the Cosmos SDK. An in-process test network can be configured with any
+number of validators as well as account funds and even custom genesis state.
+
+When creating a test network, a series of Validator objects are returned. Each
+Validator object has useful information such as their address and public key. A
+Validator will also provide its RPC, P2P, and API addresses that can be useful
+for integration testing. In addition, a Tendermint local RPC client is also provided
+which can be handy for making direct RPC calls to Tendermint.
+
+Note, due to limitations in concurrency and the design of the RPC layer in
+Tendermint, only the first Validator object will have an RPC and API client
+exposed. Due to this exact same limitation, only a single test network can exist
+at a time. A caller must be certain it calls Cleanup after it no longer needs
+the network.
+
+A typical testing flow might look like the following:
+
+	type IntegrationTestSuite struct {
+		suite.Suite
+
+		cfg     testutil.Config
+		network *testutil.Network
+	}
+
+	func (s *IntegrationTestSuite) SetupSuite() {
+		s.T().Log("setting up integration test suite")
+
+		cfg := testutil.DefaultConfig()
+		cfg.NumValidators = 1
+
+		s.cfg = cfg
+		s.network = testutil.New(s.T(), cfg)
+
+		_, err := s.network.WaitForHeight(1)
+		s.Require().NoError(err)
+	}
+
+	func (s *IntegrationTestSuite) TearDownSuite() {
+		s.T().Log("tearing down integration test suite")
+
+		// This is important and must be called to ensure other tests can create
+		// a network!
+		s.network.Cleanup()
+	}
+
+	func (s *IntegrationTestSuite) TestQueryBalancesRequestHandlerFn() {
+		val := s.network.Validators[0]
+		baseURL := val.APIAddress
+
+		// Use baseURL to make API HTTP requests or use val.RPCClient to make direct
+		// Tendermint RPC calls.
+		// ...
+	}
+
+	func TestIntegrationTestSuite(t *testing.T) {
+		suite.Run(t, new(IntegrationTestSuite))
+	}
+*/
+package network

testutil/network/network.go

@@ -0,0 +1,511 @@
+package network
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethclient"
+	jsonrpc "github.com/ethereum/go-ethereum/rpc"
+
+	"google.golang.org/grpc"
+
+	"github.com/stretchr/testify/require"
+	tmcfg "github.com/tendermint/tendermint/config"
+	tmflags "github.com/tendermint/tendermint/libs/cli/flags"
+	"github.com/tendermint/tendermint/libs/log"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/node"
+	tmclient "github.com/tendermint/tendermint/rpc/client"
+	dbm "github.com/tendermint/tm-db"
+
+	"github.com/cosmos/cosmos-sdk/baseapp"
+	"github.com/cosmos/cosmos-sdk/client"
+	"github.com/cosmos/cosmos-sdk/client/tx"
+	"github.com/cosmos/cosmos-sdk/codec"
+	codectypes "github.com/cosmos/cosmos-sdk/codec/types"
+	"github.com/cosmos/cosmos-sdk/crypto/keyring"
+	cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
+	"github.com/cosmos/cosmos-sdk/server"
+	"github.com/cosmos/cosmos-sdk/server/api"
+	servertypes "github.com/cosmos/cosmos-sdk/server/types"
+	"github.com/cosmos/cosmos-sdk/simapp"
+	"github.com/cosmos/cosmos-sdk/simapp/params"
+	storetypes "github.com/cosmos/cosmos-sdk/store/types"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
+	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
+	"github.com/cosmos/cosmos-sdk/x/genutil"
+	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+
+	"github.com/tharsis/ethermint/app"
+	srvconfig "github.com/tharsis/ethermint/cmd/ethermintd/config"
+	"github.com/tharsis/ethermint/crypto/hd"
+	"github.com/tharsis/ethermint/encoding"
+	ethermint "github.com/tharsis/ethermint/types"
+)
+
+// package-wide network lock to only allow one test network at a time
+var lock = new(sync.Mutex)
+
+// AppConstructor defines a function which accepts a network configuration and
+// creates an ABCI Application to provide to Tendermint.
+type AppConstructor = func(val Validator) servertypes.Application
+
+// NewAppConstructor returns a new simapp AppConstructor
+func NewAppConstructor(encodingCfg params.EncodingConfig) AppConstructor {
+	return func(val Validator) servertypes.Application {
+		return app.NewEthermintApp(
+			val.Ctx.Logger, dbm.NewMemDB(), nil, true, make(map[int64]bool), val.Ctx.Config.RootDir, 0,
+			encodingCfg,
+			simapp.EmptyAppOptions{},
+			baseapp.SetPruning(storetypes.NewPruningOptionsFromString(val.AppConfig.Pruning)),
+			baseapp.SetMinGasPrices(val.AppConfig.MinGasPrices),
+			baseapp.SetTrace(true),
+		)
+	}
+}
+
+// Config defines the necessary configuration used to bootstrap and start an
+// in-process local testing network.
+type Config struct {
+	Codec             codec.Marshaler
+	LegacyAmino       *codec.LegacyAmino // TODO: Remove!
+	InterfaceRegistry codectypes.InterfaceRegistry
+
+	TxConfig         client.TxConfig
+	AccountRetriever client.AccountRetriever
+	AppConstructor   AppConstructor             // the ABCI application constructor
+	GenesisState     map[string]json.RawMessage // custom gensis state to provide
+	TimeoutCommit    time.Duration              // the consensus commitment timeout
+	ChainID          string                     // the network chain-id
+	NumValidators    int                        // the total number of validators to create and bond
+	BondDenom        string                     // the staking bond denomination
+	MinGasPrices     string                     // the minimum gas prices each validator will accept
+	AccountTokens    sdk.Int                    // the amount of unique validator tokens (e.g. 1000node0)
+	StakingTokens    sdk.Int                    // the amount of tokens each validator has available to stake
+	BondedTokens     sdk.Int                    // the amount of tokens each validator stakes
+	PruningStrategy  string                     // the pruning strategy each validator will have
+	EnableLogging    bool                       // enable Tendermint logging to STDOUT
+	CleanupDir       bool                       // remove base temporary directory during cleanup
+	SigningAlgo      string                     // signing algorithm for keys
+	KeyringOptions   []keyring.Option
+}
+
+// DefaultConfig returns a sane default configuration suitable for nearly all
+// testing requirements.
+func DefaultConfig() Config {
+	encCfg := encoding.MakeConfig(app.ModuleBasics)
+
+	return Config{
+		Codec:             encCfg.Marshaler,
+		TxConfig:          encCfg.TxConfig,
+		LegacyAmino:       encCfg.Amino,
+		InterfaceRegistry: encCfg.InterfaceRegistry,
+		AccountRetriever:  authtypes.AccountRetriever{},
+		AppConstructor:    NewAppConstructor(encCfg),
+		GenesisState:      app.NewDefaultGenesisState(),
+		TimeoutCommit:     2 * time.Second,
+		ChainID:           "ethermint-" + fmt.Sprintf("%d", tmrand.NewRand().Int63n(1000)),
+		NumValidators:     4,
+		BondDenom:         ethermint.AttoPhoton,
+		MinGasPrices:      fmt.Sprintf("0.000006%s", ethermint.AttoPhoton),
+		AccountTokens:     sdk.TokensFromConsensusPower(1000),
+		StakingTokens:     sdk.TokensFromConsensusPower(500),
+		BondedTokens:      sdk.TokensFromConsensusPower(100),
+		PruningStrategy:   storetypes.PruningOptionNothing,
+		CleanupDir:        true,
+		SigningAlgo:       string(hd.EthSecp256k1Type),
+		KeyringOptions:    []keyring.Option{hd.EthSecp256k1Option()},
+	}
+}
+
+type (
+	// Network defines a local in-process testing network using SimApp. It can be
+	// configured to start any number of validators, each with its own RPC and API
+	// clients. Typically, this test network would be used in client and integration
+	// testing where user input is expected.
+	//
+	// Note, due to Tendermint constraints in regards to RPC functionality, there
+	// may only be one test network running at a time. Thus, any caller must be
+	// sure to Cleanup after testing is finished in order to allow other tests
+	// to create networks. In addition, only the first validator will have a valid
+	// RPC and API server/client.
+	Network struct {
+		T          *testing.T
+		BaseDir    string
+		Validators []*Validator
+
+		Config Config
+	}
+
+	// Validator defines an in-process Tendermint validator node. Through this object,
+	// a client can make RPC and API calls and interact with any client command
+	// or handler.
+	Validator struct {
+		AppConfig       *srvconfig.Config
+		ClientCtx       client.Context
+		Ctx             *server.Context
+		Dir             string
+		NodeID          string
+		PubKey          cryptotypes.PubKey
+		Moniker         string
+		APIAddress      string
+		RPCAddress      string
+		P2PAddress      string
+		JSONRPCAddress  string
+		EthereumAddress common.Address
+		Address         sdk.AccAddress
+		ValAddress      sdk.ValAddress
+		RPCClient       tmclient.Client
+		JSONRPCClient   *ethclient.Client
+
+		tmNode  *node.Node
+		api     *api.Server
+		grpc    *grpc.Server
+		jsonRPC *jsonrpc.Server
+	}
+)
+
+// New creates a new Network for integration tests.
+func New(t *testing.T, cfg Config) *Network {
+	// only one caller/test can create and use a network at a time
+	t.Log("acquiring test network lock")
+	lock.Lock()
+
+	baseDir, err := ioutil.TempDir(t.TempDir(), cfg.ChainID)
+	require.NoError(t, err)
+	t.Logf("created temporary directory: %s", baseDir)
+
+	network := &Network{
+		T:          t,
+		BaseDir:    baseDir,
+		Validators: make([]*Validator, cfg.NumValidators),
+		Config:     cfg,
+	}
+
+	t.Log("preparing test network...")
+
+	var (
+		monikers   = make([]string, cfg.NumValidators)
+		nodeIDs    = make([]string, cfg.NumValidators)
+		valPubKeys = make([]cryptotypes.PubKey, cfg.NumValidators)
+	)
+
+	var (
+		genAccounts []authtypes.GenesisAccount
+		genBalances []banktypes.Balance
+		genFiles    []string
+	)
+
+	buf := bufio.NewReader(os.Stdin)
+
+	// generate private keys, node IDs, and initial transactions
+	for i := 0; i < cfg.NumValidators; i++ {
+		appCfg := srvconfig.DefaultConfig()
+		appCfg.Pruning = cfg.PruningStrategy
+		appCfg.MinGasPrices = cfg.MinGasPrices
+		appCfg.API.Enable = true
+		appCfg.API.Swagger = false
+		appCfg.Telemetry.Enabled = false
+
+		ctx := server.NewDefaultContext()
+		tmCfg := ctx.Config
+		tmCfg.Consensus.TimeoutCommit = cfg.TimeoutCommit
+
+		// Only allow the first validator to expose an RPC, API and gRPC
+		// server/client due to Tendermint in-process constraints.
+		apiAddr := ""
+		jsonRPCAddr := ""
+		tmCfg.RPC.ListenAddress = ""
+		appCfg.GRPC.Enable = false
+		appCfg.EVMRPC.Enable = false
+
+		if i == 0 {
+			apiListenAddr, _, err := server.FreeTCPAddr()
+			require.NoError(t, err)
+			appCfg.API.Address = apiListenAddr
+
+			apiURL, err := url.Parse(apiListenAddr)
+			require.NoError(t, err)
+			apiAddr = fmt.Sprintf("http://%s:%s", apiURL.Hostname(), apiURL.Port())
+
+			jsonRPCListenAddr, _, err := server.FreeTCPAddr()
+			require.NoError(t, err)
+			t.Log(jsonRPCListenAddr)
+			appCfg.EVMRPC.RPCAddress = jsonRPCListenAddr
+			appCfg.EVMRPC.Enable = true
+
+			jsonRPCAPIURL, err := url.Parse(jsonRPCListenAddr)
+			require.NoError(t, err)
+			jsonRPCAddr = fmt.Sprintf("http://%s:%s", jsonRPCAPIURL.Hostname(), jsonRPCAPIURL.Port())
+
+			rpcAddr, _, err := server.FreeTCPAddr()
+			require.NoError(t, err)
+			tmCfg.RPC.ListenAddress = rpcAddr
+
+			_, grpcPort, err := server.FreeTCPAddr()
+			require.NoError(t, err)
+			appCfg.GRPC.Address = fmt.Sprintf("0.0.0.0:%s", grpcPort)
+			appCfg.GRPC.Enable = true
+		}
+
+		logger := log.NewNopLogger()
+		if cfg.EnableLogging {
+			logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+			logger, _ = tmflags.ParseLogLevel("info", logger, tmcfg.DefaultLogLevel)
+		}
+
+		ctx.Logger = logger
+
+		nodeDirName := fmt.Sprintf("node%d", i)
+		nodeDir := filepath.Join(network.BaseDir, nodeDirName, "ethermintd")
+		gentxsDir := filepath.Join(network.BaseDir, "gentxs")
+
+		require.NoError(t, os.MkdirAll(filepath.Join(nodeDir, "config"), 0755))
+
+		tmCfg.SetRoot(nodeDir)
+		tmCfg.Moniker = nodeDirName
+		monikers[i] = nodeDirName
+
+		proxyAddr, _, err := server.FreeTCPAddr()
+		require.NoError(t, err)
+		tmCfg.ProxyApp = proxyAddr
+
+		p2pAddr, _, err := server.FreeTCPAddr()
+		require.NoError(t, err)
+		tmCfg.P2P.ListenAddress = p2pAddr
+		tmCfg.P2P.AddrBookStrict = false
+		tmCfg.P2P.AllowDuplicateIP = true
+
+		nodeID, pubKey, err := genutil.InitializeNodeValidatorFiles(tmCfg)
+		require.NoError(t, err)
+		nodeIDs[i] = nodeID
+		valPubKeys[i] = pubKey
+
+		kb, err := keyring.New(sdk.KeyringServiceName(), keyring.BackendTest, nodeDir, buf, cfg.KeyringOptions...)
+		require.NoError(t, err)
+
+		keyringAlgos, _ := kb.SupportedAlgorithms()
+		algo, err := keyring.NewSigningAlgoFromString(cfg.SigningAlgo, keyringAlgos)
+		require.NoError(t, err)
+
+		addr, secret, err := server.GenerateSaveCoinKey(kb, nodeDirName, true, algo)
+		require.NoError(t, err)
+
+		info := map[string]string{"secret": secret}
+		infoBz, err := json.Marshal(info)
+		require.NoError(t, err)
+
+		// save private key seed words
+		require.NoError(t, writeFile(fmt.Sprintf("%v.json", "key_seed"), nodeDir, infoBz))
+
+		balances := sdk.NewCoins(
+			sdk.NewCoin(fmt.Sprintf("%stoken", nodeDirName), cfg.AccountTokens),
+			sdk.NewCoin(cfg.BondDenom, cfg.StakingTokens),
+		)
+
+		genFiles = append(genFiles, tmCfg.GenesisFile())
+		genBalances = append(genBalances, banktypes.Balance{Address: addr.String(), Coins: balances.Sort()})
+		genAccounts = append(genAccounts, authtypes.NewBaseAccount(addr, nil, 0, 0))
+
+		commission, err := sdk.NewDecFromStr("0.5")
+		require.NoError(t, err)
+
+		createValMsg, err := stakingtypes.NewMsgCreateValidator(
+			sdk.ValAddress(addr),
+			valPubKeys[i],
+			sdk.NewCoin(cfg.BondDenom, cfg.BondedTokens),
+			stakingtypes.NewDescription(nodeDirName, "", "", "", ""),
+			stakingtypes.NewCommissionRates(commission, sdk.OneDec(), sdk.OneDec()),
+			sdk.OneInt(),
+		)
+		require.NoError(t, err)
+
+		p2pURL, err := url.Parse(p2pAddr)
+		require.NoError(t, err)
+
+		memo := fmt.Sprintf("%s@%s:%s", nodeIDs[i], p2pURL.Hostname(), p2pURL.Port())
+		fee := sdk.NewCoins(sdk.NewCoin(fmt.Sprintf("%stoken", nodeDirName), sdk.NewInt(0)))
+		txBuilder := cfg.TxConfig.NewTxBuilder()
+		require.NoError(t, txBuilder.SetMsgs(createValMsg))
+		txBuilder.SetFeeAmount(fee)    // Arbitrary fee
+		txBuilder.SetGasLimit(1000000) // Need at least 100386
+		txBuilder.SetMemo(memo)
+
+		txFactory := tx.Factory{}
+		txFactory = txFactory.
+			WithChainID(cfg.ChainID).
+			WithMemo(memo).
+			WithKeybase(kb).
+			WithTxConfig(cfg.TxConfig)
+
+		err = tx.Sign(txFactory, nodeDirName, txBuilder, true)
+		require.NoError(t, err)
+
+		txBz, err := cfg.TxConfig.TxJSONEncoder()(txBuilder.GetTx())
+		require.NoError(t, err)
+		require.NoError(t, writeFile(fmt.Sprintf("%v.json", nodeDirName), gentxsDir, txBz))
+
+		srvconfig.WriteConfigFile(filepath.Join(nodeDir, "config/app.toml"), appCfg)
+
+		clientCtx := client.Context{}.
+			WithKeyringDir(nodeDir).
+			WithKeyring(kb).
+			WithHomeDir(tmCfg.RootDir).
+			WithChainID(cfg.ChainID).
+			WithInterfaceRegistry(cfg.InterfaceRegistry).
+			WithJSONMarshaler(cfg.Codec).
+			WithLegacyAmino(cfg.LegacyAmino).
+			WithTxConfig(cfg.TxConfig).
+			WithAccountRetriever(cfg.AccountRetriever)
+
+		network.Validators[i] = &Validator{
+			AppConfig:       appCfg,
+			ClientCtx:       clientCtx,
+			Ctx:             ctx,
+			Dir:             filepath.Join(network.BaseDir, nodeDirName),
+			NodeID:          nodeID,
+			PubKey:          pubKey,
+			Moniker:         nodeDirName,
+			RPCAddress:      tmCfg.RPC.ListenAddress,
+			P2PAddress:      tmCfg.P2P.ListenAddress,
+			APIAddress:      apiAddr,
+			JSONRPCAddress:  jsonRPCAddr,
+			EthereumAddress: common.BytesToAddress(addr),
+			Address:         addr,
+			ValAddress:      sdk.ValAddress(addr),
+		}
+	}
+
+	require.NoError(t, initGenFiles(cfg, genAccounts, genBalances, genFiles))
+	require.NoError(t, collectGenFiles(cfg, network.Validators, network.BaseDir))
+
+	t.Log("starting test network...")
+	for _, v := range network.Validators {
+		require.NoError(t, startInProcess(cfg, v))
+	}
+
+	t.Log("started test network")
+
+	// Ensure we cleanup incase any test was abruptly halted (e.g. SIGINT) as any
+	// defer in a test would not be called.
+	server.TrapSignal(network.Cleanup)
+
+	return network
+}
+
+// LatestHeight returns the latest height of the network or an error if the
+// query fails or no validators exist.
+func (n *Network) LatestHeight() (int64, error) {
+	if len(n.Validators) == 0 {
+		return 0, errors.New("no validators available")
+	}
+
+	status, err := n.Validators[0].RPCClient.Status(context.Background())
+	if err != nil {
+		return 0, err
+	}
+
+	return status.SyncInfo.LatestBlockHeight, nil
+}
+
+// WaitForHeight performs a blocking check where it waits for a block to be
+// committed after a given block. If that height is not reached within a timeout,
+// an error is returned. Regardless, the latest height queried is returned.
+func (n *Network) WaitForHeight(h int64) (int64, error) {
+	return n.WaitForHeightWithTimeout(h, 10*time.Second)
+}
+
+// WaitForHeightWithTimeout is the same as WaitForHeight except the caller can
+// provide a custom timeout.
+func (n *Network) WaitForHeightWithTimeout(h int64, t time.Duration) (int64, error) {
+	ticker := time.NewTicker(time.Second)
+	timeout := time.After(t)
+
+	if len(n.Validators) == 0 {
+		return 0, errors.New("no validators available")
+	}
+
+	var latestHeight int64
+	val := n.Validators[0]
+
+	for {
+		select {
+		case <-timeout:
+			ticker.Stop()
+			return latestHeight, errors.New("timeout exceeded waiting for block")
+		case <-ticker.C:
+			status, err := val.RPCClient.Status(context.Background())
+			if err == nil && status != nil {
+				latestHeight = status.SyncInfo.LatestBlockHeight
+				if latestHeight >= h {
+					return latestHeight, nil
+				}
+			}
+		}
+	}
+}
+
+// WaitForNextBlock waits for the next block to be committed, returning an error
+// upon failure.
+func (n *Network) WaitForNextBlock() error {
+	lastBlock, err := n.LatestHeight()
+	if err != nil {
+		return err
+	}
+
+	_, err = n.WaitForHeight(lastBlock + 1)
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+// Cleanup removes the root testing (temporary) directory and stops both the
+// Tendermint and API services. It allows other callers to create and start
+// test networks. This method must be called when a test is finished, typically
+// in a defer.
+func (n *Network) Cleanup() {
+	defer func() {
+		lock.Unlock()
+		n.T.Log("released test network lock")
+	}()
+
+	n.T.Log("cleaning up test network...")
+
+	for _, v := range n.Validators {
+		if v.tmNode != nil && v.tmNode.IsRunning() {
+			_ = v.tmNode.Stop()
+		}
+
+		if v.api != nil {
+			_ = v.api.Close()
+		}
+
+		if v.grpc != nil {
+			v.grpc.Stop()
+		}
+
+		if v.jsonRPC != nil {
+			v.jsonRPC.Stop()
+		}
+	}
+
+	if n.Config.CleanupDir {
+		_ = os.RemoveAll(n.BaseDir)
+	}
+
+	n.T.Log("finished cleaning up test network")
+}

testutil/network/network_test.go

@@ -0,0 +1,46 @@
+// +build norace
+
+package network_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/tharsis/ethermint/testutil/network"
+)
+
+type IntegrationTestSuite struct {
+	suite.Suite
+
+	network *network.Network
+}
+
+func (s *IntegrationTestSuite) SetupSuite() {
+	s.T().Log("setting up integration test suite")
+
+	s.network = network.New(s.T(), network.DefaultConfig())
+	s.Require().NotNil(s.network)
+
+	_, err := s.network.WaitForHeight(1)
+	s.Require().NoError(err)
+}
+
+func (s *IntegrationTestSuite) TearDownSuite() {
+	s.T().Log("tearing down integration test suite")
+	s.network.Cleanup()
+}
+
+func (s *IntegrationTestSuite) TestNetwork_Liveness() {
+	h, err := s.network.WaitForHeightWithTimeout(10, time.Minute)
+	s.Require().NoError(err, "expected to reach 10 blocks; got %d", h)
+
+	latestHeight, err := s.network.LatestHeight()
+	s.Require().NoError(err, "latest height failed")
+	s.Require().GreaterOrEqual(latestHeight, h)
+}
+
+func TestIntegrationTestSuite(t *testing.T) {
+	suite.Run(t, new(IntegrationTestSuite))
+}

testutil/network/util.go

@@ -0,0 +1,276 @@
+package network
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"time"
+
+	jsonrpc "github.com/ethereum/go-ethereum/rpc"
+	"github.com/gorilla/mux"
+	"github.com/improbable-eng/grpc-web/go/grpcweb"
+	"github.com/rs/cors"
+
+	tmos "github.com/tendermint/tendermint/libs/os"
+	"github.com/tendermint/tendermint/node"
+	"github.com/tendermint/tendermint/p2p"
+	pvm "github.com/tendermint/tendermint/privval"
+	"github.com/tendermint/tendermint/proxy"
+	"github.com/tendermint/tendermint/rpc/client/local"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+
+	"github.com/cosmos/cosmos-sdk/server/api"
+	servergrpc "github.com/cosmos/cosmos-sdk/server/grpc"
+	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
+	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
+	"github.com/cosmos/cosmos-sdk/x/genutil"
+	genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
+	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+
+	"github.com/tharsis/ethermint/ethereum/rpc"
+	ethsrv "github.com/tharsis/ethermint/server"
+	ethermint "github.com/tharsis/ethermint/types"
+)
+
+func startInProcess(cfg Config, val *Validator) error {
+	logger := val.Ctx.Logger
+	tmCfg := val.Ctx.Config
+	tmCfg.Instrumentation.Prometheus = false
+
+	nodeKey, err := p2p.LoadOrGenNodeKey(tmCfg.NodeKeyFile())
+	if err != nil {
+		return fmt.Errorf("failed to load or generate node key: %w", err)
+	}
+
+	app := cfg.AppConstructor(*val)
+
+	genDocProvider := node.DefaultGenesisDocProviderFunc(tmCfg)
+	tmNode, err := node.NewNode(
+		tmCfg,
+		pvm.LoadOrGenFilePV(tmCfg.PrivValidatorKeyFile(), tmCfg.PrivValidatorStateFile()),
+		nodeKey,
+		proxy.NewLocalClientCreator(app),
+		genDocProvider,
+		node.DefaultDBProvider,
+		node.DefaultMetricsProvider(tmCfg.Instrumentation),
+		logger.With("module", val.Moniker),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create node: %w", err)
+	}
+
+	if err := tmNode.Start(); err != nil {
+		return fmt.Errorf("failed to start node: %w", err)
+	}
+
+	val.tmNode = tmNode
+
+	if val.RPCAddress != "" {
+		val.RPCClient = local.New(tmNode)
+	}
+
+	// We'll need a RPC client if the validator exposes a gRPC or REST endpoint.
+	if val.APIAddress != "" || val.AppConfig.GRPC.Enable {
+		val.ClientCtx = val.ClientCtx.
+			WithClient(val.RPCClient)
+
+		// Add the tx service in the gRPC router.
+		app.RegisterTxService(val.ClientCtx)
+
+		// Add the tendermint queries service in the gRPC router.
+		app.RegisterTendermintService(val.ClientCtx)
+	}
+
+	if val.APIAddress != "" {
+		apiSrv := api.New(val.ClientCtx, logger.With("module", "api-server"))
+		app.RegisterAPIRoutes(apiSrv, val.AppConfig.API)
+
+		errCh := make(chan error)
+
+		go func() {
+			if err := apiSrv.Start(*val.AppConfig.ToSDKConfig()); err != nil {
+				errCh <- err
+			}
+		}()
+
+		select {
+		case err := <-errCh:
+			return fmt.Errorf("failed to start API server: %w", err)
+		case <-time.After(5 * time.Second): // assume server started successfully
+		}
+
+		val.api = apiSrv
+	}
+
+	if val.AppConfig.GRPC.Enable {
+		grpcSrv, err := servergrpc.StartGRPCServer(val.ClientCtx, app, val.AppConfig.GRPC.Address)
+		if err != nil {
+			return fmt.Errorf("failed to start gRPC server: %w", err)
+		}
+
+		val.grpc = grpcSrv
+	}
+
+	if val.AppConfig.EVMRPC.Enable {
+		tmEndpoint := "/websocket"
+		tmRPCAddr := val.Ctx.Config.RPC.ListenAddress
+		tmWsClient := ethsrv.ConnectTmWS(tmRPCAddr, tmEndpoint)
+
+		val.jsonRPC = jsonrpc.NewServer()
+
+		apis := rpc.GetRPCAPIs(val.ClientCtx, tmWsClient)
+		for _, api := range apis {
+			if err := val.jsonRPC.RegisterName(api.Namespace, api.Service); err != nil {
+				return fmt.Errorf("failed to register JSON-RPC namespace %s: %w", api.Namespace, err)
+			}
+		}
+
+		r := mux.NewRouter()
+		r.HandleFunc("/", val.jsonRPC.ServeHTTP).Methods("POST")
+		if val.grpc != nil {
+			grpcWeb := grpcweb.WrapServer(val.grpc)
+			ethsrv.MountGRPCWebServices(r, grpcWeb, grpcweb.ListGRPCResources(val.grpc))
+		}
+
+		handlerWithCors := cors.New(cors.Options{
+			AllowedOrigins: []string{"*"},
+			AllowedMethods: []string{
+				http.MethodHead,
+				http.MethodGet,
+				http.MethodPost,
+				http.MethodPut,
+				http.MethodPatch,
+				http.MethodDelete,
+			},
+			AllowedHeaders:     []string{"*"},
+			AllowCredentials:   false,
+			OptionsPassthrough: false,
+		})
+
+		httpSrv := &http.Server{
+			Addr: strings.TrimPrefix(val.AppConfig.EVMRPC.RPCAddress, "tcp://"), // FIXME: timeouts
+			// Addr:    val.AppConfig.EVMRPC.RPCAddress, // FIXME: address has too many colons
+			Handler: handlerWithCors.Handler(r),
+		}
+
+		httpSrvDone := make(chan struct{}, 1)
+
+		errCh := make(chan error)
+		go func() {
+			if err := httpSrv.ListenAndServe(); err != nil {
+				if err == http.ErrServerClosed {
+					close(httpSrvDone)
+					return
+				}
+
+				errCh <- err
+			}
+		}()
+
+		select {
+		case err := <-errCh:
+			return fmt.Errorf("JSON-RPC go routine failed to start: %w", err)
+		case <-time.After(1 * time.Second): // assume EVM RPC server started successfully
+		}
+	}
+
+	return nil
+}
+
+func collectGenFiles(cfg Config, vals []*Validator, outputDir string) error {
+	genTime := tmtime.Now()
+
+	for i := 0; i < cfg.NumValidators; i++ {
+		tmCfg := vals[i].Ctx.Config
+
+		nodeDir := filepath.Join(outputDir, vals[i].Moniker, "ethermintd")
+		gentxsDir := filepath.Join(outputDir, "gentxs")
+
+		tmCfg.Moniker = vals[i].Moniker
+		tmCfg.SetRoot(nodeDir)
+
+		initCfg := genutiltypes.NewInitConfig(cfg.ChainID, gentxsDir, vals[i].NodeID, vals[i].PubKey)
+
+		genFile := tmCfg.GenesisFile()
+		genDoc, err := types.GenesisDocFromFile(genFile)
+		if err != nil {
+			return fmt.Errorf("failed to create genesis doc: %w", err)
+		}
+
+		appState, err := genutil.GenAppStateFromConfig(cfg.Codec, cfg.TxConfig,
+			tmCfg, initCfg, *genDoc, banktypes.GenesisBalancesIterator{})
+		if err != nil {
+			return fmt.Errorf("failed to create app state: %w", err)
+		}
+
+		// overwrite each validator's genesis file to have a canonical genesis time
+		if err := genutil.ExportGenesisFileWithTime(genFile, cfg.ChainID, nil, appState, genTime); err != nil {
+			return fmt.Errorf("failed to export genesis: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func initGenFiles(cfg Config, genAccounts []authtypes.GenesisAccount, genBalances []banktypes.Balance, genFiles []string) error {
+
+	// set the accounts in the genesis state
+	var authGenState authtypes.GenesisState
+	cfg.Codec.MustUnmarshalJSON(cfg.GenesisState[authtypes.ModuleName], &authGenState)
+
+	accounts, err := authtypes.PackAccounts(genAccounts)
+	if err != nil {
+		return err
+	}
+
+	authGenState.Accounts = accounts
+	cfg.GenesisState[authtypes.ModuleName] = cfg.Codec.MustMarshalJSON(&authGenState)
+
+	// set the balances in the genesis state
+	var bankGenState banktypes.GenesisState
+	cfg.Codec.MustUnmarshalJSON(cfg.GenesisState[banktypes.ModuleName], &bankGenState)
+
+	bankGenState.Balances = genBalances
+	cfg.GenesisState[banktypes.ModuleName] = cfg.Codec.MustMarshalJSON(&bankGenState)
+
+	var stakingGenState stakingtypes.GenesisState
+	cfg.Codec.MustUnmarshalJSON(cfg.GenesisState[stakingtypes.ModuleName], &stakingGenState)
+
+	stakingGenState.Params.BondDenom = ethermint.AttoPhoton
+	cfg.GenesisState[stakingtypes.ModuleName] = cfg.Codec.MustMarshalJSON(&stakingGenState)
+
+	appGenStateJSON, err := json.MarshalIndent(cfg.GenesisState, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	genDoc := types.GenesisDoc{
+		ChainID:    cfg.ChainID,
+		AppState:   appGenStateJSON,
+		Validators: nil,
+	}
+
+	// generate empty genesis files for each validator and save
+	for i := 0; i < cfg.NumValidators; i++ {
+		if err := genDoc.SaveAs(genFiles[i]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func writeFile(name string, dir string, contents []byte) error {
+	writePath := filepath.Join(dir)
+	file := filepath.Join(writePath, name)
+
+	err := tmos.EnsureDir(writePath, 0755)
+	if err != nil {
+		return err
+	}
+
+	return tmos.WriteFile(file, contents, 0644)
+}