Remaining Nitro config and refactoring (#259)

Part of [Implement remaining ipld-eth-server config and load payment channels on startup](https://www.notion.so/Implement-remaining-ipld-eth-server-config-and-load-payment-channels-on-startup-5fcb33e49d504d618169a2089740eb67)

Co-authored-by: neeraj <neeraj.rtly@gmail.com>
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Reviewed-on: #259
Co-authored-by: Prathamesh Musale <prathamesh@noreply.git.vdb.to>
Co-committed-by: Prathamesh Musale <prathamesh@noreply.git.vdb.to>

Dockerfile

@@ -1,9 +1,9 @@
-FROM golang:1.19-alpine as debugger
+FROM golang:1.21-alpine as debugger
 
 # Include dlv
 RUN go install github.com/go-delve/delve/cmd/dlv@latest
 
-FROM golang:1.19-alpine as builder
+FROM golang:1.21-alpine as builder
 
 RUN apk --update --no-cache add gcc musl-dev binutils-gold git
 
@@ -28,6 +28,8 @@ RUN GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -
 # app container
 FROM alpine
 
+RUN apk --update --no-cache add bash jq curl
+
 ARG USER="vdm"
 ARG CONFIG_FILE="./environments/example.toml"
 
@@ -41,6 +43,7 @@ USER $USER
 COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/$CONFIG_FILE config.toml
 COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/entrypoint.sh .
 
+RUN mkdir -p nitro-data && chown -R 5000:5000 nitro-data
 
 # keep binaries immutable
 COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-server/ipld-eth-server ipld-eth-server

cmd/common.go

@@ -35,3 +35,49 @@ func addDatabaseFlags(command *cobra.Command) {
 	viper.BindPFlag("database.user", command.PersistentFlags().Lookup("database-user"))
 	viper.BindPFlag("database.password", command.PersistentFlags().Lookup("database-password"))
 }
+
+func addNitroFlags(command *cobra.Command) {
+	// nitro flags
+	command.PersistentFlags().Bool("nitro-run-node-in-process", false, "nitro run node in process")
+	command.PersistentFlags().String("nitro-rpc-query-rates-file", "", "nitro rpcQueryRatesFile")
+
+	command.PersistentFlags().String("nitro-pk", "", "nitro pk")
+	command.PersistentFlags().String("nitro-chain-pk", "", "nitro chainPk")
+	command.PersistentFlags().String("nitro-chain-url", "ws://127.0.0.1:8545", "nitro chainUrl")
+	command.PersistentFlags().String("nitro-na-address", "", "nitro naAddress")
+	command.PersistentFlags().String("nitro-vpa-address", "", "nitro vpaAddress")
+	command.PersistentFlags().String("nitro-ca-address", "", "nitro caAddress")
+	command.PersistentFlags().Bool("nitro-use-durable-store", false, "nitro useDurableStore")
+	command.PersistentFlags().String("nitro-durable-store-folder", "", "nitro durableStoreFolder")
+	command.PersistentFlags().Int("nitro-msg-port", 3005, "nitro msgPort")
+	command.PersistentFlags().Int("nitro-rpc-port", 4005, "nitro rpcPort")
+	command.PersistentFlags().Int("nitro-ws-msg-port", 5005, "nitro wsMsgPort")
+	command.PersistentFlags().Uint("nitro-chain-start-block", 0, "nitro chainStartBlock")
+	command.PersistentFlags().String("nitro-tls-cert-filepath", "", "nitro tlsCertFilepath")
+	command.PersistentFlags().String("nitro-tls-key-filepath", "", "nitro tlsKeyFilepath")
+
+	command.PersistentFlags().String("nitro-endpoint", "", "nitro endpoint")
+	command.PersistentFlags().Bool("nitro-is-secure", false, "nitro isSecure")
+
+	// nitro flag bindings
+	viper.BindPFlag("nitro.runNodeInProcess", command.PersistentFlags().Lookup("nitro-run-node-in-process"))
+	viper.BindPFlag("nitro.rpcQueryRatesFile", command.PersistentFlags().Lookup("nitro-rpc-query-rates-file"))
+
+	viper.BindPFlag("nitro.inProcesssNode.pk", command.PersistentFlags().Lookup("nitro-pk"))
+	viper.BindPFlag("nitro.inProcesssNode.chainPk", command.PersistentFlags().Lookup("nitro-chain-pk"))
+	viper.BindPFlag("nitro.inProcesssNode.chainUrl", command.PersistentFlags().Lookup("nitro-chain-url"))
+	viper.BindPFlag("nitro.inProcesssNode.naAddress", command.PersistentFlags().Lookup("nitro-na-address"))
+	viper.BindPFlag("nitro.inProcesssNode.vpaAddress", command.PersistentFlags().Lookup("nitro-vpa-address"))
+	viper.BindPFlag("nitro.inProcesssNode.caAddress", command.PersistentFlags().Lookup("nitro-ca-address"))
+	viper.BindPFlag("nitro.inProcesssNode.useDurableStore", command.PersistentFlags().Lookup("nitro-use-durable-store"))
+	viper.BindPFlag("nitro.inProcesssNode.durableStoreFolder", command.PersistentFlags().Lookup("nitro-durable-store"))
+	viper.BindPFlag("nitro.inProcesssNode.msgPort", command.PersistentFlags().Lookup("nitro-msg-port"))
+	viper.BindPFlag("nitro.inProcesssNode.rpcPort", command.PersistentFlags().Lookup("nitro-rpc-port"))
+	viper.BindPFlag("nitro.inProcesssNode.wsMsgPort", command.PersistentFlags().Lookup("nitro-ws-msg-port"))
+	viper.BindPFlag("nitro.inProcesssNode.chainStartBlock", command.PersistentFlags().Lookup("nitro-chain-start-block"))
+	viper.BindPFlag("nitro.inProcesssNode.tlsCertFilepath", command.PersistentFlags().Lookup("nitro-tls-cert-filepath"))
+	viper.BindPFlag("nitro.inProcesssNode.tlsKeyFilepath", command.PersistentFlags().Lookup("nitro-tls-key-filepath"))
+
+	viper.BindPFlag("nitro.remoteNode.nitroEndpoint", command.PersistentFlags().Lookup("nitro-endpoint"))
+	viper.BindPFlag("nitro.remoteNode.isSecure", command.PersistentFlags().Lookup("nitro-is-secure"))
+}

cmd/serve.go

@@ -16,7 +16,12 @@
 package cmd
 
 import (
+	"crypto/tls"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"io/fs"
+	"math/big"
 	"net/http"
 	"net/url"
 	"os"
@@ -26,15 +31,26 @@ import (
 	"time"
 
 	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/mailgun/groupcache/v2"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"github.com/statechannels/go-nitro/node/engine"
+	"github.com/statechannels/go-nitro/node/engine/chainservice"
+	nitroStore "github.com/statechannels/go-nitro/node/engine/store"
+	"github.com/statechannels/go-nitro/paymentsmanager"
+	"github.com/statechannels/go-nitro/rpc/transport"
+	"golang.org/x/exp/slog"
 
 	"github.com/cerc-io/ipld-eth-server/v5/pkg/graphql"
 	srpc "github.com/cerc-io/ipld-eth-server/v5/pkg/rpc"
 	s "github.com/cerc-io/ipld-eth-server/v5/pkg/serve"
 	v "github.com/cerc-io/ipld-eth-server/v5/version"
+	nitroNode "github.com/statechannels/go-nitro/node"
+	nitrop2pms "github.com/statechannels/go-nitro/node/engine/messageservice/p2p-message-service"
+	nitroRpc "github.com/statechannels/go-nitro/rpc"
+	nitroHttpTransport "github.com/statechannels/go-nitro/rpc/transport/http"
 )
 
 var ErrNoRpcEndpoints = errors.New("no rpc endpoints is available")
@@ -55,6 +71,8 @@ func serve() {
 	logWithCommand.Infof("ipld-eth-server version: %s", v.VersionWithMeta)
 
 	wg := new(sync.WaitGroup)
+	defer wg.Wait()
+
 	serverConfig, err := s.NewConfig()
 	if err != nil {
 		logWithCommand.Fatal(err)
@@ -75,7 +93,41 @@ func serve() {
 	}
 
 	server.Serve(wg)
-	if err := startServers(server, serverConfig); err != nil {
+
+	var voucherValidator paymentsmanager.VoucherValidator
+
+	nitroConfig := serverConfig.Nitro
+	if nitroConfig.RunNodeInProcess {
+		log.Info("Running an in-process Nitro node")
+
+		pm, nitroRpcServer := initNitroInProcess(wg, nitroConfig)
+		defer pm.Stop()
+		defer nitroRpcServer.Close()
+
+		voucherValidator = paymentsmanager.InProcessVoucherValidator{PaymentsManager: *pm}
+	} else {
+		log.Info("Connecting to a remote Nitro node")
+
+		isSecure := nitroConfig.RemoteNode.IsSecure
+		nitroRpcClient, err := nitroRpc.NewHttpRpcClient(nitroConfig.RemoteNode.NitroEndpoint, isSecure)
+		if err != nil {
+			logWithCommand.Fatal(err)
+		}
+		defer nitroRpcClient.Close()
+
+		voucherValidator = nitroRpc.RemoteVoucherValidator{Client: nitroRpcClient}
+	}
+
+	queryRates, err := readRpcQueryRates(nitroConfig.RpcQueryRatesFile)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	paymentMiddleware := func(next http.Handler) http.Handler {
+		return paymentsmanager.HTTPMiddleware(next, voucherValidator, queryRates)
+	}
+
+	if err := startServers(server, serverConfig, [](func(next http.Handler) http.Handler){paymentMiddleware}); err != nil {
 		logWithCommand.Fatal(err)
 	}
 	graphQL, err := startEthGraphQL(server, serverConfig)
@@ -102,10 +154,9 @@ func serve() {
 		graphQL.Stop()
 	}
 	server.Stop()
-	wg.Wait()
 }
 
-func startServers(server s.Server, settings *s.Config) error {
+func startServers(server s.Server, settings *s.Config, httpMiddlewares [](func(next http.Handler) http.Handler)) error {
 	if settings.IPCEnabled {
 		logWithCommand.Debug("starting up IPC server")
 		_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
@@ -128,7 +179,7 @@ func startServers(server s.Server, settings *s.Config) error {
 
 	if settings.HTTPEnabled {
 		logWithCommand.Debug("starting up HTTP server")
-		_, err := srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"vdb", "eth", "debug", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
+		_, err := srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"vdb", "eth", "debug", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{}, httpMiddlewares)
 		if err != nil {
 			return err
 		}
@@ -260,6 +311,8 @@ func init() {
 
 	addDatabaseFlags(serveCmd)
 
+	addNitroFlags(serveCmd)
+
 	// flags for all config variables
 	// eth graphql and json-rpc parameters
 	serveCmd.PersistentFlags().Bool("server-graphql", false, "turn on the eth graphql server")
@@ -339,3 +392,158 @@ func init() {
 	viper.BindPFlag("validator.enabled", serveCmd.PersistentFlags().Lookup("validator-enabled"))
 	viper.BindPFlag("validator.everyNthBlock", serveCmd.PersistentFlags().Lookup("validator-every-nth-block"))
 }
+
+// Initializes an in-process Nitro node, payments manager and a Nitro RPC server
+func initNitroInProcess(wg *sync.WaitGroup, nitroConfig *s.NitroConfig) (*paymentsmanager.PaymentsManager, *nitroRpc.RpcServer) {
+	nitroNode, err := initNitroNode(&nitroConfig.InProcessNode)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	pm, err := paymentsmanager.NewPaymentsManager(nitroNode)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	pm.Start(wg)
+
+	tlsCertFilepath := nitroConfig.InProcessNode.TlsCertFilepath
+	tlsKeyFilepath := nitroConfig.InProcessNode.TlsKeyFilepath
+
+	var cert *tls.Certificate
+	if tlsCertFilepath != "" && tlsKeyFilepath != "" {
+		*cert, err = tls.LoadX509KeyPair(tlsCertFilepath, tlsKeyFilepath)
+		if err != nil {
+			logWithCommand.Fatal(err)
+		}
+	}
+
+	nitroRpcServer, err := initNitroRpcServer(nitroNode, pm, cert, nitroConfig.InProcessNode.RpcPort)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	return &pm, nitroRpcServer
+}
+
+// https://github.com/cerc-io/go-nitro/blob/release-v0.1.1-ts-port-0.1.7/internal/node/node.go#L17
+func initNitroNode(config *s.InProcessNitroNodeConfig) (*nitroNode.Node, error) {
+	pkString := config.Pk
+	useDurableStore := config.UseDurableStore
+	durableStoreFolder := config.DurableStoreFolder
+	msgPort := config.MsgPort
+	wsMsgPort := config.WsMsgPort
+	chainUrl := config.ChainUrl
+	chainStartBlock := config.ChainStartBlock
+	chainPk := config.ChainPk
+	naAddress := config.NaAddress
+	vpaAddress := config.VpaAddress
+	caAddress := config.CaAddress
+
+	chainAuthToken := ""
+	publicIp := "0.0.0.0"
+
+	chainOpts := chainservice.ChainOpts{
+		ChainUrl:        chainUrl,
+		ChainStartBlock: chainStartBlock,
+		ChainAuthToken:  chainAuthToken,
+		ChainPk:         chainPk,
+		NaAddress:       common.HexToAddress(naAddress),
+		VpaAddress:      common.HexToAddress(vpaAddress),
+		CaAddress:       common.HexToAddress(caAddress),
+	}
+
+	storeOpts := nitroStore.StoreOpts{
+		PkBytes:            common.Hex2Bytes(pkString),
+		UseDurableStore:    useDurableStore,
+		DurableStoreFolder: durableStoreFolder,
+	}
+
+	bootPeers := []string{}
+	messageOpts := nitrop2pms.MessageOpts{
+		PkBytes:   common.Hex2Bytes(pkString),
+		TcpPort:   msgPort,
+		WsMsgPort: wsMsgPort,
+		BootPeers: bootPeers,
+		PublicIp:  publicIp,
+	}
+
+	ourStore, err := nitroStore.NewStore(storeOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Info("Initializing message service...", " tcp port=", msgPort, " web socket port=", wsMsgPort)
+	messageService := nitrop2pms.NewMessageService(messageOpts)
+
+	// Compare chainOpts.ChainStartBlock to lastBlockNum seen in store. The larger of the two
+	// gets passed as an argument when creating NewEthChainService
+	storeBlockNum, err := ourStore.GetLastBlockNumSeen()
+	if err != nil {
+		return nil, err
+	}
+	if storeBlockNum > chainOpts.ChainStartBlock {
+		chainOpts.ChainStartBlock = storeBlockNum
+	}
+
+	log.Info("Initializing chain service...")
+	ourChain, err := chainservice.NewEthChainService(chainOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	node := nitroNode.New(
+		messageService,
+		ourChain,
+		ourStore,
+		&engine.PermissivePolicy{},
+	)
+
+	return &node, nil
+}
+
+func initNitroRpcServer(node *nitroNode.Node, pm paymentsmanager.PaymentsManager, cert *tls.Certificate, rpcPort int) (*nitroRpc.RpcServer, error) {
+	var transport transport.Responder
+	var err error
+
+	slog.Info("Initializing Nitro HTTP RPC transport...")
+	transport, err = nitroHttpTransport.NewHttpTransportAsServer(fmt.Sprint(rpcPort), cert)
+	if err != nil {
+		return nil, err
+	}
+
+	rpcServer, err := nitroRpc.NewRpcServer(node, pm, transport)
+	if err != nil {
+		return nil, err
+	}
+
+	slog.Info("Completed Nitro RPC server initialization")
+	return rpcServer, nil
+}
+
+func readRpcQueryRates(filepath string) (map[string]*big.Int, error) {
+	result := make(map[string]*big.Int)
+
+	if filepath == "" {
+		logWithCommand.Warn("RPC query rates file path not provided")
+		return result, nil
+	}
+
+	jsonFile, err := os.Open(filepath)
+	defer jsonFile.Close()
+
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			logWithCommand.Warn("RPC query rates file does not exist")
+			return result, nil
+		}
+		return nil, err
+	}
+
+	decoder := json.NewDecoder(jsonFile)
+	err = decoder.Decode(&result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}

environments/example.toml

@@ -31,3 +31,27 @@
     clientName = "Geth" # $ETH_CLIENT_NAME
     genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
     networkID = "1" # $ETH_NETWORK_ID
+
+[nitro]
+    runNodeInProcess = false # NITRO_RUN_NODE_IN_PROCESS
+    rpcQueryRatesFile = "environments/rpcQueryRates.json" # NITRO_RPC_QUERY_RATES_FILE
+
+    [nitro.inProcesssNode]
+        pk = "" # NITRO_PK
+        chainPk = "" # NITRO_CHAIN_PK
+        chainUrl = "ws://127.0.0.1:8545" # NITRO_CHAIN_URL
+        naAddress = "" # NITRO_NA_ADDRESS
+        vpaAddress = "" # NITRO_VPA_ADDRESS
+        caAddress = "" # NITRO_CA_ADDRESS
+        useDurableStore = true # NITRO_USE_DURABLE_STORE
+        durableStoreFolder = "./data/nitronode" # NITRO_DURABLE_STORE_FOLDER
+        msgPort = 3005 # NITRO_MSG_PORT
+        rpcPort = 4005 # NITRO_RPC_PORT
+        wsMsgPort = 5005 # NITRO_WS_MSG_PORT
+        chainStartBlock = 0 # NITRO_CHAIN_START_BLOCK
+        tlsCertFilepath = "" # NITRO_TLS_CERT_FILEPATH
+        tlsKeyFilepath = "" # NITRO_TLS_KEY_FILEPATH
+
+    [nitro.remoteNode]
+        nitroEndpoint = "127.0.0.1:4005/api/v1" # NITRO_ENDPOINT
+        isSecure = false # NITRO_IS_SECURE

environments/rpcQueryRates.json

@@ -0,0 +1,6 @@
+{
+  "eth_getBlockByNumber": 50,
+  "eth_getBlockByHash": 50,
+  "eth_getStorageAt": 50,
+  "eth_getLogs": 50
+}

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/cerc-io/ipfs-ethdb/v5 v5.0.0-alpha
 	github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha
 	github.com/cerc-io/plugeth-statediff v0.1.1
-	github.com/ethereum/go-ethereum v1.11.6
+	github.com/ethereum/go-ethereum v1.12.0 // TODO: Investigate
 	github.com/google/uuid v1.3.0
 	github.com/graph-gophers/graphql-go v1.3.0
 	github.com/ipfs/go-cid v0.4.1
@@ -23,6 +23,8 @@ require (
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.11.0
+	github.com/statechannels/go-nitro v0.1.2
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
 	gorm.io/driver/postgres v1.3.7
 	gorm.io/gorm v1.23.5
 )
@@ -77,6 +79,7 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
@@ -230,6 +233,14 @@ require (
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
 	github.com/thoas/go-funk v0.9.3 // indirect
+	github.com/tidwall/btree v1.6.0 // indirect
+	github.com/tidwall/buntdb v1.2.10 // indirect
+	github.com/tidwall/gjson v1.14.4 // indirect
+	github.com/tidwall/grect v0.1.4 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/rtred v0.1.2 // indirect
+	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
@@ -254,7 +265,6 @@ require (
 	go.uber.org/zap v1.25.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
-	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
@@ -291,3 +301,5 @@ replace (
 	// https://git.vdb.to/cerc-io/ipfs-ethdb/src/branch/v5-go-upgrade
 	github.com/cerc-io/ipfs-ethdb/v5 => github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha.0.20231013070931-0b1a36562a28
 )
+
+replace github.com/statechannels/go-nitro v0.1.2 => github.com/cerc-io/go-nitro v0.1.2-ts-port-0.1.9

go.sum

@@ -91,12 +91,16 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
+github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo=
+github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA=
 github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U=
 github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 h1:KdUfX2zKommPRa+PD0sWZUyXe9w277ABlgELO7H04IM=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
+github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ=
+github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o=
 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
 github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
 github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
@@ -110,6 +114,8 @@ github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2
 github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.1.1-alpha.0.20231013075659-56aa03028c43 h1:pkGCN+VWo5Qmu4iDjA7noGrE6wM8VOVeX1Mn6ucYhPg=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.1.1-alpha.0.20231013075659-56aa03028c43/go.mod h1:snThUFpyCrpZhTuz3HibJRLL2XaS+lKNsM3XAE0gB/4=
+github.com/cerc-io/go-nitro v0.1.2-ts-port-0.1.9 h1:Q4hptIWBsSRQ9A16xQuPog2eihhspqwzP1Hm5TvahgA=
+github.com/cerc-io/go-nitro v0.1.2-ts-port-0.1.9/go.mod h1:YYQvj9es00ZfLTwxZLM1M0ihUrqz8+lU2c10G06My3A=
 github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha.0.20231013070931-0b1a36562a28 h1:5FXtMuZXTIXjjzzLdqgyzx9pjD22FB5os2vXayRn+BQ=
 github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha.0.20231013070931-0b1a36562a28/go.mod h1:W1C6qTXGsPcsK1HKUYPsXmBORjO2ekdm+101sJkpdNI=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
@@ -286,6 +292,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
+github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -789,6 +797,8 @@ github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJys
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo=
 github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
+github.com/miguelmota/go-ethereum-hdwallet v0.1.1 h1:zdXGlHao7idpCBjEGTXThVAtMKs+IxAgivZ75xqkWK0=
+github.com/miguelmota/go-ethereum-hdwallet v0.1.1/go.mod h1:f9m9uXokAHA6WNoYOPjj4AqjJS5pquQRiYYj/XSyPYc=
 github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8=
 github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms=
 github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc=
@@ -1052,6 +1062,28 @@ github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJ
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
 github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
+github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
+github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
+github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
+github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/tidwall/buntdb v1.2.10 h1:U/ebfkmYPBnyiNZIirUiWFcxA/mgzjbKlyPynFsPtyM=
+github.com/tidwall/buntdb v1.2.10/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
+github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
+github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
+github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
+github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
+github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
+github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
+github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
 github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
 github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
 github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=

pkg/rpc/http.go

@@ -18,6 +18,7 @@ package rpc
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -28,16 +29,23 @@ import (
 )
 
 // StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
-func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
-
+func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts, httpMiddlewares [](func(next http.Handler) http.Handler)) (*rpc.Server, error) {
 	srv := rpc.NewServer()
 	err := node.RegisterApis(apis, modules, srv)
 	if err != nil {
 		utils.Fatalf("Could not register HTTP API: %w", err)
 	}
-	handler := prom.HTTPMiddleware(node.NewHTTPHandlerStack(srv, cors, vhosts, nil))
+
+	promHandler := prom.HTTPMiddleware(node.NewHTTPHandlerStack(srv, cors, vhosts, nil))
+
+	// Chain the HTTP middlewares
+	handler := promHandler
+	for _, middleware := range httpMiddlewares {
+		handler = middleware(handler)
+	}
 
 	// start http server
+	// request -> payments -> metrics -> server
 	_, addr, err := node.StartHTTPEndpoint(endpoint, rpc.DefaultHTTPTimeouts, handler)
 	if err != nil {
 		utils.Fatalf("Could not start RPC api: %v", err)

pkg/serve/config.go

@@ -79,8 +79,56 @@ const (
 	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
 	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
 	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
+
+	NITRO_RUN_NODE_IN_PROCESS  = "NITRO_RUN_NODE_IN_PROCESS"
+	NITRO_RPC_QUERY_RATES_FILE = "NITRO_RPC_QUERY_RATES_FILE"
+	NITRO_PK                   = "NITRO_PK"
+	NITRO_CHAIN_PK             = "NITRO_CHAIN_PK"
+	NITRO_CHAIN_URL            = "NITRO_CHAIN_URL"
+	NITRO_NA_ADDRESS           = "NITRO_NA_ADDRESS"
+	NITRO_VPA_ADDRESS          = "NITRO_VPA_ADDRESS"
+	NITRO_CA_ADDRESS           = "NITRO_CA_ADDRESS"
+	NITRO_USE_DURABLE_STORE    = "NITRO_USE_DURABLE_STORE"
+	NITRO_DURABLE_STORE_FOLDER = "NITRO_DURABLE_STORE_FOLDER"
+	NITRO_ENDPOINT             = "NITRO_ENDPOINT"
+	NITRO_IS_SECURE            = "NITRO_IS_SECURE"
+	NITRO_MSG_PORT             = "NITRO_MSG_PORT"
+	NITRO_WS_MSG_PORT          = "NITRO_WS_MSG_PORT"
+	NITRO_RPC_PORT             = "NITRO_RPC_PORT"
+	NITRO_CHAIN_START_BLOCK    = "NITRO_CHAIN_START_BLOCK"
+	NITRO_TLS_CERT_FILEPATH    = "NITRO_TLS_CERT_FILEPATH"
+	NITRO_TLS_KEY_FILEPATH     = "NITRO_TLS_KEY_FILEPATH"
 )
 
+type InProcessNitroNodeConfig struct {
+	Pk                 string
+	ChainPk            string
+	ChainUrl           string
+	NaAddress          string
+	VpaAddress         string
+	CaAddress          string
+	UseDurableStore    bool
+	DurableStoreFolder string
+	RpcPort            int
+	MsgPort            int
+	WsMsgPort          int
+	ChainStartBlock    uint64
+	TlsCertFilepath    string
+	TlsKeyFilepath     string
+}
+
+type RemoteNitroNodeConfig struct {
+	NitroEndpoint string
+	IsSecure      bool
+}
+
+type NitroConfig struct {
+	RunNodeInProcess  bool
+	RpcQueryRatesFile string
+	InProcessNode     InProcessNitroNodeConfig
+	RemoteNode        RemoteNitroNodeConfig
+}
+
 // Config struct
 type Config struct {
 	DB       *sqlx.DB
@@ -116,6 +164,8 @@ type Config struct {
 
 	StateValidationEnabled       bool
 	StateValidationEveryNthBlock uint64
+
+	Nitro *NitroConfig
 }
 
 // NewConfig is used to initialize a watcher config from a .toml file
@@ -248,6 +298,8 @@ func NewConfig() (*Config, error) {
 
 	c.loadValidatorConfig()
 
+	c.loadNitroConfig()
+
 	return c, err
 }
 
@@ -280,6 +332,52 @@ func (c *Config) dbInit() {
 	c.DBConfig.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime"))
 }
 
+func (c *Config) loadNitroConfig() {
+	c.Nitro = &NitroConfig{InProcessNode: InProcessNitroNodeConfig{}, RemoteNode: RemoteNitroNodeConfig{}}
+
+	viper.BindEnv("nitro.runNodeInProcess", NITRO_RUN_NODE_IN_PROCESS)
+	viper.BindEnv("nitro.rpcQueryRatesFile", NITRO_RPC_QUERY_RATES_FILE)
+
+	viper.BindEnv("nitro.inProcesssNode.pk", NITRO_PK)
+	viper.BindEnv("nitro.inProcesssNode.chainPk", NITRO_CHAIN_PK)
+	viper.BindEnv("nitro.inProcesssNode.chainUrl", NITRO_CHAIN_URL)
+	viper.BindEnv("nitro.inProcesssNode.naAddress", NITRO_NA_ADDRESS)
+	viper.BindEnv("nitro.inProcesssNode.vpaAddress", NITRO_VPA_ADDRESS)
+	viper.BindEnv("nitro.inProcesssNode.caAddress", NITRO_CA_ADDRESS)
+	viper.BindEnv("nitro.inProcesssNode.useDurableStore", NITRO_USE_DURABLE_STORE)
+	viper.BindEnv("nitro.inProcesssNode.durableStoreFolder", NITRO_DURABLE_STORE_FOLDER)
+	viper.BindEnv("nitro.inProcesssNode.msgPort", NITRO_MSG_PORT)
+	viper.BindEnv("nitro.inProcesssNode.rpcPort", NITRO_RPC_PORT)
+	viper.BindEnv("nitro.inProcesssNode.wsMsgPort", NITRO_WS_MSG_PORT)
+	viper.BindEnv("nitro.inProcesssNode.chainStartBlock", NITRO_CHAIN_START_BLOCK)
+	viper.BindEnv("nitro.inProcesssNode.tlsCertFilepath", NITRO_TLS_CERT_FILEPATH)
+	viper.BindEnv("nitro.inProcesssNode.tlsKeyFilepath", NITRO_TLS_KEY_FILEPATH)
+
+	viper.BindEnv("nitro.remoteNode.nitroEndpoint", NITRO_ENDPOINT)
+	viper.BindEnv("nitro.remoteNode.isSecure", NITRO_IS_SECURE)
+
+	c.Nitro.RunNodeInProcess = viper.GetBool("nitro.runNodeInProcess")
+	c.Nitro.RpcQueryRatesFile = viper.GetString("nitro.rpcQueryRatesFile")
+
+	c.Nitro.InProcessNode.Pk = viper.GetString("nitro.inProcesssNode.pk")
+	c.Nitro.InProcessNode.ChainPk = viper.GetString("nitro.inProcesssNode.chainPk")
+	c.Nitro.InProcessNode.ChainUrl = viper.GetString("nitro.inProcesssNode.chainUrl")
+	c.Nitro.InProcessNode.NaAddress = viper.GetString("nitro.inProcesssNode.naAddress")
+	c.Nitro.InProcessNode.VpaAddress = viper.GetString("nitro.inProcesssNode.vpaAddress")
+	c.Nitro.InProcessNode.CaAddress = viper.GetString("nitro.inProcesssNode.caAddress")
+	c.Nitro.InProcessNode.UseDurableStore = viper.GetBool("nitro.inProcesssNode.useDurableStore")
+	c.Nitro.InProcessNode.DurableStoreFolder = viper.GetString("nitro.inProcesssNode.durableStoreFolder")
+	c.Nitro.InProcessNode.MsgPort = viper.GetInt("nitro.inProcesssNode.msgPort")
+	c.Nitro.InProcessNode.RpcPort = viper.GetInt("nitro.inProcesssNode.rpcPort")
+	c.Nitro.InProcessNode.WsMsgPort = viper.GetInt("nitro.inProcesssNode.wsMsgPort")
+	c.Nitro.InProcessNode.ChainStartBlock = viper.GetUint64("nitro.inProcesssNode.chainStartBlock")
+	c.Nitro.InProcessNode.TlsCertFilepath = viper.GetString("nitro.inProcesssNode.tlsCertFilepath")
+	c.Nitro.InProcessNode.TlsKeyFilepath = viper.GetString("nitro.inProcesssNode.tlsKeyFilepath")
+
+	c.Nitro.RemoteNode.NitroEndpoint = viper.GetString("nitro.remoteNode.nitroEndpoint")
+	c.Nitro.RemoteNode.IsSecure = viper.GetBool("nitro.remoteNode.isSecure")
+}
+
 func (c *Config) loadGroupCacheConfig() {
 	viper.BindEnv("groupcache.pool.enabled", ethServerShared.GcachePoolEnabled)
 	viper.BindEnv("groupcache.pool.httpEndpoint", ethServerShared.GcachePoolHttpPath)