From 8d10dc98ee0ac98cb3b06722e623c643d042ee35 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 5 Mar 2021 10:37:44 -0600 Subject: [PATCH] proxy command for graphQL endpoints directly ontop of Postgres indexes --- cmd/proxy.go | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 5 +- go.sum | 15 ++++++ 3 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 cmd/proxy.go diff --git a/cmd/proxy.go b/cmd/proxy.go new file mode 100644 index 00000000..9dd743f8 --- /dev/null +++ b/cmd/proxy.go @@ -0,0 +1,136 @@ +package cmd + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/ethereum/go-ethereum/rpc" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/vulcanize/gap-filler/pkg/mux" + "github.com/vulcanize/gap-filler/pkg/qlservices" +) + +var ErrNoRpcEndpoints = errors.New("no rpc endpoints is available") + +// proxyCmd represents the proxy command +var proxyCmd = &cobra.Command{ + Use: "proxy", + Short: "serve chain data from PG-IPFS or proxy geths", + Long: `This command configures a VulcanizeDB ipld-eth-server graphql server. + +`, + Run: func(cmd *cobra.Command, args []string) { + subCommand = cmd.CalledAs() + logWithCommand = *logrus.WithField("SubCommand", subCommand) + proxy() + }, +} + +func proxy() { + gqlDefaultAddr, err := url.Parse(viper.GetString("gql.default")) + if err != nil { + logWithCommand.Fatal(err) + } + + gqlTracingAPIAddr, err := url.Parse(viper.GetString("gql.tracing")) + if err != nil { + logWithCommand.Fatal(err) + } + + rpcClients, err := parseRpcAddresses(viper.GetString("rpc.eth")) + if err != nil { + logrus.Error("bad rpc.eth addresses") + logWithCommand.Fatal(err) + } + + rpcBalancer, err := qlservices.NewBalancer(rpcClients) + if err != nil { + logWithCommand.Fatal(err) + } + + tracingClients, err := parseRpcAddresses(viper.GetString("rpc.tracing")) + if err != nil { + logrus.Error("bad rpc.tracing addresses") + logWithCommand.Fatal(err) + } + + tracingBalancer, err := qlservices.NewBalancer(tracingClients) + if err != nil { + logWithCommand.Fatal(err) + } + + router, err := mux.NewServeMux(&mux.Options{ + BasePath: viper.GetString("http.path"), + EnableGraphiQL: viper.GetBool("gql.gui"), + Postgraphile: mux.PostgraphileOptions{ + Default: gqlDefaultAddr, + TracingAPI: gqlTracingAPIAddr, + }, + RPC: mux.RPCOptions{ + DefaultBalancer: rpcBalancer, + TracingBalancer: tracingBalancer, + }, + }) + if err != nil { + logWithCommand.Fatal(err) + } + + addr := fmt.Sprintf("%s:%s", viper.GetString("http.host"), viper.GetString("http.port")) + if err := http.ListenAndServe(addr, router); err != nil { + logWithCommand.Fatal(err) + } +} + +func parseRpcAddresses(value string) ([]*rpc.Client, error) { + rpcAddresses := strings.Split(value, ",") + rpcClients := make([]*rpc.Client, 0, len(rpcAddresses)) + for _, address := range rpcAddresses { + rpcClient, err := rpc.Dial(address) + if err != nil { + logWithCommand.Errorf("couldn't connect to %s. Error: %s", address, err) + continue + } + + rpcClients = append(rpcClients, rpcClient) + } + + if len(rpcClients) == 0 { + logWithCommand.Error(ErrNoRpcEndpoints) + return nil, ErrNoRpcEndpoints + } + + return rpcClients, nil +} + +func init() { + rootCmd.AddCommand(proxyCmd) + + // flags + proxyCmd.PersistentFlags().String("http-host", "127.0.0.1", "http host") + proxyCmd.PersistentFlags().String("http-port", "8080", "http port") + proxyCmd.PersistentFlags().String("http-path", "/", "http base path") + + proxyCmd.PersistentFlags().String("rpc-eth", "http://127.0.0.1:8545", "comma separated ethereum rpc addresses. Example http://127.0.0.1:8545,http://127.0.0.2:8545") + proxyCmd.PersistentFlags().String("rpc-tracing", "http://127.0.0.1:8000", "comma separated traicing api addresses") + + proxyCmd.PersistentFlags().String("gql-default", "http://127.0.0.1:5020/graphql", "postgraphile address") + proxyCmd.PersistentFlags().String("gql-tracing", "http://127.0.0.1:5020/graphql", "tracing api postgraphile address") + proxyCmd.PersistentFlags().Bool("gql-gui", false, "enable graphiql interface") + + // and their .toml config bindings + viper.BindPFlag("http.host", proxyCmd.PersistentFlags().Lookup("http-host")) + viper.BindPFlag("http.port", proxyCmd.PersistentFlags().Lookup("http-port")) + viper.BindPFlag("http.path", proxyCmd.PersistentFlags().Lookup("http-path")) + + viper.BindPFlag("rpc.eth", proxyCmd.PersistentFlags().Lookup("rpc-eth")) + viper.BindPFlag("rpc.tracing", proxyCmd.PersistentFlags().Lookup("rpc-tracing")) + + viper.BindPFlag("gql.default", proxyCmd.PersistentFlags().Lookup("gql-default")) + viper.BindPFlag("gql.tracing", proxyCmd.PersistentFlags().Lookup("gql-tracing")) + viper.BindPFlag("gql.gui", proxyCmd.PersistentFlags().Lookup("gql-gui")) +} diff --git a/go.mod b/go.mod index e03e8e1f..9bcb2c7f 100644 --- a/go.mod +++ b/go.mod @@ -16,10 +16,11 @@ require ( github.com/onsi/ginkgo v1.15.0 github.com/onsi/gomega v1.10.1 github.com/prometheus/client_golang v1.5.1 - github.com/sirupsen/logrus v1.6.0 - github.com/spf13/cobra v1.0.0 + github.com/sirupsen/logrus v1.7.0 + github.com/spf13/cobra v1.1.1 github.com/spf13/viper v1.7.0 github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha + github.com/vulcanize/gap-filler v0.3.0 github.com/vulcanize/ipfs-ethdb v0.0.2-alpha golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect ) diff --git a/go.sum b/go.sum index fa5e40a5..4049c556 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepB github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/friendsofgo/graphiql v0.2.2 h1:ccnuxpjgIkB+Lr9YB2ZouiZm7wvciSfqwpa9ugWzmn0= +github.com/friendsofgo/graphiql v0.2.2/go.mod h1:8Y2kZ36AoTGWs78+VRpvATyt3LJBx0SZXmay80ZTRWo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -237,6 +239,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e h1:IpssFbpfPSx/3c7x601Npx+UOQ4tqd0Rk4sObCQ+zlQ= github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graphql-go/graphql v0.7.9 h1:5Va/Rt4l5g3YjwDnid3vFfn43faaQBq7rMcIZ0VnV34= +github.com/graphql-go/graphql v0.7.9/go.mod h1:k6yrAYQaSP59DC5UVxbgxESlmVyojThKdORUqGDGmrI= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -929,6 +933,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= @@ -952,10 +958,14 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= @@ -989,8 +999,12 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= +github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/vulcanize/gap-filler v0.3.0 h1:SaBrOxlxCDPbSA902K/46iELH8yM3BqhvBuXK1ktFDI= +github.com/vulcanize/gap-filler v0.3.0/go.mod h1:4odsXyckNU1xdXk37qXY2SbxUM6oDpQrpWZtEHfq+0w= github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5 h1:U+BqhjRLR22e9OEm8cgWC3Eq3bh8G6azjNpXeenfCG4= github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ= github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.8 h1:7TK52k55uvSl+1SCKYYFelzH1NrpvEcDrpeU9nUDIpI= @@ -1338,6 +1352,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=