From 42923d15c1d35dbbc0e63b1520966bc9d15cb4af Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 09:25:47 -0400 Subject: [PATCH 1/6] Update cobra to require `head` or `historic` when using `capture`. --- README.md | 14 ++++++++++++++ cmd/capture.go | 17 +++++------------ cmd/head.go | 9 ++------- cmd/historic.go | 9 ++------- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 77b377e..88998e9 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,19 @@ This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. +# Running the Application + +To run the application, utilize the following command, and update the values as needed. + +``` +go run main.go capture head --db.address localhost \ + --db.password password \ + --db.port 8077 \ + --db.username username \ + --lh.address localhost \ + --lh.port 5052 +``` + # Contribution If you want to contribute please make sure you do the following: @@ -9,6 +22,7 @@ If you want to contribute please make sure you do the following: - Create a Github issue before starting your work. - Follow the branching structure. - Delete your branch once it has been merged. + - Do not delete the `develop` branch. We can add branch protection once we make the branch public. ## Branching Structure diff --git a/cmd/capture.go b/cmd/capture.go index b0e35c9..52b5315 100644 --- a/cmd/capture.go +++ b/cmd/capture.go @@ -5,24 +5,17 @@ Copyright © 2022 NAME HERE package cmd import ( - "fmt" - "github.com/spf13/cobra" ) // captureCmd represents the capture command var captureCmd = &cobra.Command{ Use: "capture", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("capture called") - }, + Short: "Capture the SignedBeaconBlocks and BeaconStates from the Beacon Chain", + Long: `Capture SignedBeaconBlocks and BeaconStates from the Beacon Chain. + These blocks and states will be captured in + Postgres. They require a lighthouse client to be connected. You can run this to + capture blocks and states at head or historic blocks.`, } func init() { diff --git a/cmd/head.go b/cmd/head.go index d5b7e2f..f0e4675 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -13,13 +13,8 @@ import ( // headCmd represents the head command var headCmd = &cobra.Command{ Use: "head", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "Capture only the blocks and state at head.", + Long: `Capture only the blocks and state at head.`, Run: func(cmd *cobra.Command, args []string) { fmt.Println("head called") }, diff --git a/cmd/historic.go b/cmd/historic.go index e29f167..7b6feeb 100644 --- a/cmd/historic.go +++ b/cmd/historic.go @@ -13,13 +13,8 @@ import ( // historicCmd represents the historic command var historicCmd = &cobra.Command{ Use: "historic", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "Capture the historic blocks and states.", + Long: `Capture the historic blocks and states.`, Run: func(cmd *cobra.Command, args []string) { fmt.Println("historic called") }, -- 2.45.2 From ed3d0be2b78c22639d632b27fd579bfe95d3d630 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 12:12:55 -0400 Subject: [PATCH 2/6] A very generic package for implementing PGX driver I copied most of the code from the `statediff` service from within geth. The idea is that I can create formal DB packages, that can be utilized in other projects down the road. --- .gitignore | 2 + go.mod | 15 +++ go.sum | 159 ++++++++++++++++++++++ pkg/dbtools/sql/error.go | 22 ++++ pkg/dbtools/sql/interfaces.go | 54 ++++++++ pkg/dbtools/sql/postgres/config.go | 75 +++++++++++ pkg/dbtools/sql/postgres/pgx.go | 190 +++++++++++++++++++++++++++ pkg/dbtools/sql/postgres/pgx_test.go | 89 +++++++++++++ pkg/testhelpers/testhelper.go | 23 ++++ 9 files changed, 629 insertions(+) create mode 100644 .gitignore create mode 100644 pkg/dbtools/sql/error.go create mode 100644 pkg/dbtools/sql/interfaces.go create mode 100644 pkg/dbtools/sql/postgres/config.go create mode 100644 pkg/dbtools/sql/postgres/pgx.go create mode 100644 pkg/dbtools/sql/postgres/pgx_test.go create mode 100644 pkg/testhelpers/testhelper.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..48e5153 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +ipld-ethcl-indexer diff --git a/go.mod b/go.mod index 5b9b916..b96803c 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,25 @@ go 1.18 require github.com/sirupsen/logrus v1.8.1 +require ( + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.10.0 // indirect + github.com/jackc/puddle v1.2.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect +) + require ( github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/georgysavva/scany v0.3.0 github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jackc/pgx/v4 v4.15.0 github.com/magiconair/properties v1.8.6 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/pelletier/go-toml v1.9.4 // indirect diff --git a/go.sum b/go.sum index 855bce8..2e87236 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -46,7 +47,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cockroach-go/v2 v2.2.0/go.mod h1:u3MiKYGupPPjkn3ozknpMUpxPaNLTFWAya419/zv6eI= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -57,9 +63,18 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/georgysavva/scany v0.3.0 h1:MA1aEqPbnNuiek59gMpNPqQrXXroyFj5jCADlETdxiA= +github.com/georgysavva/scany v0.3.0/go.mod h1:q8QyrfXjmBk9iJD00igd4lbkAKEXAH/zIYoZ0z/Wan4= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -122,27 +137,123 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.0-beta.8 h1:dy81yyLYJDwMTifq24Oi/IslOslRrDSb3jwDggjz3Z0= github.com/pelletier/go-toml/v2 v2.0.0-beta.8/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= @@ -158,7 +269,10 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.11.0 h1:7OX/1FS6n7jHD1zGrZTM7WtY13ZELRyosK4k93oPr44= github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -169,19 +283,41 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -227,6 +363,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -266,20 +403,27 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -302,8 +446,10 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -311,6 +457,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -323,14 +471,18 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -338,6 +490,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -363,6 +516,8 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -458,6 +613,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -466,6 +622,9 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= +gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/dbtools/sql/error.go b/pkg/dbtools/sql/error.go new file mode 100644 index 0000000..9701968 --- /dev/null +++ b/pkg/dbtools/sql/error.go @@ -0,0 +1,22 @@ +package sql + +import ( + "fmt" +) + +const ( + DbConnectionFailedMsg = "db connection failed" + SettingNodeFailedMsg = "unable to set db node" +) + +func ErrDBConnectionFailed(connectErr error) error { + return formatError(DbConnectionFailedMsg, connectErr.Error()) +} + +func ErrUnableToSetNode(setErr error) error { + return formatError(SettingNodeFailedMsg, setErr.Error()) +} + +func formatError(msg, err string) error { + return fmt.Errorf("%s: %s", msg, err) +} diff --git a/pkg/dbtools/sql/interfaces.go b/pkg/dbtools/sql/interfaces.go new file mode 100644 index 0000000..095b89b --- /dev/null +++ b/pkg/dbtools/sql/interfaces.go @@ -0,0 +1,54 @@ +package sql + +import ( + "context" + "io" + "time" +) + +// Database interfaces required by the sql indexer +type Database interface { + Driver +} + +// Driver interface has all the methods required by a driver implementation to support the sql indexer +type Driver interface { + QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow + Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) + Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error + Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error + Begin(ctx context.Context) (Tx, error) + Stats() Stats + Context() context.Context + io.Closer +} + +// Tx interface to accommodate different concrete SQL transaction types +type Tx interface { + QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow + Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) + Commit(ctx context.Context) error + Rollback(ctx context.Context) error +} + +// ScannableRow interface to accommodate different concrete row types +type ScannableRow interface { + Scan(dest ...interface{}) error +} + +// Result interface to accommodate different concrete result types +type Result interface { + RowsAffected() (int64, error) +} + +// Stats interface to accommodate different concrete sql stats types +type Stats interface { + MaxOpen() int64 + Open() int64 + InUse() int64 + Idle() int64 + WaitCount() int64 + WaitDuration() time.Duration + MaxIdleClosed() int64 + MaxLifetimeClosed() int64 +} diff --git a/pkg/dbtools/sql/postgres/config.go b/pkg/dbtools/sql/postgres/config.go new file mode 100644 index 0000000..2a908d8 --- /dev/null +++ b/pkg/dbtools/sql/postgres/config.go @@ -0,0 +1,75 @@ +package postgres + +import ( + "fmt" + "strings" + "time" +) + +// DriverType to explicitly type the kind of sql driver we are using +type DriverType string + +const ( + PGX DriverType = "PGX" + SQLX DriverType = "SQLX" + Unknown DriverType = "Unknown" +) + +// DefaultConfig are default parameters for connecting to a Postgres sql +var DefaultConfig = Config{ + Hostname: "localhost", + Port: 8077, + DatabaseName: "vulcanize_testing", + Username: "vdbm", + Password: "password", +} + +// ResolveDriverType resolves a DriverType from a provided string +func ResolveDriverType(str string) (DriverType, error) { + switch strings.ToLower(str) { + case "pgx", "pgxpool": + return PGX, nil + case "sqlx": + return SQLX, nil + default: + return Unknown, fmt.Errorf("unrecognized driver type string: %s", str) + } +} + +// Config holds params for a Postgres db +type Config struct { + // conn string params + Hostname string + Port int + DatabaseName string + Username string + Password string + + // conn settings + MaxConns int + MaxIdle int + MinConns int + MaxConnIdleTime time.Duration + MaxConnLifetime time.Duration + ConnTimeout time.Duration + + // node info params + ID string + ClientName string + + // driver type + Driver DriverType +} + +// DbConnectionString constructs and returns the connection string from the config +func (c Config) DbConnectionString() string { + if len(c.Username) > 0 && len(c.Password) > 0 { + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable", + c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName) + } + if len(c.Username) > 0 && len(c.Password) == 0 { + return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable", + c.Username, c.Hostname, c.Port, c.DatabaseName) + } + return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName) +} diff --git a/pkg/dbtools/sql/postgres/pgx.go b/pkg/dbtools/sql/postgres/pgx.go new file mode 100644 index 0000000..4749168 --- /dev/null +++ b/pkg/dbtools/sql/postgres/pgx.go @@ -0,0 +1,190 @@ +package postgres + +import ( + "context" + "time" + + "github.com/georgysavva/scany/pgxscan" + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" +) + +// PGXDriver driver, implements sql.Driver +type PGXDriver struct { + ctx context.Context + pool *pgxpool.Pool +} + +// NewPGXDriver returns a new pgx driver. +// It initializes the connection pool. +func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) { + pgConf, err := MakeConfig(config) + if err != nil { + return nil, err + } + dbPool, err := pgxpool.ConnectConfig(ctx, pgConf) + if err != nil { + return nil, sql.ErrDBConnectionFailed(err) + } + pg := &PGXDriver{ctx: ctx, pool: dbPool} + return pg, nil +} + +// MakeConfig creates a pgxpool.Config from the provided Config +func MakeConfig(config Config) (*pgxpool.Config, error) { + conf, err := pgxpool.ParseConfig("") + if err != nil { + return nil, err + } + + //conf.ConnConfig.BuildStatementCache = nil + conf.ConnConfig.Config.Host = config.Hostname + conf.ConnConfig.Config.Port = uint16(config.Port) + conf.ConnConfig.Config.Database = config.DatabaseName + conf.ConnConfig.Config.User = config.Username + conf.ConnConfig.Config.Password = config.Password + + if config.ConnTimeout != 0 { + conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout + } + if config.MaxConns != 0 { + conf.MaxConns = int32(config.MaxConns) + } + if config.MinConns != 0 { + conf.MinConns = int32(config.MinConns) + } + if config.MaxConnLifetime != 0 { + conf.MaxConnLifetime = config.MaxConnLifetime + } + if config.MaxConnIdleTime != 0 { + conf.MaxConnIdleTime = config.MaxConnIdleTime + } + return conf, nil +} + +// QueryRow satisfies sql.Database +func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { + return pgx.pool.QueryRow(ctx, sql, args...) +} + +// Exec satisfies sql.Database +func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { + res, err := pgx.pool.Exec(ctx, sql, args...) + return resultWrapper{ct: res}, err +} + +// Select satisfies sql.Database +func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return pgxscan.Select(ctx, pgx.pool, dest, query, args...) +} + +// Get satisfies sql.Database +func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return pgxscan.Get(ctx, pgx.pool, dest, query, args...) +} + +// Begin satisfies sql.Database +func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) { + tx, err := pgx.pool.Begin(ctx) + if err != nil { + return nil, err + } + return pgxTxWrapper{tx: tx}, nil +} + +func (pgx *PGXDriver) Stats() sql.Stats { + stats := pgx.pool.Stat() + return pgxStatsWrapper{stats: stats} +} + +// Close satisfies sql.Database/io.Closer +func (pgx *PGXDriver) Close() error { + pgx.pool.Close() + return nil +} + +// Context satisfies sql.Database +func (pgx *PGXDriver) Context() context.Context { + return pgx.ctx +} + +type resultWrapper struct { + ct pgconn.CommandTag +} + +// RowsAffected satisfies sql.Result +func (r resultWrapper) RowsAffected() (int64, error) { + return r.ct.RowsAffected(), nil +} + +type pgxStatsWrapper struct { + stats *pgxpool.Stat +} + +// MaxOpen satisfies sql.Stats +func (s pgxStatsWrapper) MaxOpen() int64 { + return int64(s.stats.MaxConns()) +} + +// Open satisfies sql.Stats +func (s pgxStatsWrapper) Open() int64 { + return int64(s.stats.TotalConns()) +} + +// InUse satisfies sql.Stats +func (s pgxStatsWrapper) InUse() int64 { + return int64(s.stats.AcquiredConns()) +} + +// Idle satisfies sql.Stats +func (s pgxStatsWrapper) Idle() int64 { + return int64(s.stats.IdleConns()) +} + +// WaitCount satisfies sql.Stats +func (s pgxStatsWrapper) WaitCount() int64 { + return s.stats.EmptyAcquireCount() +} + +// WaitDuration satisfies sql.Stats +func (s pgxStatsWrapper) WaitDuration() time.Duration { + return s.stats.AcquireDuration() +} + +// MaxIdleClosed satisfies sql.Stats +func (s pgxStatsWrapper) MaxIdleClosed() int64 { + // this stat isn't supported by pgxpool, but we don't want to panic + return 0 +} + +// MaxLifetimeClosed satisfies sql.Stats +func (s pgxStatsWrapper) MaxLifetimeClosed() int64 { + return s.stats.CanceledAcquireCount() +} + +type pgxTxWrapper struct { + tx pgx.Tx +} + +// QueryRow satisfies sql.Tx +func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { + return t.tx.QueryRow(ctx, sql, args...) +} + +// Exec satisfies sql.Tx +func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { + res, err := t.tx.Exec(ctx, sql, args...) + return resultWrapper{ct: res}, err +} + +// Commit satisfies sql.Tx +func (t pgxTxWrapper) Commit(ctx context.Context) error { + return t.tx.Commit(ctx) +} + +// Rollback satisfies sql.Tx +func (t pgxTxWrapper) Rollback(ctx context.Context) error { + return t.tx.Rollback(ctx) +} diff --git a/pkg/dbtools/sql/postgres/pgx_test.go b/pkg/dbtools/sql/postgres/pgx_test.go new file mode 100644 index 0000000..782c426 --- /dev/null +++ b/pkg/dbtools/sql/postgres/pgx_test.go @@ -0,0 +1,89 @@ +package postgres + +import ( + "context" + "math/big" + "strings" + "testing" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers" +) + +var ( + pgConfig, _ = MakeConfig(DefaultConfig) + ctx = context.Background() +) + +func expectContainsSubstring(t *testing.T, full string, sub string) { + if !strings.Contains(full, sub) { + t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub) + } +} + +func TestPostgresPGX(t *testing.T) { + t.Run("connects to the sql", func(t *testing.T) { + dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err) + } + if dbPool == nil { + t.Fatal("DB pool is nil") + } + dbPool.Close() + }) + + t.Run("serializes big.Int to db", func(t *testing.T) { + // postgres driver doesn't support go big.Int type + // various casts in golang uint64, int64, overflow for + // transaction value (in wei) even though + // postgres numeric can handle an arbitrary + // sized int, so use string representation of big.Int + // and cast on insert + + dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err) + } + defer dbPool.Close() + + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + testhelpers.ExpectEqual(t, bi.String(), "34940183920000000000") + + defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) + _, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )") + if err != nil { + t.Fatal(err) + } + + sqlStatement := ` + INSERT INTO example (id, data) + VALUES (1, cast($1 AS NUMERIC))` + _, err = dbPool.Exec(ctx, sqlStatement, bi.String()) + if err != nil { + t.Fatal(err) + } + + var data string + err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data) + if err != nil { + t.Fatal(err) + } + + testhelpers.ExpectEqual(t, data, bi.String()) + actual := new(big.Int) + actual.SetString(data, 10) + testhelpers.ExpectEqual(t, actual, bi) + }) + + t.Run("throws error when can't connect to the database", func(t *testing.T) { + _, err := NewPGXDriver(ctx, Config{}) + if err == nil { + t.Fatal("Expected an error") + } + + expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) + }) +} diff --git a/pkg/testhelpers/testhelper.go b/pkg/testhelpers/testhelper.go new file mode 100644 index 0000000..4e3c917 --- /dev/null +++ b/pkg/testhelpers/testhelper.go @@ -0,0 +1,23 @@ +package testhelpers + +import ( + "reflect" + "testing" +) + +// ExpectEqual asserts the provided interfaces are deep equal +func ExpectEqual(t *testing.T, got interface{}, want interface{}) { + if !reflect.DeepEqual(got, want) { + t.Fatalf("Expected: %v\nActual: %v", want, got) + } +} + +// ListContainsString used to check if a list of strings contains a particular string +func ListContainsString(sss []string, s string) bool { + for _, str := range sss { + if s == str { + return true + } + } + return false +} -- 2.45.2 From d0d4f2498ef4382ecd675549167855d495661a49 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 13:06:00 -0400 Subject: [PATCH 3/6] Put PGX and other future Postgres Drivers behind SQL package This PR makes allows users to provide a config, along with a driver string. It will then provide the respective driver. --- pkg/dbtools/sql/postgres/database.go | 51 ++++++++++++++++++++++++++++ pkg/dbtools/sql/postgres/pgx.go | 32 ++++++++--------- pkg/dbtools/sql/postgres/pgx_test.go | 13 +++++-- 3 files changed, 78 insertions(+), 18 deletions(-) create mode 100644 pkg/dbtools/sql/postgres/database.go diff --git a/pkg/dbtools/sql/postgres/database.go b/pkg/dbtools/sql/postgres/database.go new file mode 100644 index 0000000..3297911 --- /dev/null +++ b/pkg/dbtools/sql/postgres/database.go @@ -0,0 +1,51 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" +) + +var _ sql.Database = &DB{} + +// TODO: Make NewPostgresDB accept a string and Config. IT should +// Create a driver of its own. +// This will make sure that if you want a driver, it conforms to the interface. + +// NewPostgresDB returns a postgres.DB using the provided Config and driver type. +func NewPostgresDB(c Config, driverName string) (*DB, error) { + var driver *pgxDriver + + driverType, err := ResolveDriverType(driverName) + if err != nil { + return nil, err + } + + driver, err = createDriver(c, driverType) + + if err != nil { + return nil, err + } + + return &DB{driver}, nil +} + +func createDriver(c Config, driverType DriverType) (*pgxDriver, error) { + switch driverType { + case PGX: + driver, err := newPGXDriver(context.Background(), c) + if err != nil { + return nil, fmt.Errorf("Error Creating Driver, err: %e", err) + } + return driver, nil + default: + return nil, fmt.Errorf("Can't find a driver to create") + } + +} + +// DB implements sql.Database using a configured driver and Postgres statement syntax +type DB struct { + sql.Driver +} diff --git a/pkg/dbtools/sql/postgres/pgx.go b/pkg/dbtools/sql/postgres/pgx.go index 4749168..bd0388f 100644 --- a/pkg/dbtools/sql/postgres/pgx.go +++ b/pkg/dbtools/sql/postgres/pgx.go @@ -11,16 +11,16 @@ import ( "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" ) -// PGXDriver driver, implements sql.Driver -type PGXDriver struct { +// pgxDriver driver, implements sql.Driver +type pgxDriver struct { ctx context.Context pool *pgxpool.Pool } -// NewPGXDriver returns a new pgx driver. +// newPGXDriver returns a new pgx driver. // It initializes the connection pool. -func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) { - pgConf, err := MakeConfig(config) +func newPGXDriver(ctx context.Context, config Config) (*pgxDriver, error) { + pgConf, err := makeConfig(config) if err != nil { return nil, err } @@ -28,12 +28,12 @@ func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) { if err != nil { return nil, sql.ErrDBConnectionFailed(err) } - pg := &PGXDriver{ctx: ctx, pool: dbPool} + pg := &pgxDriver{ctx: ctx, pool: dbPool} return pg, nil } -// MakeConfig creates a pgxpool.Config from the provided Config -func MakeConfig(config Config) (*pgxpool.Config, error) { +// makeConfig creates a pgxpool.Config from the provided Config +func makeConfig(config Config) (*pgxpool.Config, error) { conf, err := pgxpool.ParseConfig("") if err != nil { return nil, err @@ -65,28 +65,28 @@ func MakeConfig(config Config) (*pgxpool.Config, error) { } // QueryRow satisfies sql.Database -func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { +func (pgx *pgxDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { return pgx.pool.QueryRow(ctx, sql, args...) } // Exec satisfies sql.Database -func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { +func (pgx *pgxDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { res, err := pgx.pool.Exec(ctx, sql, args...) return resultWrapper{ct: res}, err } // Select satisfies sql.Database -func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error { +func (pgx *pgxDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error { return pgxscan.Select(ctx, pgx.pool, dest, query, args...) } // Get satisfies sql.Database -func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error { +func (pgx *pgxDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error { return pgxscan.Get(ctx, pgx.pool, dest, query, args...) } // Begin satisfies sql.Database -func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) { +func (pgx *pgxDriver) Begin(ctx context.Context) (sql.Tx, error) { tx, err := pgx.pool.Begin(ctx) if err != nil { return nil, err @@ -94,19 +94,19 @@ func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) { return pgxTxWrapper{tx: tx}, nil } -func (pgx *PGXDriver) Stats() sql.Stats { +func (pgx *pgxDriver) Stats() sql.Stats { stats := pgx.pool.Stat() return pgxStatsWrapper{stats: stats} } // Close satisfies sql.Database/io.Closer -func (pgx *PGXDriver) Close() error { +func (pgx *pgxDriver) Close() error { pgx.pool.Close() return nil } // Context satisfies sql.Database -func (pgx *PGXDriver) Context() context.Context { +func (pgx *pgxDriver) Context() context.Context { return pgx.ctx } diff --git a/pkg/dbtools/sql/postgres/pgx_test.go b/pkg/dbtools/sql/postgres/pgx_test.go index 782c426..65daf96 100644 --- a/pkg/dbtools/sql/postgres/pgx_test.go +++ b/pkg/dbtools/sql/postgres/pgx_test.go @@ -12,7 +12,7 @@ import ( ) var ( - pgConfig, _ = MakeConfig(DefaultConfig) + pgConfig, _ = makeConfig(DefaultConfig) ctx = context.Background() ) @@ -79,11 +79,20 @@ func TestPostgresPGX(t *testing.T) { }) t.Run("throws error when can't connect to the database", func(t *testing.T) { - _, err := NewPGXDriver(ctx, Config{}) + _, err := NewPostgresDB(Config{}, "PGX") if err == nil { t.Fatal("Expected an error") } expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) }) + t.Run("Connect to the database", func(t *testing.T) { + driver, err := NewPostgresDB(DefaultConfig, "pgx") + defer driver.Close() + + if err != nil { + t.Fatal("Error creating the postgres driver") + } + + }) } -- 2.45.2 From 827475f029a5f2f0586de61facd22a7d1cd2a45d Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 15:44:15 -0400 Subject: [PATCH 4/6] Add DB Connection and Logging * Utilize LogRus * Create a DB connection using PGX. * Create an internal boot package for starting the application. --- .gitignore | 1 + cmd/head.go | 11 +++ cmd/root.go | 76 ++++++++++++++++--- internal/boot/boot.go | 23 ++++++ internal/boot/setup_database.go | 29 +++++++ internal/boot/startup.md | 3 + pkg/{dbtools => database}/sql/error.go | 0 pkg/{dbtools => database}/sql/interfaces.go | 0 .../sql/postgres/config.go | 4 - .../sql/postgres/database.go | 19 +++-- pkg/{dbtools => database}/sql/postgres/pgx.go | 2 +- .../sql/postgres/pgx_test.go | 2 +- .../{testhelper.go => test_helper.go} | 0 13 files changed, 143 insertions(+), 27 deletions(-) create mode 100644 internal/boot/boot.go create mode 100644 internal/boot/setup_database.go create mode 100644 internal/boot/startup.md rename pkg/{dbtools => database}/sql/error.go (100%) rename pkg/{dbtools => database}/sql/interfaces.go (100%) rename pkg/{dbtools => database}/sql/postgres/config.go (96%) rename pkg/{dbtools => database}/sql/postgres/database.go (67%) rename pkg/{dbtools => database}/sql/postgres/pgx.go (98%) rename pkg/{dbtools => database}/sql/postgres/pgx_test.go (97%) rename pkg/testhelpers/{testhelper.go => test_helper.go} (100%) diff --git a/.gitignore b/.gitignore index 48e5153..a0fdb30 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ ipld-ethcl-indexer +ipld-ethcl-indexer.log diff --git a/cmd/head.go b/cmd/head.go index f0e4675..2d297a3 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -7,7 +7,10 @@ package cmd import ( "fmt" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" ) // headCmd represents the head command @@ -17,9 +20,17 @@ var headCmd = &cobra.Command{ Long: `Capture only the blocks and state at head.`, Run: func(cmd *cobra.Command, args []string) { fmt.Println("head called") + startHeadTracking() }, } +func startHeadTracking() { + _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver) + if err != nil { + log.Fatal("Unable to Start application with error: ", err) + } +} + func init() { captureCmd.AddCommand(headCmd) diff --git a/cmd/root.go b/cmd/root.go index ef743fd..05b9682 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,18 +8,22 @@ import ( "fmt" "os" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - dbUserName string - dbPassword string - dbAddress string - dbPort uint16 - lhAddress string - lhPort uint16 + cfgFile string + dbUsername string + dbPassword string + dbName string + dbAddress string + dbDriver string + dbPort int + lhAddress string + lhPort uint16 + logWithCommand log.Entry ) // rootCmd represents the base command when called without any subcommands @@ -28,6 +32,7 @@ var rootCmd = &cobra.Command{ Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.", Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain. It can either do this will keeping track of head, or backfilling historic data.`, + PersistentPreRun: initFuncs, // Uncomment the following line if your bare application // has an action associated with it: // Run: func(cmd *cobra.Command, args []string) {}, @@ -42,6 +47,43 @@ func Execute() { } } +// Prerun for Cobra +func initFuncs(cmd *cobra.Command, args []string) { + viper.BindEnv("log.file", "LOGRUS_FILE") + logfile := viper.GetString("log.file") + if logfile != "" { + file, err := os.OpenFile(logfile, + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + log.Infof("Directing output to %s", logfile) + log.SetOutput(file) + } else { + log.SetOutput(os.Stdout) + log.Info("Failed to log to file, using default stdout") + } + } else { + log.SetOutput(os.Stdout) + } + if err := logLevel(); err != nil { + log.Fatal("Could not set log level: ", err) + } +} + +// Set the log level for the application +func logLevel() error { + viper.BindEnv("log.level", "LOGRUS_LEVEL") + lvl, err := log.ParseLevel(viper.GetString("log.level")) + if err != nil { + return err + } + log.SetLevel(lvl) + if lvl > log.InfoLevel { + log.SetReportCaller(true) + } + log.Info("Log level set to ", lvl.String()) + return nil +} + func init() { cobra.OnInitialize(initConfig) @@ -51,18 +93,24 @@ func init() { // Optional Flags rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") + rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") + rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging") // Required Flags //// DB Specific - rootCmd.PersistentFlags().StringVarP(&dbUserName, "db.username", "u", "", "Database username (required)") - rootCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "p", "", "Database Password (required)") - rootCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "a", "", "Port to connect to DB(required)") - rootCmd.PersistentFlags().Uint16VarP(&dbPort, "db.port", "o", 0, "Port to connect to DB(required)") + rootCmd.PersistentFlags().StringVarP(&dbUsername, "db.username", "", "", "Database username (required)") + rootCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "", "", "Database Password (required)") + rootCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "", "", "Port to connect to DB(required)") + rootCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)") + rootCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)") + rootCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)") rootCmd.MarkPersistentFlagRequired("db.username") rootCmd.MarkPersistentFlagRequired("db.password") rootCmd.MarkPersistentFlagRequired("db.address") rootCmd.MarkPersistentFlagRequired("db.port") + rootCmd.MarkPersistentFlagRequired("db.name") + rootCmd.MarkPersistentFlagRequired("db.driver") //// Lighthouse Specific rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)") @@ -71,11 +119,17 @@ func init() { rootCmd.MarkPersistentFlagRequired("lh.port") // Bind Flags with Viper + // Optional + viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) + viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file")) + //// DB Flags viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username")) viper.BindPFlag("db.password", rootCmd.PersistentFlags().Lookup("db.password")) viper.BindPFlag("db.address", rootCmd.PersistentFlags().Lookup("db.address")) viper.BindPFlag("db.port", rootCmd.PersistentFlags().Lookup("db.port")) + viper.BindPFlag("db.name", rootCmd.PersistentFlags().Lookup("db.name")) + viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver")) // LH specific viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address")) diff --git a/internal/boot/boot.go b/internal/boot/boot.go new file mode 100644 index 0000000..e251a5b --- /dev/null +++ b/internal/boot/boot.go @@ -0,0 +1,23 @@ +package boot + +import ( + log "github.com/sirupsen/logrus" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" +) + +func setUpLightHouse() { + +} + +// This function will perform some boot operations. +// 1. Setup a logger +// 2. Connect to the database. +// 3. Connect to to the lighthouse client. +func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { + log.Debug("Setting up DB connection") + DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) + if err != nil { + return nil, err + } + return DB, nil +} diff --git a/internal/boot/setup_database.go b/internal/boot/setup_database.go new file mode 100644 index 0000000..a1efaf0 --- /dev/null +++ b/internal/boot/setup_database.go @@ -0,0 +1,29 @@ +// This file will allow users to setup a new DB based on the user provided inputs. + +package boot + +import ( + log "github.com/sirupsen/logrus" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" +) + +func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { + log.Debug("Resolving Driver Type") + DbDriver, err := postgres.ResolveDriverType(driverName) + if err != nil { + log.Fatal("Can't Connect to DB") + } + log.Info("Using Driver:", DbDriver) + + postgresConfig := postgres.Config{ + Hostname: dbHostname, + Port: dbPort, + DatabaseName: dbName, + Username: dbUsername, + Password: dbPassword, + Driver: DbDriver, + } + DB, err := postgres.NewPostgresDB(postgresConfig) + return DB, err + +} diff --git a/internal/boot/startup.md b/internal/boot/startup.md new file mode 100644 index 0000000..72ec31f --- /dev/null +++ b/internal/boot/startup.md @@ -0,0 +1,3 @@ +# Overview + +This small package will group all the functions needed to start the application. The functions listed here can be utilized regardless of whether the application is meant to track `head` or for `historic` processing. diff --git a/pkg/dbtools/sql/error.go b/pkg/database/sql/error.go similarity index 100% rename from pkg/dbtools/sql/error.go rename to pkg/database/sql/error.go diff --git a/pkg/dbtools/sql/interfaces.go b/pkg/database/sql/interfaces.go similarity index 100% rename from pkg/dbtools/sql/interfaces.go rename to pkg/database/sql/interfaces.go diff --git a/pkg/dbtools/sql/postgres/config.go b/pkg/database/sql/postgres/config.go similarity index 96% rename from pkg/dbtools/sql/postgres/config.go rename to pkg/database/sql/postgres/config.go index 2a908d8..a518bed 100644 --- a/pkg/dbtools/sql/postgres/config.go +++ b/pkg/database/sql/postgres/config.go @@ -53,10 +53,6 @@ type Config struct { MaxConnLifetime time.Duration ConnTimeout time.Duration - // node info params - ID string - ClientName string - // driver type Driver DriverType } diff --git a/pkg/dbtools/sql/postgres/database.go b/pkg/database/sql/postgres/database.go similarity index 67% rename from pkg/dbtools/sql/postgres/database.go rename to pkg/database/sql/postgres/database.go index 3297911..9d44823 100644 --- a/pkg/dbtools/sql/postgres/database.go +++ b/pkg/database/sql/postgres/database.go @@ -4,7 +4,8 @@ import ( "context" "fmt" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" + log "github.com/sirupsen/logrus" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" ) var _ sql.Database = &DB{} @@ -14,15 +15,10 @@ var _ sql.Database = &DB{} // This will make sure that if you want a driver, it conforms to the interface. // NewPostgresDB returns a postgres.DB using the provided Config and driver type. -func NewPostgresDB(c Config, driverName string) (*DB, error) { +func NewPostgresDB(c Config) (*DB, error) { var driver *pgxDriver - driverType, err := ResolveDriverType(driverName) - if err != nil { - return nil, err - } - - driver, err = createDriver(c, driverType) + driver, err := createDriver(c) if err != nil { return nil, err @@ -31,15 +27,18 @@ func NewPostgresDB(c Config, driverName string) (*DB, error) { return &DB{driver}, nil } -func createDriver(c Config, driverType DriverType) (*pgxDriver, error) { - switch driverType { +func createDriver(c Config) (*pgxDriver, error) { + switch c.Driver { case PGX: + log.Debug("Creating New Driver") driver, err := newPGXDriver(context.Background(), c) if err != nil { return nil, fmt.Errorf("Error Creating Driver, err: %e", err) } + log.Info("Successfully created a driver for PGX") return driver, nil default: + log.Fatal("Couldnt find a driver to create for: ", c.Driver) return nil, fmt.Errorf("Can't find a driver to create") } diff --git a/pkg/dbtools/sql/postgres/pgx.go b/pkg/database/sql/postgres/pgx.go similarity index 98% rename from pkg/dbtools/sql/postgres/pgx.go rename to pkg/database/sql/postgres/pgx.go index bd0388f..3981b33 100644 --- a/pkg/dbtools/sql/postgres/pgx.go +++ b/pkg/database/sql/postgres/pgx.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" ) // pgxDriver driver, implements sql.Driver diff --git a/pkg/dbtools/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go similarity index 97% rename from pkg/dbtools/sql/postgres/pgx_test.go rename to pkg/database/sql/postgres/pgx_test.go index 65daf96..ab75a7e 100644 --- a/pkg/dbtools/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/jackc/pgx/v4/pgxpool" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers" ) diff --git a/pkg/testhelpers/testhelper.go b/pkg/testhelpers/test_helper.go similarity index 100% rename from pkg/testhelpers/testhelper.go rename to pkg/testhelpers/test_helper.go -- 2.45.2 From d7ad4108a7b28624f00d141debbead0dbe20bab9 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 18:12:44 -0400 Subject: [PATCH 5/6] Code clean up + Beacon Chain Connection This concludes all the code needed to connect to the DB and beacon node. We will no longer reference the lighthouse client because this application should work interchangeably with any beacon node. I have also standardized logging. --- README.md | 33 +++++++-- cmd/capture.go | 2 +- cmd/head.go | 10 +-- cmd/root.go | 97 ++++++++++++++++----------- internal/boot/boot.go | 79 ++++++++++++++++++++-- internal/boot/{startup.md => boot.md} | 0 internal/boot/setup_database.go | 29 -------- pkg/database/sql/postgres/config.go | 1 + pkg/database/sql/postgres/database.go | 2 +- pkg/database/sql/postgres/pgx_test.go | 6 +- pkg/loghelper/log_error.go | 13 ++++ 11 files changed, 184 insertions(+), 88 deletions(-) rename internal/boot/{startup.md => boot.md} (100%) delete mode 100644 internal/boot/setup_database.go create mode 100644 pkg/loghelper/log_error.go diff --git a/README.md b/README.md index 88998e9..1025ccc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ipld-ethcl-indexer -This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. +This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node. # Running the Application @@ -10,11 +10,36 @@ To run the application, utilize the following command, and update the values as go run main.go capture head --db.address localhost \ --db.password password \ --db.port 8077 \ - --db.username username \ - --lh.address localhost \ - --lh.port 5052 + --db.username vdbm \ + --db.name vulcanize_testing \ + --db.driver PGX \ + --bc.address localhost \ + --bc.port 5052 \ + --log.level info ``` +# Development Patterns + +This section will cover some generic development patterns utilizes. + +## Logging + +For logging, please keep the following in mind: + +- Utilize logrus. +- Use `log.Debug` to highlight that you are **about** to do something. +- Use `log.Info-Fatal` when the thing you were about to do has been completed, along with the result. + +``` +log.Debug("Adding 1 + 2") +a := 1 + 2 +log.Info("1 + 2 successfully Added, outcome is: ", a) +``` + +## Boot + +The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start. + # Contribution If you want to contribute please make sure you do the following: diff --git a/cmd/capture.go b/cmd/capture.go index 52b5315..6bda50a 100644 --- a/cmd/capture.go +++ b/cmd/capture.go @@ -14,7 +14,7 @@ var captureCmd = &cobra.Command{ Short: "Capture the SignedBeaconBlocks and BeaconStates from the Beacon Chain", Long: `Capture SignedBeaconBlocks and BeaconStates from the Beacon Chain. These blocks and states will be captured in - Postgres. They require a lighthouse client to be connected. You can run this to + Postgres. They require a beacon client to be connected. You can run this to capture blocks and states at head or historic blocks.`, } diff --git a/cmd/head.go b/cmd/head.go index 2d297a3..86e4f9f 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -5,12 +5,9 @@ Copyright © 2022 NAME HERE package cmd import ( - "fmt" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) // headCmd represents the head command @@ -19,15 +16,14 @@ var headCmd = &cobra.Command{ Short: "Capture only the blocks and state at head.", Long: `Capture only the blocks and state at head.`, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("head called") startHeadTracking() }, } func startHeadTracking() { - _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver) + _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) if err != nil { - log.Fatal("Unable to Start application with error: ", err) + loghelper.LogError(err).Error("Unable to Start application") } } diff --git a/cmd/root.go b/cmd/root.go index 05b9682..b06c224 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,31 +6,32 @@ package cmd import ( "fmt" + "io" "os" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - dbUsername string - dbPassword string - dbName string - dbAddress string - dbDriver string - dbPort int - lhAddress string - lhPort uint16 - logWithCommand log.Entry + cfgFile string + dbUsername string + dbPassword string + dbName string + dbAddress string + dbDriver string + dbPort int + bcAddress string + bcPort int ) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "ipld-ethcl-indexer", - Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.", - Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain. + Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.", + Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain. It can either do this will keeping track of head, or backfilling historic data.`, PersistentPreRun: initFuncs, // Uncomment the following line if your bare application @@ -49,23 +50,10 @@ func Execute() { // Prerun for Cobra func initFuncs(cmd *cobra.Command, args []string) { - viper.BindEnv("log.file", "LOGRUS_FILE") - logfile := viper.GetString("log.file") - if logfile != "" { - file, err := os.OpenFile(logfile, - os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err == nil { - log.Infof("Directing output to %s", logfile) - log.SetOutput(file) - } else { - log.SetOutput(os.Stdout) - log.Info("Failed to log to file, using default stdout") - } - } else { - log.SetOutput(os.Stdout) - } + logFormat() + logFile() if err := logLevel(); err != nil { - log.Fatal("Could not set log level: ", err) + log.WithField("err", err).Error("Could not set log level") } } @@ -84,6 +72,35 @@ func logLevel() error { return nil } +// Create a log file +func logFile() { + viper.BindEnv("log.file", "LOGRUS_FILE") + logfile := viper.GetString("log.file") + if logfile != "" { + file, err := os.OpenFile(logfile, + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + log.Infof("Directing output to %s", logfile) + mw := io.MultiWriter(os.Stdout, file) + logrus.SetOutput(mw) + } else { + log.SetOutput(os.Stdout) + log.Info("Failed to log to file, using default stdout") + } + } else { + log.SetOutput(os.Stdout) + } +} + +func logFormat() { + logFormat := viper.GetString("log.format") + + if logFormat == "json" { + log.SetFormatter(&log.JSONFormatter{}) + + } +} + func init() { cobra.OnInitialize(initConfig) @@ -93,8 +110,9 @@ func init() { // Optional Flags rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") - rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") - rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") + rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().String("log.format", "json", "json or text") // Required Flags @@ -112,16 +130,17 @@ func init() { rootCmd.MarkPersistentFlagRequired("db.name") rootCmd.MarkPersistentFlagRequired("db.driver") - //// Lighthouse Specific - rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)") - rootCmd.PersistentFlags().Uint16VarP(&lhPort, "lh.port", "r", 0, "Port to connect to lighthouse node (required if username is set)") - rootCmd.MarkPersistentFlagRequired("lh.address") - rootCmd.MarkPersistentFlagRequired("lh.port") + //// Beacon Client Specific + rootCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)") + rootCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)") + rootCmd.MarkPersistentFlagRequired("bc.address") + rootCmd.MarkPersistentFlagRequired("bc.port") // Bind Flags with Viper // Optional - viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) - viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file")) + viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level")) + viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file")) + viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format")) //// DB Flags viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username")) @@ -132,8 +151,8 @@ func init() { viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver")) // LH specific - viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address")) - viper.BindPFlag("lh.port", rootCmd.PersistentFlags().Lookup("lh.port")) + viper.BindPFlag("bc.address", rootCmd.PersistentFlags().Lookup("bc.address")) + viper.BindPFlag("bc.port", rootCmd.PersistentFlags().Lookup("bc.port")) // Cobra also supports local flags, which will only run // when this action is called directly. diff --git a/internal/boot/boot.go b/internal/boot/boot.go index e251a5b..13420ef 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -1,19 +1,88 @@ package boot import ( + "fmt" + "net/http" + "strconv" + log "github.com/sirupsen/logrus" "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) -func setUpLightHouse() { +var ( + bcHealthEndpoint = "/eth/v1/node/health" +) + +// This function will ensure that we can connect to the beacon client. +// Keep in mind, the beacon client will allow you to connect to it but it might +// Not allow you to make http requests. This is part of its built in logic, and you will have +// to follow their provided guidelines. https://lighthouse-book.sigmaprime.io/api-bn.html#security +func checkBeaconClient(bcAddress string, bcPort int) error { + log.Debug("Attempting to connect to the beacon client") + bcEndpoint := "http://" + bcAddress + ":" + strconv.Itoa(bcPort) + bcHealthEndpoint + resp, err := http.Get(bcEndpoint) + if err != nil { + loghelper.LogError(err).Error("Unable to get bc endpoint: ", bcEndpoint) + return err + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + log.Error("We recieved a non 2xx status code when checking the health of the beacon node.") + log.Error("Health Endpoint Status Code: ", resp.StatusCode) + return fmt.Errorf("beacon Node Provided a non 2xx status code, code provided: %d", resp.StatusCode) + } + + log.Info("We can successfully reach the beacon client.") + return nil } -// This function will perform some boot operations. -// 1. Setup a logger +// A simple wrapper to create a DB object to use. +func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { + log.Debug("Resolving Driver Type") + DbDriver, err := postgres.ResolveDriverType(driverName) + if err != nil { + log.WithFields(log.Fields{ + "err": err, + "driver_name_provided": driverName, + }).Error("Can't resolve driver type") + } + log.Info("Using Driver:", DbDriver) + + postgresConfig := postgres.Config{ + Hostname: dbHostname, + Port: dbPort, + DatabaseName: dbName, + Username: dbUsername, + Password: dbPassword, + Driver: DbDriver, + } + DB, err := postgres.NewPostgresDB(postgresConfig) + + if err != nil { + loghelper.LogError(err).Error("Unable to connect to the DB") + return nil, err + } + return DB, err + +} + +// This function will perform some boot operations. If any steps fail, the application will fail to start. +// Keep in mind that the DB connection can be lost later in the lifecycle of the application or +// it might not be able to connect to the beacon client. +// +// 1. Make sure the Beacon client is up. +// // 2. Connect to the database. -// 3. Connect to to the lighthouse client. -func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { +// +func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { + log.Debug("Checking beacon Client") + err := checkBeaconClient(bcAddress, bcPort) + if err != nil { + return nil, err + } + log.Debug("Setting up DB connection") DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) if err != nil { diff --git a/internal/boot/startup.md b/internal/boot/boot.md similarity index 100% rename from internal/boot/startup.md rename to internal/boot/boot.md diff --git a/internal/boot/setup_database.go b/internal/boot/setup_database.go deleted file mode 100644 index a1efaf0..0000000 --- a/internal/boot/setup_database.go +++ /dev/null @@ -1,29 +0,0 @@ -// This file will allow users to setup a new DB based on the user provided inputs. - -package boot - -import ( - log "github.com/sirupsen/logrus" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" -) - -func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { - log.Debug("Resolving Driver Type") - DbDriver, err := postgres.ResolveDriverType(driverName) - if err != nil { - log.Fatal("Can't Connect to DB") - } - log.Info("Using Driver:", DbDriver) - - postgresConfig := postgres.Config{ - Hostname: dbHostname, - Port: dbPort, - DatabaseName: dbName, - Username: dbUsername, - Password: dbPassword, - Driver: DbDriver, - } - DB, err := postgres.NewPostgresDB(postgresConfig) - return DB, err - -} diff --git a/pkg/database/sql/postgres/config.go b/pkg/database/sql/postgres/config.go index a518bed..efe5eb9 100644 --- a/pkg/database/sql/postgres/config.go +++ b/pkg/database/sql/postgres/config.go @@ -22,6 +22,7 @@ var DefaultConfig = Config{ DatabaseName: "vulcanize_testing", Username: "vdbm", Password: "password", + Driver: "PGX", } // ResolveDriverType resolves a DriverType from a provided string diff --git a/pkg/database/sql/postgres/database.go b/pkg/database/sql/postgres/database.go index 9d44823..d79821a 100644 --- a/pkg/database/sql/postgres/database.go +++ b/pkg/database/sql/postgres/database.go @@ -38,7 +38,7 @@ func createDriver(c Config) (*pgxDriver, error) { log.Info("Successfully created a driver for PGX") return driver, nil default: - log.Fatal("Couldnt find a driver to create for: ", c.Driver) + log.Error("Couldnt find a driver to create for: ", c.Driver) return nil, fmt.Errorf("Can't find a driver to create") } diff --git a/pkg/database/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go index ab75a7e..b66deeb 100644 --- a/pkg/database/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -79,7 +79,9 @@ func TestPostgresPGX(t *testing.T) { }) t.Run("throws error when can't connect to the database", func(t *testing.T) { - _, err := NewPostgresDB(Config{}, "PGX") + _, err := NewPostgresDB(Config{ + Driver: "PGX", + }) if err == nil { t.Fatal("Expected an error") } @@ -87,7 +89,7 @@ func TestPostgresPGX(t *testing.T) { expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) }) t.Run("Connect to the database", func(t *testing.T) { - driver, err := NewPostgresDB(DefaultConfig, "pgx") + driver, err := NewPostgresDB(DefaultConfig) defer driver.Close() if err != nil { diff --git a/pkg/loghelper/log_error.go b/pkg/loghelper/log_error.go new file mode 100644 index 0000000..41d0149 --- /dev/null +++ b/pkg/loghelper/log_error.go @@ -0,0 +1,13 @@ +// A simple function to help with logging errors. +package loghelper + +import ( + log "github.com/sirupsen/logrus" +) + +// A simple helper function that will help wrap the error message. +func LogError(err error) *log.Entry { + return log.WithFields(log.Fields{ + "err": err, + }) +} -- 2.45.2 From 7a2c4b3cefcab51450ccf517d13e9c20cbf861fc Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 18:23:21 -0400 Subject: [PATCH 6/6] Last second clean ups --- README.md | 2 ++ cmd/root.go | 1 + pkg/database/sql/postgres/database.go | 5 +---- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1025ccc..a63cac9 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ a := 1 + 2 log.Info("1 + 2 successfully Added, outcome is: ", a) ``` +- `loghelper.LogError(err)` is a pretty wrapper to output errors. + ## Boot The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start. diff --git a/cmd/root.go b/cmd/root.go index b06c224..860dc78 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -92,6 +92,7 @@ func logFile() { } } +// Format the logger func logFormat() { logFormat := viper.GetString("log.format") diff --git a/pkg/database/sql/postgres/database.go b/pkg/database/sql/postgres/database.go index d79821a..a28d2ad 100644 --- a/pkg/database/sql/postgres/database.go +++ b/pkg/database/sql/postgres/database.go @@ -10,10 +10,6 @@ import ( var _ sql.Database = &DB{} -// TODO: Make NewPostgresDB accept a string and Config. IT should -// Create a driver of its own. -// This will make sure that if you want a driver, it conforms to the interface. - // NewPostgresDB returns a postgres.DB using the provided Config and driver type. func NewPostgresDB(c Config) (*DB, error) { var driver *pgxDriver @@ -27,6 +23,7 @@ func NewPostgresDB(c Config) (*DB, error) { return &DB{driver}, nil } +// Create a driver based on the config func createDriver(c Config) (*pgxDriver, error) { switch c.Driver { case PGX: -- 2.45.2