From 42923d15c1d35dbbc0e63b1520966bc9d15cb4af Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 09:25:47 -0400 Subject: [PATCH 01/11] Update cobra to require `head` or `historic` when using `capture`. --- README.md | 14 ++++++++++++++ cmd/capture.go | 17 +++++------------ cmd/head.go | 9 ++------- cmd/historic.go | 9 ++------- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 77b377e..88998e9 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,19 @@ This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. +# Running the Application + +To run the application, utilize the following command, and update the values as needed. + +``` +go run main.go capture head --db.address localhost \ + --db.password password \ + --db.port 8077 \ + --db.username username \ + --lh.address localhost \ + --lh.port 5052 +``` + # Contribution If you want to contribute please make sure you do the following: @@ -9,6 +22,7 @@ If you want to contribute please make sure you do the following: - Create a Github issue before starting your work. - Follow the branching structure. - Delete your branch once it has been merged. + - Do not delete the `develop` branch. We can add branch protection once we make the branch public. ## Branching Structure diff --git a/cmd/capture.go b/cmd/capture.go index b0e35c9..52b5315 100644 --- a/cmd/capture.go +++ b/cmd/capture.go @@ -5,24 +5,17 @@ Copyright © 2022 NAME HERE package cmd import ( - "fmt" - "github.com/spf13/cobra" ) // captureCmd represents the capture command var captureCmd = &cobra.Command{ Use: "capture", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("capture called") - }, + Short: "Capture the SignedBeaconBlocks and BeaconStates from the Beacon Chain", + Long: `Capture SignedBeaconBlocks and BeaconStates from the Beacon Chain. + These blocks and states will be captured in + Postgres. They require a lighthouse client to be connected. You can run this to + capture blocks and states at head or historic blocks.`, } func init() { diff --git a/cmd/head.go b/cmd/head.go index d5b7e2f..f0e4675 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -13,13 +13,8 @@ import ( // headCmd represents the head command var headCmd = &cobra.Command{ Use: "head", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "Capture only the blocks and state at head.", + Long: `Capture only the blocks and state at head.`, Run: func(cmd *cobra.Command, args []string) { fmt.Println("head called") }, diff --git a/cmd/historic.go b/cmd/historic.go index e29f167..7b6feeb 100644 --- a/cmd/historic.go +++ b/cmd/historic.go @@ -13,13 +13,8 @@ import ( // historicCmd represents the historic command var historicCmd = &cobra.Command{ Use: "historic", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "Capture the historic blocks and states.", + Long: `Capture the historic blocks and states.`, Run: func(cmd *cobra.Command, args []string) { fmt.Println("historic called") }, -- 2.45.2 From ed3d0be2b78c22639d632b27fd579bfe95d3d630 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 12:12:55 -0400 Subject: [PATCH 02/11] A very generic package for implementing PGX driver I copied most of the code from the `statediff` service from within geth. The idea is that I can create formal DB packages, that can be utilized in other projects down the road. --- .gitignore | 2 + go.mod | 15 +++ go.sum | 159 ++++++++++++++++++++++ pkg/dbtools/sql/error.go | 22 ++++ pkg/dbtools/sql/interfaces.go | 54 ++++++++ pkg/dbtools/sql/postgres/config.go | 75 +++++++++++ pkg/dbtools/sql/postgres/pgx.go | 190 +++++++++++++++++++++++++++ pkg/dbtools/sql/postgres/pgx_test.go | 89 +++++++++++++ pkg/testhelpers/testhelper.go | 23 ++++ 9 files changed, 629 insertions(+) create mode 100644 .gitignore create mode 100644 pkg/dbtools/sql/error.go create mode 100644 pkg/dbtools/sql/interfaces.go create mode 100644 pkg/dbtools/sql/postgres/config.go create mode 100644 pkg/dbtools/sql/postgres/pgx.go create mode 100644 pkg/dbtools/sql/postgres/pgx_test.go create mode 100644 pkg/testhelpers/testhelper.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..48e5153 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +ipld-ethcl-indexer diff --git a/go.mod b/go.mod index 5b9b916..b96803c 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,25 @@ go 1.18 require github.com/sirupsen/logrus v1.8.1 +require ( + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.10.0 // indirect + github.com/jackc/puddle v1.2.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect +) + require ( github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/georgysavva/scany v0.3.0 github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jackc/pgx/v4 v4.15.0 github.com/magiconair/properties v1.8.6 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/pelletier/go-toml v1.9.4 // indirect diff --git a/go.sum b/go.sum index 855bce8..2e87236 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -46,7 +47,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cockroach-go/v2 v2.2.0/go.mod h1:u3MiKYGupPPjkn3ozknpMUpxPaNLTFWAya419/zv6eI= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -57,9 +63,18 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/georgysavva/scany v0.3.0 h1:MA1aEqPbnNuiek59gMpNPqQrXXroyFj5jCADlETdxiA= +github.com/georgysavva/scany v0.3.0/go.mod h1:q8QyrfXjmBk9iJD00igd4lbkAKEXAH/zIYoZ0z/Wan4= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -122,27 +137,123 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.0-beta.8 h1:dy81yyLYJDwMTifq24Oi/IslOslRrDSb3jwDggjz3Z0= github.com/pelletier/go-toml/v2 v2.0.0-beta.8/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= @@ -158,7 +269,10 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.11.0 h1:7OX/1FS6n7jHD1zGrZTM7WtY13ZELRyosK4k93oPr44= github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -169,19 +283,41 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -227,6 +363,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -266,20 +403,27 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -302,8 +446,10 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -311,6 +457,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -323,14 +471,18 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -338,6 +490,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -363,6 +516,8 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -458,6 +613,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -466,6 +622,9 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= +gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/dbtools/sql/error.go b/pkg/dbtools/sql/error.go new file mode 100644 index 0000000..9701968 --- /dev/null +++ b/pkg/dbtools/sql/error.go @@ -0,0 +1,22 @@ +package sql + +import ( + "fmt" +) + +const ( + DbConnectionFailedMsg = "db connection failed" + SettingNodeFailedMsg = "unable to set db node" +) + +func ErrDBConnectionFailed(connectErr error) error { + return formatError(DbConnectionFailedMsg, connectErr.Error()) +} + +func ErrUnableToSetNode(setErr error) error { + return formatError(SettingNodeFailedMsg, setErr.Error()) +} + +func formatError(msg, err string) error { + return fmt.Errorf("%s: %s", msg, err) +} diff --git a/pkg/dbtools/sql/interfaces.go b/pkg/dbtools/sql/interfaces.go new file mode 100644 index 0000000..095b89b --- /dev/null +++ b/pkg/dbtools/sql/interfaces.go @@ -0,0 +1,54 @@ +package sql + +import ( + "context" + "io" + "time" +) + +// Database interfaces required by the sql indexer +type Database interface { + Driver +} + +// Driver interface has all the methods required by a driver implementation to support the sql indexer +type Driver interface { + QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow + Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) + Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error + Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error + Begin(ctx context.Context) (Tx, error) + Stats() Stats + Context() context.Context + io.Closer +} + +// Tx interface to accommodate different concrete SQL transaction types +type Tx interface { + QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow + Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) + Commit(ctx context.Context) error + Rollback(ctx context.Context) error +} + +// ScannableRow interface to accommodate different concrete row types +type ScannableRow interface { + Scan(dest ...interface{}) error +} + +// Result interface to accommodate different concrete result types +type Result interface { + RowsAffected() (int64, error) +} + +// Stats interface to accommodate different concrete sql stats types +type Stats interface { + MaxOpen() int64 + Open() int64 + InUse() int64 + Idle() int64 + WaitCount() int64 + WaitDuration() time.Duration + MaxIdleClosed() int64 + MaxLifetimeClosed() int64 +} diff --git a/pkg/dbtools/sql/postgres/config.go b/pkg/dbtools/sql/postgres/config.go new file mode 100644 index 0000000..2a908d8 --- /dev/null +++ b/pkg/dbtools/sql/postgres/config.go @@ -0,0 +1,75 @@ +package postgres + +import ( + "fmt" + "strings" + "time" +) + +// DriverType to explicitly type the kind of sql driver we are using +type DriverType string + +const ( + PGX DriverType = "PGX" + SQLX DriverType = "SQLX" + Unknown DriverType = "Unknown" +) + +// DefaultConfig are default parameters for connecting to a Postgres sql +var DefaultConfig = Config{ + Hostname: "localhost", + Port: 8077, + DatabaseName: "vulcanize_testing", + Username: "vdbm", + Password: "password", +} + +// ResolveDriverType resolves a DriverType from a provided string +func ResolveDriverType(str string) (DriverType, error) { + switch strings.ToLower(str) { + case "pgx", "pgxpool": + return PGX, nil + case "sqlx": + return SQLX, nil + default: + return Unknown, fmt.Errorf("unrecognized driver type string: %s", str) + } +} + +// Config holds params for a Postgres db +type Config struct { + // conn string params + Hostname string + Port int + DatabaseName string + Username string + Password string + + // conn settings + MaxConns int + MaxIdle int + MinConns int + MaxConnIdleTime time.Duration + MaxConnLifetime time.Duration + ConnTimeout time.Duration + + // node info params + ID string + ClientName string + + // driver type + Driver DriverType +} + +// DbConnectionString constructs and returns the connection string from the config +func (c Config) DbConnectionString() string { + if len(c.Username) > 0 && len(c.Password) > 0 { + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable", + c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName) + } + if len(c.Username) > 0 && len(c.Password) == 0 { + return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable", + c.Username, c.Hostname, c.Port, c.DatabaseName) + } + return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName) +} diff --git a/pkg/dbtools/sql/postgres/pgx.go b/pkg/dbtools/sql/postgres/pgx.go new file mode 100644 index 0000000..4749168 --- /dev/null +++ b/pkg/dbtools/sql/postgres/pgx.go @@ -0,0 +1,190 @@ +package postgres + +import ( + "context" + "time" + + "github.com/georgysavva/scany/pgxscan" + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" +) + +// PGXDriver driver, implements sql.Driver +type PGXDriver struct { + ctx context.Context + pool *pgxpool.Pool +} + +// NewPGXDriver returns a new pgx driver. +// It initializes the connection pool. +func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) { + pgConf, err := MakeConfig(config) + if err != nil { + return nil, err + } + dbPool, err := pgxpool.ConnectConfig(ctx, pgConf) + if err != nil { + return nil, sql.ErrDBConnectionFailed(err) + } + pg := &PGXDriver{ctx: ctx, pool: dbPool} + return pg, nil +} + +// MakeConfig creates a pgxpool.Config from the provided Config +func MakeConfig(config Config) (*pgxpool.Config, error) { + conf, err := pgxpool.ParseConfig("") + if err != nil { + return nil, err + } + + //conf.ConnConfig.BuildStatementCache = nil + conf.ConnConfig.Config.Host = config.Hostname + conf.ConnConfig.Config.Port = uint16(config.Port) + conf.ConnConfig.Config.Database = config.DatabaseName + conf.ConnConfig.Config.User = config.Username + conf.ConnConfig.Config.Password = config.Password + + if config.ConnTimeout != 0 { + conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout + } + if config.MaxConns != 0 { + conf.MaxConns = int32(config.MaxConns) + } + if config.MinConns != 0 { + conf.MinConns = int32(config.MinConns) + } + if config.MaxConnLifetime != 0 { + conf.MaxConnLifetime = config.MaxConnLifetime + } + if config.MaxConnIdleTime != 0 { + conf.MaxConnIdleTime = config.MaxConnIdleTime + } + return conf, nil +} + +// QueryRow satisfies sql.Database +func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { + return pgx.pool.QueryRow(ctx, sql, args...) +} + +// Exec satisfies sql.Database +func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { + res, err := pgx.pool.Exec(ctx, sql, args...) + return resultWrapper{ct: res}, err +} + +// Select satisfies sql.Database +func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return pgxscan.Select(ctx, pgx.pool, dest, query, args...) +} + +// Get satisfies sql.Database +func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return pgxscan.Get(ctx, pgx.pool, dest, query, args...) +} + +// Begin satisfies sql.Database +func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) { + tx, err := pgx.pool.Begin(ctx) + if err != nil { + return nil, err + } + return pgxTxWrapper{tx: tx}, nil +} + +func (pgx *PGXDriver) Stats() sql.Stats { + stats := pgx.pool.Stat() + return pgxStatsWrapper{stats: stats} +} + +// Close satisfies sql.Database/io.Closer +func (pgx *PGXDriver) Close() error { + pgx.pool.Close() + return nil +} + +// Context satisfies sql.Database +func (pgx *PGXDriver) Context() context.Context { + return pgx.ctx +} + +type resultWrapper struct { + ct pgconn.CommandTag +} + +// RowsAffected satisfies sql.Result +func (r resultWrapper) RowsAffected() (int64, error) { + return r.ct.RowsAffected(), nil +} + +type pgxStatsWrapper struct { + stats *pgxpool.Stat +} + +// MaxOpen satisfies sql.Stats +func (s pgxStatsWrapper) MaxOpen() int64 { + return int64(s.stats.MaxConns()) +} + +// Open satisfies sql.Stats +func (s pgxStatsWrapper) Open() int64 { + return int64(s.stats.TotalConns()) +} + +// InUse satisfies sql.Stats +func (s pgxStatsWrapper) InUse() int64 { + return int64(s.stats.AcquiredConns()) +} + +// Idle satisfies sql.Stats +func (s pgxStatsWrapper) Idle() int64 { + return int64(s.stats.IdleConns()) +} + +// WaitCount satisfies sql.Stats +func (s pgxStatsWrapper) WaitCount() int64 { + return s.stats.EmptyAcquireCount() +} + +// WaitDuration satisfies sql.Stats +func (s pgxStatsWrapper) WaitDuration() time.Duration { + return s.stats.AcquireDuration() +} + +// MaxIdleClosed satisfies sql.Stats +func (s pgxStatsWrapper) MaxIdleClosed() int64 { + // this stat isn't supported by pgxpool, but we don't want to panic + return 0 +} + +// MaxLifetimeClosed satisfies sql.Stats +func (s pgxStatsWrapper) MaxLifetimeClosed() int64 { + return s.stats.CanceledAcquireCount() +} + +type pgxTxWrapper struct { + tx pgx.Tx +} + +// QueryRow satisfies sql.Tx +func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { + return t.tx.QueryRow(ctx, sql, args...) +} + +// Exec satisfies sql.Tx +func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { + res, err := t.tx.Exec(ctx, sql, args...) + return resultWrapper{ct: res}, err +} + +// Commit satisfies sql.Tx +func (t pgxTxWrapper) Commit(ctx context.Context) error { + return t.tx.Commit(ctx) +} + +// Rollback satisfies sql.Tx +func (t pgxTxWrapper) Rollback(ctx context.Context) error { + return t.tx.Rollback(ctx) +} diff --git a/pkg/dbtools/sql/postgres/pgx_test.go b/pkg/dbtools/sql/postgres/pgx_test.go new file mode 100644 index 0000000..782c426 --- /dev/null +++ b/pkg/dbtools/sql/postgres/pgx_test.go @@ -0,0 +1,89 @@ +package postgres + +import ( + "context" + "math/big" + "strings" + "testing" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers" +) + +var ( + pgConfig, _ = MakeConfig(DefaultConfig) + ctx = context.Background() +) + +func expectContainsSubstring(t *testing.T, full string, sub string) { + if !strings.Contains(full, sub) { + t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub) + } +} + +func TestPostgresPGX(t *testing.T) { + t.Run("connects to the sql", func(t *testing.T) { + dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err) + } + if dbPool == nil { + t.Fatal("DB pool is nil") + } + dbPool.Close() + }) + + t.Run("serializes big.Int to db", func(t *testing.T) { + // postgres driver doesn't support go big.Int type + // various casts in golang uint64, int64, overflow for + // transaction value (in wei) even though + // postgres numeric can handle an arbitrary + // sized int, so use string representation of big.Int + // and cast on insert + + dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err) + } + defer dbPool.Close() + + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + testhelpers.ExpectEqual(t, bi.String(), "34940183920000000000") + + defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) + _, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )") + if err != nil { + t.Fatal(err) + } + + sqlStatement := ` + INSERT INTO example (id, data) + VALUES (1, cast($1 AS NUMERIC))` + _, err = dbPool.Exec(ctx, sqlStatement, bi.String()) + if err != nil { + t.Fatal(err) + } + + var data string + err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data) + if err != nil { + t.Fatal(err) + } + + testhelpers.ExpectEqual(t, data, bi.String()) + actual := new(big.Int) + actual.SetString(data, 10) + testhelpers.ExpectEqual(t, actual, bi) + }) + + t.Run("throws error when can't connect to the database", func(t *testing.T) { + _, err := NewPGXDriver(ctx, Config{}) + if err == nil { + t.Fatal("Expected an error") + } + + expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) + }) +} diff --git a/pkg/testhelpers/testhelper.go b/pkg/testhelpers/testhelper.go new file mode 100644 index 0000000..4e3c917 --- /dev/null +++ b/pkg/testhelpers/testhelper.go @@ -0,0 +1,23 @@ +package testhelpers + +import ( + "reflect" + "testing" +) + +// ExpectEqual asserts the provided interfaces are deep equal +func ExpectEqual(t *testing.T, got interface{}, want interface{}) { + if !reflect.DeepEqual(got, want) { + t.Fatalf("Expected: %v\nActual: %v", want, got) + } +} + +// ListContainsString used to check if a list of strings contains a particular string +func ListContainsString(sss []string, s string) bool { + for _, str := range sss { + if s == str { + return true + } + } + return false +} -- 2.45.2 From d0d4f2498ef4382ecd675549167855d495661a49 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 13:06:00 -0400 Subject: [PATCH 03/11] Put PGX and other future Postgres Drivers behind SQL package This PR makes allows users to provide a config, along with a driver string. It will then provide the respective driver. --- pkg/dbtools/sql/postgres/database.go | 51 ++++++++++++++++++++++++++++ pkg/dbtools/sql/postgres/pgx.go | 32 ++++++++--------- pkg/dbtools/sql/postgres/pgx_test.go | 13 +++++-- 3 files changed, 78 insertions(+), 18 deletions(-) create mode 100644 pkg/dbtools/sql/postgres/database.go diff --git a/pkg/dbtools/sql/postgres/database.go b/pkg/dbtools/sql/postgres/database.go new file mode 100644 index 0000000..3297911 --- /dev/null +++ b/pkg/dbtools/sql/postgres/database.go @@ -0,0 +1,51 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" +) + +var _ sql.Database = &DB{} + +// TODO: Make NewPostgresDB accept a string and Config. IT should +// Create a driver of its own. +// This will make sure that if you want a driver, it conforms to the interface. + +// NewPostgresDB returns a postgres.DB using the provided Config and driver type. +func NewPostgresDB(c Config, driverName string) (*DB, error) { + var driver *pgxDriver + + driverType, err := ResolveDriverType(driverName) + if err != nil { + return nil, err + } + + driver, err = createDriver(c, driverType) + + if err != nil { + return nil, err + } + + return &DB{driver}, nil +} + +func createDriver(c Config, driverType DriverType) (*pgxDriver, error) { + switch driverType { + case PGX: + driver, err := newPGXDriver(context.Background(), c) + if err != nil { + return nil, fmt.Errorf("Error Creating Driver, err: %e", err) + } + return driver, nil + default: + return nil, fmt.Errorf("Can't find a driver to create") + } + +} + +// DB implements sql.Database using a configured driver and Postgres statement syntax +type DB struct { + sql.Driver +} diff --git a/pkg/dbtools/sql/postgres/pgx.go b/pkg/dbtools/sql/postgres/pgx.go index 4749168..bd0388f 100644 --- a/pkg/dbtools/sql/postgres/pgx.go +++ b/pkg/dbtools/sql/postgres/pgx.go @@ -11,16 +11,16 @@ import ( "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" ) -// PGXDriver driver, implements sql.Driver -type PGXDriver struct { +// pgxDriver driver, implements sql.Driver +type pgxDriver struct { ctx context.Context pool *pgxpool.Pool } -// NewPGXDriver returns a new pgx driver. +// newPGXDriver returns a new pgx driver. // It initializes the connection pool. -func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) { - pgConf, err := MakeConfig(config) +func newPGXDriver(ctx context.Context, config Config) (*pgxDriver, error) { + pgConf, err := makeConfig(config) if err != nil { return nil, err } @@ -28,12 +28,12 @@ func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) { if err != nil { return nil, sql.ErrDBConnectionFailed(err) } - pg := &PGXDriver{ctx: ctx, pool: dbPool} + pg := &pgxDriver{ctx: ctx, pool: dbPool} return pg, nil } -// MakeConfig creates a pgxpool.Config from the provided Config -func MakeConfig(config Config) (*pgxpool.Config, error) { +// makeConfig creates a pgxpool.Config from the provided Config +func makeConfig(config Config) (*pgxpool.Config, error) { conf, err := pgxpool.ParseConfig("") if err != nil { return nil, err @@ -65,28 +65,28 @@ func MakeConfig(config Config) (*pgxpool.Config, error) { } // QueryRow satisfies sql.Database -func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { +func (pgx *pgxDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { return pgx.pool.QueryRow(ctx, sql, args...) } // Exec satisfies sql.Database -func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { +func (pgx *pgxDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { res, err := pgx.pool.Exec(ctx, sql, args...) return resultWrapper{ct: res}, err } // Select satisfies sql.Database -func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error { +func (pgx *pgxDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error { return pgxscan.Select(ctx, pgx.pool, dest, query, args...) } // Get satisfies sql.Database -func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error { +func (pgx *pgxDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error { return pgxscan.Get(ctx, pgx.pool, dest, query, args...) } // Begin satisfies sql.Database -func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) { +func (pgx *pgxDriver) Begin(ctx context.Context) (sql.Tx, error) { tx, err := pgx.pool.Begin(ctx) if err != nil { return nil, err @@ -94,19 +94,19 @@ func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) { return pgxTxWrapper{tx: tx}, nil } -func (pgx *PGXDriver) Stats() sql.Stats { +func (pgx *pgxDriver) Stats() sql.Stats { stats := pgx.pool.Stat() return pgxStatsWrapper{stats: stats} } // Close satisfies sql.Database/io.Closer -func (pgx *PGXDriver) Close() error { +func (pgx *pgxDriver) Close() error { pgx.pool.Close() return nil } // Context satisfies sql.Database -func (pgx *PGXDriver) Context() context.Context { +func (pgx *pgxDriver) Context() context.Context { return pgx.ctx } diff --git a/pkg/dbtools/sql/postgres/pgx_test.go b/pkg/dbtools/sql/postgres/pgx_test.go index 782c426..65daf96 100644 --- a/pkg/dbtools/sql/postgres/pgx_test.go +++ b/pkg/dbtools/sql/postgres/pgx_test.go @@ -12,7 +12,7 @@ import ( ) var ( - pgConfig, _ = MakeConfig(DefaultConfig) + pgConfig, _ = makeConfig(DefaultConfig) ctx = context.Background() ) @@ -79,11 +79,20 @@ func TestPostgresPGX(t *testing.T) { }) t.Run("throws error when can't connect to the database", func(t *testing.T) { - _, err := NewPGXDriver(ctx, Config{}) + _, err := NewPostgresDB(Config{}, "PGX") if err == nil { t.Fatal("Expected an error") } expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) }) + t.Run("Connect to the database", func(t *testing.T) { + driver, err := NewPostgresDB(DefaultConfig, "pgx") + defer driver.Close() + + if err != nil { + t.Fatal("Error creating the postgres driver") + } + + }) } -- 2.45.2 From 827475f029a5f2f0586de61facd22a7d1cd2a45d Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 15:44:15 -0400 Subject: [PATCH 04/11] Add DB Connection and Logging * Utilize LogRus * Create a DB connection using PGX. * Create an internal boot package for starting the application. --- .gitignore | 1 + cmd/head.go | 11 +++ cmd/root.go | 76 ++++++++++++++++--- internal/boot/boot.go | 23 ++++++ internal/boot/setup_database.go | 29 +++++++ internal/boot/startup.md | 3 + pkg/{dbtools => database}/sql/error.go | 0 pkg/{dbtools => database}/sql/interfaces.go | 0 .../sql/postgres/config.go | 4 - .../sql/postgres/database.go | 19 +++-- pkg/{dbtools => database}/sql/postgres/pgx.go | 2 +- .../sql/postgres/pgx_test.go | 2 +- .../{testhelper.go => test_helper.go} | 0 13 files changed, 143 insertions(+), 27 deletions(-) create mode 100644 internal/boot/boot.go create mode 100644 internal/boot/setup_database.go create mode 100644 internal/boot/startup.md rename pkg/{dbtools => database}/sql/error.go (100%) rename pkg/{dbtools => database}/sql/interfaces.go (100%) rename pkg/{dbtools => database}/sql/postgres/config.go (96%) rename pkg/{dbtools => database}/sql/postgres/database.go (67%) rename pkg/{dbtools => database}/sql/postgres/pgx.go (98%) rename pkg/{dbtools => database}/sql/postgres/pgx_test.go (97%) rename pkg/testhelpers/{testhelper.go => test_helper.go} (100%) diff --git a/.gitignore b/.gitignore index 48e5153..a0fdb30 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ ipld-ethcl-indexer +ipld-ethcl-indexer.log diff --git a/cmd/head.go b/cmd/head.go index f0e4675..2d297a3 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -7,7 +7,10 @@ package cmd import ( "fmt" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" ) // headCmd represents the head command @@ -17,9 +20,17 @@ var headCmd = &cobra.Command{ Long: `Capture only the blocks and state at head.`, Run: func(cmd *cobra.Command, args []string) { fmt.Println("head called") + startHeadTracking() }, } +func startHeadTracking() { + _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver) + if err != nil { + log.Fatal("Unable to Start application with error: ", err) + } +} + func init() { captureCmd.AddCommand(headCmd) diff --git a/cmd/root.go b/cmd/root.go index ef743fd..05b9682 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,18 +8,22 @@ import ( "fmt" "os" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - dbUserName string - dbPassword string - dbAddress string - dbPort uint16 - lhAddress string - lhPort uint16 + cfgFile string + dbUsername string + dbPassword string + dbName string + dbAddress string + dbDriver string + dbPort int + lhAddress string + lhPort uint16 + logWithCommand log.Entry ) // rootCmd represents the base command when called without any subcommands @@ -28,6 +32,7 @@ var rootCmd = &cobra.Command{ Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.", Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain. It can either do this will keeping track of head, or backfilling historic data.`, + PersistentPreRun: initFuncs, // Uncomment the following line if your bare application // has an action associated with it: // Run: func(cmd *cobra.Command, args []string) {}, @@ -42,6 +47,43 @@ func Execute() { } } +// Prerun for Cobra +func initFuncs(cmd *cobra.Command, args []string) { + viper.BindEnv("log.file", "LOGRUS_FILE") + logfile := viper.GetString("log.file") + if logfile != "" { + file, err := os.OpenFile(logfile, + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + log.Infof("Directing output to %s", logfile) + log.SetOutput(file) + } else { + log.SetOutput(os.Stdout) + log.Info("Failed to log to file, using default stdout") + } + } else { + log.SetOutput(os.Stdout) + } + if err := logLevel(); err != nil { + log.Fatal("Could not set log level: ", err) + } +} + +// Set the log level for the application +func logLevel() error { + viper.BindEnv("log.level", "LOGRUS_LEVEL") + lvl, err := log.ParseLevel(viper.GetString("log.level")) + if err != nil { + return err + } + log.SetLevel(lvl) + if lvl > log.InfoLevel { + log.SetReportCaller(true) + } + log.Info("Log level set to ", lvl.String()) + return nil +} + func init() { cobra.OnInitialize(initConfig) @@ -51,18 +93,24 @@ func init() { // Optional Flags rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") + rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") + rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging") // Required Flags //// DB Specific - rootCmd.PersistentFlags().StringVarP(&dbUserName, "db.username", "u", "", "Database username (required)") - rootCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "p", "", "Database Password (required)") - rootCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "a", "", "Port to connect to DB(required)") - rootCmd.PersistentFlags().Uint16VarP(&dbPort, "db.port", "o", 0, "Port to connect to DB(required)") + rootCmd.PersistentFlags().StringVarP(&dbUsername, "db.username", "", "", "Database username (required)") + rootCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "", "", "Database Password (required)") + rootCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "", "", "Port to connect to DB(required)") + rootCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)") + rootCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)") + rootCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)") rootCmd.MarkPersistentFlagRequired("db.username") rootCmd.MarkPersistentFlagRequired("db.password") rootCmd.MarkPersistentFlagRequired("db.address") rootCmd.MarkPersistentFlagRequired("db.port") + rootCmd.MarkPersistentFlagRequired("db.name") + rootCmd.MarkPersistentFlagRequired("db.driver") //// Lighthouse Specific rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)") @@ -71,11 +119,17 @@ func init() { rootCmd.MarkPersistentFlagRequired("lh.port") // Bind Flags with Viper + // Optional + viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) + viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file")) + //// DB Flags viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username")) viper.BindPFlag("db.password", rootCmd.PersistentFlags().Lookup("db.password")) viper.BindPFlag("db.address", rootCmd.PersistentFlags().Lookup("db.address")) viper.BindPFlag("db.port", rootCmd.PersistentFlags().Lookup("db.port")) + viper.BindPFlag("db.name", rootCmd.PersistentFlags().Lookup("db.name")) + viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver")) // LH specific viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address")) diff --git a/internal/boot/boot.go b/internal/boot/boot.go new file mode 100644 index 0000000..e251a5b --- /dev/null +++ b/internal/boot/boot.go @@ -0,0 +1,23 @@ +package boot + +import ( + log "github.com/sirupsen/logrus" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" +) + +func setUpLightHouse() { + +} + +// This function will perform some boot operations. +// 1. Setup a logger +// 2. Connect to the database. +// 3. Connect to to the lighthouse client. +func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { + log.Debug("Setting up DB connection") + DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) + if err != nil { + return nil, err + } + return DB, nil +} diff --git a/internal/boot/setup_database.go b/internal/boot/setup_database.go new file mode 100644 index 0000000..a1efaf0 --- /dev/null +++ b/internal/boot/setup_database.go @@ -0,0 +1,29 @@ +// This file will allow users to setup a new DB based on the user provided inputs. + +package boot + +import ( + log "github.com/sirupsen/logrus" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" +) + +func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { + log.Debug("Resolving Driver Type") + DbDriver, err := postgres.ResolveDriverType(driverName) + if err != nil { + log.Fatal("Can't Connect to DB") + } + log.Info("Using Driver:", DbDriver) + + postgresConfig := postgres.Config{ + Hostname: dbHostname, + Port: dbPort, + DatabaseName: dbName, + Username: dbUsername, + Password: dbPassword, + Driver: DbDriver, + } + DB, err := postgres.NewPostgresDB(postgresConfig) + return DB, err + +} diff --git a/internal/boot/startup.md b/internal/boot/startup.md new file mode 100644 index 0000000..72ec31f --- /dev/null +++ b/internal/boot/startup.md @@ -0,0 +1,3 @@ +# Overview + +This small package will group all the functions needed to start the application. The functions listed here can be utilized regardless of whether the application is meant to track `head` or for `historic` processing. diff --git a/pkg/dbtools/sql/error.go b/pkg/database/sql/error.go similarity index 100% rename from pkg/dbtools/sql/error.go rename to pkg/database/sql/error.go diff --git a/pkg/dbtools/sql/interfaces.go b/pkg/database/sql/interfaces.go similarity index 100% rename from pkg/dbtools/sql/interfaces.go rename to pkg/database/sql/interfaces.go diff --git a/pkg/dbtools/sql/postgres/config.go b/pkg/database/sql/postgres/config.go similarity index 96% rename from pkg/dbtools/sql/postgres/config.go rename to pkg/database/sql/postgres/config.go index 2a908d8..a518bed 100644 --- a/pkg/dbtools/sql/postgres/config.go +++ b/pkg/database/sql/postgres/config.go @@ -53,10 +53,6 @@ type Config struct { MaxConnLifetime time.Duration ConnTimeout time.Duration - // node info params - ID string - ClientName string - // driver type Driver DriverType } diff --git a/pkg/dbtools/sql/postgres/database.go b/pkg/database/sql/postgres/database.go similarity index 67% rename from pkg/dbtools/sql/postgres/database.go rename to pkg/database/sql/postgres/database.go index 3297911..9d44823 100644 --- a/pkg/dbtools/sql/postgres/database.go +++ b/pkg/database/sql/postgres/database.go @@ -4,7 +4,8 @@ import ( "context" "fmt" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" + log "github.com/sirupsen/logrus" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" ) var _ sql.Database = &DB{} @@ -14,15 +15,10 @@ var _ sql.Database = &DB{} // This will make sure that if you want a driver, it conforms to the interface. // NewPostgresDB returns a postgres.DB using the provided Config and driver type. -func NewPostgresDB(c Config, driverName string) (*DB, error) { +func NewPostgresDB(c Config) (*DB, error) { var driver *pgxDriver - driverType, err := ResolveDriverType(driverName) - if err != nil { - return nil, err - } - - driver, err = createDriver(c, driverType) + driver, err := createDriver(c) if err != nil { return nil, err @@ -31,15 +27,18 @@ func NewPostgresDB(c Config, driverName string) (*DB, error) { return &DB{driver}, nil } -func createDriver(c Config, driverType DriverType) (*pgxDriver, error) { - switch driverType { +func createDriver(c Config) (*pgxDriver, error) { + switch c.Driver { case PGX: + log.Debug("Creating New Driver") driver, err := newPGXDriver(context.Background(), c) if err != nil { return nil, fmt.Errorf("Error Creating Driver, err: %e", err) } + log.Info("Successfully created a driver for PGX") return driver, nil default: + log.Fatal("Couldnt find a driver to create for: ", c.Driver) return nil, fmt.Errorf("Can't find a driver to create") } diff --git a/pkg/dbtools/sql/postgres/pgx.go b/pkg/database/sql/postgres/pgx.go similarity index 98% rename from pkg/dbtools/sql/postgres/pgx.go rename to pkg/database/sql/postgres/pgx.go index bd0388f..3981b33 100644 --- a/pkg/dbtools/sql/postgres/pgx.go +++ b/pkg/database/sql/postgres/pgx.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" ) // pgxDriver driver, implements sql.Driver diff --git a/pkg/dbtools/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go similarity index 97% rename from pkg/dbtools/sql/postgres/pgx_test.go rename to pkg/database/sql/postgres/pgx_test.go index 65daf96..ab75a7e 100644 --- a/pkg/dbtools/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/jackc/pgx/v4/pgxpool" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/dbtools/sql" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers" ) diff --git a/pkg/testhelpers/testhelper.go b/pkg/testhelpers/test_helper.go similarity index 100% rename from pkg/testhelpers/testhelper.go rename to pkg/testhelpers/test_helper.go -- 2.45.2 From d7ad4108a7b28624f00d141debbead0dbe20bab9 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 18:12:44 -0400 Subject: [PATCH 05/11] Code clean up + Beacon Chain Connection This concludes all the code needed to connect to the DB and beacon node. We will no longer reference the lighthouse client because this application should work interchangeably with any beacon node. I have also standardized logging. --- README.md | 33 +++++++-- cmd/capture.go | 2 +- cmd/head.go | 10 +-- cmd/root.go | 97 ++++++++++++++++----------- internal/boot/boot.go | 79 ++++++++++++++++++++-- internal/boot/{startup.md => boot.md} | 0 internal/boot/setup_database.go | 29 -------- pkg/database/sql/postgres/config.go | 1 + pkg/database/sql/postgres/database.go | 2 +- pkg/database/sql/postgres/pgx_test.go | 6 +- pkg/loghelper/log_error.go | 13 ++++ 11 files changed, 184 insertions(+), 88 deletions(-) rename internal/boot/{startup.md => boot.md} (100%) delete mode 100644 internal/boot/setup_database.go create mode 100644 pkg/loghelper/log_error.go diff --git a/README.md b/README.md index 88998e9..1025ccc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ipld-ethcl-indexer -This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. +This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node. # Running the Application @@ -10,11 +10,36 @@ To run the application, utilize the following command, and update the values as go run main.go capture head --db.address localhost \ --db.password password \ --db.port 8077 \ - --db.username username \ - --lh.address localhost \ - --lh.port 5052 + --db.username vdbm \ + --db.name vulcanize_testing \ + --db.driver PGX \ + --bc.address localhost \ + --bc.port 5052 \ + --log.level info ``` +# Development Patterns + +This section will cover some generic development patterns utilizes. + +## Logging + +For logging, please keep the following in mind: + +- Utilize logrus. +- Use `log.Debug` to highlight that you are **about** to do something. +- Use `log.Info-Fatal` when the thing you were about to do has been completed, along with the result. + +``` +log.Debug("Adding 1 + 2") +a := 1 + 2 +log.Info("1 + 2 successfully Added, outcome is: ", a) +``` + +## Boot + +The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start. + # Contribution If you want to contribute please make sure you do the following: diff --git a/cmd/capture.go b/cmd/capture.go index 52b5315..6bda50a 100644 --- a/cmd/capture.go +++ b/cmd/capture.go @@ -14,7 +14,7 @@ var captureCmd = &cobra.Command{ Short: "Capture the SignedBeaconBlocks and BeaconStates from the Beacon Chain", Long: `Capture SignedBeaconBlocks and BeaconStates from the Beacon Chain. These blocks and states will be captured in - Postgres. They require a lighthouse client to be connected. You can run this to + Postgres. They require a beacon client to be connected. You can run this to capture blocks and states at head or historic blocks.`, } diff --git a/cmd/head.go b/cmd/head.go index 2d297a3..86e4f9f 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -5,12 +5,9 @@ Copyright © 2022 NAME HERE package cmd import ( - "fmt" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) // headCmd represents the head command @@ -19,15 +16,14 @@ var headCmd = &cobra.Command{ Short: "Capture only the blocks and state at head.", Long: `Capture only the blocks and state at head.`, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("head called") startHeadTracking() }, } func startHeadTracking() { - _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver) + _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) if err != nil { - log.Fatal("Unable to Start application with error: ", err) + loghelper.LogError(err).Error("Unable to Start application") } } diff --git a/cmd/root.go b/cmd/root.go index 05b9682..b06c224 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,31 +6,32 @@ package cmd import ( "fmt" + "io" "os" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - dbUsername string - dbPassword string - dbName string - dbAddress string - dbDriver string - dbPort int - lhAddress string - lhPort uint16 - logWithCommand log.Entry + cfgFile string + dbUsername string + dbPassword string + dbName string + dbAddress string + dbDriver string + dbPort int + bcAddress string + bcPort int ) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "ipld-ethcl-indexer", - Short: "This application will keep track of all BeaconState's and SginedBeaconBlock's on the Beacon Chain.", - Long: `This is an application that will capture the BeaconState's and SginedBeaconBlock's on the Beacon Chain. + Short: "This application will keep track of all BeaconState's and SignedBeaconBlock's on the Beacon Chain.", + Long: `This is an application that will capture the BeaconState's and SignedBeaconBlock's on the Beacon Chain. It can either do this will keeping track of head, or backfilling historic data.`, PersistentPreRun: initFuncs, // Uncomment the following line if your bare application @@ -49,23 +50,10 @@ func Execute() { // Prerun for Cobra func initFuncs(cmd *cobra.Command, args []string) { - viper.BindEnv("log.file", "LOGRUS_FILE") - logfile := viper.GetString("log.file") - if logfile != "" { - file, err := os.OpenFile(logfile, - os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err == nil { - log.Infof("Directing output to %s", logfile) - log.SetOutput(file) - } else { - log.SetOutput(os.Stdout) - log.Info("Failed to log to file, using default stdout") - } - } else { - log.SetOutput(os.Stdout) - } + logFormat() + logFile() if err := logLevel(); err != nil { - log.Fatal("Could not set log level: ", err) + log.WithField("err", err).Error("Could not set log level") } } @@ -84,6 +72,35 @@ func logLevel() error { return nil } +// Create a log file +func logFile() { + viper.BindEnv("log.file", "LOGRUS_FILE") + logfile := viper.GetString("log.file") + if logfile != "" { + file, err := os.OpenFile(logfile, + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + log.Infof("Directing output to %s", logfile) + mw := io.MultiWriter(os.Stdout, file) + logrus.SetOutput(mw) + } else { + log.SetOutput(os.Stdout) + log.Info("Failed to log to file, using default stdout") + } + } else { + log.SetOutput(os.Stdout) + } +} + +func logFormat() { + logFormat := viper.GetString("log.format") + + if logFormat == "json" { + log.SetFormatter(&log.JSONFormatter{}) + + } +} + func init() { cobra.OnInitialize(initConfig) @@ -93,8 +110,9 @@ func init() { // Optional Flags rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") - rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") - rootCmd.PersistentFlags().String("log-file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") + rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().String("log.format", "json", "json or text") // Required Flags @@ -112,16 +130,17 @@ func init() { rootCmd.MarkPersistentFlagRequired("db.name") rootCmd.MarkPersistentFlagRequired("db.driver") - //// Lighthouse Specific - rootCmd.PersistentFlags().StringVarP(&lhAddress, "lh.address", "l", "", "Address to connect to lighthouse node (required if username is set)") - rootCmd.PersistentFlags().Uint16VarP(&lhPort, "lh.port", "r", 0, "Port to connect to lighthouse node (required if username is set)") - rootCmd.MarkPersistentFlagRequired("lh.address") - rootCmd.MarkPersistentFlagRequired("lh.port") + //// Beacon Client Specific + rootCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)") + rootCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)") + rootCmd.MarkPersistentFlagRequired("bc.address") + rootCmd.MarkPersistentFlagRequired("bc.port") // Bind Flags with Viper // Optional - viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) - viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file")) + viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level")) + viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file")) + viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format")) //// DB Flags viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username")) @@ -132,8 +151,8 @@ func init() { viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver")) // LH specific - viper.BindPFlag("lh.address", rootCmd.PersistentFlags().Lookup("lh.address")) - viper.BindPFlag("lh.port", rootCmd.PersistentFlags().Lookup("lh.port")) + viper.BindPFlag("bc.address", rootCmd.PersistentFlags().Lookup("bc.address")) + viper.BindPFlag("bc.port", rootCmd.PersistentFlags().Lookup("bc.port")) // Cobra also supports local flags, which will only run // when this action is called directly. diff --git a/internal/boot/boot.go b/internal/boot/boot.go index e251a5b..13420ef 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -1,19 +1,88 @@ package boot import ( + "fmt" + "net/http" + "strconv" + log "github.com/sirupsen/logrus" "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) -func setUpLightHouse() { +var ( + bcHealthEndpoint = "/eth/v1/node/health" +) + +// This function will ensure that we can connect to the beacon client. +// Keep in mind, the beacon client will allow you to connect to it but it might +// Not allow you to make http requests. This is part of its built in logic, and you will have +// to follow their provided guidelines. https://lighthouse-book.sigmaprime.io/api-bn.html#security +func checkBeaconClient(bcAddress string, bcPort int) error { + log.Debug("Attempting to connect to the beacon client") + bcEndpoint := "http://" + bcAddress + ":" + strconv.Itoa(bcPort) + bcHealthEndpoint + resp, err := http.Get(bcEndpoint) + if err != nil { + loghelper.LogError(err).Error("Unable to get bc endpoint: ", bcEndpoint) + return err + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + log.Error("We recieved a non 2xx status code when checking the health of the beacon node.") + log.Error("Health Endpoint Status Code: ", resp.StatusCode) + return fmt.Errorf("beacon Node Provided a non 2xx status code, code provided: %d", resp.StatusCode) + } + + log.Info("We can successfully reach the beacon client.") + return nil } -// This function will perform some boot operations. -// 1. Setup a logger +// A simple wrapper to create a DB object to use. +func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { + log.Debug("Resolving Driver Type") + DbDriver, err := postgres.ResolveDriverType(driverName) + if err != nil { + log.WithFields(log.Fields{ + "err": err, + "driver_name_provided": driverName, + }).Error("Can't resolve driver type") + } + log.Info("Using Driver:", DbDriver) + + postgresConfig := postgres.Config{ + Hostname: dbHostname, + Port: dbPort, + DatabaseName: dbName, + Username: dbUsername, + Password: dbPassword, + Driver: DbDriver, + } + DB, err := postgres.NewPostgresDB(postgresConfig) + + if err != nil { + loghelper.LogError(err).Error("Unable to connect to the DB") + return nil, err + } + return DB, err + +} + +// This function will perform some boot operations. If any steps fail, the application will fail to start. +// Keep in mind that the DB connection can be lost later in the lifecycle of the application or +// it might not be able to connect to the beacon client. +// +// 1. Make sure the Beacon client is up. +// // 2. Connect to the database. -// 3. Connect to to the lighthouse client. -func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { +// +func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { + log.Debug("Checking beacon Client") + err := checkBeaconClient(bcAddress, bcPort) + if err != nil { + return nil, err + } + log.Debug("Setting up DB connection") DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) if err != nil { diff --git a/internal/boot/startup.md b/internal/boot/boot.md similarity index 100% rename from internal/boot/startup.md rename to internal/boot/boot.md diff --git a/internal/boot/setup_database.go b/internal/boot/setup_database.go deleted file mode 100644 index a1efaf0..0000000 --- a/internal/boot/setup_database.go +++ /dev/null @@ -1,29 +0,0 @@ -// This file will allow users to setup a new DB based on the user provided inputs. - -package boot - -import ( - log "github.com/sirupsen/logrus" - "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" -) - -func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { - log.Debug("Resolving Driver Type") - DbDriver, err := postgres.ResolveDriverType(driverName) - if err != nil { - log.Fatal("Can't Connect to DB") - } - log.Info("Using Driver:", DbDriver) - - postgresConfig := postgres.Config{ - Hostname: dbHostname, - Port: dbPort, - DatabaseName: dbName, - Username: dbUsername, - Password: dbPassword, - Driver: DbDriver, - } - DB, err := postgres.NewPostgresDB(postgresConfig) - return DB, err - -} diff --git a/pkg/database/sql/postgres/config.go b/pkg/database/sql/postgres/config.go index a518bed..efe5eb9 100644 --- a/pkg/database/sql/postgres/config.go +++ b/pkg/database/sql/postgres/config.go @@ -22,6 +22,7 @@ var DefaultConfig = Config{ DatabaseName: "vulcanize_testing", Username: "vdbm", Password: "password", + Driver: "PGX", } // ResolveDriverType resolves a DriverType from a provided string diff --git a/pkg/database/sql/postgres/database.go b/pkg/database/sql/postgres/database.go index 9d44823..d79821a 100644 --- a/pkg/database/sql/postgres/database.go +++ b/pkg/database/sql/postgres/database.go @@ -38,7 +38,7 @@ func createDriver(c Config) (*pgxDriver, error) { log.Info("Successfully created a driver for PGX") return driver, nil default: - log.Fatal("Couldnt find a driver to create for: ", c.Driver) + log.Error("Couldnt find a driver to create for: ", c.Driver) return nil, fmt.Errorf("Can't find a driver to create") } diff --git a/pkg/database/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go index ab75a7e..b66deeb 100644 --- a/pkg/database/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -79,7 +79,9 @@ func TestPostgresPGX(t *testing.T) { }) t.Run("throws error when can't connect to the database", func(t *testing.T) { - _, err := NewPostgresDB(Config{}, "PGX") + _, err := NewPostgresDB(Config{ + Driver: "PGX", + }) if err == nil { t.Fatal("Expected an error") } @@ -87,7 +89,7 @@ func TestPostgresPGX(t *testing.T) { expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) }) t.Run("Connect to the database", func(t *testing.T) { - driver, err := NewPostgresDB(DefaultConfig, "pgx") + driver, err := NewPostgresDB(DefaultConfig) defer driver.Close() if err != nil { diff --git a/pkg/loghelper/log_error.go b/pkg/loghelper/log_error.go new file mode 100644 index 0000000..41d0149 --- /dev/null +++ b/pkg/loghelper/log_error.go @@ -0,0 +1,13 @@ +// A simple function to help with logging errors. +package loghelper + +import ( + log "github.com/sirupsen/logrus" +) + +// A simple helper function that will help wrap the error message. +func LogError(err error) *log.Entry { + return log.WithFields(log.Fields{ + "err": err, + }) +} -- 2.45.2 From 7a2c4b3cefcab51450ccf517d13e9c20cbf861fc Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Wed, 20 Apr 2022 18:23:21 -0400 Subject: [PATCH 06/11] Last second clean ups --- README.md | 2 ++ cmd/root.go | 1 + pkg/database/sql/postgres/database.go | 5 +---- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1025ccc..a63cac9 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ a := 1 + 2 log.Info("1 + 2 successfully Added, outcome is: ", a) ``` +- `loghelper.LogError(err)` is a pretty wrapper to output errors. + ## Boot The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start. diff --git a/cmd/root.go b/cmd/root.go index b06c224..860dc78 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -92,6 +92,7 @@ func logFile() { } } +// Format the logger func logFormat() { logFormat := viper.GetString("log.format") diff --git a/pkg/database/sql/postgres/database.go b/pkg/database/sql/postgres/database.go index d79821a..a28d2ad 100644 --- a/pkg/database/sql/postgres/database.go +++ b/pkg/database/sql/postgres/database.go @@ -10,10 +10,6 @@ import ( var _ sql.Database = &DB{} -// TODO: Make NewPostgresDB accept a string and Config. IT should -// Create a driver of its own. -// This will make sure that if you want a driver, it conforms to the interface. - // NewPostgresDB returns a postgres.DB using the provided Config and driver type. func NewPostgresDB(c Config) (*DB, error) { var driver *pgxDriver @@ -27,6 +23,7 @@ func NewPostgresDB(c Config) (*DB, error) { return &DB{driver}, nil } +// Create a driver based on the config func createDriver(c Config) (*pgxDriver, error) { switch c.Driver { case PGX: -- 2.45.2 From e2f7fa381bd1f9b28e8550fb4b5da228847b1cc3 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Fri, 22 Apr 2022 08:28:01 -0400 Subject: [PATCH 07/11] Utilize Ginkgo toreplace `testing` library, and add CI/CD. (#9) * Utilize Ginkgo and replace `testing` library. * Add TOC to docs * Add Docker specific files * Remove -e * Update on-pr-manual.yml * Add depth * Add repositories * Remove $ from path * Update path for make * Setup Go and Ginkgo * Use go mod download * Use go install * Update on-pr-manual.yml * Use latest * Remove install of GINKGO * Add explicit gopath * Explicitly specify the gopath * Update on-pr-manual.yml * Update on-pr-manual.yml * Update on-pr-manual.yml * Update on-pr-manual.yml * Update on-pr-manual.yml * Update on-pr-manual.yml * Use which ginkgo * Try with make now * Final working Make --- .github/workflows/.env | 0 .github/workflows/on-pr-automated.yaml | 70 +++++++++ .github/workflows/on-pr-manual.yml | 48 +++++-- Dockerfile | 20 +++ Makefile | 36 +++++ README.md | 20 ++- application_component.md | 15 ++ entrypoint.sh | 31 ++++ go.mod | 13 +- go.sum | 57 +++++++- pkg/database/sql/postgres/pgx_test.go | 135 ++++++++---------- .../sql/postgres/postgres_suite_test.go | 13 ++ pkg/testhelpers/test_helper.go | 7 +- 13 files changed, 367 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/.env create mode 100644 .github/workflows/on-pr-automated.yaml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 application_component.md create mode 100755 entrypoint.sh create mode 100644 pkg/database/sql/postgres/postgres_suite_test.go diff --git a/.github/workflows/.env b/.github/workflows/.env new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/on-pr-automated.yaml b/.github/workflows/on-pr-automated.yaml new file mode 100644 index 0000000..2f196de --- /dev/null +++ b/.github/workflows/on-pr-automated.yaml @@ -0,0 +1,70 @@ +name: Test Application + +on: + pull_request: + +jobs: + build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker build + run: make docker-build + test: + name: Run unit tests + runs-on: ubuntu-latest + env: + foundry-test-ref: feature/build-stack + ipld-eth-db-ref: main + GOPATH: /tmp/go + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.foundry-test-ref }} + path: "./foundry-test/" + repository: vulcanize/foundry-test + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-eth-db-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + cat ./config.sh + + - name: Run docker compose + run: | + docker-compose \ + -f "$GITHUB_WORKSPACE/foundry-test/docker/local/docker-compose-db.yml" \ + -f "$GITHUB_WORKSPACE/foundry-test/docker/latest/docker-compose-lighthouse.yml" \ + --env-file ./config.sh \ + up -d --build + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + check-latest: true + + - name: Install packages + run: | + go install github.com/onsi/ginkgo/v2/ginkgo@latest + which ginkgo + + - name: Run the tests using Make + run: | + cd ipld-ethcl-indexer + make test diff --git a/.github/workflows/on-pr-manual.yml b/.github/workflows/on-pr-manual.yml index edd2fa2..26807cb 100644 --- a/.github/workflows/on-pr-manual.yml +++ b/.github/workflows/on-pr-manual.yml @@ -23,33 +23,55 @@ jobs: test: name: Run unit tests runs-on: ubuntu-latest + env: + GOPATH: /tmp/go steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.foundry-test-ref }} path: "./foundry-test/" - - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.inputs.foundry-test-ref }} - path: "/tmp/foundry-test/" + repository: vulcanize/foundry-test + fetch-depth: 0 - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.ipld-eth-db-ref }} - path: "/tmp/ipld-eth-db-ref/" + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + cat ./config.sh - name: Run docker compose run: | - docker-compose -e vulcanize_ipld_ethcl_indexer="$(pwd)/ipld-ethcl/indexer" \ - -e vulcanize_ipld_eth_db=/tmp/ipld-eth-db-ref/ - -f "/tmp/foundry-test/docker/local/docker-compose-db.yml" - -f "/tmp/foundry-test/docker/latest/docker-compose-lighthouse.yml" - -d up --build + docker-compose \ + -f "$GITHUB_WORKSPACE/foundry-test/docker/local/docker-compose-db.yml" \ + -f "$GITHUB_WORKSPACE/foundry-test/docker/latest/docker-compose-lighthouse.yml" \ + --env-file ./config.sh \ + up -d --build - - name: Wait and run tests + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + check-latest: true + + - name: Install packages run: | - sleep 20 + go install github.com/onsi/ginkgo/v2/ginkgo@latest + which ginkgo + + - name: Run the tests using Make + run: | + cd ipld-ethcl-indexer make test diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..46f2575 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:1.18-alpine as builder + +WORKDIR /go/src/github.com/vulcanize/ipld-ethcl-indexer +RUN apk --no-cache add ca-certificates make git g++ linux-headers + +ENV GO111MODULE=on +COPY go.mod . +COPY go.sum . +RUN go mod tidy; go mod download +COPY . . + +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-ethcl-indexer . +RUN chmod +x ipld-ethcl-indexer + +FROM frolvlad/alpine-bash:latest +RUN apk --no-cache add ca-certificates +WORKDIR /root/ +COPY --from=builder /go/src/github.com/vulcanize/ipld-ethcl-indexer/ipld-ethcl-indexer /root/ipld-ethcl-indexer +ADD entrypoint.sh . +ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..36f985f --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +BIN = $(GOPATH)/bin +BASE = $(GOPATH)/src/$(PACKAGE) +PKGS = go list ./... | grep -v "^vendor/" + +# Tools +## Testing library +GINKGO = $(BIN)/ginkgo +$(BIN)/ginkgo: + go install github.com/onsi/ginkgo/ginkgo + + +.PHONY: installtools +installtools: | $(GINKGO) + echo "Installing tools" + +.PHONY: test +test: + go vet ./... + go fmt ./... + $(GINKGO) -r + +#.PHONY: integrationtest +#integrationtest: | $(GINKGO) $(GOOSE) +# go vet ./... +# go fmt ./... +# $(GINKGO) -r test/ -v + +.PHONY: build +build: + go fmt ./... + GO111MODULE=on go build + +## Build docker image +.PHONY: docker-build +docker-build: + docker build -t vulcanize/ipld-ethcl-indexer . \ No newline at end of file diff --git a/README.md b/README.md index a63cac9..d3b7892 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,19 @@ +- [ipld-ethcl-indexer](#ipld-ethcl-indexer) +- [Running the Application](#running-the-application) +- [Development Patterns](#development-patterns) + - [Logging](#logging) + - [Testing](#testing) +- [Contribution](#contribution) + - [Branching Structure](#branching-structure) + +Table of contents generated with markdown-toc + # ipld-ethcl-indexer This application will capture all the `BeaconState`'s and `SignedBeaconBlock`'s from the consensus chain on Ethereum. This application is going to connect to the lighthouse client, but hypothetically speaking, it should be interchangeable with any eth2 beacon node. +To learn more about the applications individual components, please read the [application components](/application_component.md). + # Running the Application To run the application, utilize the following command, and update the values as needed. @@ -38,9 +50,13 @@ log.Info("1 + 2 successfully Added, outcome is: ", a) - `loghelper.LogError(err)` is a pretty wrapper to output errors. -## Boot +## Testing -The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start. +This project utilizes `ginkgo` for testing. A few notes on testing: + +- All tests within this code base will test **public methods only**. +- All test packages are named `{base_package}_test`. This ensures we only test the public methods. +- If there is a need to test a private method, please include why in the testing file. # Contribution diff --git a/application_component.md b/application_component.md new file mode 100644 index 0000000..0dbc67d --- /dev/null +++ b/application_component.md @@ -0,0 +1,15 @@ +- [Overview](#overview) +- [Components](#components) + - [Boot](#boot) + +Table of contents generated with markdown-toc + +# Overview + +This document will go through various application components + +# Components + +## Boot + +The boot package in `internal` is utilized to start the application. Everything in the boot process must complete successfully for the application to start. If it does not, the application will not start. diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..6e4608e --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,31 @@ +#!/bin/bash +echo "Starting ipld-ethcl-indexer" + +echo /root/ipld-ethcl-indexer capture head --db.address $DB_ADDRESS \ + --db.password $DB_PASSWORD \ + --db.port $DB_PORT \ + --db.username $DB_USER \ + --db.name $DB_NAME \ + --db.driver $DB_DRIVER \ + --bc.address $BC_ADDRESS \ + --bc.port $BC_PORT \ + --log.level $LOG_LEVEL + +/root/ipld-ethcl-indexer capture head --db.address $DB_ADDRESS \ + --db.password $DB_PASSWORD \ + --db.port $DB_PORT \ + --db.username $DB_USER \ + --db.name $DB_NAME \ + --db.driver $DB_DRIVER \ + --bc.address $BC_ADDRESS \ + --bc.port $BC_PORT \ + --log.level $LOG_LEVEL + +rv=$? + +if [ $rv != 0 ]; then + echo "ipld-ethcl-indexer startup failed" + exit 1 +fi + +tail -f /dev/null \ No newline at end of file diff --git a/go.mod b/go.mod index b96803c..cb608b1 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,15 @@ module github.com/vulcanize/ipld-ethcl-indexer go 1.18 -require github.com/sirupsen/logrus v1.8.1 +require ( + github.com/jackc/pgconn v1.11.0 + github.com/onsi/ginkgo/v2 v2.1.3 + github.com/onsi/gomega v1.19.0 + github.com/sirupsen/logrus v1.8.1 +) require ( github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.11.0 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.2.0 // indirect @@ -15,6 +19,7 @@ require ( github.com/jackc/puddle v1.2.1 // indirect github.com/pkg/errors v0.9.1 // indirect golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect + golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect ) require ( @@ -29,10 +34,10 @@ require ( github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect github.com/spf13/afero v1.8.2 // indirect github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/cobra v1.4.0 // indirect + github.com/spf13/cobra v1.4.0 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.11.0 // indirect + github.com/spf13/viper v1.11.0 github.com/subosito/gotenv v1.2.0 // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index 2e87236..614e88e 100644 --- a/go.sum +++ b/go.sum @@ -47,13 +47,16 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs= github.com/cockroachdb/cockroach-go/v2 v2.2.0/go.mod h1:u3MiKYGupPPjkn3ozknpMUpxPaNLTFWAya419/zv6eI= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -61,6 +64,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/georgysavva/scany v0.3.0 h1:MA1aEqPbnNuiek59gMpNPqQrXXroyFj5jCADlETdxiA= @@ -72,8 +77,11 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -100,6 +108,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -111,6 +122,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -124,6 +136,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -133,11 +146,11 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -157,10 +170,10 @@ github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -210,15 +223,18 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -233,6 +249,18 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.0-beta.8 h1:dy81yyLYJDwMTifq24Oi/IslOslRrDSb3jwDggjz3Z0= @@ -241,6 +269,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -251,6 +280,7 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -270,12 +300,14 @@ github.com/spf13/viper v1.11.0 h1:7OX/1FS6n7jHD1zGrZTM7WtY13ZELRyosK4k93oPr44= github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -353,6 +385,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -374,6 +407,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -383,6 +417,9 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -404,6 +441,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -417,9 +455,11 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -441,8 +481,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -513,6 +555,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -610,13 +653,21 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/database/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go index b66deeb..c8c1bae 100644 --- a/pkg/database/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -1,100 +1,89 @@ -package postgres +package postgres_test import ( "context" + "fmt" "math/big" "strings" - "testing" - "github.com/jackc/pgx/v4/pgxpool" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" "github.com/vulcanize/ipld-ethcl-indexer/pkg/testhelpers" ) -var ( - pgConfig, _ = makeConfig(DefaultConfig) - ctx = context.Background() -) +var _ = Describe("Pgx", func() { -func expectContainsSubstring(t *testing.T, full string, sub string) { - if !strings.Contains(full, sub) { - t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub) - } -} + var ( + ctx context.Context + ) -func TestPostgresPGX(t *testing.T) { - t.Run("connects to the sql", func(t *testing.T) { - dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig) - if err != nil { - t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err) - } - if dbPool == nil { - t.Fatal("DB pool is nil") - } - dbPool.Close() + BeforeEach(func() { + ctx = context.Background() }) - t.Run("serializes big.Int to db", func(t *testing.T) { - // postgres driver doesn't support go big.Int type - // various casts in golang uint64, int64, overflow for - // transaction value (in wei) even though - // postgres numeric can handle an arbitrary - // sized int, so use string representation of big.Int - // and cast on insert + Describe("Connecting to the DB", func() { + Context("But connection is unsucessful", func() { + It("throws error when can't connect to the database", func() { + _, err := postgres.NewPostgresDB(postgres.Config{ + Driver: "PGX", + }) + Expect(err).NotTo(BeNil()) - dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig) - if err != nil { - t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err) - } - defer dbPool.Close() + present, err := doesContainsSubstring(err.Error(), sql.DbConnectionFailedMsg) + Expect(present).To(BeTrue()) + }) + }) + Context("The connection is successful", func() { + It("Should create a DB object", func() { + db, err := postgres.NewPostgresDB(postgres.DefaultConfig) + defer db.Close() + Expect(err).To(BeNil()) + }) + }) + }) + Describe("Write to the DB", func() { + Context("Serialize big.Int to DB", func() { + It("Should serialize successfully", func() { + dbPool, err := postgres.NewPostgresDB(postgres.DefaultConfig) + Expect(err).To(BeNil()) + defer dbPool.Close() - bi := new(big.Int) - bi.SetString("34940183920000000000", 10) - testhelpers.ExpectEqual(t, bi.String(), "34940183920000000000") + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + isEqual, err := testhelpers.IsEqual(bi.String(), "34940183920000000000") + Expect(isEqual).To(BeTrue()) - defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) - _, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )") - if err != nil { - t.Fatal(err) - } + defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) + _, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )") + Expect(err).To(BeNil()) - sqlStatement := ` + sqlStatement := ` INSERT INTO example (id, data) VALUES (1, cast($1 AS NUMERIC))` - _, err = dbPool.Exec(ctx, sqlStatement, bi.String()) - if err != nil { - t.Fatal(err) - } + _, err = dbPool.Exec(ctx, sqlStatement, bi.String()) + Expect(err).To(BeNil()) - var data string - err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data) - if err != nil { - t.Fatal(err) - } + var data string + err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data) + Expect(err).To(BeNil()) - testhelpers.ExpectEqual(t, data, bi.String()) - actual := new(big.Int) - actual.SetString(data, 10) - testhelpers.ExpectEqual(t, actual, bi) - }) + isEqual, err = testhelpers.IsEqual(data, bi.String()) + Expect(isEqual).To(BeTrue()) + actual := new(big.Int) + actual.SetString(data, 10) - t.Run("throws error when can't connect to the database", func(t *testing.T) { - _, err := NewPostgresDB(Config{ - Driver: "PGX", + isEqual, err = testhelpers.IsEqual(actual, bi) + Expect(isEqual).To(BeTrue()) + }) }) - if err == nil { - t.Fatal("Expected an error") - } - - expectContainsSubstring(t, err.Error(), sql.DbConnectionFailedMsg) }) - t.Run("Connect to the database", func(t *testing.T) { - driver, err := NewPostgresDB(DefaultConfig) - defer driver.Close() +}) - if err != nil { - t.Fatal("Error creating the postgres driver") - } - - }) +func doesContainsSubstring(full string, sub string) (bool, error) { + if !strings.Contains(full, sub) { + return false, fmt.Errorf("Expected \"%v\" to contain substring \"%v\"\n", full, sub) + } + return true, nil } diff --git a/pkg/database/sql/postgres/postgres_suite_test.go b/pkg/database/sql/postgres/postgres_suite_test.go new file mode 100644 index 0000000..eb6a7f9 --- /dev/null +++ b/pkg/database/sql/postgres/postgres_suite_test.go @@ -0,0 +1,13 @@ +package postgres_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPostgres(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Postgres Suite") +} diff --git a/pkg/testhelpers/test_helper.go b/pkg/testhelpers/test_helper.go index 4e3c917..5b3cc0f 100644 --- a/pkg/testhelpers/test_helper.go +++ b/pkg/testhelpers/test_helper.go @@ -1,15 +1,16 @@ package testhelpers import ( + "fmt" "reflect" - "testing" ) // ExpectEqual asserts the provided interfaces are deep equal -func ExpectEqual(t *testing.T, got interface{}, want interface{}) { +func IsEqual(got interface{}, want interface{}) (bool, error) { if !reflect.DeepEqual(got, want) { - t.Fatalf("Expected: %v\nActual: %v", want, got) + return false, fmt.Errorf("Expected: %v\nActual: %v", want, got) } + return true, nil } // ListContainsString used to check if a list of strings contains a particular string -- 2.45.2 From 3d46a029f1469400812b44ec9f15e754cc0b6211 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Fri, 22 Apr 2022 12:27:54 -0400 Subject: [PATCH 08/11] Feature/11 testing add tests for boot (#12) * Utilize Versioning * Update Testing and CI/CD * Update Testing * Add Linting and Remove timeout * Update Lint * handle errors to make the linter happy --- .github/ISSUE_TEMPLATE/feature_request.md | 4 +- .github/workflows/on-pr-automated.yaml | 77 +++++++++++++++++++-- Makefile | 27 ++++++-- README.md | 3 + cmd/capture.go | 79 +++++++++++++++++++-- cmd/head.go | 3 +- cmd/root.go | 84 ++++++++--------------- cmd/version.go | 52 ++++++++++++++ go.mod | 3 + go.sum | 13 ++-- internal/boot/boot.go | 22 ++++++ internal/boot/boot_suite_test.go | 13 ++++ internal/boot/boot_test.go | 48 +++++++++++++ pkg/database/sql/postgres/pgx_test.go | 12 +++- pkg/version/version.go | 25 +++++++ 15 files changed, 385 insertions(+), 80 deletions(-) create mode 100644 cmd/version.go create mode 100644 internal/boot/boot_suite_test.go create mode 100644 internal/boot/boot_test.go create mode 100644 pkg/version/version.go diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index f642407..bdb347e 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,7 +7,7 @@ assignees: "" --- - [Request](#request) - - [Potentialy Solution](#potentialy-solution) + - [Potential Solution](#potential-solution) - [Alternative Solutions](#alternative-solutions) - [Additional Context](#additional-context) @@ -17,7 +17,7 @@ assignees: "" **Explain what you want and why. If this feature is related to a problem please highlight it here.** -## Potentialy Solution +## Potential Solution **Provide any details for a potential solution.** diff --git a/.github/workflows/on-pr-automated.yaml b/.github/workflows/on-pr-automated.yaml index 2f196de..d210b2e 100644 --- a/.github/workflows/on-pr-automated.yaml +++ b/.github/workflows/on-pr-automated.yaml @@ -2,17 +2,74 @@ name: Test Application on: pull_request: + paths: + - "!**.md" + - ".gitignore" + - "!LICENSE" + - "!.github/workflows/**" + - ".github/workflows/on-pr-automated.yaml" jobs: build: - name: Run docker build + name: Run Docker Build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Run docker build run: make docker-build - test: - name: Run unit tests + + unit-test: + name: Run Unit Tests + runs-on: ubuntu-latest + env: + foundry-test-ref: feature/build-stack + ipld-eth-db-ref: main + GOPATH: /tmp/go + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.foundry-test-ref }} + path: "./foundry-test/" + repository: vulcanize/foundry-test + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-eth-db-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + cat ./config.sh + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + check-latest: true + + - name: Install packages + run: | + go install github.com/onsi/ginkgo/v2/ginkgo@latest + which ginkgo + + - name: Run the tests using Make + run: | + cd ipld-ethcl-indexer + make unit-test-ci + + integration-test: + name: Run Integration Tests runs-on: ubuntu-latest env: foundry-test-ref: feature/build-stack @@ -67,4 +124,16 @@ jobs: - name: Run the tests using Make run: | cd ipld-ethcl-indexer - make test + make integration-test-ci + + golangci: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + args: --disable errcheck diff --git a/Makefile b/Makefile index 36f985f..7b97424 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,28 @@ test: go fmt ./... $(GINKGO) -r -#.PHONY: integrationtest -#integrationtest: | $(GINKGO) $(GOOSE) -# go vet ./... -# go fmt ./... -# $(GINKGO) -r test/ -v +.PHONY: integration-test-ci +integration-test-ci: + go vet ./... + go fmt ./... + $(GINKGO) -r --label-filter integration \ + --procs=4 --compilers=4 \ + --randomize-all --randomize-suites \ + --fail-on-pending --keep-going \ + --cover --coverprofile=cover.profile \ + --race --trace --json-report=report.json + +.PHONY: unit-test-ci +unit-test-ci: + go vet ./... + go fmt ./... + $(GINKGO) -r --label-filter unit \ + --procs=4 --compilers=4 \ + --randomize-all --randomize-suites \ + --fail-on-pending --keep-going \ + --cover --coverprofile=cover.profile \ + --race --trace --json-report=report.json + .PHONY: build build: diff --git a/README.md b/README.md index d3b7892..f24ade2 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,9 @@ This project utilizes `ginkgo` for testing. A few notes on testing: - All tests within this code base will test **public methods only**. - All test packages are named `{base_package}_test`. This ensures we only test the public methods. - If there is a need to test a private method, please include why in the testing file. +- Unit tests must contain the `Label("unit")`. +- Unit tests should not rely on any running service. If a running service is needed. Utilize an integration test. +- Integration tests must contain the `Label("integration")`. # Contribution diff --git a/cmd/capture.go b/cmd/capture.go index 6bda50a..54d3fb9 100644 --- a/cmd/capture.go +++ b/cmd/capture.go @@ -5,7 +5,21 @@ Copyright © 2022 NAME HERE package cmd import ( + "os" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + dbUsername string + dbPassword string + dbName string + dbAddress string + dbDriver string + dbPort int + bcAddress string + bcPort int ) // captureCmd represents the capture command @@ -21,13 +35,64 @@ var captureCmd = &cobra.Command{ func init() { rootCmd.AddCommand(captureCmd) + // Required Flags + + //// DB Specific + captureCmd.PersistentFlags().StringVarP(&dbUsername, "db.username", "", "", "Database username (required)") + captureCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "", "", "Database Password (required)") + captureCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "", "", "Port to connect to DB(required)") + captureCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)") + captureCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)") + captureCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)") + err := captureCmd.MarkPersistentFlagRequired("db.username") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.password") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.address") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.port") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.name") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("db.driver") + exitErr(err) + + //// Beacon Client Specific + captureCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)") + captureCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)") + err = captureCmd.MarkPersistentFlagRequired("bc.address") + exitErr(err) + err = captureCmd.MarkPersistentFlagRequired("bc.port") + exitErr(err) + + // Bind Flags with Viper + //// DB Flags + err = viper.BindPFlag("db.username", captureCmd.PersistentFlags().Lookup("db.username")) + exitErr(err) + err = viper.BindPFlag("db.password", captureCmd.PersistentFlags().Lookup("db.password")) + exitErr(err) + err = viper.BindPFlag("db.address", captureCmd.PersistentFlags().Lookup("db.address")) + exitErr(err) + err = viper.BindPFlag("db.port", captureCmd.PersistentFlags().Lookup("db.port")) + exitErr(err) + err = viper.BindPFlag("db.name", captureCmd.PersistentFlags().Lookup("db.name")) + exitErr(err) + err = viper.BindPFlag("db.driver", captureCmd.PersistentFlags().Lookup("db.driver")) + exitErr(err) + + // LH specific + err = viper.BindPFlag("bc.address", captureCmd.PersistentFlags().Lookup("bc.address")) + exitErr(err) + err = viper.BindPFlag("bc.port", captureCmd.PersistentFlags().Lookup("bc.port")) + exitErr(err) // Here you will define your flags and configuration settings. - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // captureCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // captureCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} + +// Helper function to catch any errors. +// We need to capture these errors for the linter. +func exitErr(err error) { + if err != nil { + os.Exit(1) + } } diff --git a/cmd/head.go b/cmd/head.go index 86e4f9f..12b432a 100644 --- a/cmd/head.go +++ b/cmd/head.go @@ -20,8 +20,9 @@ var headCmd = &cobra.Command{ }, } +// Start the application to track at head. func startHeadTracking() { - _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) + _, err := boot.BootApplicationWithRetry(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) if err != nil { loghelper.LogError(err).Error("Unable to Start application") } diff --git a/cmd/root.go b/cmd/root.go index 860dc78..6fc17c6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,22 +9,13 @@ import ( "io" "os" - "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - dbUsername string - dbPassword string - dbName string - dbAddress string - dbDriver string - dbPort int - bcAddress string - bcPort int + cfgFile string ) // rootCmd represents the base command when called without any subcommands @@ -51,7 +42,9 @@ func Execute() { // Prerun for Cobra func initFuncs(cmd *cobra.Command, args []string) { logFormat() - logFile() + if err := logFile(); err != nil { + log.WithField("err", err).Error("Could not set log file") + } if err := logLevel(); err != nil { log.WithField("err", err).Error("Could not set log level") } @@ -59,7 +52,10 @@ func initFuncs(cmd *cobra.Command, args []string) { // Set the log level for the application func logLevel() error { - viper.BindEnv("log.level", "LOGRUS_LEVEL") + err := viper.BindEnv("log.level", "LOGRUS_LEVEL") + if err != nil { + return err + } lvl, err := log.ParseLevel(viper.GetString("log.level")) if err != nil { return err @@ -73,16 +69,23 @@ func logLevel() error { } // Create a log file -func logFile() { - viper.BindEnv("log.file", "LOGRUS_FILE") +func logFile() error { + err := viper.BindEnv("log.file", "LOGRUS_FILE") + if err != nil { + return err + } logfile := viper.GetString("log.file") if logfile != "" { file, err := os.OpenFile(logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err == nil { - log.Infof("Directing output to %s", logfile) - mw := io.MultiWriter(os.Stdout, file) - logrus.SetOutput(mw) + if viper.GetBool("log.output") { + log.Infof("Directing output to %s", logfile) + mw := io.MultiWriter(os.Stdout, file) + log.SetOutput(mw) + } else { + log.SetOutput(file) + } } else { log.SetOutput(os.Stdout) log.Info("Failed to log to file, using default stdout") @@ -90,6 +93,7 @@ func logFile() { } else { log.SetOutput(os.Stdout) } + return nil } // Format the logger @@ -113,47 +117,19 @@ func init() { rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ipld-ethcl-indexer.yaml)") rootCmd.PersistentFlags().String("log.level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") rootCmd.PersistentFlags().String("log.file", "ipld-ethcl-indexer.log", "file path for logging") + rootCmd.PersistentFlags().Bool("log.output", true, "Should we log to STDOUT") rootCmd.PersistentFlags().String("log.format", "json", "json or text") - // Required Flags - - //// DB Specific - rootCmd.PersistentFlags().StringVarP(&dbUsername, "db.username", "", "", "Database username (required)") - rootCmd.PersistentFlags().StringVarP(&dbPassword, "db.password", "", "", "Database Password (required)") - rootCmd.PersistentFlags().StringVarP(&dbAddress, "db.address", "", "", "Port to connect to DB(required)") - rootCmd.PersistentFlags().StringVarP(&dbName, "db.name", "n", "", "Database name connect to DB(required)") - rootCmd.PersistentFlags().StringVarP(&dbDriver, "db.driver", "", "", "Database Driver to connect to DB(required)") - rootCmd.PersistentFlags().IntVarP(&dbPort, "db.port", "", 0, "Port to connect to DB(required)") - rootCmd.MarkPersistentFlagRequired("db.username") - rootCmd.MarkPersistentFlagRequired("db.password") - rootCmd.MarkPersistentFlagRequired("db.address") - rootCmd.MarkPersistentFlagRequired("db.port") - rootCmd.MarkPersistentFlagRequired("db.name") - rootCmd.MarkPersistentFlagRequired("db.driver") - - //// Beacon Client Specific - rootCmd.PersistentFlags().StringVarP(&bcAddress, "bc.address", "l", "", "Address to connect to beacon node (required if username is set)") - rootCmd.PersistentFlags().IntVarP(&bcPort, "bc.port", "r", 0, "Port to connect to beacon node (required if username is set)") - rootCmd.MarkPersistentFlagRequired("bc.address") - rootCmd.MarkPersistentFlagRequired("bc.port") - // Bind Flags with Viper // Optional - viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level")) - viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file")) - viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format")) - - //// DB Flags - viper.BindPFlag("db.username", rootCmd.PersistentFlags().Lookup("db.username")) - viper.BindPFlag("db.password", rootCmd.PersistentFlags().Lookup("db.password")) - viper.BindPFlag("db.address", rootCmd.PersistentFlags().Lookup("db.address")) - viper.BindPFlag("db.port", rootCmd.PersistentFlags().Lookup("db.port")) - viper.BindPFlag("db.name", rootCmd.PersistentFlags().Lookup("db.name")) - viper.BindPFlag("db.driver", rootCmd.PersistentFlags().Lookup("db.driver")) - - // LH specific - viper.BindPFlag("bc.address", rootCmd.PersistentFlags().Lookup("bc.address")) - viper.BindPFlag("bc.port", rootCmd.PersistentFlags().Lookup("bc.port")) + err := viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log.level")) + exitErr(err) + err = viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log.file")) + exitErr(err) + err = viper.BindPFlag("log.output", rootCmd.PersistentFlags().Lookup("log.output")) + exitErr(err) + err = viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("log.format")) + exitErr(err) // Cobra also supports local flags, which will only run // when this action is called directly. diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 0000000..7b70eb4 --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + v "github.com/vulcanize/ipld-ethcl-indexer/pkg/version" +) + +var ( + Major = 0 // Major version component of the current release + Minor = 0 // Minor version component of the current release + Patch = 0 // Patch version component of the current release + Meta = "" // Version metadata to append to the version string +) + +// versionCmd represents the version command +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Prints the version of ipld-eth-server", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + version := v.Version{ + Major: Major, + Minor: Minor, + Patch: Patch, + Meta: Meta, + } + log.Infof("ipld-ethcl-indexer version: %s", version.GetVersionWithMeta()) + fmt.Println(version.GetVersionWithMeta()) + }, +} + +func init() { + rootCmd.AddCommand(versionCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // versionCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // versionCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/go.mod b/go.mod index cb608b1..de583a9 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,12 @@ require ( github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.10.0 // indirect github.com/jackc/puddle v1.2.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lib/pq v1.10.4 // indirect github.com/pkg/errors v0.9.1 // indirect golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) require ( diff --git a/go.sum b/go.sum index 614e88e..fa80980 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -223,19 +224,22 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -657,8 +661,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= diff --git a/internal/boot/boot.go b/internal/boot/boot.go index 13420ef..2bbbc98 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "strconv" + "time" log "github.com/sirupsen/logrus" "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" @@ -12,6 +13,8 @@ import ( var ( bcHealthEndpoint = "/eth/v1/node/health" + maxRetry = 5 // Max times to try to connect to the DB or BC at boot. + retryInterval = 30 // The time to wait between each try. ) // This function will ensure that we can connect to the beacon client. @@ -77,6 +80,8 @@ func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, db // 2. Connect to the database. // func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { + log.Info("Booting the Application") + log.Debug("Checking beacon Client") err := checkBeaconClient(bcAddress, bcPort) if err != nil { @@ -90,3 +95,20 @@ func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername st } return DB, nil } + +// Add retry logic to ensure that we are give the Beacon Client and the DB time to start. +func BootApplicationWithRetry(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { + var db *postgres.DB + var err error + for i := 0; i < maxRetry; i++ { + db, err = BootApplication(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName, bcAddress, bcPort) + if err != nil { + log.WithFields(log.Fields{ + "retryNumber": i, + }).Warn("Unable to boot application. Going to try again") + time.Sleep(time.Duration(retryInterval) * time.Second) + continue + } + } + return db, err +} diff --git a/internal/boot/boot_suite_test.go b/internal/boot/boot_suite_test.go new file mode 100644 index 0000000..e7bcdf3 --- /dev/null +++ b/internal/boot/boot_suite_test.go @@ -0,0 +1,13 @@ +package boot_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBoot(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Boot Suite") +} diff --git a/internal/boot/boot_test.go b/internal/boot/boot_test.go new file mode 100644 index 0000000..729f30b --- /dev/null +++ b/internal/boot/boot_test.go @@ -0,0 +1,48 @@ +package boot_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vulcanize/ipld-ethcl-indexer/internal/boot" +) + +var _ = Describe("Boot", func() { + var ( + dbAddress string = "localhost" + dbPort int = 8077 + dbName string = "vulcanize_testing" + dbUsername string = "vdbm" + dbPassword string = "password" + dbDriver string = "PGX" + bcAddress string = "localhost" + bcPort int = 5052 + ) + Describe("Booting the application", Label("integration"), func() { + Context("When the DB and BC are both up and running", func() { + It("Should connect successfully", func() { + db, err := boot.BootApplicationWithRetry(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) + defer db.Close() + Expect(err).To(BeNil()) + }) + }) + Context("When the DB is running but not the BC", func() { + It("Should not connect successfully", func() { + _, err := boot.BootApplication(dbAddress, dbPort, dbName, dbUsername, dbPassword, dbDriver, "hi", 100) + Expect(err).ToNot(BeNil()) + }) + }) + Context("When the BC is running but not the DB", func() { + It("Should not connect successfully", func() { + _, err := boot.BootApplication("hi", 10, dbName, dbUsername, dbPassword, dbDriver, bcAddress, bcPort) + Expect(err).ToNot(BeNil()) + }) + }) + Context("When neither the BC or DB are running", func() { + It("Should not connect successfully", func() { + _, err := boot.BootApplication("hi", 10, dbName, dbUsername, dbPassword, dbDriver, "hi", 100) + Expect(err).ToNot(BeNil()) + }) + }) + }) + +}) diff --git a/pkg/database/sql/postgres/pgx_test.go b/pkg/database/sql/postgres/pgx_test.go index c8c1bae..c5c5422 100644 --- a/pkg/database/sql/postgres/pgx_test.go +++ b/pkg/database/sql/postgres/pgx_test.go @@ -23,7 +23,7 @@ var _ = Describe("Pgx", func() { ctx = context.Background() }) - Describe("Connecting to the DB", func() { + Describe("Connecting to the DB", Label("integration"), func() { Context("But connection is unsucessful", func() { It("throws error when can't connect to the database", func() { _, err := postgres.NewPostgresDB(postgres.Config{ @@ -43,7 +43,7 @@ var _ = Describe("Pgx", func() { }) }) }) - Describe("Write to the DB", func() { + Describe("Write to the DB", Label("integration"), func() { Context("Serialize big.Int to DB", func() { It("Should serialize successfully", func() { dbPool, err := postgres.NewPostgresDB(postgres.DefaultConfig) @@ -53,9 +53,13 @@ var _ = Describe("Pgx", func() { bi := new(big.Int) bi.SetString("34940183920000000000", 10) isEqual, err := testhelpers.IsEqual(bi.String(), "34940183920000000000") + Expect(err).To(BeNil()) Expect(isEqual).To(BeTrue()) - defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) + defer func() { + _, err := dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) + Expect(err).To(BeNil()) + }() _, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )") Expect(err).To(BeNil()) @@ -71,11 +75,13 @@ var _ = Describe("Pgx", func() { isEqual, err = testhelpers.IsEqual(data, bi.String()) Expect(isEqual).To(BeTrue()) + Expect(err).To(BeNil()) actual := new(big.Int) actual.SetString(data, 10) isEqual, err = testhelpers.IsEqual(actual, bi) Expect(isEqual).To(BeTrue()) + Expect(err).To(BeNil()) }) }) }) diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 0000000..778fc85 --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,25 @@ +package version + +import "fmt" + +// A reusable structure to allow developers to set their application versions. +type Version struct { + Major int // Major version component of the current release + Minor int // Minor version component of the current release + Patch int // Patch version component of the current release + Meta string // Version metadata to append to the version string +} + +// Provides a string with the version +func (version *Version) GetVersion() string { + return fmt.Sprintf("%d.%d.%d", version.Major, version.Minor, version.Patch) +} + +// Provides a string with the version and Meta. +func (version *Version) GetVersionWithMeta() string { + v := version.GetVersion() + if version.Meta != "" { + v += "-" + version.Meta + } + return v +} -- 2.45.2 From eba553a90578fe8a7e1aeecc654cab153334694f Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Fri, 22 Apr 2022 13:02:14 -0400 Subject: [PATCH 09/11] Utilized interfaces for DB --- internal/boot/boot.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/internal/boot/boot.go b/internal/boot/boot.go index 2bbbc98..46e3b02 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -7,14 +7,16 @@ import ( "time" log "github.com/sirupsen/logrus" + "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql" "github.com/vulcanize/ipld-ethcl-indexer/pkg/database/sql/postgres" "github.com/vulcanize/ipld-ethcl-indexer/pkg/loghelper" ) var ( - bcHealthEndpoint = "/eth/v1/node/health" - maxRetry = 5 // Max times to try to connect to the DB or BC at boot. - retryInterval = 30 // The time to wait between each try. + bcHealthEndpoint = "/eth/v1/node/health" + maxRetry = 5 // Max times to try to connect to the DB or BC at boot. + retryInterval = 30 // The time to wait between each try. + DB sql.Database = &postgres.DB{} ) // This function will ensure that we can connect to the beacon client. @@ -42,7 +44,7 @@ func checkBeaconClient(bcAddress string, bcPort int) error { } // A simple wrapper to create a DB object to use. -func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (*postgres.DB, error) { +func SetupPostgresDb(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string) (sql.Database, error) { log.Debug("Resolving Driver Type") DbDriver, err := postgres.ResolveDriverType(driverName) if err != nil { @@ -61,7 +63,7 @@ func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, db Password: dbPassword, Driver: DbDriver, } - DB, err := postgres.NewPostgresDB(postgresConfig) + DB, err = postgres.NewPostgresDB(postgresConfig) if err != nil { loghelper.LogError(err).Error("Unable to connect to the DB") @@ -79,7 +81,7 @@ func SetupDb(dbHostname string, dbPort int, dbName string, dbUsername string, db // // 2. Connect to the database. // -func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { +func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (sql.Database, error) { log.Info("Booting the Application") log.Debug("Checking beacon Client") @@ -89,7 +91,7 @@ func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername st } log.Debug("Setting up DB connection") - DB, err := SetupDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) + DB, err := SetupPostgresDb(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName) if err != nil { return nil, err } @@ -97,11 +99,10 @@ func BootApplication(dbHostname string, dbPort int, dbName string, dbUsername st } // Add retry logic to ensure that we are give the Beacon Client and the DB time to start. -func BootApplicationWithRetry(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (*postgres.DB, error) { - var db *postgres.DB +func BootApplicationWithRetry(dbHostname string, dbPort int, dbName string, dbUsername string, dbPassword string, driverName string, bcAddress string, bcPort int) (sql.Database, error) { var err error for i := 0; i < maxRetry; i++ { - db, err = BootApplication(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName, bcAddress, bcPort) + DB, err = BootApplication(dbHostname, dbPort, dbName, dbUsername, dbPassword, driverName, bcAddress, bcPort) if err != nil { log.WithFields(log.Fields{ "retryNumber": i, @@ -110,5 +111,5 @@ func BootApplicationWithRetry(dbHostname string, dbPort int, dbName string, dbUs continue } } - return db, err + return DB, err } -- 2.45.2 From ce808f417abc9c707e77978acbaf836b405a7e61 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Fri, 22 Apr 2022 13:04:45 -0400 Subject: [PATCH 10/11] Update Manual Testing CI/CD --- .github/workflows/on-pr-manual.yml | 71 ++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/.github/workflows/on-pr-manual.yml b/.github/workflows/on-pr-manual.yml index 26807cb..d5d015a 100644 --- a/.github/workflows/on-pr-manual.yml +++ b/.github/workflows/on-pr-manual.yml @@ -14,14 +14,15 @@ on: jobs: build: - name: Run docker build + name: Run Docker Build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Run docker build run: make docker-build - test: - name: Run unit tests + + unit-test: + name: Run Unit Tests runs-on: ubuntu-latest env: GOPATH: /tmp/go @@ -53,6 +54,56 @@ jobs: echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh cat ./config.sh + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + check-latest: true + + - name: Install packages + run: | + go install github.com/onsi/ginkgo/v2/ginkgo@latest + which ginkgo + + - name: Run the tests using Make + run: | + cd ipld-ethcl-indexer + make unit-test-ci + + integration-test: + name: Run Integration Tests + runs-on: ubuntu-latest + env: + foundry-test-ref: feature/build-stack + ipld-eth-db-ref: main + GOPATH: /tmp/go + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/checkout@v2 + with: + path: "./ipld-ethcl-indexer" + + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.foundry-test-ref }} + path: "./foundry-test/" + repository: vulcanize/foundry-test + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.ipld-eth-db-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh + echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh + cat ./config.sh + - name: Run docker compose run: | docker-compose \ @@ -74,4 +125,16 @@ jobs: - name: Run the tests using Make run: | cd ipld-ethcl-indexer - make test + make integration-test-ci + + golangci: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: ">=1.17.0" + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + args: --disable errcheck -- 2.45.2 From 8eb584db940d89eb37c805f145eabf55b6d35611 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Fri, 22 Apr 2022 16:54:09 -0400 Subject: [PATCH 11/11] Try combining everything into one pipeline (#13) * Try combining everything into one pipeline * Update on-pr-manual.yml * Remove PAth for testing * Combine single YML workflow file for PR * Add a comment --- .github/workflows/on-pr-automated.yaml | 139 ------------------ .../workflows/{on-pr-manual.yml => on-pr.yml} | 20 ++- 2 files changed, 15 insertions(+), 144 deletions(-) delete mode 100644 .github/workflows/on-pr-automated.yaml rename .github/workflows/{on-pr-manual.yml => on-pr.yml} (85%) diff --git a/.github/workflows/on-pr-automated.yaml b/.github/workflows/on-pr-automated.yaml deleted file mode 100644 index d210b2e..0000000 --- a/.github/workflows/on-pr-automated.yaml +++ /dev/null @@ -1,139 +0,0 @@ -name: Test Application - -on: - pull_request: - paths: - - "!**.md" - - ".gitignore" - - "!LICENSE" - - "!.github/workflows/**" - - ".github/workflows/on-pr-automated.yaml" - -jobs: - build: - name: Run Docker Build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Run docker build - run: make docker-build - - unit-test: - name: Run Unit Tests - runs-on: ubuntu-latest - env: - foundry-test-ref: feature/build-stack - ipld-eth-db-ref: main - GOPATH: /tmp/go - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - uses: actions/checkout@v2 - with: - path: "./ipld-ethcl-indexer" - - - uses: actions/checkout@v3 - with: - ref: ${{ env.foundry-test-ref }} - path: "./foundry-test/" - repository: vulcanize/foundry-test - fetch-depth: 0 - - - uses: actions/checkout@v3 - with: - ref: ${{ env.ipld-eth-db-ref }} - repository: vulcanize/ipld-eth-db - path: "./ipld-eth-db/" - fetch-depth: 0 - - - name: Create config file - run: | - echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh - echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh - cat ./config.sh - - - uses: actions/setup-go@v3 - with: - go-version: ">=1.17.0" - check-latest: true - - - name: Install packages - run: | - go install github.com/onsi/ginkgo/v2/ginkgo@latest - which ginkgo - - - name: Run the tests using Make - run: | - cd ipld-ethcl-indexer - make unit-test-ci - - integration-test: - name: Run Integration Tests - runs-on: ubuntu-latest - env: - foundry-test-ref: feature/build-stack - ipld-eth-db-ref: main - GOPATH: /tmp/go - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - uses: actions/checkout@v2 - with: - path: "./ipld-ethcl-indexer" - - - uses: actions/checkout@v3 - with: - ref: ${{ env.foundry-test-ref }} - path: "./foundry-test/" - repository: vulcanize/foundry-test - fetch-depth: 0 - - - uses: actions/checkout@v3 - with: - ref: ${{ env.ipld-eth-db-ref }} - repository: vulcanize/ipld-eth-db - path: "./ipld-eth-db/" - fetch-depth: 0 - - - name: Create config file - run: | - echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > ./config.sh - echo vulcanize_ipld_ethcl_indexer=$GITHUB_WORKSPACE/ipld-ethcl-indexer >> ./config.sh - cat ./config.sh - - - name: Run docker compose - run: | - docker-compose \ - -f "$GITHUB_WORKSPACE/foundry-test/docker/local/docker-compose-db.yml" \ - -f "$GITHUB_WORKSPACE/foundry-test/docker/latest/docker-compose-lighthouse.yml" \ - --env-file ./config.sh \ - up -d --build - - - uses: actions/setup-go@v3 - with: - go-version: ">=1.17.0" - check-latest: true - - - name: Install packages - run: | - go install github.com/onsi/ginkgo/v2/ginkgo@latest - which ginkgo - - - name: Run the tests using Make - run: | - cd ipld-ethcl-indexer - make integration-test-ci - - golangci: - runs-on: ubuntu-latest - steps: - - uses: actions/setup-go@v3 - with: - go-version: ">=1.17.0" - - uses: actions/checkout@v3 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - args: --disable errcheck diff --git a/.github/workflows/on-pr-manual.yml b/.github/workflows/on-pr.yml similarity index 85% rename from .github/workflows/on-pr-manual.yml rename to .github/workflows/on-pr.yml index d5d015a..a033463 100644 --- a/.github/workflows/on-pr-manual.yml +++ b/.github/workflows/on-pr.yml @@ -1,4 +1,4 @@ -name: Test Application Manually +name: Test Application On PR on: workflow_dispatch: @@ -11,6 +11,13 @@ on: description: "The branch, commit or sha from ipld-eth-db to checkout" required: false default: "main" + pull_request: + paths: + - "!**.md" + - ".gitignore" + - "!LICENSE" + - "!.github/workflows/**" + - ".github/workflows/on-pr.yml" jobs: build: @@ -24,7 +31,10 @@ jobs: unit-test: name: Run Unit Tests runs-on: ubuntu-latest + ## IF you want to update the default branch for `pull_request runs, do it after the ||` env: + foundry-test-ref: ${{ github.event.inputs.foundry-test-ref || 'feature/build-stack'}} + ipld-eth-db-ref: ${{ github.event.inputs.ipld-eth-db-ref || 'main' }} GOPATH: /tmp/go steps: - name: Create GOPATH @@ -36,14 +46,14 @@ jobs: - uses: actions/checkout@v3 with: - ref: ${{ github.event.inputs.foundry-test-ref }} + ref: ${{ env.foundry-test-ref }} path: "./foundry-test/" repository: vulcanize/foundry-test fetch-depth: 0 - uses: actions/checkout@v3 with: - ref: ${{ github.event.inputs.ipld-eth-db-ref }} + ref: ${{ env.ipld-eth-db-ref }} repository: vulcanize/ipld-eth-db path: "./ipld-eth-db/" fetch-depth: 0 @@ -86,14 +96,14 @@ jobs: - uses: actions/checkout@v3 with: - ref: ${{ github.event.inputs.foundry-test-ref }} + ref: ${{ env.foundry-test-ref }} path: "./foundry-test/" repository: vulcanize/foundry-test fetch-depth: 0 - uses: actions/checkout@v3 with: - ref: ${{ github.event.inputs.ipld-eth-db-ref }} + ref: ${{ env.ipld-eth-db-ref }} repository: vulcanize/ipld-eth-db path: "./ipld-eth-db/" fetch-depth: 0 -- 2.45.2