From 6bd563e3d5002959924a838651f2883cc5ecc010 Mon Sep 17 00:00:00 2001 From: nabarun Date: Fri, 3 Jun 2022 14:30:20 +0530 Subject: [PATCH] Use GORM to retrieve data from database --- go.mod | 19 ++-- go.sum | 29 +++-- pkg/eth/cid_retriever.go | 217 ++++++++++++++---------------------- pkg/eth/ipld_fetcher.go | 35 ------ pkg/graphql/graphql.go | 53 ++------- pkg/graphql/graphql_test.go | 73 ++---------- pkg/shared/functions.go | 14 --- 7 files changed, 138 insertions(+), 302 deletions(-) diff --git a/go.mod b/go.mod index cfc23a2b..a6ef0b0b 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-ds-help v1.0.0 github.com/ipfs/go-ipld-format v0.2.0 + github.com/jinzhu/now v1.1.5 // indirect github.com/jmoiron/sqlx v1.3.5 github.com/joho/godotenv v1.4.0 github.com/lib/pq v1.10.5 @@ -22,10 +23,13 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.11.0 + github.com/thoas/go-funk v0.9.2 // indirect github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.2 - github.com/thoas/go-funk v0.9.2 github.com/vulcanize/gap-filler v0.3.1 github.com/vulcanize/ipfs-ethdb/v3 v3.0.3 + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect + gorm.io/driver/postgres v1.3.7 + gorm.io/gorm v1.23.5 ) require ( @@ -124,18 +128,19 @@ require ( github.com/ipld/go-codec-dagpb v1.3.0 // indirect github.com/ipld/go-ipld-prime v0.12.2 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.10.0 // indirect + github.com/jackc/pgconn v1.12.1 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.1.1 // indirect + github.com/jackc/pgproto3/v2 v2.3.0 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.8.1 // indirect - github.com/jackc/pgx/v4 v4.13.0 // indirect - github.com/jackc/puddle v1.1.3 // indirect + github.com/jackc/pgtype v1.11.0 // indirect + github.com/jackc/pgx/v4 v4.16.1 // indirect + github.com/jackc/puddle v1.2.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/jinzhu/copier v0.2.4 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect github.com/klauspost/compress v1.11.7 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/koron/go-ssdp v0.0.2 // indirect @@ -244,7 +249,6 @@ require ( github.com/stretchr/testify v1.7.1 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect - github.com/thoas/go-funk v0.9.2 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect @@ -266,7 +270,6 @@ require ( go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.19.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect - golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect diff --git a/go.sum b/go.sum index 8df04b33..eb2c8782 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -788,8 +789,9 @@ github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7 github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8= +github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -807,8 +809,9 @@ github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= +github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= @@ -821,8 +824,9 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkAL github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= +github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= @@ -833,15 +837,17 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y= +github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -866,8 +872,12 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jinzhu/copier v0.2.4 h1:dT3tI+8GzU8DjJFCj9mLYtjfRtUmK7edauduQdcZCpI= github.com/jinzhu/copier v0.2.4/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -1920,8 +1930,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2423,6 +2433,11 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.3.7 h1:FKF6sIMDHDEvvMF/XJvbnCl0nu6KSKUaPXevJ4r+VYQ= +gorm.io/driver/postgres v1.3.7/go.mod h1:f02ympjIcgtHEGFMZvdgTxODZ9snAHDb4hXfigBVuNI= +gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.5 h1:TnlF26wScKSvknUC/Rn8t0NLLM22fypYBlvj1+aH6dM= +gorm.io/gorm v1.23.5/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go index 789ad459..04a5c719 100644 --- a/pkg/eth/cid_retriever.go +++ b/pkg/eth/cid_retriever.go @@ -26,7 +26,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/lib/pq" log "github.com/sirupsen/logrus" - "github.com/thoas/go-funk" + "gorm.io/driver/postgres" + "gorm.io/gorm" "github.com/vulcanize/ipld-eth-server/v3/pkg/shared" ) @@ -40,13 +41,70 @@ type Retriever interface { // CIDRetriever satisfies the CIDRetriever interface for ethereum type CIDRetriever struct { - db *sqlx.DB + db *sqlx.DB + gormDB *gorm.DB +} + +type IPLD struct { + models.IPLDModel +} + +// TableName overrides the table name used by IPLD +func (IPLD) TableName() string { + return "public.blocks" +} + +type HeaderCid struct { + CID string `gorm:"column:cid"` + BlockHash string `gorm:"primaryKey"` + BlockNumber string + ParentHash string + Timestamp uint64 + StateRoot string + TotalDifficulty string `gorm:"column:td"` + TxRoot string + RctRoot string `gorm:"column:receipt_root"` + UncleRoot string + Bloom []byte + MhKey string + TransactionCids []TransactionCid `gorm:"foreignKey:HeaderID;references:BlockHash"` + IPLD IPLD `gorm:"foreignKey:MhKey;references:Key"` +} + +// TableName overrides the table name used by HeaderCid +func (HeaderCid) TableName() string { + return "eth.header_cids" +} + +type TransactionCid struct { + CID string `gorm:"column:cid"` + TxHash string `gorm:"primaryKey"` + HeaderID string `gorm:"column:header_id"` + Index int64 + Src string + Dst string + MhKey string + IPLD IPLD `gorm:"foreignKey:MhKey;references:Key"` +} + +// TableName overrides the table name used by TransactionCid +func (TransactionCid) TableName() string { + return "eth.transaction_cids" } // NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface func NewCIDRetriever(db *sqlx.DB) *CIDRetriever { + gormDB, err := gorm.Open(postgres.New(postgres.Config{ + Conn: db, + }), &gorm.Config{}) + if err != nil { + log.Error(err) + return nil + } + return &CIDRetriever{ - db: db, + db: db, + gormDB: gormDB, } } @@ -594,29 +652,6 @@ func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID string) return txCIDs, tx.Select(&txCIDs, pgStr, headerID) } -// RetrieveTxCIDsByHeaderIDs retrieves all tx CIDs for the given headerIDs -func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderIDs(tx *sqlx.Tx, headerIDs []string) ([]models.TxModel, error) { - log.Debug("retrieving tx cids for headerIDs ", headerIDs) - pgStr := `SELECT header_id, index, tx_hash, cid, mh_key, - dst, src, tx_data, tx_type, value - FROM eth.transaction_cids - WHERE header_id in (?) - ORDER BY index` - - var txCIDs []models.TxModel - if len(headerIDs) < 1 { - return txCIDs, nil - } - - query, args, err := sqlx.In(pgStr, headerIDs) - if err != nil { - return txCIDs, err - } - query = tx.Rebind(query) - - return txCIDs, tx.Select(&txCIDs, query, args...) -} - // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txHashes []string) ([]models.ReceiptModel, error) { log.Debugf("retrieving receipt cids for tx hashes %v", txHashes) @@ -631,134 +666,50 @@ func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txHashes []stri } // RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number -func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]models.HeaderModel, [][]models.TxModel, error) { +func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCid, error) { log.Debug("retrieving header cids and tx cids for block number ", blockNumber) - // Begin new db tx - tx, err := ecr.db.Beginx() - if err != nil { - return nil, nil, err - } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() + var headerCIDs []HeaderCid - var headerCIDs []models.HeaderModel - headerCIDs, err = ecr.RetrieveHeaderCIDs(tx, blockNumber) + // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 + // Will use join for TransactionCids once preload for 1:N is supported. + err := ecr.gormDB.Preload("TransactionCids").Joins("IPLD").Find(&headerCIDs, "block_number = ?", blockNumber).Error if err != nil { log.Error("header cid retrieval error") - return nil, nil, err + return nil, err } - var allTxCIDs [][]models.TxModel - txHeaderIDs := funk.Map( - headerCIDs, - func(headerCID models.HeaderModel) string { - return headerCID.BlockHash - }, - ) - txCIDs, err := ecr.RetrieveTxCIDsByHeaderIDs(tx, txHeaderIDs.([]string)) - if err != nil { - log.Error("tx cid retrieval error") - return nil, nil, err - } - - txCIDsByHeaderID := funk.Reduce( - txCIDs, - func(acc map[string][]models.TxModel, txCID models.TxModel) map[string][]models.TxModel { - if _, ok := acc[txCID.HeaderID]; !ok { - acc[txCID.HeaderID] = []models.TxModel{} - } - - txCIDs = append(acc[txCID.HeaderID], txCID) - acc[txCID.HeaderID] = txCIDs - return acc - }, - make(map[string][]models.TxModel), - ) - - txCIDsByHeaderIDMap := txCIDsByHeaderID.(map[string][]models.TxModel) - - for _, headerCID := range headerCIDs { - txCIDs := txCIDsByHeaderIDMap[headerCID.BlockHash] - allTxCIDs = append(allTxCIDs, txCIDs) - } - - return headerCIDs, allTxCIDs, nil + return headerCIDs, nil } // RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash -func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash) (models.HeaderModel, []models.TxModel, error) { +func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash) (HeaderCid, error) { log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String()) - // Begin new db tx - tx, err := ecr.db.Beginx() - if err != nil { - return models.HeaderModel{}, nil, err - } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() + var headerCID HeaderCid - var headerCID models.HeaderModel - headerCID, err = ecr.RetrieveHeaderCIDByHash(tx, blockHash) + // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 + // Will use join for TransactionCids once preload for 1:N is supported. + err := ecr.gormDB.Preload("TransactionCids").Joins("IPLD").First(&headerCID, "block_hash = ?", blockHash.String()).Error if err != nil { log.Error("header cid retrieval error") - return models.HeaderModel{}, nil, err - } - if err != nil { - return models.HeaderModel{}, nil, err + return headerCID, err } - var txCIDs []models.TxModel - txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.BlockHash) - if err != nil { - log.Error("tx cid retrieval error") - return models.HeaderModel{}, nil, err - } - - return headerCID, txCIDs, nil + return headerCID, nil } // RetrieveTxCIDByHash returns the tx for the given tx hash -func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string) (models.TxModel, error) { +func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string) (TransactionCid, error) { log.Debug("retrieving tx cid for tx hash ", txHash) - // Begin new db tx - tx, err := ecr.db.Beginx() - if err != nil { - return models.TxModel{}, err - } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() + var txCID TransactionCid - pgStr := `SELECT header_id, index, tx_hash, cid, mh_key, - dst, src, tx_data, tx_type, value - FROM eth.transaction_cids - WHERE tx_hash = $1 - ORDER BY index` - var txCID models.TxModel - return txCID, tx.Get(&txCID, pgStr, txHash) + err := ecr.gormDB.Joins("IPLD").First(&txCID, "tx_hash = ?", txHash).Error + if err != nil { + log.Error("header cid retrieval error") + return txCID, err + } + + return txCID, nil } diff --git a/pkg/eth/ipld_fetcher.go b/pkg/eth/ipld_fetcher.go index fa0d4c57..751f03f5 100644 --- a/pkg/eth/ipld_fetcher.go +++ b/pkg/eth/ipld_fetcher.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" - "github.com/thoas/go-funk" "github.com/vulcanize/ipld-eth-server/v3/pkg/shared" ) @@ -113,40 +112,6 @@ func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c models.HeaderModel) (models.IPL }, nil } -// FetchHeaders fetches headers -func (f *IPLDFetcher) FetchHeaders(tx *sqlx.Tx, cids []models.HeaderModel) ([]models.IPLDModel, error) { - log.Debug("fetching header iplds") - headerIPLDs := make([]models.IPLDModel, len(cids)) - - if len(cids) < 1 { - return headerIPLDs, nil - } - - mhKeys := make([]string, len(cids)) - for i, c := range cids { - var err error - mhKeys[i] = c.MhKey - if err != nil { - return nil, err - } - } - - fetchedIPLDs, err := shared.FetchIPLDsByMhKeys(tx, mhKeys) - if err != nil { - return nil, err - } - - for i, c := range cids { - headerIPLD := funk.Find(fetchedIPLDs, func(ipld models.IPLDModel) bool { - return ipld.Key == c.MhKey - }).(models.IPLDModel) - - headerIPLDs[i] = headerIPLD - } - - return headerIPLDs, nil -} - // FetchUncles fetches uncles func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []models.UncleModel) ([]models.IPLDModel, error) { log.Debug("fetching uncle iplds") diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index 1c8cc31b..b42de133 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -33,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/vulcanize/ipld-eth-server/v3/pkg/eth" "github.com/vulcanize/ipld-eth-server/v3/pkg/shared" @@ -1264,19 +1263,17 @@ type EthHeaderCidCondition struct { func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct { Condition *EthHeaderCidCondition }) (*EthHeaderCidsConnection, error) { - var headerCIDs []models.HeaderModel - var allTxCIDs [][]models.TxModel + var headerCIDs []eth.HeaderCid var err error if args.Condition.BlockHash != nil { - headerCID, txCIDs, err := r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(common.HexToHash(*args.Condition.BlockHash)) + headerCID, err := r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(common.HexToHash(*args.Condition.BlockHash)) if err != nil { return nil, err } headerCIDs = append(headerCIDs, headerCID) - allTxCIDs = append(allTxCIDs, txCIDs) } else if args.Condition.BlockNumber != nil { - headerCIDs, allTxCIDs, err = r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockNumber(args.Condition.BlockNumber.ToInt().Int64()) + headerCIDs, err = r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockNumber(args.Condition.BlockNumber.ToInt().Int64()) if err != nil { return nil, err } @@ -1300,13 +1297,8 @@ func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct { } }() - headerIPLDs, err := r.backend.Fetcher.FetchHeaders(tx, headerCIDs) - if err != nil { - return nil, err - } - var resultNodes []*EthHeaderCid - for idx, headerCID := range headerCIDs { + for _, headerCID := range headerCIDs { var blockNumber BigInt blockNumber.UnmarshalText([]byte(headerCID.BlockNumber)) @@ -1328,10 +1320,13 @@ func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct { receiptRoot: headerCID.RctRoot, uncleRoot: headerCID.UncleRoot, bloom: Bytes(headerCID.Bloom).String(), + ipfsBlock: IPFSBlock{ + key: headerCID.IPLD.Key, + data: Bytes(headerCID.IPLD.Data).String(), + }, } - txCIDs := allTxCIDs[idx] - for _, txCID := range txCIDs { + for _, txCID := range headerCID.TransactionCids { ethHeaderCidNode.transactions = append(ethHeaderCidNode.transactions, &EthTransactionCid{ cid: txCID.CID, txHash: txCID.TxHash, @@ -1341,11 +1336,6 @@ func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct { }) } - ethHeaderCidNode.ipfsBlock = IPFSBlock{ - key: headerIPLDs[idx].Key, - data: Bytes(headerIPLDs[idx].Data).String(), - } - resultNodes = append(resultNodes, ðHeaderCidNode) } @@ -1363,27 +1353,6 @@ func (r *Resolver) EthTransactionCidByTxHash(ctx context.Context, args struct { return nil, err } - // Begin tx - tx, err := r.backend.DB.Beginx() - if err != nil { - return nil, err - } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() - - txIPLDs, err := r.backend.Fetcher.FetchTrxs(tx, []models.TxModel{txCID}) - if err != nil { - return nil, err - } - return &EthTransactionCid{ cid: txCID.CID, txHash: txCID.TxHash, @@ -1391,8 +1360,8 @@ func (r *Resolver) EthTransactionCidByTxHash(ctx context.Context, args struct { src: txCID.Src, dst: txCID.Dst, ipfsBlock: IPFSBlock{ - key: txIPLDs[0].Key, - data: Bytes(txIPLDs[0].Data).String(), + key: txCID.IPLD.Key, + data: Bytes(txCID.IPLD.Data).String(), }, }, nil } diff --git a/pkg/graphql/graphql_test.go b/pkg/graphql/graphql_test.go index fc8259a4..4744005b 100644 --- a/pkg/graphql/graphql_test.go +++ b/pkg/graphql/graphql_test.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/indexer/models" sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/jmoiron/sqlx" . "github.com/onsi/ginkgo" @@ -258,30 +257,13 @@ var _ = Describe("GraphQL", func() { allEthHeaderCidsResp, err := client.AllEthHeaderCids(ctx, graphql.EthHeaderCidCondition{BlockNumber: new(graphql.BigInt).SetUint64(2)}) Expect(err).ToNot(HaveOccurred()) - headerCIDs, txCIDs, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockNumber(2) - Expect(err).ToNot(HaveOccurred()) - - // Begin tx - tx, err := backend.DB.Beginx() - Expect(err).ToNot(HaveOccurred()) - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() - - headerIPLDs, err := backend.Fetcher.FetchHeaders(tx, headerCIDs) + headerCIDs, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockNumber(2) Expect(err).ToNot(HaveOccurred()) for idx, headerCID := range headerCIDs { ethHeaderCid := allEthHeaderCidsResp.Nodes[idx] - compareEthHeaderCid(ethHeaderCid, headerCID, txCIDs[idx], headerIPLDs[idx]) + compareEthHeaderCid(ethHeaderCid, headerCID) } }) @@ -290,30 +272,12 @@ var _ = Describe("GraphQL", func() { allEthHeaderCidsResp, err := client.AllEthHeaderCids(ctx, graphql.EthHeaderCidCondition{BlockHash: &blockHash}) Expect(err).ToNot(HaveOccurred()) - headerCID, txCIDs, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash()) + headerCID, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash()) Expect(err).ToNot(HaveOccurred()) - // Begin tx - tx, err := backend.DB.Beginx() - Expect(err).ToNot(HaveOccurred()) - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() - - headerIPLDs, err := backend.Fetcher.FetchHeaders(tx, []models.HeaderModel{headerCID}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(allEthHeaderCidsResp.Nodes)).To(Equal(1)) - Expect(len(allEthHeaderCidsResp.Nodes)).To(Equal(1)) ethHeaderCid := allEthHeaderCidsResp.Nodes[0] - compareEthHeaderCid(ethHeaderCid, headerCID, txCIDs, headerIPLDs[0]) + compareEthHeaderCid(ethHeaderCid, headerCID) }) }) @@ -328,29 +292,12 @@ var _ = Describe("GraphQL", func() { compareEthTxCid(*ethTransactionCidResp, txCID) - // Begin tx - tx, err := backend.DB.Beginx() - Expect(err).ToNot(HaveOccurred()) - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() - - txIPLDs, err := backend.Fetcher.FetchTrxs(tx, []models.TxModel{txCID}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(txIPLDs)).To(Equal(1)) - Expect(ethTransactionCidResp.BlockByMhKey.Data).To(Equal(graphql.Bytes(txIPLDs[0].Data).String())) + Expect(ethTransactionCidResp.BlockByMhKey.Data).To(Equal(graphql.Bytes(txCID.IPLD.Data).String())) }) }) }) -func compareEthHeaderCid(ethHeaderCid graphql.EthHeaderCidResp, headerCID models.HeaderModel, txCIDs []models.TxModel, headerIPLD models.IPLDModel) { +func compareEthHeaderCid(ethHeaderCid graphql.EthHeaderCidResp, headerCID eth.HeaderCid) { blockNumber, err := strconv.ParseInt(headerCID.BlockNumber, 10, 64) Expect(err).ToNot(HaveOccurred()) @@ -369,16 +316,16 @@ func compareEthHeaderCid(ethHeaderCid graphql.EthHeaderCidResp, headerCID models Expect(ethHeaderCid.UncleRoot).To(Equal(headerCID.UncleRoot)) Expect(ethHeaderCid.Bloom).To(Equal(graphql.Bytes(headerCID.Bloom).String())) - for tIdx, txCID := range txCIDs { + for tIdx, txCID := range headerCID.TransactionCids { ethTxCid := ethHeaderCid.EthTransactionCidsByHeaderId.Nodes[tIdx] compareEthTxCid(ethTxCid, txCID) } - Expect(ethHeaderCid.BlockByMhKey.Data).To(Equal(graphql.Bytes(headerIPLD.Data).String())) - Expect(ethHeaderCid.BlockByMhKey.Key).To(Equal(headerIPLD.Key)) + Expect(ethHeaderCid.BlockByMhKey.Data).To(Equal(graphql.Bytes(headerCID.IPLD.Data).String())) + Expect(ethHeaderCid.BlockByMhKey.Key).To(Equal(headerCID.IPLD.Key)) } -func compareEthTxCid(ethTxCid graphql.EthTransactionCidResp, txCID models.TxModel) { +func compareEthTxCid(ethTxCid graphql.EthTransactionCidResp, txCID eth.TransactionCid) { Expect(ethTxCid.Cid).To(Equal(txCID.CID)) Expect(ethTxCid.TxHash).To(Equal(txCID.TxHash)) Expect(ethTxCid.Index).To(Equal(int32(txCID.Index))) diff --git a/pkg/shared/functions.go b/pkg/shared/functions.go index eede20bc..b1ac4550 100644 --- a/pkg/shared/functions.go +++ b/pkg/shared/functions.go @@ -19,7 +19,6 @@ package shared import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/statediff/indexer/ipld" - "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" @@ -78,19 +77,6 @@ func FetchIPLDByMhKey(tx *sqlx.Tx, mhKey string) ([]byte, error) { return block, tx.Get(&block, pgStr, mhKey) } -// FetchIPLDByMhKeysAndBlockNumbers is used to retrieve iplds from Postgres blockstore with the provided tx, mhkey strings and blockNumbers -func FetchIPLDsByMhKeys(tx *sqlx.Tx, mhKeys []string) ([]models.IPLDModel, error) { - var blocks []models.IPLDModel - pgStr := `SELECT key, data FROM public.blocks WHERE key IN (?)` - query, args, err := sqlx.In(pgStr, mhKeys) - if err != nil { - return blocks, err - } - query = tx.Rebind(query) - - return blocks, tx.Select(&blocks, query, args...) -} - // MultihashKeyFromCID converts a cid into a blockstore-prefixed multihash db key string func MultihashKeyFromCID(c cid.Cid) string { dbKey := dshelp.MultihashToDsKey(c.Hash())