Merge pull request #233 from cerc-io/ian/v5_dev

v5 Checkpoint

cmd/serve.go

@@ -25,7 +25,6 @@ import (
 	"sync"
 	"time"
 
-
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/mailgun/groupcache/v2"

docker-compose.yml

@@ -5,7 +5,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v4.2.3-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.0-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "vulcanize_testing"

go.mod

@@ -6,9 +6,10 @@ require (
 	github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha
 	github.com/cerc-io/go-eth-state-node-iterator v1.1.9
 	github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha
+	github.com/cerc-io/ipld-eth-statedb v0.0.3-alpha
 	github.com/ethereum/go-ethereum v1.10.26
+	github.com/google/uuid v1.3.0
 	github.com/graph-gophers/graphql-go v1.3.0
-	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-cid v0.2.0
 	github.com/ipfs/go-ipfs-blockstore v1.2.0
 	github.com/ipfs/go-ipfs-ds-help v1.1.0
@@ -17,7 +18,6 @@ require (
 	github.com/lib/pq v1.10.6
 	github.com/machinebox/graphql v0.2.2
 	github.com/mailgun/groupcache/v2 v2.3.0
-	github.com/multiformats/go-multihash v0.2.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
 	github.com/prometheus/client_golang v1.12.1
@@ -77,7 +77,6 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/graphql-go/graphql v0.7.9 // indirect
@@ -97,6 +96,7 @@ require (
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-bitfield v1.0.0 // indirect
 	github.com/ipfs/go-bitswap v0.8.0 // indirect
+	github.com/ipfs/go-block-format v0.0.3 // indirect
 	github.com/ipfs/go-blockservice v0.4.0 // indirect
 	github.com/ipfs/go-cidutil v0.1.0 // indirect
 	github.com/ipfs/go-datastore v0.5.1 // indirect
@@ -139,14 +139,14 @@ require (
 	github.com/ipld/go-codec-dagpb v1.4.0 // indirect
 	github.com/ipld/go-ipld-prime v0.17.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.12.1 // indirect
+	github.com/jackc/pgconn v1.14.0 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.3.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.2 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgtype v1.11.0 // indirect
+	github.com/jackc/pgtype v1.14.0 // indirect
-	github.com/jackc/pgx/v4 v4.16.1 // indirect
+	github.com/jackc/pgx/v4 v4.18.1 // indirect
-	github.com/jackc/puddle v1.2.1 // indirect
+	github.com/jackc/puddle v1.3.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
@@ -211,6 +211,7 @@ require (
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.1.0 // indirect
 	github.com/multiformats/go-multicodec v0.5.0 // indirect
+	github.com/multiformats/go-multihash v0.2.0 // indirect
 	github.com/multiformats/go-multistream v0.3.3 // indirect
 	github.com/multiformats/go-varint v0.0.6 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
@@ -244,8 +245,8 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
-	github.com/stretchr/testify v1.7.2 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
 	github.com/thoas/go-funk v0.9.2 // indirect
@@ -275,14 +276,14 @@ require (
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
-	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
+	golang.org/x/crypto v0.6.0 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	golang.org/x/net v0.6.0 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/tools v0.1.12 // indirect
 	golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect

go.sum

@@ -153,6 +153,8 @@ github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha h1:gesMZEbNU+fcAMc
 github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg=
 github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha h1:5iqNXeitkj3g7FxyKK/Pz+1HN7Ac9JZzCRj3Lv+uHiw=
 github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha/go.mod h1:dPscFRMvTWPKnoZ4U0D9v4bsrw6XdH7sOp8hUrVzOWA=
+github.com/cerc-io/ipld-eth-statedb v0.0.3-alpha h1:7NCJpF2XmOzD0gmQUkvCf00mylkJ/gBfULhsELV2Awk=
+github.com/cerc-io/ipld-eth-statedb v0.0.3-alpha/go.mod h1:Vx8auhxWh7V3qMgx8qo3bdp3ftSiytvXqffEmXvgpIA=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -769,8 +771,9 @@ github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7
 github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
 github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
 github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
-github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8=
 github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono=
+github.com/jackc/pgconn v1.14.0 h1:vrbA9Ud87g6JdFWkHTJXppVce58qPIdP7N8y0Ml/A7Q=
+github.com/jackc/pgconn v1.14.0/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E=
 github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
 github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
@@ -789,11 +792,13 @@ github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX
 github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
 github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
 github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
-github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y=
 github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.3.2 h1:7eY55bdBeCz1F2fTzSz69QC+pG46jYq9/jtSPiJ5nn0=
+github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
 github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
-github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
 github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
 github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
 github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
 github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
@@ -803,8 +808,9 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkAL
 github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
 github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
 github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
-github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs=
 github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
+github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw=
+github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
 github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
 github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
 github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
@@ -815,16 +821,18 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6
 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
 github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0=
 github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
-github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y=
 github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ=
+github.com/jackc/pgx/v4 v4.18.1 h1:YP7G1KABtKpB5IHrO9vYwSrCOhs7p3uqhvhhQBptya0=
+github.com/jackc/pgx/v4 v4.18.1/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw=
 github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
+github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@@ -1734,8 +1742,10 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -1744,8 +1754,10 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@@ -1829,6 +1841,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
@@ -1956,8 +1969,8 @@ golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
+golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
-golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2002,8 +2015,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
-golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2067,8 +2080,10 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220517181318-183a9ca12b87/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -2093,8 +2108,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2197,11 +2213,14 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2210,8 +2229,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2290,8 +2310,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
-golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/debug/backend.go

@@ -42,6 +42,7 @@ type Backend struct {
 }
 
 // StateAtBlock retrieves the state database associated with a certain block
+// We can't sub in our ipld-eth-statedb here because to match the expected interface we need to return *state.StateDB not vm.StateDB
 func (b *Backend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive, preferDisk bool) (*state.StateDB, error) {
 	rpcBlockNumber := rpc.BlockNumber(block.NumberU64())
 	statedb, _, err := b.StateAndHeaderByNumberOrHash(ctx, rpc.BlockNumberOrHashWithNumber(rpcBlockNumber))

pkg/eth/api.go

@@ -28,13 +28,13 @@ import (
 	"time"
 
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
+	ipld_eth_statedb "github.com/cerc-io/ipld-eth-statedb"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/filters"
@@ -720,7 +720,7 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log
 
 	// If we have a blockHash to filter on, fire off single retrieval query
 	if crit.BlockHash != nil {
-		filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, 0, crit.BlockHash)
+		filteredLogs, err := pea.B.Retriever.RetrieveFilteredLogs(tx, filter, 0, crit.BlockHash)
 		if err != nil {
 			return nil, err
 		}
@@ -748,7 +748,7 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log
 	end := endingBlock.Int64()
 	var logs []*types.Log
 	for i := start; i <= end; i++ {
-		filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, i, nil)
+		filteredLogs, err := pea.B.Retriever.RetrieveFilteredLogs(tx, filter, i, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -875,6 +875,7 @@ func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, s
 	return nil, err
 }
 
+// this continues to use ipfs-ethdb based geth StateDB as it requires trie access
 func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) {
 	state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash)
 	if state == nil || err != nil {
@@ -977,7 +978,7 @@ type OverrideAccount struct {
 type StateOverride map[common.Address]OverrideAccount
 
 // Apply overrides the fields of specified accounts into the given state.
-func (diff *StateOverride) Apply(state *state.StateDB) error {
+func (diff *StateOverride) Apply(state *ipld_eth_statedb.StateDB) error {
 	if diff == nil {
 		return nil
 	}
@@ -1054,7 +1055,7 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 		log.Debugxf(ctx, "Executing EVM call finished %s runtime %s", time.Now().String(), time.Since(start).String())
 	}(time.Now())
 
-	state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash)
+	state, header, err := b.IPLDStateDBAndHeaderByNumberOrHash(ctx, blockNrOrHash)
 	if state == nil || err != nil {
 		return nil, err
 	}

pkg/eth/backend.go

@@ -23,12 +23,13 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
-	"strconv"
 	"time"
 
 	validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg"
 	ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres"
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
+	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
+	ipld_eth_statedb "github.com/cerc-io/ipld-eth-statedb"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
@@ -45,12 +46,8 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
-	sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers"
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/jmoiron/sqlx"
-
-	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
 )
 
 var (
@@ -61,43 +58,6 @@ var (
 	errMultipleHeadersForHash = errors.New("more than one headers for the given hash")
 	errTxHashNotFound         = errors.New("transaction for hash not found")
 	errTxHashInMultipleBlocks = errors.New("transaction for hash found in more than one canonical block")
-
-	// errMissingSignature is returned if a block's extra-data section doesn't seem
-	// to contain a 65 byte secp256k1 signature.
-)
-
-const (
-	RetrieveCanonicalBlockHashByNumber = `SELECT block_hash
-									FROM canonical_header_hash($1) AS block_hash
-									WHERE block_hash IS NOT NULL`
-	RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids
-									INNER JOIN public.blocks ON (
-										header_cids.mh_key = blocks.key
-										AND header_cids.block_number = blocks.block_number
-									)
-									WHERE block_hash = (SELECT canonical_header_hash($1))`
-	RetrieveTD = `SELECT CAST(td as Text) FROM eth.header_cids
-			WHERE header_cids.block_hash = $1`
-	RetrieveRPCTransaction = `SELECT blocks.data, header_id, transaction_cids.block_number, index
-			FROM public.blocks, eth.transaction_cids
-			WHERE blocks.key = transaction_cids.mh_key
-			AND blocks.block_number = transaction_cids.block_number
-			AND transaction_cids.tx_hash = $1
-			AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))`
-	RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids
-											WHERE state_accounts.header_id = state_cids.header_id
-											AND state_accounts.state_path = state_cids.state_path
-											AND state_accounts.block_number = state_cids.block_number
-											AND state_cids.header_id = header_cids.block_hash
-											AND state_cids.block_number = header_cids.block_number
-											AND state_leaf_key = $1
-											AND header_cids.block_number <= (SELECT block_number
-																FROM eth.header_cids
-																WHERE block_hash = $2)
-											AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number))
-											ORDER BY header_cids.block_number DESC
-											LIMIT 1`
-	RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1`
 )
 
 const (
@@ -109,12 +69,13 @@ type Backend struct {
 	DB *sqlx.DB
 
 	// postgres db interfaces
-	Retriever     *CIDRetriever
+	Retriever *Retriever
-	IPLDRetriever *IPLDRetriever
 
 	// ethereum interfaces
 	EthDB         ethdb.Database
 	StateDatabase state.Database
+	// We'll use this state.Database for eth_call and any place we don't need trie access
+	IpldStateDatabase ipld_eth_statedb.StateDatabase
 
 	Config *Config
 }
@@ -135,7 +96,7 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) {
 		groupName = StateDBGroupCacheName
 	}
 
-	r := NewCIDRetriever(db)
+	r := NewRetriever(db)
 	ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{
 		Name:           groupName,
 		Size:           gcc.StateDB.CacheSizeInMB * 1024 * 1024,
@@ -143,13 +104,16 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) {
 	})
 
 	logStateDBStatsOnTimer(ethDB.(*ipfsethdb.Database), gcc)
-
+	ipldStateDB, err := ipld_eth_statedb.NewStateDatabaseWithSqlxPool(db)
+	if err != nil {
+		return nil, err
+	}
 	return &Backend{
 		DB:                db,
 		Retriever:         r,
-		IPLDRetriever: NewIPLDRetriever(db),
 		EthDB:             ethDB,
 		StateDatabase:     state.NewDatabase(ethDB),
+		IpldStateDatabase: ipldStateDB,
 		Config:            c,
 	}, nil
 }
@@ -208,7 +172,7 @@ func (b *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.He
 		}
 	}()
 
-	_, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(tx, hash)
+	_, headerRLP, err := b.Retriever.RetrieveHeaderByHash(tx, hash)
 	if err != nil {
 		return nil, err
 	}
@@ -374,22 +338,12 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 		return nil, err
 	}
 
-	// When num. of uncles = 2,
+	// We should not have any non-determinism in the ordering of the uncles returned to us now
-	// Check if calculated uncle hash matches the one in header
-	// If not, re-order the two uncles
-	// Assumption: Max num. of uncles in mainnet = 2
-	if len(uncles) == 2 {
 	uncleHash := types.CalcUncleHash(uncles)
-		if uncleHash != header.UncleHash {
+	// Check if uncle hash matches expected hash
-			uncles[0], uncles[1] = uncles[1], uncles[0]
-
-			uncleHash = types.CalcUncleHash(uncles)
-			// Check if uncle hash matches after re-ordering
 	if uncleHash != header.UncleHash {
 		log.Error("uncle hash mismatch for block hash: ", hash.Hex())
 	}
-		}
-	}
 
 	// Fetch transactions
 	transactions, err := b.GetTransactionsByBlockHashAndNumber(tx, hash, blockNumber)
@@ -411,7 +365,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 
 // GetHeaderByBlockHash retrieves header for a provided block hash
 func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.Header, error) {
-	_, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(tx, hash)
+	_, headerRLP, err := b.Retriever.RetrieveHeaderByHash(tx, hash)
 	if err != nil {
 		return nil, err
 	}
@@ -422,49 +376,37 @@ func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.He
 
 // GetUnclesByBlockHash retrieves uncles for a provided block hash
 func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types.Header, error) {
-	_, uncleBytes, err := b.IPLDRetriever.RetrieveUnclesByBlockHash(tx, hash)
+	_, uncleBytes, err := b.Retriever.RetrieveUnclesByBlockHash(tx, hash)
 	if err != nil {
 		return nil, err
 	}
 
-	uncles := make([]*types.Header, len(uncleBytes))
+	uncles := make([]*types.Header, 0)
-	for i, bytes := range uncleBytes {
+	if err := rlp.DecodeBytes(uncleBytes, uncles); err != nil {
-		var uncle types.Header
-		err = rlp.DecodeBytes(bytes, &uncle)
-		if err != nil {
 		return nil, err
 	}
 
-		uncles[i] = &uncle
-	}
-
 	return uncles, nil
 }
 
 // GetUnclesByBlockHashAndNumber retrieves uncles for a provided block hash and number
 func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) ([]*types.Header, error) {
-	_, uncleBytes, err := b.IPLDRetriever.RetrieveUncles(tx, hash, number)
+	_, uncleBytes, err := b.Retriever.RetrieveUncles(tx, hash, number)
 	if err != nil {
 		return nil, err
 	}
 
-	uncles := make([]*types.Header, len(uncleBytes))
+	uncles := make([]*types.Header, 0)
-	for i, bytes := range uncleBytes {
+	if err := rlp.DecodeBytes(uncleBytes, uncles); err != nil {
-		var uncle types.Header
-		err = rlp.DecodeBytes(bytes, &uncle)
-		if err != nil {
 		return nil, err
 	}
 
-		uncles[i] = &uncle
-	}
-
 	return uncles, nil
 }
 
 // GetTransactionsByBlockHash retrieves transactions for a provided block hash
 func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Transactions, error) {
-	_, transactionBytes, err := b.IPLDRetriever.RetrieveTransactionsByBlockHash(tx, hash)
+	_, transactionBytes, err := b.Retriever.RetrieveTransactionsByBlockHash(tx, hash)
 	if err != nil {
 		return nil, err
 	}
@@ -484,7 +426,7 @@ func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (typ
 
 // GetTransactionsByBlockHashAndNumber retrieves transactions for a provided block hash and number
 func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Transactions, error) {
-	_, transactionBytes, err := b.IPLDRetriever.RetrieveTransactions(tx, hash, number)
+	_, transactionBytes, err := b.Retriever.RetrieveTransactions(tx, hash, number)
 	if err != nil {
 		return nil, err
 	}
@@ -504,7 +446,7 @@ func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.H
 
 // GetReceiptsByBlockHash retrieves receipts for a provided block hash
 func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Receipts, error) {
-	_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash)
+	_, receiptBytes, txs, err := b.Retriever.RetrieveReceiptsByBlockHash(tx, hash)
 	if err != nil {
 		return nil, err
 	}
@@ -522,7 +464,7 @@ func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.R
 
 // GetReceiptsByBlockHashAndNumber retrieves receipts for a provided block hash and number
 func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Receipts, error) {
-	_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number)
+	_, receiptBytes, txs, err := b.Retriever.RetrieveReceipts(tx, hash, number)
 	if err != nil {
 		return nil, err
 	}
@@ -585,11 +527,10 @@ func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Rece
 		}
 	}()
 
-	headerCID, err := b.Retriever.RetrieveHeaderCIDByHash(tx, hash)
+	blockNumber, err := b.Retriever.RetrieveBlockNumberByHash(tx, hash)
 	if err != nil {
 		return nil, err
 	}
-	blockNumber, _ := strconv.ParseUint(string(headerCID.BlockNumber), 10, 64)
 
 	return b.GetReceiptsByBlockHashAndNumber(tx, hash, blockNumber)
 }
@@ -612,7 +553,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash, number uint64)
 		}
 	}()
 
-	_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number)
+	_, receiptBytes, txs, err := b.Retriever.RetrieveReceipts(tx, hash, number)
 	if err != nil {
 		return nil, err
 	}
@@ -658,6 +599,32 @@ func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHas
 	return nil, nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
+// IPLDStateDBAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash
+func (b *Backend) IPLDStateDBAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*ipld_eth_statedb.StateDB, *types.Header, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.IPLDStateDBAndHeaderByNumber(ctx, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		header, err := b.HeaderByHash(ctx, hash)
+		if err != nil {
+			return nil, nil, err
+		}
+		if header == nil {
+			return nil, nil, errors.New("header for hash not found")
+		}
+		canonicalHash, err := b.GetCanonicalHash(header.Number.Uint64())
+		if err != nil {
+			return nil, nil, err
+		}
+		if blockNrOrHash.RequireCanonical && canonicalHash != hash {
+			return nil, nil, errors.New("hash is not currently canonical")
+		}
+		stateDB, err := ipld_eth_statedb.New(header.Root, b.IpldStateDatabase)
+		return stateDB, header, err
+	}
+	return nil, nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
 // StateAndHeaderByNumber returns the statedb and header for a provided block number
 func (b *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) {
 	// Pending state is only known by the miner
@@ -676,6 +643,24 @@ func (b *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNu
 	return stateDb, header, err
 }
 
+// IPLDStateDBAndHeaderByNumber returns the statedb and header for a provided block number
+func (b *Backend) IPLDStateDBAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*ipld_eth_statedb.StateDB, *types.Header, error) {
+	// Pending state is only known by the miner
+	if number == rpc.PendingBlockNumber {
+		return nil, nil, errPendingBlockNumber
+	}
+	// Otherwise resolve the block number and return its state
+	header, err := b.HeaderByNumber(ctx, number)
+	if err != nil {
+		return nil, nil, err
+	}
+	if header == nil {
+		return nil, nil, errors.New("header not found")
+	}
+	stateDb, err := ipld_eth_statedb.New(header.Root, b.IpldStateDatabase)
+	return stateDb, header, err
+}
+
 // GetCanonicalHash gets the canonical hash for the provided number, if there is one
 func (b *Backend) GetCanonicalHash(number uint64) (common.Hash, error) {
 	var hashResult string
@@ -697,11 +682,11 @@ func (b *Backend) GetCanonicalHeader(number uint64) (string, []byte, error) {
 }
 
 // GetEVM constructs and returns a vm.EVM
-func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) {
+func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state vm.StateDB, header *types.Header) (*vm.EVM, func() error, error) {
 	vmError := func() error { return nil }
 	txContext := core.NewEVMTxContext(msg)
-	context := core.NewEVMBlockContext(header, b, nil)
+	blockContext := core.NewEVMBlockContext(header, b, nil)
-	return vm.NewEVM(context, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil
+	return vm.NewEVM(blockContext, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil
 }
 
 // GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash
@@ -753,7 +738,7 @@ func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address,
 		return nil, err
 	}
 
-	_, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash)
+	_, accountRlp, err := b.Retriever.RetrieveAccountByAddressAndBlockHash(address, hash)
 	if err != nil {
 		return nil, err
 	}
@@ -884,7 +869,7 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address,
 		return nil, err
 	}
 
-	_, _, storageRlp, err := b.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash)
+	_, _, storageRlp, err := b.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash)
 	return storageRlp, err
 }
 
@@ -961,7 +946,7 @@ func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSlice
 		if depthReached > metaData.maxDepth {
 			metaData.maxDepth = depthReached
 		}
-		if node.NodeType == sdtypes.Leaf {
+		if node.NodeType == Leaf {
 			metaData.leafCount++
 		}
 		leavesFetchTime += leafFetchTime
@@ -1003,7 +988,7 @@ func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSlice
 	if depthReached > metaData.maxDepth {
 		metaData.maxDepth = depthReached
 	}
-	if node.NodeType == sdtypes.Leaf {
+	if node.NodeType == Leaf {
 		metaData.leafCount++
 	}
 
@@ -1045,7 +1030,7 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice
 			continue
 		}
 
-		node, nodeElements, err := sdtrie.ResolveNode(it, b.StateDatabase.TrieDB())
+		node, nodeElements, err := ResolveNodeIt(it, b.StateDatabase.TrieDB())
 		if err != nil {
 			return err
 		}
@@ -1060,7 +1045,7 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice
 		if depthReached > metaData.maxDepth {
 			metaData.maxDepth = depthReached
 		}
-		if node.NodeType == sdtypes.Leaf {
+		if node.NodeType == Leaf {
 			metaData.leafCount++
 		}
 		leavesFetchTime += leafFetchTime

pkg/eth/backend_utils.go

@@ -35,8 +35,6 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers"
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/ethereum/go-ethereum/trie"
 )
 
@@ -334,7 +332,7 @@ func fillSliceNodeData(
 	ethDB ethdb.KeyValueReader,
 	nodesMap map[string]string,
 	leavesMap map[string]GetSliceResponseAccount,
-	node sdtypes.StateNode,
+	node StateNode,
 	nodeElements []interface{},
 	storage bool,
 ) (int64, error) {
@@ -344,7 +342,7 @@ func fillSliceNodeData(
 
 	// Extract account data if it's a Leaf node
 	leafStartTime := makeTimestamp()
-	if node.NodeType == sdtypes.Leaf && !storage {
+	if node.NodeType == Leaf && !storage {
 		stateLeafKey, storageRoot, code, err := extractContractAccountInfo(ethDB, node, nodeElements)
 		if err != nil {
 			return 0, fmt.Errorf("GetSlice account lookup error: %s", err.Error())
@@ -362,7 +360,7 @@ func fillSliceNodeData(
 	return makeTimestamp() - leafStartTime, nil
 }
 
-func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNode, nodeElements []interface{}) (string, string, []byte, error) {
+func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node StateNode, nodeElements []interface{}) (string, string, []byte, error) {
 	var account types.StateAccount
 	if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
 		return "", "", nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
@@ -387,23 +385,3 @@ func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNo
 
 	return stateLeafKeyString, storageRootString, codeBytes, nil
 }
-
-func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) {
-	nodePath := make([]byte, len(path))
-	copy(nodePath, path)
-
-	var nodeElements []interface{}
-	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
-		return sdtypes.StateNode{}, nil, err
-	}
-
-	ty, err := sdtrie.CheckKeyType(nodeElements)
-	if err != nil {
-		return sdtypes.StateNode{}, nil, err
-	}
-	return sdtypes.StateNode{
-		NodeType:  ty,
-		Path:      nodePath,
-		NodeValue: node,
-	}, nodeElements, nil
-}

pkg/eth/cid_retriever.go

@@ -1,374 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"fmt"
-	"math/big"
-
-	"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/statediff/indexer/models"
-	"github.com/jmoiron/sqlx"
-	"github.com/lib/pq"
-	"gorm.io/driver/postgres"
-	"gorm.io/gorm"
-)
-
-// CIDRetriever satisfies the CIDRetriever interface for ethereum
-type CIDRetriever struct {
-	db     *sqlx.DB
-	gormDB *gorm.DB
-}
-
-type IPLDModelRecord struct {
-	models.IPLDModel
-}
-
-// TableName overrides the table name used by IPLD
-func (IPLDModelRecord) TableName() string {
-	return "public.blocks"
-}
-
-type HeaderCIDRecord struct {
-	CID             string `gorm:"column:cid"`
-	BlockHash       string `gorm:"primaryKey"`
-	BlockNumber     string `gorm:"primaryKey"`
-	ParentHash      string
-	Timestamp       uint64
-	StateRoot       string
-	TotalDifficulty string `gorm:"column:td"`
-	TxRoot          string
-	RctRoot         string `gorm:"column:receipt_root"`
-	UncleRoot       string
-	Bloom           []byte
-	MhKey           string
-
-	// gorm doesn't check if foreign key exists in database.
-	// It is required to eager load relations using preload.
-	TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"`
-	IPLD            IPLDModelRecord        `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"`
-}
-
-// TableName overrides the table name used by HeaderCIDRecord
-func (HeaderCIDRecord) TableName() string {
-	return "eth.header_cids"
-}
-
-type TransactionCIDRecord struct {
-	CID         string `gorm:"column:cid"`
-	TxHash      string `gorm:"primaryKey"`
-	BlockNumber string `gorm:"primaryKey"`
-	HeaderID    string `gorm:"column:header_id"`
-	Index       int64
-	Src         string
-	Dst         string
-	MhKey       string
-	IPLD        IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"`
-}
-
-// TableName overrides the table name used by TransactionCIDRecord
-func (TransactionCIDRecord) TableName() string {
-	return "eth.transaction_cids"
-}
-
-// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface
-func NewCIDRetriever(db *sqlx.DB) *CIDRetriever {
-	gormDB, err := gorm.Open(postgres.New(postgres.Config{
-		Conn: db,
-	}), &gorm.Config{})
-
-	if err != nil {
-		log.Error(err)
-		return nil
-	}
-
-	return &CIDRetriever{
-		db:     db,
-		gormDB: gormDB,
-	}
-}
-
-// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
-func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
-	var blockNumber int64
-	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1")
-	return blockNumber, err
-}
-
-// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
-func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
-	var blockNumber int64
-	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1")
-	return blockNumber, err
-}
-
-func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) {
-	for i, topicSet := range topics {
-		if len(topicSet) == 0 {
-			continue
-		}
-
-		if !first {
-			pgStr += " AND"
-		} else {
-			first = false
-		}
-		pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id)
-		args = append(args, pq.Array(topicSet))
-		*id++
-	}
-	return pgStr, args
-}
-
-func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) {
-	if len(rctFilter.LogAddresses) > 0 {
-		pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id)
-		args = append(args, pq.Array(rctFilter.LogAddresses))
-		*id++
-	}
-
-	// Filter on topics if there are any
-	if hasTopics(rctFilter.Topics) {
-		pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
-	}
-
-	return pgStr, args
-}
-
-func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) {
-	rctCond := " AND (receipt_cids.tx_id = ANY ( "
-	logQuery := "SELECT rct_id FROM eth.log_cids WHERE"
-	if len(rctFilter.LogAddresses) > 0 {
-		// Filter on log contract addresses if there are any
-		pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id)
-		args = append(args, pq.Array(rctFilter.LogAddresses))
-		*id++
-
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
-		}
-
-		pgStr += ")"
-
-		// Filter on txHashes if there are any, and we are matching txs
-		if rctFilter.MatchTxs && len(txHashes) > 0 {
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
-			args = append(args, pq.Array(txHashes))
-		}
-		pgStr += ")"
-	} else { // If there are no contract addresses to filter on
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += rctCond + logQuery
-			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true)
-			pgStr += ")"
-			// Filter on txHashes if there are any, and we are matching txs
-			if rctFilter.MatchTxs && len(txHashes) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
-				args = append(args, pq.Array(txHashes))
-			}
-			pgStr += ")"
-		} else if rctFilter.MatchTxs && len(txHashes) > 0 {
-			// If there are no contract addresses or topics to filter on,
-			// Filter on txHashes if there are any, and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id)
-			args = append(args, pq.Array(txHashes))
-		}
-	}
-
-	return pgStr, args
-}
-
-// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided
-// filter parameters.
-func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) {
-	log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String())
-	args := make([]interface{}, 0, 4)
-	id := 1
-	pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.header_id as block_hash,
-			eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address,
-			eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data,
-			data, eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash
-				FROM eth.log_cids, eth.receipt_cids, public.blocks
-				WHERE eth.log_cids.rct_id = receipt_cids.tx_id
-				AND eth.log_cids.header_id = receipt_cids.header_id
-				AND eth.log_cids.block_number = receipt_cids.block_number
-				AND log_cids.leaf_mh_key = blocks.key
-				AND log_cids.block_number = blocks.block_number
-				AND receipt_cids.header_id = $1`
-
-	args = append(args, blockHash.String())
-	id++
-
-	if blockNumber != nil {
-		pgStr += ` AND receipt_cids.block_number = $2`
-		id++
-		args = append(args, blockNumber.Int64())
-	}
-
-	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
-	pgStr += ` ORDER BY log_cids.index`
-
-	logCIDs := make([]LogResult, 0)
-	err := tx.Select(&logCIDs, pgStr, args...)
-	if err != nil {
-		return nil, err
-	}
-
-	return logCIDs, nil
-}
-
-// RetrieveFilteredLog retrieves and returns all the log CIDs provided blockHeight or blockHash that conform to the provided
-// filter parameters.
-func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) {
-	log.Debug("retrieving log cids for receipt ids")
-	args := make([]interface{}, 0, 4)
-	pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id,
-			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
-			eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
-			eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status, header_cids.block_hash
-							FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids
-							WHERE eth.log_cids.rct_id = receipt_cids.tx_id
-							AND eth.log_cids.header_id = eth.receipt_cids.header_id
-							AND eth.log_cids.block_number = eth.receipt_cids.block_number
-							AND receipt_cids.tx_id = transaction_cids.tx_hash
-							AND receipt_cids.header_id = transaction_cids.header_id
-							AND receipt_cids.block_number = transaction_cids.block_number
-							AND transaction_cids.header_id = header_cids.block_hash
-							AND transaction_cids.block_number = header_cids.block_number`
-	id := 1
-	if blockNumber > 0 {
-		pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
-		args = append(args, blockNumber)
-		id++
-	}
-	if blockHash != nil {
-		pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id)
-		args = append(args, blockHash.String())
-		id++
-	}
-
-	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
-	pgStr += ` ORDER BY log_cids.index`
-
-	logCIDs := make([]LogResult, 0)
-	err := tx.Select(&logCIDs, pgStr, args...)
-	if err != nil {
-		return nil, err
-	}
-
-	return logCIDs, nil
-}
-
-func hasTopics(topics [][]string) bool {
-	for _, topicSet := range topics {
-		if len(topicSet) > 0 {
-			return true
-		}
-	}
-	return false
-}
-
-// RetrieveHeaderCIDByHash returns the header for the given block hash
-func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (models.HeaderModel, error) {
-	log.Debug("retrieving header cids for block hash ", blockHash.String())
-	pgStr := `SELECT block_hash, CAST(block_number as Text), parent_hash, cid, mh_key, CAST(td as Text),
-			state_root, uncle_root, tx_root, receipt_root, bloom, timestamp FROM eth.header_cids
-			WHERE block_hash = $1`
-	var headerCID models.HeaderModel
-	return headerCID, tx.Get(&headerCID, pgStr, blockHash.String())
-}
-
-// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number
-func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) {
-	log.Debug("retrieving header cids and tx cids for block number ", blockNumber)
-
-	var headerCIDs []HeaderCIDRecord
-
-	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
-	// Will use join for TransactionCIDs once preload for 1:N is supported.
-	err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
-		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
-	}).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error
-
-	if err != nil {
-		log.Error("header cid retrieval error")
-		return nil, err
-	}
-
-	return headerCIDs, nil
-}
-
-// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number)
-func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) {
-	log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String())
-
-	var headerCIDs []HeaderCIDRecord
-
-	conditions := map[string]interface{}{"block_hash": blockHash.String()}
-	if blockNumber != nil {
-		conditions["header_cids.block_number"] = blockNumber.Int64()
-	}
-
-	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
-	// Will use join for TransactionCIDs once preload for 1:N is supported.
-	err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
-		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
-	}).Joins("IPLD").Find(&headerCIDs, conditions).Error
-
-	if err != nil {
-		log.Error("header cid retrieval error")
-		return HeaderCIDRecord{}, err
-	}
-
-	if len(headerCIDs) == 0 {
-		return HeaderCIDRecord{}, errHeaderHashNotFound
-	} else if len(headerCIDs) > 1 {
-		return HeaderCIDRecord{}, errMultipleHeadersForHash
-	}
-
-	return headerCIDs[0], nil
-}
-
-// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number)
-func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) {
-	log.Debug("retrieving tx cid for tx hash ", txHash)
-
-	var txCIDs []TransactionCIDRecord
-
-	var err error
-	if blockNumber != nil {
-		err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error
-	} else {
-		err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error
-	}
-	if err != nil {
-		log.Error("tx retrieval error")
-		return TransactionCIDRecord{}, err
-	}
-
-	if len(txCIDs) == 0 {
-		return TransactionCIDRecord{}, errTxHashNotFound
-	} else if len(txCIDs) > 1 {
-		// a transaction can be part of a only one canonical block
-		return TransactionCIDRecord{}, errTxHashInMultipleBlocks
-	}
-
-	return txCIDs[0], nil
-}

pkg/eth/helpers.go

@@ -18,25 +18,8 @@ package eth
 
 import (
 	"time"
-
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 )
 
-func ResolveToNodeType(nodeType int) sdtypes.NodeType {
-	switch nodeType {
-	case 0:
-		return sdtypes.Branch
-	case 1:
-		return sdtypes.Extension
-	case 2:
-		return sdtypes.Leaf
-	case 3:
-		return sdtypes.Removed
-	default:
-		return sdtypes.Unknown
-	}
-}
-
 // Timestamp in milliseconds
 func makeTimestamp() int64 {
 	return time.Now().UnixNano() / int64(time.Millisecond)

pkg/eth/interfaces.go

@@ -1,47 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"context"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/bloombits"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/event"
-	"github.com/ethereum/go-ethereum/rpc"
-)
-
-// FilterBackend is the geth interface we need to satisfy to use their filters
-type FilterBackend interface {
-	ChainDb() ethdb.Database
-	HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
-	HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error)
-	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
-	GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error)
-
-	SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
-	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
-	SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
-	SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
-	SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
-
-	BloomStatus() (uint64, uint64)
-	ServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
-}

pkg/eth/ipld_retriever.go

@@ -1,368 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"fmt"
-	"strconv"
-
-	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
-	"github.com/ethereum/go-ethereum/statediff/trie_helpers"
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
-	"github.com/jmoiron/sqlx"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/rlp"
-)
-
-const (
-	RetrieveHeaderByHashPgStr = `SELECT cid, data
-								FROM eth.header_cids
-									INNER JOIN public.blocks ON (
-										header_cids.mh_key = blocks.key
-										AND header_cids.block_number = blocks.block_number
-									)
-								WHERE block_hash = $1`
-	RetrieveUnclesPgStr = `SELECT uncle_cids.cid, data
-										FROM eth.uncle_cids
-											INNER JOIN eth.header_cids ON (
-												uncle_cids.header_id = header_cids.block_hash
-												AND uncle_cids.block_number = header_cids.block_number
-											)
-											INNER JOIN public.blocks ON (
-												uncle_cids.mh_key = blocks.key
-												AND uncle_cids.block_number = blocks.block_number
-											)
-										WHERE header_cids.block_hash = $1
-										AND header_cids.block_number = $2
-										ORDER BY uncle_cids.parent_hash`
-	RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
-										FROM eth.uncle_cids
-											INNER JOIN eth.header_cids ON (
-												uncle_cids.header_id = header_cids.block_hash
-												AND uncle_cids.block_number = header_cids.block_number
-											)
-											INNER JOIN public.blocks ON (
-												uncle_cids.mh_key = blocks.key
-												AND uncle_cids.block_number = blocks.block_number
-											)
-										WHERE header_cids.block_hash = $1
-										ORDER BY uncle_cids.parent_hash`
-	RetrieveTransactionsPgStr = `SELECT transaction_cids.cid, data
-											FROM eth.transaction_cids
-												INNER JOIN eth.header_cids ON (
-													transaction_cids.header_id = header_cids.block_hash
-													AND transaction_cids.block_number = header_cids.block_number
-												)
-												INNER JOIN public.blocks ON (
-													transaction_cids.mh_key = blocks.key
-													AND transaction_cids.block_number = blocks.block_number
-												)
-											WHERE block_hash = $1
-											AND header_cids.block_number = $2
-											ORDER BY eth.transaction_cids.index ASC`
-	RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data
-											FROM eth.transaction_cids
-												INNER JOIN eth.header_cids ON (
-													transaction_cids.header_id = header_cids.block_hash
-													AND transaction_cids.block_number = header_cids.block_number
-												)
-												INNER JOIN public.blocks ON (
-													transaction_cids.mh_key = blocks.key
-													AND transaction_cids.block_number = blocks.block_number
-												)
-											WHERE block_hash = $1
-											ORDER BY eth.transaction_cids.index ASC`
-	RetrieveReceiptsPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
-										FROM eth.receipt_cids
-											INNER JOIN eth.transaction_cids ON (
-												receipt_cids.tx_id = transaction_cids.tx_hash
-												AND receipt_cids.header_id = transaction_cids.header_id
-												AND receipt_cids.block_number = transaction_cids.block_number
-											)
-											INNER JOIN eth.header_cids ON (
-												transaction_cids.header_id = header_cids.block_hash
-												AND transaction_cids.block_number = header_cids.block_number
-											)
-											INNER JOIN public.blocks ON (
-												receipt_cids.leaf_mh_key = blocks.key
-												AND receipt_cids.block_number = blocks.block_number
-											)
-										WHERE block_hash = $1
-										AND header_cids.block_number = $2
-										ORDER BY eth.transaction_cids.index ASC`
-	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
-										FROM eth.receipt_cids
-											INNER JOIN eth.transaction_cids ON (
-												receipt_cids.tx_id = transaction_cids.tx_hash
-												AND receipt_cids.header_id = transaction_cids.header_id
-												AND receipt_cids.block_number = transaction_cids.block_number
-											)
-											INNER JOIN eth.header_cids ON (
-												transaction_cids.header_id = header_cids.block_hash
-												AND transaction_cids.block_number = header_cids.block_number
-											)
-											INNER JOIN public.blocks ON (
-												receipt_cids.leaf_mh_key = blocks.key
-												AND receipt_cids.block_number = blocks.block_number
-											)
-										WHERE block_hash = $1
-										ORDER BY eth.transaction_cids.index ASC`
-	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type
-												FROM eth.state_cids
-													INNER JOIN eth.header_cids ON (
-														state_cids.header_id = header_cids.block_hash
-														AND state_cids.block_number = header_cids.block_number
-													)
-												WHERE state_leaf_key = $1
-												AND header_cids.block_number <= (SELECT block_number
-																	FROM eth.header_cids
-																	WHERE block_hash = $2)
-												AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number))
-												ORDER BY header_cids.block_number DESC
-												LIMIT 1`
-	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, mh_key, block_number, node_type, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)`
-)
-
-var EmptyNodeValue = make([]byte, common.HashLength)
-
-type rctIpldResult struct {
-	LeafCID string `db:"leaf_cid"`
-	Data    []byte `db:"data"`
-	TxHash  string `db:"tx_hash"`
-}
-
-type ipldResult struct {
-	CID    string `db:"cid"`
-	Data   []byte `db:"data"`
-	TxHash string `db:"tx_hash"`
-}
-
-type IPLDRetriever struct {
-	db *sqlx.DB
-}
-
-func NewIPLDRetriever(db *sqlx.DB) *IPLDRetriever {
-	return &IPLDRetriever{
-		db: db,
-	}
-}
-
-// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
-func (r *IPLDRetriever) RetrieveHeaderByHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) {
-	headerResult := new(ipldResult)
-	return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
-}
-
-// RetrieveUncles returns the cids and rlp bytes for the uncles corresponding to the provided block hash, number (of non-omner root block)
-func (r *IPLDRetriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
-	uncleResults := make([]ipldResult, 0)
-	if err := tx.Select(&uncleResults, RetrieveUnclesPgStr, hash.Hex(), number); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(uncleResults))
-	uncles := make([][]byte, len(uncleResults))
-	for i, res := range uncleResults {
-		cids[i] = res.CID
-		uncles[i] = res.Data
-	}
-	return cids, uncles, nil
-}
-
-// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block)
-func (r *IPLDRetriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
-	uncleResults := make([]ipldResult, 0)
-	if err := tx.Select(&uncleResults, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(uncleResults))
-	uncles := make([][]byte, len(uncleResults))
-	for i, res := range uncleResults {
-		cids[i] = res.CID
-		uncles[i] = res.Data
-	}
-	return cids, uncles, nil
-}
-
-// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number
-func (r *IPLDRetriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
-	txResults := make([]ipldResult, 0)
-	if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(txResults))
-	txs := make([][]byte, len(txResults))
-	for i, res := range txResults {
-		cids[i] = res.CID
-		txs[i] = res.Data
-	}
-	return cids, txs, nil
-}
-
-// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash
-func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
-	txResults := make([]ipldResult, 0)
-	if err := tx.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(txResults))
-	txs := make([][]byte, len(txResults))
-	for i, res := range txResults {
-		cids[i] = res.CID
-		txs[i] = res.Data
-	}
-	return cids, txs, nil
-}
-
-// DecodeLeafNode decodes the leaf node data
-func DecodeLeafNode(node []byte) ([]byte, error) {
-	var nodeElements []interface{}
-	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
-		return nil, err
-	}
-	ty, err := trie_helpers.CheckKeyType(nodeElements)
-	if err != nil {
-		return nil, err
-	}
-
-	if ty != sdtypes.Leaf {
-		return nil, fmt.Errorf("expected leaf node but found %s", ty)
-	}
-	return nodeElements[1].([]byte), nil
-}
-
-// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number.
-// cid returned corresponds to the leaf node data which contains the receipt.
-func (r *IPLDRetriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) {
-	rctResults := make([]rctIpldResult, 0)
-	if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil {
-		return nil, nil, nil, err
-	}
-	cids := make([]string, len(rctResults))
-	rcts := make([][]byte, len(rctResults))
-	txs := make([]common.Hash, len(rctResults))
-
-	for i, res := range rctResults {
-		cids[i] = res.LeafCID
-		nodeVal, err := DecodeLeafNode(res.Data)
-		if err != nil {
-			return nil, nil, nil, err
-		}
-		rcts[i] = nodeVal
-		txs[i] = common.HexToHash(res.TxHash)
-	}
-
-	return cids, rcts, txs, nil
-}
-
-// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
-// cid returned corresponds to the leaf node data which contains the receipt.
-func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
-	rctResults := make([]rctIpldResult, 0)
-	if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
-		return nil, nil, nil, err
-	}
-	cids := make([]string, len(rctResults))
-	rcts := make([][]byte, len(rctResults))
-	txs := make([]common.Hash, len(rctResults))
-
-	for i, res := range rctResults {
-		cids[i] = res.LeafCID
-		nodeVal, err := DecodeLeafNode(res.Data)
-		if err != nil {
-			return nil, nil, nil, err
-		}
-		rcts[i] = nodeVal
-		txs[i] = common.HexToHash(res.TxHash)
-	}
-
-	return cids, rcts, txs, nil
-}
-
-type nodeInfo struct {
-	CID              string `db:"cid"`
-	MhKey            string `db:"mh_key"`
-	BlockNumber      string `db:"block_number"`
-	Data             []byte `db:"data"`
-	NodeType         int    `db:"node_type"`
-	StateLeafRemoved bool   `db:"state_leaf_removed"`
-}
-
-// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
-// TODO: ensure this handles deleted accounts appropriately
-func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) {
-	accountResult := new(nodeInfo)
-	leafKey := crypto.Keccak256Hash(address.Bytes())
-	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
-		return "", nil, err
-	}
-
-	if accountResult.NodeType == sdtypes.Removed.Int() {
-		return "", EmptyNodeValue, nil
-	}
-
-	blockNumber, err := strconv.ParseUint(accountResult.BlockNumber, 10, 64)
-	if err != nil {
-		return "", nil, err
-	}
-	accountResult.Data, err = shared.FetchIPLD(r.db, accountResult.MhKey, blockNumber)
-	if err != nil {
-		return "", nil, err
-	}
-
-	var i []interface{}
-	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
-		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
-	}
-	if len(i) != 2 {
-		return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
-	}
-	return accountResult.CID, i[1].([]byte), nil
-}
-
-// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash
-func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) (string, []byte, []byte, error) {
-	storageResult := new(nodeInfo)
-	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
-	storageHash := crypto.Keccak256Hash(key.Bytes())
-	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
-		return "", nil, nil, err
-	}
-	if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() {
-		return "", EmptyNodeValue, EmptyNodeValue, nil
-	}
-
-	blockNumber, err := strconv.ParseUint(storageResult.BlockNumber, 10, 64)
-	if err != nil {
-		return "", nil, nil, err
-	}
-	storageResult.Data, err = shared.FetchIPLD(r.db, storageResult.MhKey, blockNumber)
-	if err != nil {
-		return "", nil, nil, err
-	}
-
-	var i []interface{}
-	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
-		err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
-		return "", nil, nil, err
-	}
-	if len(i) != 2 {
-		return "", nil, nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
-	}
-	return storageResult.CID, storageResult.Data, i[1].([]byte), nil
-}

pkg/eth/node_types.go

@@ -0,0 +1,116 @@
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+)
+
+// NodeType for explicitly setting type of node
+type NodeType string
+
+const (
+	Unknown   NodeType = "Unknown"
+	Branch    NodeType = "Branch"
+	Extension NodeType = "Extension"
+	Leaf      NodeType = "Leaf"
+	Removed   NodeType = "Removed" // used to represent paths which have been emptied
+)
+
+func (n NodeType) Int() int {
+	switch n {
+	case Branch:
+		return 0
+	case Extension:
+		return 1
+	case Leaf:
+		return 2
+	case Removed:
+		return 3
+	default:
+		return -1
+	}
+}
+
+// CheckKeyType checks what type of key we have
+func CheckKeyType(elements []interface{}) (NodeType, error) {
+	if len(elements) > 2 {
+		return Branch, nil
+	}
+	if len(elements) < 2 {
+		return Unknown, fmt.Errorf("node cannot be less than two elements in length")
+	}
+	switch elements[0].([]byte)[0] / 16 {
+	case '\x00':
+		return Extension, nil
+	case '\x01':
+		return Extension, nil
+	case '\x02':
+		return Leaf, nil
+	case '\x03':
+		return Leaf, nil
+	default:
+		return Unknown, fmt.Errorf("unknown hex prefix")
+	}
+}
+
+// StateNode holds the data for a single state diff node
+type StateNode struct {
+	NodeType     NodeType      `json:"nodeType"        gencodec:"required"`
+	Path         []byte        `json:"path"            gencodec:"required"`
+	NodeValue    []byte        `json:"value"           gencodec:"required"`
+	StorageNodes []StorageNode `json:"storage"`
+	LeafKey      []byte        `json:"leafKey"`
+}
+
+// StorageNode holds the data for a single storage diff node
+type StorageNode struct {
+	NodeType  NodeType `json:"nodeType"        gencodec:"required"`
+	Path      []byte   `json:"path"            gencodec:"required"`
+	NodeValue []byte   `json:"value"           gencodec:"required"`
+	LeafKey   []byte   `json:"leafKey"`
+}
+
+func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (StateNode, []interface{}, error) {
+	nodePath := make([]byte, len(path))
+	copy(nodePath, path)
+
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return StateNode{}, nil, err
+	}
+
+	ty, err := CheckKeyType(nodeElements)
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+	return StateNode{
+		NodeType:  ty,
+		Path:      nodePath,
+		NodeValue: node,
+	}, nodeElements, nil
+}
+
+// ResolveNodeIt return the state diff node pointed by the iterator.
+func ResolveNodeIt(it trie.NodeIterator, trieDB *trie.Database) (StateNode, []interface{}, error) {
+	nodePath := make([]byte, len(it.Path()))
+	copy(nodePath, it.Path())
+	node, err := trieDB.Node(it.Hash())
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+	var nodeElements []interface{}
+	if err = rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return StateNode{}, nil, err
+	}
+	ty, err := CheckKeyType(nodeElements)
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+	return StateNode{
+		NodeType:  ty,
+		Path:      nodePath,
+		NodeValue: node,
+	}, nodeElements, nil
+}

pkg/eth/retriever.go

@@ -0,0 +1,543 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+	"math/big"
+	"strconv"
+
+	"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
+	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/trie_helpers"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+// Retriever is used for fetching
+type Retriever struct {
+	db     *sqlx.DB
+	gormDB *gorm.DB
+}
+
+type IPLDModelRecord struct {
+	models.IPLDModel
+}
+
+// TableName overrides the table name used by IPLD
+func (IPLDModelRecord) TableName() string {
+	return "ipld.blocks"
+}
+
+type HeaderCIDRecord struct {
+	CID             string `gorm:"column:cid"`
+	BlockHash       string `gorm:"primaryKey"`
+	BlockNumber     string `gorm:"primaryKey"`
+	ParentHash      string
+	Timestamp       uint64
+	StateRoot       string
+	TotalDifficulty string `gorm:"column:td"`
+	TxRoot          string
+	RctRoot         string `gorm:"column:receipt_root"`
+	UncleRoot       string
+	Bloom           []byte
+	MhKey           string
+
+	// gorm doesn't check if foreign key exists in database.
+	// It is required to eager load relations using preload.
+	TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"`
+	IPLD            IPLDModelRecord        `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"`
+}
+
+// TableName overrides the table name used by HeaderCIDRecord
+func (HeaderCIDRecord) TableName() string {
+	return "eth.header_cids"
+}
+
+type TransactionCIDRecord struct {
+	CID         string `gorm:"column:cid"`
+	TxHash      string `gorm:"primaryKey"`
+	BlockNumber string `gorm:"primaryKey"`
+	HeaderID    string `gorm:"column:header_id"`
+	Index       int64
+	Src         string
+	Dst         string
+	MhKey       string
+	IPLD        IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"`
+}
+
+// TableName overrides the table name used by TransactionCIDRecord
+func (TransactionCIDRecord) TableName() string {
+	return "eth.transaction_cids"
+}
+
+// NewRetriever returns a pointer to a new Retriever which supports the Retriever interface
+func NewRetriever(db *sqlx.DB) *Retriever {
+	gormDB, err := gorm.Open(postgres.New(postgres.Config{
+		Conn: db,
+	}), &gorm.Config{})
+
+	if err != nil {
+		log.Error(err)
+		return nil
+	}
+
+	return &Retriever{
+		db:     db,
+		gormDB: gormDB,
+	}
+}
+
+// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
+func (r *Retriever) RetrieveFirstBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := r.db.Get(&blockNumber, "SELECT block_number FROM ipld.blocks ORDER BY block_number ASC LIMIT 1")
+	return blockNumber, err
+}
+
+// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
+func (r *Retriever) RetrieveLastBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := r.db.Get(&blockNumber, "SELECT block_number FROM ipld.blocks ORDER BY block_number DESC LIMIT 1")
+	return blockNumber, err
+}
+
+func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) {
+	for i, topicSet := range topics {
+		if len(topicSet) == 0 {
+			continue
+		}
+
+		if !first {
+			pgStr += " AND"
+		} else {
+			first = false
+		}
+		pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id)
+		args = append(args, pq.Array(topicSet))
+		*id++
+	}
+	return pgStr, args
+}
+
+func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) {
+	if len(rctFilter.LogAddresses) > 0 {
+		pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id)
+		args = append(args, pq.Array(rctFilter.LogAddresses))
+		*id++
+	}
+
+	// Filter on topics if there are any
+	if hasTopics(rctFilter.Topics) {
+		pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
+	}
+
+	return pgStr, args
+}
+
+func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) {
+	rctCond := " AND (receipt_cids.tx_id = ANY ( "
+	logQuery := "SELECT rct_id FROM eth.log_cids WHERE"
+	if len(rctFilter.LogAddresses) > 0 {
+		// Filter on log contract addresses if there are any
+		pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id)
+		args = append(args, pq.Array(rctFilter.LogAddresses))
+		*id++
+
+		// Filter on topics if there are any
+		if hasTopics(rctFilter.Topics) {
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
+		}
+
+		pgStr += ")"
+
+		// Filter on txHashes if there are any, and we are matching txs
+		if rctFilter.MatchTxs && len(txHashes) > 0 {
+			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
+			args = append(args, pq.Array(txHashes))
+		}
+		pgStr += ")"
+	} else { // If there are no contract addresses to filter on
+		// Filter on topics if there are any
+		if hasTopics(rctFilter.Topics) {
+			pgStr += rctCond + logQuery
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true)
+			pgStr += ")"
+			// Filter on txHashes if there are any, and we are matching txs
+			if rctFilter.MatchTxs && len(txHashes) > 0 {
+				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
+				args = append(args, pq.Array(txHashes))
+			}
+			pgStr += ")"
+		} else if rctFilter.MatchTxs && len(txHashes) > 0 {
+			// If there are no contract addresses or topics to filter on,
+			// Filter on txHashes if there are any, and we are matching txs
+			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id)
+			args = append(args, pq.Array(txHashes))
+		}
+	}
+
+	return pgStr, args
+}
+
+// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided
+// filter parameters.
+func (r *Retriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String())
+	args := make([]interface{}, 0, 4)
+	id := 1
+	pgStr := RetrieveFilteredGQLLogs
+	args = append(args, blockHash.String())
+	id++
+
+	if blockNumber != nil {
+		pgStr += ` AND receipt_cids.block_number = $2`
+		id++
+		args = append(args, blockNumber.Int64())
+	}
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY log_cids.index`
+
+	logCIDs := make([]LogResult, 0)
+	err := tx.Select(&logCIDs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return logCIDs, nil
+}
+
+// RetrieveFilteredLogs retrieves and returns all the log CIDs provided blockHeight or blockHash that conform to the provided
+// filter parameters.
+func (r *Retriever) RetrieveFilteredLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids")
+	args := make([]interface{}, 0, 4)
+	pgStr := RetrieveFilteredLogs
+	id := 1
+	if blockNumber > 0 {
+		pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
+		args = append(args, blockNumber)
+		id++
+	}
+	if blockHash != nil {
+		pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id)
+		args = append(args, blockHash.String())
+		id++
+	}
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY log_cids.index`
+
+	logCIDs := make([]LogResult, 0)
+	err := tx.Select(&logCIDs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return logCIDs, nil
+}
+
+func hasTopics(topics [][]string) bool {
+	for _, topicSet := range topics {
+		if len(topicSet) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// RetrieveBlockNumberByHash returns the block number for the given block hash
+func (r *Retriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) {
+	log.Debug("retrieving block number for block hash ", blockHash.String())
+	pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1`
+	var blockNumberStr string
+	if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil {
+		return 0, err
+	}
+	return strconv.ParseUint(blockNumberStr, 10, 64)
+}
+
+// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number
+func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) {
+	log.Debug("retrieving header cids and tx cids for block number ", blockNumber)
+
+	var headerCIDs []HeaderCIDRecord
+
+	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
+	// Will use join for TransactionCIDs once preload for 1:N is supported.
+	err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
+		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
+	}).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error
+
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return nil, err
+	}
+
+	return headerCIDs, nil
+}
+
+// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number)
+func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) {
+	log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String())
+
+	var headerCIDs []HeaderCIDRecord
+
+	conditions := map[string]interface{}{"block_hash": blockHash.String()}
+	if blockNumber != nil {
+		conditions["header_cids.block_number"] = blockNumber.Int64()
+	}
+
+	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
+	// Will use join for TransactionCIDs once preload for 1:N is supported.
+	err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
+		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
+	}).Joins("IPLD").Find(&headerCIDs, conditions).Error
+
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return HeaderCIDRecord{}, err
+	}
+
+	if len(headerCIDs) == 0 {
+		return HeaderCIDRecord{}, errHeaderHashNotFound
+	} else if len(headerCIDs) > 1 {
+		return HeaderCIDRecord{}, errMultipleHeadersForHash
+	}
+
+	return headerCIDs[0], nil
+}
+
+// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number)
+func (r *Retriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) {
+	log.Debug("retrieving tx cid for tx hash ", txHash)
+
+	var txCIDs []TransactionCIDRecord
+
+	var err error
+	if blockNumber != nil {
+		err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error
+	} else {
+		err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error
+	}
+	if err != nil {
+		log.Error("tx retrieval error")
+		return TransactionCIDRecord{}, err
+	}
+
+	if len(txCIDs) == 0 {
+		return TransactionCIDRecord{}, errTxHashNotFound
+	} else if len(txCIDs) > 1 {
+		// a transaction can be part of a only one canonical block
+		return TransactionCIDRecord{}, errTxHashInMultipleBlocks
+	}
+
+	return txCIDs[0], nil
+}
+
+var EmptyNodeValue = make([]byte, common.HashLength)
+
+// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
+func (r *Retriever) RetrieveHeaderByHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) {
+	headerResult := new(ipldResult)
+	return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
+}
+
+// RetrieveUncles returns the cid and rlp bytes for the uncle list corresponding to the provided block hash, number (of non-omner root block)
+func (r *Retriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) (string, []byte, error) {
+	uncleResult := new(ipldResult)
+	if err := tx.Select(uncleResult, RetrieveUnclesPgStr, hash.Hex(), number); err != nil {
+		return "", nil, err
+	}
+	return uncleResult.CID, uncleResult.Data, nil
+}
+
+// RetrieveUnclesByBlockHash returns the cid and rlp bytes for the uncle list corresponding to the provided block hash (of non-omner root block)
+func (r *Retriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) {
+	uncleResult := new(ipldResult)
+	if err := tx.Select(uncleResult, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil {
+		return "", nil, err
+	}
+	return uncleResult.CID, uncleResult.Data, nil
+}
+
+// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number
+func (r *Retriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash
+func (r *Retriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	if err := tx.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// DecodeLeafNode decodes the leaf node data
+func DecodeLeafNode(node []byte) ([]byte, error) {
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return nil, err
+	}
+	ty, err := trie_helpers.CheckKeyType(nodeElements)
+	if err != nil {
+		return nil, err
+	}
+
+	if ty != sdtypes.Leaf {
+		return nil, fmt.Errorf("expected leaf node but found %s", ty)
+	}
+	return nodeElements[1].([]byte), nil
+}
+
+// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number.
+// cid returned corresponds to the leaf node data which contains the receipt.
+func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) {
+	rctResults := make([]ipldResult, 0)
+	if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil {
+		return nil, nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	txs := make([]common.Hash, len(rctResults))
+
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		rcts[i] = nodeVal
+		txs[i] = common.HexToHash(res.TxHash)
+	}
+
+	return cids, rcts, txs, nil
+}
+
+// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
+// cid returned corresponds to the leaf node data which contains the receipt.
+func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
+	rctResults := make([]ipldResult, 0)
+	if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	txs := make([]common.Hash, len(rctResults))
+
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		rcts[i] = nodeVal
+		txs[i] = common.HexToHash(res.TxHash)
+	}
+
+	return cids, rcts, txs, nil
+}
+
+// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
+// TODO: ensure this handles deleted accounts appropriately
+func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) {
+	accountResult := new(nodeInfo)
+	leafKey := crypto.Keccak256Hash(address.Bytes())
+	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
+		return "", nil, err
+	}
+
+	if accountResult.Removed {
+		return "", EmptyNodeValue, nil
+	}
+
+	blockNumber, err := strconv.ParseUint(accountResult.BlockNumber, 10, 64)
+	if err != nil {
+		return "", nil, err
+	}
+	accountResult.Data, err = shared.FetchIPLD(r.db, accountResult.CID, blockNumber)
+	if err != nil {
+		return "", nil, err
+	}
+
+	var i []interface{}
+	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
+		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
+	}
+	if len(i) != 2 {
+		return "", nil, fmt.Errorf("eth Retriever expected state leaf node rlp to decode into two elements")
+	}
+	return accountResult.CID, i[1].([]byte), nil
+}
+
+// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash
+func (r *Retriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) (string, []byte, []byte, error) {
+	storageResult := new(nodeInfo)
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	storageHash := crypto.Keccak256Hash(key.Bytes())
+	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
+		return "", nil, nil, err
+	}
+	if storageResult.StateLeafRemoved || storageResult.Removed {
+		return "", EmptyNodeValue, EmptyNodeValue, nil
+	}
+
+	blockNumber, err := strconv.ParseUint(storageResult.BlockNumber, 10, 64)
+	if err != nil {
+		return "", nil, nil, err
+	}
+	storageResult.Data, err = shared.FetchIPLD(r.db, storageResult.CID, blockNumber)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	var i []interface{}
+	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
+		err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
+		return "", nil, nil, err
+	}
+	if len(i) != 2 {
+		return "", nil, nil, fmt.Errorf("eth Retriever expected storage leaf node rlp to decode into two elements")
+	}
+	return storageResult.CID, storageResult.Data, i[1].([]byte), nil
+}

pkg/eth/retriever_test.go

@@ -33,13 +33,13 @@ var _ = Describe("Retriever", func() {
 	var (
 		db          *sqlx.DB
 		diffIndexer interfaces.StateDiffIndexer
-		retriever   *eth.CIDRetriever
+		retriever   *eth.Retriever
 	)
 	BeforeEach(func() {
 		db = shared.SetupDB()
 		diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
 
-		retriever = eth.NewCIDRetriever(db)
+		retriever = eth.NewRetriever(db)
 	})
 	AfterEach(func() {
 		shared.TearDownDB(db)

pkg/eth/sql.go

@@ -0,0 +1,183 @@
+package eth
+
+const (
+	RetrieveHeaderByHashPgStr = `SELECT cid, data
+								FROM eth.header_cids
+									INNER JOIN ipld.blocks ON (
+										header_cids.cid = blocks.key
+										AND header_cids.block_number = blocks.block_number
+									)
+								WHERE block_hash = $1`
+	RetrieveUnclesPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (
+												uncle_cids.header_id = header_cids.block_hash
+												AND uncle_cids.block_number = header_cids.block_number
+											)
+											INNER JOIN ipld.blocks ON (
+												uncle_cids.cid = blocks.key
+												AND uncle_cids.block_number = blocks.block_number
+											)
+										WHERE header_cids.block_hash = $1
+										AND header_cids.block_number = $2
+										ORDER BY uncle_cids.parent_hash
+										LIMIT 1`
+	RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (
+												uncle_cids.header_id = header_cids.block_hash
+												AND uncle_cids.block_number = header_cids.block_number
+											)
+											INNER JOIN ipld.blocks ON (
+												uncle_cids.cid = blocks.key
+												AND uncle_cids.block_number = blocks.block_number
+											)
+										WHERE header_cids.block_hash = $1
+										ORDER BY uncle_cids.parent_hash
+										LIMIT 1`
+	RetrieveTransactionsPgStr = `SELECT transaction_cids.cid, data
+											FROM eth.transaction_cids
+												INNER JOIN eth.header_cids ON (
+													transaction_cids.header_id = header_cids.block_hash
+													AND transaction_cids.block_number = header_cids.block_number
+												)
+												INNER JOIN ipld.blocks ON (
+													transaction_cids.cid = blocks.key
+													AND transaction_cids.block_number = blocks.block_number
+												)
+											WHERE block_hash = $1
+											AND header_cids.block_number = $2
+											ORDER BY eth.transaction_cids.index ASC`
+	RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data
+											FROM eth.transaction_cids
+												INNER JOIN eth.header_cids ON (
+													transaction_cids.header_id = header_cids.block_hash
+													AND transaction_cids.block_number = header_cids.block_number
+												)
+												INNER JOIN ipld.blocks ON (
+													transaction_cids.cid = blocks.key
+													AND transaction_cids.block_number = blocks.block_number
+												)
+											WHERE block_hash = $1
+											ORDER BY eth.transaction_cids.index ASC`
+	RetrieveReceiptsPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash
+										FROM eth.receipt_cids
+											INNER JOIN eth.transaction_cids ON (
+												receipt_cids.tx_id = transaction_cids.tx_hash
+												AND receipt_cids.header_id = transaction_cids.header_id
+												AND receipt_cids.block_number = transaction_cids.block_number
+											)
+											INNER JOIN eth.header_cids ON (
+												transaction_cids.header_id = header_cids.block_hash
+												AND transaction_cids.block_number = header_cids.block_number
+											)
+											INNER JOIN ipld.blocks ON (
+												receipt_cids.cid = blocks.key
+												AND receipt_cids.block_number = blocks.block_number
+											)
+										WHERE block_hash = $1
+										AND header_cids.block_number = $2
+										ORDER BY eth.transaction_cids.index ASC`
+	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash
+										FROM eth.receipt_cids
+											INNER JOIN eth.transaction_cids ON (
+												receipt_cids.tx_id = transaction_cids.tx_hash
+												AND receipt_cids.header_id = transaction_cids.header_id
+												AND receipt_cids.block_number = transaction_cids.block_number
+											)
+											INNER JOIN eth.header_cids ON (
+												transaction_cids.header_id = header_cids.block_hash
+												AND transaction_cids.block_number = header_cids.block_number
+											)
+											INNER JOIN ipld.blocks ON (
+												receipt_cids.cid = blocks.key
+												AND receipt_cids.block_number = blocks.block_number
+											)
+										WHERE block_hash = $1
+										ORDER BY eth.transaction_cids.index ASC`
+	// TODO: join on ipld.blocks and return IPLD object in this query instead of round tripping back to ipld.blocks
+	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.block_number, state_cids.removed
+												FROM eth.state_cids
+													INNER JOIN eth.header_cids ON (
+														state_cids.header_id = header_cids.block_hash
+														AND state_cids.block_number = header_cids.block_number
+													)
+												WHERE state_leaf_key = $1
+												AND header_cids.block_number <= (SELECT block_number
+																	FROM eth.header_cids
+																	WHERE block_hash = $2)
+												AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number))
+												ORDER BY header_cids.block_number DESC
+												LIMIT 1`
+	RetrieveFilteredGQLLogs = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.header_id as block_hash,
+			eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address,
+			eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data,
+			data, eth.receipt_cids.cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash
+				FROM eth.log_cids, eth.receipt_cids, ipld.blocks
+				WHERE eth.log_cids.rct_id = receipt_cids.tx_id
+				AND eth.log_cids.header_id = receipt_cids.header_id
+				AND eth.log_cids.block_number = receipt_cids.block_number
+				AND log_cids.cid = blocks.key
+				AND log_cids.block_number = blocks.block_number
+				AND receipt_cids.header_id = $1`
+	RetrieveFilteredLogs = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id,
+			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+			eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
+			eth.receipt_cids.cid as cid, eth.receipt_cids.post_status, header_cids.block_hash
+							FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids
+							WHERE eth.log_cids.rct_id = receipt_cids.tx_id
+							AND eth.log_cids.header_id = eth.receipt_cids.header_id
+							AND eth.log_cids.block_number = eth.receipt_cids.block_number
+							AND receipt_cids.tx_id = transaction_cids.tx_hash
+							AND receipt_cids.header_id = transaction_cids.header_id
+							AND receipt_cids.block_number = transaction_cids.block_number
+							AND transaction_cids.header_id = header_cids.block_hash
+							AND transaction_cids.block_number = header_cids.block_number`
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, val, block_number, removed, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)`
+	RetrieveCanonicalBlockHashByNumber                          = `SELECT block_hash
+									FROM canonical_header_hash($1) AS block_hash
+									WHERE block_hash IS NOT NULL`
+	RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids
+									INNER JOIN ipld.blocks ON (
+										header_cids.cid = blocks.key
+										AND header_cids.block_number = blocks.block_number
+									)
+									WHERE block_hash = (SELECT canonical_header_hash($1))`
+	RetrieveTD = `SELECT CAST(td as TEXT) FROM eth.header_cids
+			WHERE header_cids.block_hash = $1`
+	RetrieveRPCTransaction = `SELECT blocks.data, header_id, transaction_cids.block_number, index
+			FROM ipld.blocks, eth.transaction_cids
+			WHERE blocks.key = transaction_cids.cid
+			AND blocks.block_number = transaction_cids.block_number
+			AND transaction_cids.tx_hash = $1
+			AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))`
+	RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids
+											WHERE state_accounts.header_id = state_cids.header_id
+											AND state_accounts.state_path = state_cids.state_path
+											AND state_accounts.block_number = state_cids.block_number
+											AND state_cids.header_id = header_cids.block_hash
+											AND state_cids.block_number = header_cids.block_number
+											AND state_leaf_key = $1
+											AND header_cids.block_number <= (SELECT block_number
+																FROM eth.header_cids
+																WHERE block_hash = $2)
+											AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number))
+											ORDER BY header_cids.block_number DESC
+											LIMIT 1`
+	RetrieveCodeByMhKey = `SELECT data FROM ipld.blocks WHERE key = $1`
+)
+
+type ipldResult struct {
+	CID    string `db:"cid"`
+	Data   []byte `db:"data"`
+	TxHash string `db:"tx_hash"`
+}
+
+type nodeInfo struct {
+	CID              string `db:"cid"`
+	Value            []byte `db:"val"`
+	BlockNumber      string `db:"block_number"`
+	Data             []byte `db:"data"`
+	Removed          bool   `db:"removed"`
+	StateLeafRemoved bool   `db:"state_leaf_removed"`
+}

pkg/eth/test_helpers/test_data.go

@@ -17,28 +17,22 @@
 package test_helpers
 
 import (
-	"bytes"
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
 	"math/big"
 
+	"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 	"github.com/ethereum/go-ethereum/statediff/indexer/models"
-	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	testhelpers "github.com/ethereum/go-ethereum/statediff/test_helpers"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/ethereum/go-ethereum/trie"
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/multiformats/go-multihash"
-
-	"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
 )
 
 // Test variables
@@ -75,9 +69,7 @@ var (
 			Extra:       []byte{},
 		},
 	}
-	ReceiptsRlp, _   = rlp.EncodeToBytes(MockReceipts)
 	MockBlock       = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
-	MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header())
 	MockChildHeader = types.Header{
 		Time:        0,
 		Number:      new(big.Int).Add(BlockNumber, common.Big1),
@@ -89,7 +81,6 @@ var (
 		ParentHash:  MockBlock.Header().Hash(),
 	}
 	MockChild            = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
-	MockChildRlp, _      = rlp.EncodeToBytes(MockChild.Header())
 	Address              = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
 	AnotherAddress       = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
 	AnotherAddress1      = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594")
@@ -157,40 +148,9 @@ var (
 		Index:       5,
 	}
 
-	Tx1 = GetTxnRlp(0, MockTransactions)
+	rctCIDs, _, _ = GetRctLeafNodeData(MockReceipts)
-	Tx2 = GetTxnRlp(1, MockTransactions)
-	Tx3 = GetTxnRlp(2, MockTransactions)
-	Tx4 = GetTxnRlp(3, MockTransactions)
-
-	rctCIDs, rctIPLDData, _ = GetRctLeafNodeData(MockReceipts)
-	HeaderCID, _            = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
-	HeaderMhKey             = shared.MultihashKeyFromCID(HeaderCID)
-	Trx1CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
-	Trx1MhKey               = shared.MultihashKeyFromCID(Trx1CID)
-	Trx2CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
-	Trx2MhKey               = shared.MultihashKeyFromCID(Trx2CID)
-	Trx3CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
-	Trx3MhKey               = shared.MultihashKeyFromCID(Trx3CID)
-	Trx4CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
-	Trx4MhKey               = shared.MultihashKeyFromCID(Trx4CID)
 	Rct1CID       = rctCIDs[0]
-	Rct1MhKey               = shared.MultihashKeyFromCID(Rct1CID)
-	Rct2CID                 = rctCIDs[1]
-	Rct2MhKey               = shared.MultihashKeyFromCID(Rct2CID)
-	Rct3CID                 = rctCIDs[2]
-	Rct3MhKey               = shared.MultihashKeyFromCID(Rct3CID)
 	Rct4CID       = rctCIDs[3]
-	Rct4MhKey               = shared.MultihashKeyFromCID(Rct4CID)
-	State1CID, _            = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
-	State1MhKey             = shared.MultihashKeyFromCID(State1CID)
-	State2CID, _            = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
-	State2MhKey             = shared.MultihashKeyFromCID(State2CID)
-	StorageCID, _           = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
-	StorageMhKey            = shared.MultihashKeyFromCID(StorageCID)
-	Rct1IPLD                = rctIPLDData[0]
-	Rct2IPLD                = rctIPLDData[1]
-	Rct3IPLD                = rctIPLDData[2]
-	Rct4IPLD                = rctIPLDData[3]
 	MockTrxMeta   = []models.TxModel{
 		{
 			CID:    "", // This is empty until we go to publish to ipfs
@@ -229,48 +189,6 @@ var (
 			Data:   []byte{},
 		},
 	}
-	MockTrxMetaPostPublsh = []models.TxModel{
-		{
-			BlockNumber: "1",
-			CID:         Trx1CID.String(), // This is empty until we go to publish to ipfs
-			MhKey:       Trx1MhKey,
-			Src:         SenderAddr.Hex(),
-			Dst:         Address.String(),
-			Index:       0,
-			TxHash:      MockTransactions[0].Hash().String(),
-			Data:        []byte{},
-		},
-		{
-			BlockNumber: "1",
-			CID:         Trx2CID.String(),
-			MhKey:       Trx2MhKey,
-			Src:         SenderAddr.Hex(),
-			Dst:         AnotherAddress.String(),
-			Index:       1,
-			TxHash:      MockTransactions[1].Hash().String(),
-			Data:        []byte{},
-		},
-		{
-			BlockNumber: "1",
-			CID:         Trx3CID.String(),
-			MhKey:       Trx3MhKey,
-			Src:         SenderAddr.Hex(),
-			Dst:         "",
-			Index:       2,
-			TxHash:      MockTransactions[2].Hash().String(),
-			Data:        MockContractByteCode,
-		},
-		{
-			BlockNumber: "1",
-			CID:         Trx4CID.String(),
-			MhKey:       Trx4MhKey,
-			Src:         SenderAddr.Hex(),
-			Dst:         AnotherAddress1.String(),
-			Index:       3,
-			TxHash:      MockTransactions[3].Hash().String(),
-			Data:        []byte{},
-		},
-	}
 	MockRctMeta = []models.ReceiptModel{
 		{
 			LeafCID:      "",
@@ -298,41 +216,6 @@ var (
 		},
 	}
 
-	MockRctMetaPostPublish = []models.ReceiptModel{
-		{
-			BlockNumber:  "1",
-			HeaderID:     MockBlock.Hash().String(),
-			LeafCID:      Rct1CID.String(),
-			LeafMhKey:    Rct1MhKey,
-			Contract:     "",
-			ContractHash: "",
-		},
-		{
-			BlockNumber:  "1",
-			HeaderID:     MockBlock.Hash().String(),
-			LeafCID:      Rct2CID.String(),
-			LeafMhKey:    Rct2MhKey,
-			Contract:     "",
-			ContractHash: "",
-		},
-		{
-			BlockNumber:  "1",
-			HeaderID:     MockBlock.Hash().String(),
-			LeafCID:      Rct3CID.String(),
-			LeafMhKey:    Rct3MhKey,
-			Contract:     ContractAddress.String(),
-			ContractHash: ContractHash,
-		},
-		{
-			BlockNumber:  "1",
-			HeaderID:     MockBlock.Hash().String(),
-			LeafCID:      Rct4CID.String(),
-			LeafMhKey:    Rct4MhKey,
-			Contract:     "",
-			ContractHash: "",
-		},
-	}
-
 	// statediff data
 	storageLocation    = common.HexToHash("0")
 	StorageLeafKey     = crypto.Keccak256Hash(storageLocation[:]).Bytes()
@@ -401,24 +284,6 @@ var (
 			StorageNodes: []sdtypes.StorageNode{},
 		},
 	}
-	MockStateMetaPostPublish = []models.StateNodeModel{
-		{
-			BlockNumber: "1",
-			CID:         State1CID.String(),
-			MhKey:       State1MhKey,
-			Path:        []byte{'\x06'},
-			NodeType:    2,
-			StateKey:    common.BytesToHash(ContractLeafKey).Hex(),
-		},
-		{
-			BlockNumber: "1",
-			CID:         State2CID.String(),
-			MhKey:       State2MhKey,
-			Path:        []byte{'\x0c'},
-			NodeType:    2,
-			StateKey:    common.BytesToHash(AccountLeafKey).Hex(),
-		},
-	}
 	MockStorageNodes = map[string][]sdtypes.StorageNode{
 		contractPath: {
 			{
@@ -439,149 +304,6 @@ var (
 		StorageNodes:    MockStorageNodes,
 		StateNodes:      MockStateNodes,
 	}
-	MockConvertedPayloadForChild = eth.ConvertedPayload{
-		TotalDifficulty: MockChild.Difficulty(),
-		Block:           MockChild,
-		Receipts:        MockReceipts,
-		TxMetaData:      MockTrxMeta,
-		ReceiptMetaData: MockRctMeta,
-		StorageNodes:    MockStorageNodes,
-		StateNodes:      MockStateNodes,
-	}
-
-	Reward         = shared.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
-	MockCIDWrapper = &eth.CIDWrapper{
-		BlockNumber: new(big.Int).Set(BlockNumber),
-		Header: models.HeaderModel{
-			BlockNumber:     "1",
-			BlockHash:       MockBlock.Hash().String(),
-			ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
-			CID:             HeaderCID.String(),
-			MhKey:           HeaderMhKey,
-			TotalDifficulty: MockBlock.Difficulty().String(),
-			Reward:          Reward.String(),
-			StateRoot:       MockBlock.Root().String(),
-			RctRoot:         MockBlock.ReceiptHash().String(),
-			TxRoot:          MockBlock.TxHash().String(),
-			UncleRoot:       MockBlock.UncleHash().String(),
-			Bloom:           MockBlock.Bloom().Bytes(),
-			Timestamp:       MockBlock.Time(),
-			TimesValidated:  1,
-			Coinbase:        "0x0000000000000000000000000000000000000000",
-		},
-		Transactions: MockTrxMetaPostPublsh,
-		Receipts:     MockRctMetaPostPublish,
-		Uncles:       []models.UncleModel{},
-		StateNodes:   MockStateMetaPostPublish,
-		StorageNodes: []models.StorageNodeWithStateKeyModel{
-			{
-				BlockNumber: "1",
-				Path:        []byte{},
-				CID:         StorageCID.String(),
-				MhKey:       StorageMhKey,
-				NodeType:    2,
-				StateKey:    common.BytesToHash(ContractLeafKey).Hex(),
-				StorageKey:  common.BytesToHash(StorageLeafKey).Hex(),
-			},
-		},
-	}
-
-	HeaderIPLD, _  = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID)
-	Trx1IPLD, _    = blocks.NewBlockWithCid(Tx1, Trx1CID)
-	Trx2IPLD, _    = blocks.NewBlockWithCid(Tx2, Trx2CID)
-	Trx3IPLD, _    = blocks.NewBlockWithCid(Tx3, Trx3CID)
-	Trx4IPLD, _    = blocks.NewBlockWithCid(Tx4, Trx4CID)
-	State1IPLD, _  = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
-	State2IPLD, _  = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
-	StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
-
-	MockIPLDs = eth.IPLDs{
-		BlockNumber: new(big.Int).Set(BlockNumber),
-		Header: models.IPLDModel{
-			BlockNumber: BlockNumber.String(),
-			Data:        HeaderIPLD.RawData(),
-			Key:         HeaderIPLD.Cid().String(),
-		},
-		Transactions: []models.IPLDModel{
-			{
-				BlockNumber: BlockNumber.String(),
-				Data:        Trx1IPLD.RawData(),
-				Key:         Trx1IPLD.Cid().String(),
-			},
-			{
-				BlockNumber: BlockNumber.String(),
-				Data:        Trx2IPLD.RawData(),
-				Key:         Trx2IPLD.Cid().String(),
-			},
-			{
-				BlockNumber: BlockNumber.String(),
-				Data:        Trx3IPLD.RawData(),
-				Key:         Trx3IPLD.Cid().String(),
-			},
-			{
-				BlockNumber: BlockNumber.String(),
-				Data:        Trx4IPLD.RawData(),
-				Key:         Trx4IPLD.Cid().String(),
-			},
-		},
-		Receipts: []models.IPLDModel{
-			{
-				BlockNumber: BlockNumber.String(),
-				Data:        Rct1IPLD,
-				Key:         Rct1CID.String(),
-			},
-			{
-				BlockNumber: BlockNumber.String(),
-				Data:        Rct2IPLD,
-				Key:         Rct2CID.String(),
-			},
-			{
-				BlockNumber: BlockNumber.String(),
-				Data:        Rct3IPLD,
-				Key:         Rct3CID.String(),
-			},
-			{
-				BlockNumber: BlockNumber.String(),
-				Data:        Rct4IPLD,
-				Key:         Rct4CID.String(),
-			},
-		},
-		StateNodes: []eth.StateNode{
-			{
-				StateLeafKey: common.BytesToHash(ContractLeafKey),
-				Type:         sdtypes.Leaf,
-				IPLD: models.IPLDModel{
-					BlockNumber: BlockNumber.String(),
-					Data:        State1IPLD.RawData(),
-					Key:         State1IPLD.Cid().String(),
-				},
-				Path: []byte{'\x06'},
-			},
-			{
-				StateLeafKey: common.BytesToHash(AccountLeafKey),
-				Type:         sdtypes.Leaf,
-				IPLD: models.IPLDModel{
-					BlockNumber: BlockNumber.String(),
-					Data:        State2IPLD.RawData(),
-					Key:         State2IPLD.Cid().String(),
-				},
-				Path: []byte{'\x0c'},
-			},
-		},
-		StorageNodes: []eth.StorageNode{
-			{
-				StateLeafKey:   common.BytesToHash(ContractLeafKey),
-				StorageLeafKey: common.BytesToHash(StorageLeafKey),
-				Type:           sdtypes.Leaf,
-				IPLD: models.IPLDModel{
-					BlockNumber: BlockNumber.String(),
-					Data:        StorageIPLD.RawData(),
-					Key:         StorageIPLD.Cid().String(),
-				},
-				Path: []byte{},
-			},
-		},
-	}
 
 	LondonBlockNum   = new(big.Int).Add(BlockNumber, big.NewInt(2))
 	MockLondonHeader = types.Header{
@@ -741,21 +463,3 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts,
 
 	return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr
 }
-
-func GetTxnRlp(num int, txs types.Transactions) []byte {
-	buf := new(bytes.Buffer)
-	txs.EncodeIndex(num, buf)
-	tx := make([]byte, buf.Len())
-	copy(tx, buf.Bytes())
-	buf.Reset()
-	return tx
-}
-
-func GetRctRlp(num int, rcts types.Receipts) []byte {
-	buf := new(bytes.Buffer)
-	rcts.EncodeIndex(num, buf)
-	rct := make([]byte, buf.Len())
-	copy(rct, buf.Bytes())
-	buf.Reset()
-	return rct
-}

pkg/eth/types.go

@@ -193,47 +193,6 @@ func (arg *CallArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (types.Mes
 	return msg, nil
 }
 
-// IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server
-// Returned by IPLDFetcher and ResponseFilterer
-type IPLDs struct {
-	BlockNumber     *big.Int
-	TotalDifficulty *big.Int
-	Header          models.IPLDModel
-	Uncles          []models.IPLDModel
-	Transactions    []models.IPLDModel
-	Receipts        []models.IPLDModel
-	StateNodes      []StateNode
-	StorageNodes    []StorageNode
-}
-
-type StateNode struct {
-	Type         sdtypes.NodeType
-	StateLeafKey common.Hash
-	Path         []byte
-	IPLD         models.IPLDModel
-}
-
-type StorageNode struct {
-	Type           sdtypes.NodeType
-	StateLeafKey   common.Hash
-	StorageLeafKey common.Hash
-	Path           []byte
-	IPLD           models.IPLDModel
-}
-
-// CIDWrapper is used to direct fetching of IPLDs from IPFS
-// Returned by CIDRetriever
-// Passed to IPLDFetcher
-type CIDWrapper struct {
-	BlockNumber  *big.Int
-	Header       models.HeaderModel
-	Uncles       []models.UncleModel
-	Transactions []models.TxModel
-	Receipts     []models.ReceiptModel
-	StateNodes   []models.StateNodeModel
-	StorageNodes []models.StorageNodeWithStateKeyModel
-}
-
 // ConvertedPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers
 // Returned by PayloadConverter
 // Passed to IPLDPublisher and ResponseFilterer
@@ -249,7 +208,7 @@ type ConvertedPayload struct {
 
 // LogResult represent a log.
 type LogResult struct {
-	LeafCID     string `db:"leaf_cid"`
+	LeafCID     string `db:"cid"`
 	ReceiptID   string `db:"rct_id"`
 	Address     string `db:"address"`
 	Index       int64  `db:"index"`

pkg/graphql/graphql.go

@@ -29,7 +29,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/filters"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -37,6 +36,7 @@ import (
 
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
+	ipld_eth_statedb "github.com/cerc-io/ipld-eth-statedb"
 )
 
 var (
@@ -51,8 +51,8 @@ type Account struct {
 }
 
 // getState fetches the StateDB object for an account.
-func (a *Account) getState(ctx context.Context) (*state.StateDB, error) {
+func (a *Account) getState(ctx context.Context) (*ipld_eth_statedb.StateDB, error) {
-	state, _, err := a.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash)
+	state, _, err := a.backend.IPLDStateDBAndHeaderByNumberOrHash(ctx, a.blockNrOrHash)
 	return state, err
 }
 
@@ -1009,7 +1009,7 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct {
 	Contract  common.Address
 	Slot      common.Hash
 }) (*StorageResult, error) {
-	cid, ipldBlock, rlpValue, err := r.backend.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash)
+	cid, ipldBlock, rlpValue, err := r.backend.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash)
 
 	if err != nil {
 		if err == sql.ErrNoRows {

pkg/graphql/service.go

@@ -81,7 +81,7 @@ func (s *Service) Start(server *p2p.Server) error {
 	return nil
 }
 
-// newHandler returns a new `http.Handler` that will answer GraphQL queries.
+// NewHandler returns a new `http.Handler` that will answer GraphQL queries.
 // It additionally exports an interactive query browser on the / endpoint.
 func NewHandler(backend *eth.Backend) (http.Handler, error) {
 	q := Resolver{backend}

pkg/serve/service.go

@@ -58,7 +58,7 @@ type Service struct {
 	sync.Mutex
 	// Used to signal shutdown of the service
 	QuitChan chan bool
-	// Underlying db
+	// Underlying db connection pool
 	db *sqlx.DB
 	// rpc client for forwarding cache misses
 	client *rpc.Client

pkg/shared/functions.go

@@ -50,14 +50,14 @@ func Rollback(tx *sqlx.Tx) {
 
 // FetchIPLDByMhKeyAndBlockNumber is used to retrieve an ipld from Postgres blockstore with the provided tx, mhkey string and blockNumber
 func FetchIPLDByMhKeyAndBlockNumber(tx *sqlx.Tx, mhKey string, blockNumber uint64) ([]byte, error) {
-	pgStr := `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2`
+	pgStr := `SELECT data FROM ipld.blocks WHERE key = $1 AND block_number = $2`
 	var block []byte
 	return block, tx.Get(&block, pgStr, mhKey, blockNumber)
 }
 
 // FetchIPLD is used to retrieve an IPLD from Postgres mhkey and blockNumber
 func FetchIPLD(db *sqlx.DB, mhKey string, blockNumber uint64) ([]byte, error) {
-	pgStr := `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2`
+	pgStr := `SELECT data FROM ipld.blocks WHERE key = $1 AND block_number = $2`
 	var block []byte
 	return block, db.Get(&block, pgStr, mhKey, blockNumber)
 }

test_config/test_config.go

@@ -18,6 +18,7 @@ package test_config
 
 import (
 	"errors"
+
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/spf13/viper"