go.mod

@@ -40,11 +40,7 @@ require (
 	github.com/influxdata/influxdb v1.8.3
 	github.com/influxdata/influxdb-client-go/v2 v2.4.0
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
-	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-cid v0.2.0
-	github.com/ipfs/go-ipfs-blockstore v1.2.0
-	github.com/ipfs/go-ipfs-ds-help v1.1.0
-	github.com/ipfs/go-ipld-format v0.4.0
 	github.com/jackc/pgconn v1.10.0
 	github.com/jackc/pgx/v4 v4.13.0
 	github.com/jackpal/go-nat-pmp v1.0.2
@@ -67,7 +63,7 @@ require (
 	github.com/rs/cors v1.7.0
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.8.0
 	github.com/supranational/blst v0.3.8
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/thoas/go-funk v0.9.2
@@ -101,12 +97,6 @@ require (
 	github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
-	github.com/gogo/protobuf v1.3.1 // indirect
-	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/go-datastore v0.5.0 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
-	github.com/ipfs/go-log v0.0.1 // indirect
-	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -114,7 +104,6 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
 	github.com/jackc/pgtype v1.8.1 // indirect
 	github.com/jackc/puddle v1.1.3 // indirect
-	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
@@ -131,12 +120,10 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/objx v0.4.0 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
-	github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.uber.org/atomic v1.6.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
 	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
 	golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect

go.sum

@@ -175,8 +175,6 @@ github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
 github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog=
 github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@@ -226,7 +224,6 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -237,13 +234,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0=
 github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
-github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
-github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
 github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
@@ -270,36 +264,8 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y
 github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE=
 github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po=
-github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
-github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
-github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc=
-github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
-github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
-github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
-github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
-github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
 github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0=
 github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro=
-github.com/ipfs/go-datastore v0.5.0 h1:rQicVCEacWyk4JZ6G5bD9TKR7lZEG1MWcG7UdWYrFAU=
-github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk=
-github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
-github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
-github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw=
-github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE=
-github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
-github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q=
-github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU=
-github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
-github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
-github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
-github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM=
-github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ=
-github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM=
-github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
-github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
-github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
-github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -374,9 +340,6 @@ github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
-github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
-github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
-github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b h1:ZGiXF8sz7PDk6RgkP+A/SFfUD0ZR/AgG6SpRNEDKZy8=
 github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b/go.mod h1:hQmNrgofl+IY/8L+n20H6E6PWBBTokdsv+q49j0QhsU=
 github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
@@ -397,7 +360,6 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
 github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4=
 github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
@@ -412,7 +374,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -464,8 +425,6 @@ github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4f
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
-github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
-github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
@@ -476,7 +435,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
-github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
@@ -484,16 +442,11 @@ github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
 github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
 github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
-github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
 github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
-github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
-github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
-github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
 github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA=
 github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84=
-github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
 github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -583,15 +536,18 @@ github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57N
 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/supranational/blst v0.3.8 h1:glwLF4oBRSJOTr05lRBgNwGQST0ndP2wg29fSeTRKCY=
 github.com/supranational/blst v0.3.8/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
@@ -611,8 +567,6 @@ github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhA
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
-github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
-github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
@@ -628,7 +582,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
@@ -638,13 +591,11 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -683,7 +634,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs=
@@ -702,7 +652,6 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -746,7 +695,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -804,7 +752,6 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -897,7 +844,6 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

statediff/builder.go

@@ -102,7 +102,7 @@ func (sdb *StateDiffBuilder) BuildStateDiffObject(args Args, params Params) (typ
 	}, nil
 }
 
-// WriteStateDiffObject writes a statediff object to output callback
+// WriteStateDiffObject writes a statediff object to output sinks
 func (sdb *StateDiffBuilder) WriteStateDiffObject(args types2.StateRoots, params Params, output types2.StateNodeSink,
 	ipldOutput types2.IPLDSink) error {
 	// Load tries for old and new states

statediff/indexer/database/file/csv_indexer_legacy_test.go

@@ -28,8 +28,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/file/types"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
 	"github.com/ethereum/go-ethereum/statediff/indexer/test"
 	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 )
@@ -90,7 +90,7 @@ func resetAndDumpWatchedAddressesCSVFileData(t *testing.T) {
 	test_helpers.TearDownDB(t, db)
 
 	outputFilePath := filepath.Join(dbDirectory, file.CSVTestConfig.WatchedAddressesFilePath)
-	stmt := fmt.Sprintf(pgCopyStatement, types.TableWatchedAddresses.Name, outputFilePath)
+	stmt := fmt.Sprintf(pgCopyStatement, schema.TableWatchedAddresses.Name, outputFilePath)
 
 	_, err = db.Exec(context.Background(), stmt)
 	require.NoError(t, err)

statediff/indexer/database/file/csv_writer.go

@@ -28,29 +28,29 @@ import (
 	"github.com/thoas/go-funk"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/file/types"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 	"github.com/ethereum/go-ethereum/statediff/indexer/models"
 	nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 )
 
 var (
-	Tables = []*types.Table{
+	Tables = []*schema.Table{
-		&types.TableIPLDBlock,
+		&schema.TableIPLDBlock,
-		&types.TableNodeInfo,
+		&schema.TableNodeInfo,
-		&types.TableHeader,
+		&schema.TableHeader,
-		&types.TableStateNode,
+		&schema.TableStateNode,
-		&types.TableStorageNode,
+		&schema.TableStorageNode,
-		&types.TableUncle,
+		&schema.TableUncle,
-		&types.TableTransaction,
+		&schema.TableTransaction,
-		&types.TableReceipt,
+		&schema.TableReceipt,
-		&types.TableLog,
+		&schema.TableLog,
 	}
 )
 
 type tableRow struct {
-	table  types.Table
+	table  schema.Table
 	values []interface{}
 }
 
@@ -90,7 +90,7 @@ func newFileWriter(path string) (ret fileWriter, err error) {
 	return
 }
 
-func makeFileWriters(dir string, tables []*types.Table) (fileWriters, error) {
+func makeFileWriters(dir string, tables []*schema.Table) (fileWriters, error) {
 	if err := os.MkdirAll(dir, 0755); err != nil {
 		return nil, err
 	}
@@ -105,7 +105,7 @@ func makeFileWriters(dir string, tables []*types.Table) (fileWriters, error) {
 	return writers, nil
 }
 
-func (tx fileWriters) write(tbl *types.Table, args ...interface{}) error {
+func (tx fileWriters) write(tbl *schema.Table, args ...interface{}) error {
 	row := tbl.ToCsvRow(args...)
 	return tx[tbl.Name].Write(row)
 }
@@ -204,13 +204,13 @@ func (csw *CSVWriter) Close() error {
 func (csw *CSVWriter) upsertNode(node nodeinfo.Info) {
 	var values []interface{}
 	values = append(values, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID)
-	csw.rows <- tableRow{types.TableNodeInfo, values}
+	csw.rows <- tableRow{schema.TableNodeInfo, values}
 }
 
 func (csw *CSVWriter) upsertIPLD(ipld models.IPLDModel) {
 	var values []interface{}
 	values = append(values, ipld.BlockNumber, ipld.Key, ipld.Data)
-	csw.rows <- tableRow{types.TableIPLDBlock, values}
+	csw.rows <- tableRow{schema.TableIPLDBlock, values}
 }
 
 func (csw *CSVWriter) upsertIPLDDirect(blockNumber, key string, value []byte) {
@@ -234,7 +234,7 @@ func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) {
 	values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
 		header.TotalDifficulty, header.NodeIDs, header.Reward, header.StateRoot, header.TxRoot,
 		header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.Coinbase)
-	csw.rows <- tableRow{types.TableHeader, values}
+	csw.rows <- tableRow{schema.TableHeader, values}
 	indexerMetrics.blocks.Inc(1)
 }
 
@@ -242,14 +242,14 @@ func (csw *CSVWriter) upsertUncleCID(uncle models.UncleModel) {
 	var values []interface{}
 	values = append(values, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
 		uncle.Reward, uncle.Index)
-	csw.rows <- tableRow{types.TableUncle, values}
+	csw.rows <- tableRow{schema.TableUncle, values}
 }
 
 func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) {
 	var values []interface{}
 	values = append(values, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
 		transaction.Src, transaction.Index, transaction.Type, transaction.Value)
-	csw.rows <- tableRow{types.TableTransaction, values}
+	csw.rows <- tableRow{schema.TableTransaction, values}
 	indexerMetrics.transactions.Inc(1)
 }
 
@@ -257,7 +257,7 @@ func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
 	var values []interface{}
 	values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
 		rct.PostState, rct.PostStatus)
-	csw.rows <- tableRow{types.TableReceipt, values}
+	csw.rows <- tableRow{schema.TableReceipt, values}
 	indexerMetrics.receipts.Inc(1)
 }
 
@@ -266,7 +266,7 @@ func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) {
 		var values []interface{}
 		values = append(values, l.BlockNumber, l.HeaderID, l.CID, l.ReceiptID, l.Address, l.Index, l.Topic0,
 			l.Topic1, l.Topic2, l.Topic3)
-		csw.rows <- tableRow{types.TableLog, values}
+		csw.rows <- tableRow{schema.TableLog, values}
 		indexerMetrics.logs.Inc(1)
 	}
 }
@@ -280,7 +280,7 @@ func (csw *CSVWriter) upsertStateCID(stateNode models.StateNodeModel) {
 	var values []interface{}
 	values = append(values, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID,
 		true, stateNode.Balance, strconv.FormatUint(stateNode.Nonce, 10), stateNode.CodeHash, stateNode.StorageRoot, stateNode.Removed)
-	csw.rows <- tableRow{types.TableStateNode, values}
+	csw.rows <- tableRow{schema.TableStateNode, values}
 }
 
 func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
@@ -292,7 +292,7 @@ func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
 	var values []interface{}
 	values = append(values, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageKey, storageCID.CID,
 		true, storageCID.Value, storageCID.Removed)
-	csw.rows <- tableRow{types.TableStorageNode, values}
+	csw.rows <- tableRow{schema.TableStorageNode, values}
 }
 
 // LoadWatchedAddresses loads watched addresses from a file
@@ -332,7 +332,7 @@ func (csw *CSVWriter) insertWatchedAddresses(args []sdtypes.WatchAddressArg, cur
 
 		var values []interface{}
 		values = append(values, arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0")
-		row := types.TableWatchedAddresses.ToCsvRow(values...)
+		row := schema.TableWatchedAddresses.ToCsvRow(values...)
 
 		// writing directly instead of using rows channel as it needs to be flushed immediately
 		err = csw.watchedAddressesWriter.Write(row)
@@ -375,7 +375,7 @@ func (csw *CSVWriter) removeWatchedAddresses(args []sdtypes.WatchAddressArg) err
 func (csw *CSVWriter) setWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
 	var rows [][]string
 	for _, arg := range args {
-		row := types.TableWatchedAddresses.ToCsvRow(arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0")
+		row := schema.TableWatchedAddresses.ToCsvRow(arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0")
 		rows = append(rows, row)
 	}
 

statediff/indexer/database/file/indexer.go

@@ -443,7 +443,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
 		storageModel := models.StorageNodeModel{
 			BlockNumber: tx.BlockNumber,
 			HeaderID:    headerID,
-			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).Hex(),
+			StateKey:    common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
 			StorageKey:  common.BytesToHash(storageNode.LeafKey).String(),
 			CID:         storageNode.CID,
 			Removed:     false,

statediff/indexer/database/file/types/schema.go

@@ -1,154 +0,0 @@
-// Copyright 2022 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package types
-
-var TableIPLDBlock = Table{
-	`ipld.blocks`,
-	[]column{
-		{name: "block_number", dbType: bigint},
-		{name: "key", dbType: text},
-		{name: "data", dbType: bytea},
-	},
-}
-
-var TableNodeInfo = Table{
-	Name: `public.nodes`,
-	Columns: []column{
-		{name: "genesis_block", dbType: varchar},
-		{name: "network_id", dbType: varchar},
-		{name: "node_id", dbType: varchar},
-		{name: "client_name", dbType: varchar},
-		{name: "chain_id", dbType: integer},
-	},
-}
-
-var TableHeader = Table{
-	"eth.header_cids",
-	[]column{
-		{name: "block_number", dbType: bigint},
-		{name: "block_hash", dbType: varchar},
-		{name: "parent_hash", dbType: varchar},
-		{name: "cid", dbType: text},
-		{name: "td", dbType: numeric},
-		{name: "node_ids", dbType: varchar, isArray: true},
-		{name: "reward", dbType: numeric},
-		{name: "state_root", dbType: varchar},
-		{name: "tx_root", dbType: varchar},
-		{name: "receipt_root", dbType: varchar},
-		{name: "uncles_hash", dbType: varchar},
-		{name: "bloom", dbType: bytea},
-		{name: "timestamp", dbType: numeric},
-		{name: "coinbase", dbType: varchar},
-	},
-}
-
-var TableStateNode = Table{
-	"eth.state_cids",
-	[]column{
-		{name: "block_number", dbType: bigint},
-		{name: "header_id", dbType: varchar},
-		{name: "state_leaf_key", dbType: varchar},
-		{name: "cid", dbType: text},
-		{name: "removed", dbType: boolean},
-		{name: "diff", dbType: boolean},
-		{name: "balance", dbType: numeric},
-		{name: "nonce", dbType: bigint},
-		{name: "code_hash", dbType: varchar},
-		{name: "storage_root", dbType: varchar},
-	},
-}
-
-var TableStorageNode = Table{
-	"eth.storage_cids",
-	[]column{
-		{name: "block_number", dbType: bigint},
-		{name: "header_id", dbType: varchar},
-		{name: "state_leaf_key", dbType: varchar},
-		{name: "storage_leaf_key", dbType: varchar},
-		{name: "cid", dbType: text},
-		{name: "removed", dbType: boolean},
-		{name: "diff", dbType: boolean},
-		{name: "val", dbType: bytea},
-	},
-}
-
-var TableUncle = Table{
-	"eth.uncle_cids",
-	[]column{
-		{name: "block_number", dbType: bigint},
-		{name: "block_hash", dbType: varchar},
-		{name: "header_id", dbType: varchar},
-		{name: "parent_hash", dbType: varchar},
-		{name: "cid", dbType: text},
-		{name: "reward", dbType: numeric},
-		{name: "index", dbType: integer},
-	},
-}
-
-var TableTransaction = Table{
-	"eth.transaction_cids",
-	[]column{
-		{name: "block_number", dbType: bigint},
-		{name: "header_id", dbType: varchar},
-		{name: "tx_hash", dbType: varchar},
-		{name: "cid", dbType: text},
-		{name: "dst", dbType: varchar},
-		{name: "src", dbType: varchar},
-		{name: "index", dbType: integer},
-		{name: "tx_type", dbType: integer},
-		{name: "value", dbType: numeric},
-	},
-}
-
-var TableReceipt = Table{
-	"eth.receipt_cids",
-	[]column{
-		{name: "block_number", dbType: bigint},
-		{name: "header_id", dbType: varchar},
-		{name: "tx_id", dbType: varchar},
-		{name: "cid", dbType: text},
-		{name: "contract", dbType: varchar},
-		{name: "post_state", dbType: varchar},
-		{name: "post_status", dbType: integer},
-	},
-}
-
-var TableLog = Table{
-	"eth.log_cids",
-	[]column{
-		{name: "block_number", dbType: bigint},
-		{name: "header_id", dbType: varchar},
-		{name: "cid", dbType: text},
-		{name: "rct_id", dbType: varchar},
-		{name: "address", dbType: varchar},
-		{name: "index", dbType: integer},
-		{name: "topic0", dbType: varchar},
-		{name: "topic1", dbType: varchar},
-		{name: "topic2", dbType: varchar},
-		{name: "topic3", dbType: varchar},
-	},
-}
-
-var TableWatchedAddresses = Table{
-	"eth_meta.watched_addresses",
-	[]column{
-		{name: "address", dbType: varchar},
-		{name: "created_at", dbType: bigint},
-		{name: "watched_at", dbType: bigint},
-		{name: "last_filled_at", dbType: bigint},
-	},
-}

statediff/indexer/database/file/types/table.go

@@ -1,104 +0,0 @@
-// Copyright 2022 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package types
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/thoas/go-funk"
-)
-
-type colType int
-
-const (
-	integer colType = iota
-	boolean
-	bigint
-	numeric
-	bytea
-	varchar
-	text
-)
-
-type column struct {
-	name    string
-	dbType  colType
-	isArray bool
-}
-type Table struct {
-	Name    string
-	Columns []column
-}
-
-func (tbl *Table) ToCsvRow(args ...interface{}) []string {
-	var row []string
-	for i, col := range tbl.Columns {
-		value := col.dbType.formatter()(args[i])
-
-		if col.isArray {
-			valueList := funk.Map(args[i], col.dbType.formatter()).([]string)
-			value = fmt.Sprintf("{%s}", strings.Join(valueList, ","))
-		}
-
-		row = append(row, value)
-	}
-	return row
-}
-
-func (tbl *Table) VarcharColumns() []string {
-	columns := funk.Filter(tbl.Columns, func(col column) bool {
-		return col.dbType == varchar
-	}).([]column)
-
-	columnNames := funk.Map(columns, func(col column) string {
-		return col.name
-	}).([]string)
-
-	return columnNames
-}
-
-type colfmt = func(interface{}) string
-
-func sprintf(f string) colfmt {
-	return func(x interface{}) string { return fmt.Sprintf(f, x) }
-}
-
-func (typ colType) formatter() colfmt {
-	switch typ {
-	case integer:
-		return sprintf("%d")
-	case boolean:
-		return func(x interface{}) string {
-			if x.(bool) {
-				return "t"
-			}
-			return "f"
-		}
-	case bigint:
-		return sprintf("%s")
-	case numeric:
-		return sprintf("%s")
-	case bytea:
-		return sprintf(`\x%x`)
-	case varchar:
-		return sprintf("%s")
-	case text:
-		return sprintf("%s")
-	}
-	panic("unreachable")
-}

statediff/indexer/database/sql/interfaces.go

@@ -52,7 +52,6 @@ type Statements interface {
 	InsertStorageStm() string
 	InsertIPLDStm() string
 	InsertIPLDsStm() string
-	InsertKnownGapsStm() string
 }
 
 // Tx interface to accommodate different concrete SQL transaction types

statediff/indexer/database/sql/postgres/config.go

@@ -18,6 +18,8 @@ package postgres
 
 import (
 	"fmt"
+	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -33,6 +35,15 @@ const (
 	Unknown DriverType = "Unknown"
 )
 
+// Env variables
+const (
+	DATABASE_NAME     = "DATABASE_NAME"
+	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
+	DATABASE_PORT     = "DATABASE_PORT"
+	DATABASE_USER     = "DATABASE_USER"
+	DATABASE_PASSWORD = "DATABASE_PASSWORD"
+)
+
 // ResolveDriverType resolves a DriverType from a provided string
 func ResolveDriverType(str string) (DriverType, error) {
 	switch strings.ToLower(str) {
@@ -49,7 +60,7 @@ func ResolveDriverType(str string) (DriverType, error) {
 var DefaultConfig = Config{
 	Hostname:     "localhost",
 	Port:         8077,
-	DatabaseName: "vulcanize_testing",
+	DatabaseName: "cerc_testing",
 	Username:     "vdbm",
 	Password:     "password",
 }
@@ -100,3 +111,26 @@ func (c Config) DbConnectionString() string {
 	}
 	return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
 }
+
+func (c Config) WithEnv() (Config, error) {
+	if val := os.Getenv(DATABASE_NAME); val != "" {
+		c.DatabaseName = val
+	}
+	if val := os.Getenv(DATABASE_HOSTNAME); val != "" {
+		c.Hostname = val
+	}
+	if val := os.Getenv(DATABASE_PORT); val != "" {
+		port, err := strconv.Atoi(val)
+		if err != nil {
+			return c, err
+		}
+		c.Port = port
+	}
+	if val := os.Getenv(DATABASE_USER); val != "" {
+		c.Username = val
+	}
+	if val := os.Getenv(DATABASE_PASSWORD); val != "" {
+		c.Password = val
+	}
+	return c, nil
+}

statediff/indexer/database/sql/postgres/database.go

@@ -16,7 +16,10 @@
 
 package postgres
 
-import "github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+import (
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
+)
 
 var _ sql.Database = &DB{}
 
@@ -39,73 +42,51 @@ type DB struct {
 // InsertHeaderStm satisfies the sql.Statements interface
 // Stm == Statement
 func (db *DB) InsertHeaderStm() string {
-	if db.upsert {
+	return schema.TableHeader.ToInsertStatement(db.upsert)
-		return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase)
+
-			VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
-			ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
-	}
-	return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase)
-			VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
-			ON CONFLICT (block_hash, block_number) DO NOTHING`
 }
 
 // InsertUncleStm satisfies the sql.Statements interface
 func (db *DB) InsertUncleStm() string {
-	return `INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, index) VALUES ($1, $2, $3, $4, $5, $6, $7)
+	return schema.TableUncle.ToInsertStatement(db.upsert)
-			ON CONFLICT (block_hash, block_number, index) DO NOTHING`
 }
 
 // InsertTxStm satisfies the sql.Statements interface
 func (db *DB) InsertTxStm() string {
-	return `INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+	return schema.TableTransaction.ToInsertStatement(db.upsert)
-			ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING`
+
 }
 
 // InsertRctStm satisfies the sql.Statements interface
 func (db *DB) InsertRctStm() string {
-	return `INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, post_status) VALUES ($1, $2, $3, $4, $5, $6, $7)
+	return schema.TableReceipt.ToInsertStatement(db.upsert)
-			ON CONFLICT (tx_id, header_id, block_number) DO NOTHING`
+
 }
 
 // InsertLogStm satisfies the sql.Statements interface
 func (db *DB) InsertLogStm() string {
-	return `INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, topic3) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+	return schema.TableLog.ToInsertStatement(db.upsert)
-			ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING`
+
 }
 
 // InsertStateStm satisfies the sql.Statements interface
 func (db *DB) InsertStateStm() string {
-	if db.upsert {
+	return schema.TableStateNode.ToInsertStatement(db.upsert)
-		return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+
-			ON CONFLICT (header_id, state_leaf_key, block_number) DO UPDATE SET (cid, removed, diff, balance, nonce, code_hash, storage_root) = ($4, $5, $6, $7, $8, $9, $10)`
-	}
-	return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
-			ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING`
 }
 
 // InsertStorageStm satisfies the sql.Statements interface
 func (db *DB) InsertStorageStm() string {
-	if db.upsert {
+	return schema.TableStorageNode.ToInsertStatement(db.upsert)
-		return `INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+
-			ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO UPDATE SET (cid, removed, diff, val) = ($4, $5, $6, $7, $8)`
-	}
-	return `INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
-			ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHING`
 }
 
 // InsertIPLDStm satisfies the sql.Statements interface
 func (db *DB) InsertIPLDStm() string {
-	return `INSERT INTO ipld.blocks (block_number, key, data) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`
+	return schema.TableIPLDBlock.ToInsertStatement(db.upsert)
 }
 
 // InsertIPLDsStm satisfies the sql.Statements interface
 func (db *DB) InsertIPLDsStm() string {
 	return `INSERT INTO ipld.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING`
 }
-
-// InsertKnownGapsStm satisfies the sql.Statements interface
-func (db *DB) InsertKnownGapsStm() string {
-	return `INSERT INTO eth_meta.known_gaps (starting_block_number, ending_block_number, checked_out, processing_key) VALUES ($1, $2, $3, $4)
-			ON CONFLICT (starting_block_number) DO UPDATE SET (ending_block_number, processing_key) = ($2, $4)
-			WHERE eth_meta.known_gaps.ending_block_number <= $2`
-}

statediff/indexer/database/sql/postgres/errors.go

@@ -26,13 +26,13 @@ const (
 )
 
 func ErrDBConnectionFailed(connectErr error) error {
-	return formatError(DbConnectionFailedMsg, connectErr.Error())
+	return formatError(DbConnectionFailedMsg, connectErr)
 }
 
 func ErrUnableToSetNode(setErr error) error {
-	return formatError(SettingNodeFailedMsg, setErr.Error())
+	return formatError(SettingNodeFailedMsg, setErr)
 }
 
-func formatError(msg, err string) error {
+func formatError(msg string, err error) error {
-	return fmt.Errorf("%s: %s", msg, err)
+	return fmt.Errorf("%s: %w", msg, err)
 }

statediff/indexer/database/sql/postgres/pgx.go

@@ -39,14 +39,19 @@ type PGXDriver struct {
 	nodeID   string
 }
 
-// NewPGXDriver returns a new pgx driver
+// ConnectPGX initializes and returns a PGX connection pool
-// it initializes the connection pool and creates the node info table
+func ConnectPGX(ctx context.Context, config Config) (*pgxpool.Pool, error) {
-func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
 	pgConf, err := MakeConfig(config)
 	if err != nil {
 		return nil, err
 	}
-	dbPool, err := pgxpool.ConnectConfig(ctx, pgConf)
+	return pgxpool.ConnectConfig(ctx, pgConf)
+}
+
+// NewPGXDriver returns a new pgx driver
+// it initializes the connection pool and creates the node info table
+func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
+	dbPool, err := ConnectPGX(ctx, config)
 	if err != nil {
 		return nil, ErrDBConnectionFailed(err)
 	}

statediff/indexer/database/sql/postgres/sqlx.go

@@ -35,12 +35,11 @@ type SQLXDriver struct {
 	nodeID   string
 }
 
-// NewSQLXDriver returns a new sqlx driver for Postgres
+// ConnectSQLX initializes and returns a SQLX connection pool for postgres
-// it initializes the connection pool and creates the node info table
+func ConnectSQLX(ctx context.Context, config Config) (*sqlx.DB, error) {
-func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
 	db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString())
 	if err != nil {
-		return &SQLXDriver{}, ErrDBConnectionFailed(err)
+		return nil, ErrDBConnectionFailed(err)
 	}
 	if config.MaxConns > 0 {
 		db.SetMaxOpenConns(config.MaxConns)
@@ -49,9 +48,19 @@ func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDri
 		db.SetConnMaxLifetime(config.MaxConnLifetime)
 	}
 	db.SetMaxIdleConns(config.MaxIdle)
+	return db, nil
+}
+
+// NewSQLXDriver returns a new sqlx driver for Postgres
+// it initializes the connection pool and creates the node info table
+func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
+	db, err := ConnectSQLX(ctx, config)
+	if err != nil {
+		return nil, err
+	}
 	driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node}
 	if err := driver.createNode(); err != nil {
-		return &SQLXDriver{}, ErrUnableToSetNode(err)
+		return nil, err
 	}
 	return driver, nil
 }
@@ -59,8 +68,10 @@ func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDri
 func (driver *SQLXDriver) createNode() error {
 	_, err := driver.db.Exec(
 		createNodeStm,
-		driver.nodeInfo.GenesisBlock, driver.nodeInfo.NetworkID,
+		driver.nodeInfo.GenesisBlock,
-		driver.nodeInfo.ID, driver.nodeInfo.ClientName,
+		driver.nodeInfo.NetworkID,
+		driver.nodeInfo.ID,
+		driver.nodeInfo.ClientName,
 		driver.nodeInfo.ChainID)
 	if err != nil {
 		return ErrUnableToSetNode(err)

statediff/indexer/database/sql/writer.go

@@ -47,10 +47,22 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
 ON CONFLICT (block_hash, block_number) DO NOTHING
 */
 func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error {
+	nodeIDs := pq.StringArray([]string{w.db.NodeID()})
 	_, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(),
-		header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, pq.StringArray([]string{w.db.NodeID()}),
+		header.BlockNumber,
-		header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UnclesHash, header.Bloom,
+		header.BlockHash,
-		header.Timestamp, header.Coinbase)
+		header.ParentHash,
+		header.CID,
+		header.TotalDifficulty,
+		nodeIDs,
+		header.Reward,
+		header.StateRoot,
+		header.TxRoot,
+		header.RctRoot,
+		header.UnclesHash,
+		header.Bloom,
+		header.Timestamp,
+		header.Coinbase)
 	if err != nil {
 		return insertError{"eth.header_cids", err, w.db.InsertHeaderStm(), header}
 	}
@@ -64,7 +76,13 @@ ON CONFLICT (block_hash, block_number) DO NOTHING
 */
 func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error {
 	_, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(),
-		uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.Index)
+		uncle.BlockNumber,
+		uncle.BlockHash,
+		uncle.HeaderID,
+		uncle.ParentHash,
+		uncle.CID,
+		uncle.Reward,
+		uncle.Index)
 	if err != nil {
 		return insertError{"eth.uncle_cids", err, w.db.InsertUncleStm(), uncle}
 	}
@@ -77,8 +95,15 @@ ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING
 */
 func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
 	_, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(),
-		transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src,
+		transaction.BlockNumber,
-		transaction.Index, transaction.Type, transaction.Value)
+		transaction.HeaderID,
+		transaction.TxHash,
+		transaction.CID,
+		transaction.Dst,
+		transaction.Src,
+		transaction.Index,
+		transaction.Type,
+		transaction.Value)
 	if err != nil {
 		return insertError{"eth.transaction_cids", err, w.db.InsertTxStm(), transaction}
 	}
@@ -92,7 +117,12 @@ ON CONFLICT (tx_id, header_id, block_number) DO NOTHING
 */
 func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
 	_, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(),
-		rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.PostState,
+		rct.BlockNumber,
+		rct.HeaderID,
+		rct.TxID,
+		rct.CID,
+		rct.Contract,
+		rct.PostState,
 		rct.PostStatus)
 	if err != nil {
 		return insertError{"eth.receipt_cids", err, w.db.InsertRctStm(), *rct}
@@ -108,8 +138,16 @@ ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING
 func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
 	for _, log := range logs {
 		_, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(),
-			log.BlockNumber, log.HeaderID, log.CID, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1,
+			log.BlockNumber,
-			log.Topic2, log.Topic3)
+			log.HeaderID,
+			log.CID,
+			log.ReceiptID,
+			log.Address,
+			log.Index,
+			log.Topic0,
+			log.Topic1,
+			log.Topic2,
+			log.Topic3)
 		if err != nil {
 			return insertError{"eth.log_cids", err, w.db.InsertLogStm(), *log}
 		}
@@ -123,20 +161,24 @@ INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, remove
 ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING
 */
 func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
+	balance := stateNode.Balance
 	if stateNode.Removed {
-		_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
+		balance = "0"
-			stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Removed, true,
+	}
-			"0", stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot)
+	_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
-		if err != nil {
+		stateNode.BlockNumber,
-			return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
+		stateNode.HeaderID,
-		}
+		stateNode.StateKey,
-	} else {
+		stateNode.CID,
-		_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
+		true,
-			stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Removed, true,
+		balance,
-			stateNode.Balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot)
+		stateNode.Nonce,
-		if err != nil {
+		stateNode.CodeHash,
-			return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
+		stateNode.StorageRoot,
-		}
+		stateNode.Removed,
+	)
+	if err != nil {
+		return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
 	}
 	return nil
 }
@@ -147,8 +189,15 @@ ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHI
 */
 func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error {
 	_, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(),
-		storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID,
+		storageCID.BlockNumber,
-		storageCID.Removed, true, storageCID.Value)
+		storageCID.HeaderID,
+		storageCID.StateKey,
+		storageCID.StorageKey,
+		storageCID.CID,
+		true,
+		storageCID.Value,
+		storageCID.Removed,
+	)
 	if err != nil {
 		return insertError{"eth.storage_cids", err, w.db.InsertStorageStm(), storageCID}
 	}

statediff/indexer/shared/schema/schema.go

@@ -0,0 +1,173 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package schema
+
+var TableIPLDBlock = Table{
+	Name: `ipld.blocks`,
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "key", Type: Dtext},
+		{Name: "data", Type: Dbytea},
+	},
+}
+
+var TableNodeInfo = Table{
+	Name: `public.nodes`,
+	Columns: []Column{
+		{Name: "genesis_block", Type: Dvarchar},
+		{Name: "network_id", Type: Dvarchar},
+		{Name: "node_id", Type: Dvarchar},
+		{Name: "client_name", Type: Dvarchar},
+		{Name: "chain_id", Type: Dinteger},
+	},
+}
+
+var TableHeader = Table{
+	Name: "eth.header_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "block_hash", Type: Dvarchar},
+		{Name: "parent_hash", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "td", Type: Dnumeric},
+		{Name: "node_ids", Type: Dvarchar, Array: true},
+		{Name: "reward", Type: Dnumeric},
+		{Name: "state_root", Type: Dvarchar},
+		{Name: "tx_root", Type: Dvarchar},
+		{Name: "receipt_root", Type: Dvarchar},
+		{Name: "uncles_hash", Type: Dvarchar},
+		{Name: "bloom", Type: Dbytea},
+		{Name: "timestamp", Type: Dnumeric},
+		{Name: "coinbase", Type: Dvarchar},
+	},
+	UpsertClause: OnConflict("block_number", "block_hash").Set(
+		"parent_hash",
+		"cid",
+		"td",
+		"node_ids",
+		"reward",
+		"state_root",
+		"tx_root",
+		"receipt_root",
+		"uncles_hash",
+		"bloom",
+		"timestamp",
+		"coinbase",
+	)}
+
+var TableStateNode = Table{
+	Name: "eth.state_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "header_id", Type: Dvarchar},
+		{Name: "state_leaf_key", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "diff", Type: Dboolean},
+		{Name: "balance", Type: Dnumeric},
+		{Name: "nonce", Type: Dbigint},
+		{Name: "code_hash", Type: Dvarchar},
+		{Name: "storage_root", Type: Dvarchar},
+		{Name: "removed", Type: Dboolean},
+	},
+	UpsertClause: OnConflict("block_number", "header_id", "state_leaf_key"),
+}
+
+var TableStorageNode = Table{
+	Name: "eth.storage_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "header_id", Type: Dvarchar},
+		{Name: "state_leaf_key", Type: Dvarchar},
+		{Name: "storage_leaf_key", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "diff", Type: Dboolean},
+		{Name: "val", Type: Dbytea},
+		{Name: "removed", Type: Dboolean},
+	},
+	UpsertClause: OnConflict("block_number", "header_id", "state_leaf_key", "storage_leaf_key"),
+}
+
+var TableUncle = Table{
+	Name: "eth.uncle_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "block_hash", Type: Dvarchar},
+		{Name: "header_id", Type: Dvarchar},
+		{Name: "parent_hash", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "reward", Type: Dnumeric},
+		{Name: "index", Type: Dinteger},
+	},
+	UpsertClause: OnConflict("block_number", "block_hash"),
+}
+
+var TableTransaction = Table{
+	Name: "eth.transaction_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "header_id", Type: Dvarchar},
+		{Name: "tx_hash", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "dst", Type: Dvarchar},
+		{Name: "src", Type: Dvarchar},
+		{Name: "index", Type: Dinteger},
+		{Name: "tx_type", Type: Dinteger},
+		{Name: "value", Type: Dnumeric},
+	},
+	UpsertClause: OnConflict("block_number", "header_id", "tx_hash"),
+}
+
+var TableReceipt = Table{
+	Name: "eth.receipt_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "header_id", Type: Dvarchar},
+		{Name: "tx_id", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "contract", Type: Dvarchar},
+		{Name: "post_state", Type: Dvarchar},
+		{Name: "post_status", Type: Dinteger},
+	},
+	UpsertClause: OnConflict("block_number", "header_id", "tx_id"),
+}
+
+var TableLog = Table{
+	Name: "eth.log_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "header_id", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "rct_id", Type: Dvarchar},
+		{Name: "address", Type: Dvarchar},
+		{Name: "index", Type: Dinteger},
+		{Name: "topic0", Type: Dvarchar},
+		{Name: "topic1", Type: Dvarchar},
+		{Name: "topic2", Type: Dvarchar},
+		{Name: "topic3", Type: Dvarchar},
+	},
+	UpsertClause: OnConflict("block_number", "header_id", "rct_id", "index"),
+}
+
+var TableWatchedAddresses = Table{
+	Name: "eth_meta.watched_addresses",
+	Columns: []Column{
+		{Name: "address", Type: Dvarchar},
+		{Name: "created_at", Type: Dbigint},
+		{Name: "watched_at", Type: Dbigint},
+		{Name: "last_filled_at", Type: Dbigint},
+	},
+}

statediff/indexer/shared/schema/table.go

@@ -0,0 +1,147 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package schema
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/thoas/go-funk"
+)
+
+type colType int
+
+const (
+	Dinteger colType = iota
+	Dboolean
+	Dbigint
+	Dnumeric
+	Dbytea
+	Dvarchar
+	Dtext
+)
+
+type ConflictClause struct {
+	Target []string
+	Update []string
+}
+
+type Column struct {
+	Name  string
+	Type  colType
+	Array bool
+}
+type Table struct {
+	Name    string
+	Columns []Column
+
+	UpsertClause ConflictClause
+}
+
+type colfmt = func(interface{}) string
+
+func (tbl *Table) ToCsvRow(args ...interface{}) []string {
+	var row []string
+	for i, col := range tbl.Columns {
+		value := col.Type.formatter()(args[i])
+
+		if col.Array {
+			valueList := funk.Map(args[i], col.Type.formatter()).([]string)
+			value = fmt.Sprintf("{%s}", strings.Join(valueList, ","))
+		}
+
+		row = append(row, value)
+	}
+	return row
+}
+
+func (tbl *Table) VarcharColumns() []string {
+	columns := funk.Filter(tbl.Columns, func(col Column) bool {
+		return col.Type == Dvarchar
+	}).([]Column)
+
+	columnNames := funk.Map(columns, func(col Column) string {
+		return col.Name
+	}).([]string)
+	return columnNames
+}
+
+func OnConflict(target ...string) ConflictClause {
+	return ConflictClause{Target: target}
+}
+func (c ConflictClause) Set(fields ...string) ConflictClause {
+	c.Update = fields
+	return c
+}
+
+// ToInsertStatement returns a Postgres-compatible SQL insert statement for the table
+// using positional placeholders
+func (tbl *Table) ToInsertStatement(upsert bool) string {
+	var colnames, placeholders []string
+	for i, col := range tbl.Columns {
+		colnames = append(colnames, col.Name)
+		placeholders = append(placeholders, fmt.Sprintf("$%d", i+1))
+	}
+	suffix := fmt.Sprintf("ON CONFLICT (%s)", strings.Join(tbl.UpsertClause.Target, ", "))
+	if upsert && len(tbl.UpsertClause.Update) != 0 {
+		var update_placeholders []string
+		for _, name := range tbl.UpsertClause.Update {
+			i := funk.IndexOf(tbl.Columns, func(col Column) bool { return col.Name == name })
+			update_placeholders = append(update_placeholders, fmt.Sprintf("$%d", i+1))
+		}
+		suffix += fmt.Sprintf(
+			" DO UPDATE SET (%s) = (%s)",
+			strings.Join(tbl.UpsertClause.Update, ", "), strings.Join(update_placeholders, ", "),
+		)
+	} else {
+		suffix += " DO NOTHING"
+	}
+
+	return fmt.Sprintf(
+		"INSERT INTO %s (%s) VALUES (%s) %s",
+		tbl.Name, strings.Join(colnames, ", "), strings.Join(placeholders, ", "), suffix,
+	)
+}
+
+func sprintf(f string) colfmt {
+	return func(x interface{}) string { return fmt.Sprintf(f, x) }
+}
+
+func (typ colType) formatter() colfmt {
+	switch typ {
+	case Dinteger:
+		return sprintf("%d")
+	case Dboolean:
+		return func(x interface{}) string {
+			if x.(bool) {
+				return "t"
+			}
+			return "f"
+		}
+	case Dbigint:
+		return sprintf("%s")
+	case Dnumeric:
+		return sprintf("%s")
+	case Dbytea:
+		return sprintf(`\x%x`)
+	case Dvarchar:
+		return sprintf("%s")
+	case Dtext:
+		return sprintf("%s")
+	}
+	panic("unreachable")
+}

statediff/indexer/shared/schema/table_test.go

@@ -0,0 +1,56 @@
+package schema_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	. "github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
+)
+
+var testHeaderTable = Table{
+	Name: "eth.header_cids",
+	Columns: []Column{
+		{Name: "block_number", Type: Dbigint},
+		{Name: "block_hash", Type: Dvarchar},
+		{Name: "parent_hash", Type: Dvarchar},
+		{Name: "cid", Type: Dtext},
+		{Name: "td", Type: Dnumeric},
+		{Name: "node_id", Type: Dvarchar},
+		{Name: "reward", Type: Dnumeric},
+		{Name: "state_root", Type: Dvarchar},
+		{Name: "tx_root", Type: Dvarchar},
+		{Name: "receipt_root", Type: Dvarchar},
+		{Name: "uncle_root", Type: Dvarchar},
+		{Name: "bloom", Type: Dbytea},
+		{Name: "timestamp", Type: Dnumeric},
+		{Name: "mh_key", Type: Dtext},
+		{Name: "times_validated", Type: Dinteger},
+		{Name: "coinbase", Type: Dvarchar},
+	},
+	UpsertClause: OnConflict("block_hash", "block_number").Set(
+		"parent_hash",
+		"cid",
+		"td",
+		"node_id",
+		"reward",
+		"state_root",
+		"tx_root",
+		"receipt_root",
+		"uncle_root",
+		"bloom",
+		"timestamp",
+		"mh_key",
+		"times_validated",
+		"coinbase",
+	)}
+
+func TestTable(t *testing.T) {
+
+	headerUpsert := `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)`
+
+	headerNoUpsert := `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_hash, block_number) DO NOTHING`
+
+	require.Equal(t, headerNoUpsert, testHeaderTable.ToInsertStatement(false))
+	require.Equal(t, headerUpsert, testHeaderTable.ToInsertStatement(true))
+}

statediff/test_helpers/helpers.go

@@ -50,15 +50,15 @@ func TestSelfDestructChainGen(i int, block *core.BlockGen) {
 	signer := types.HomesteadSigner{}
 	switch i {
 	case 0:
-		// Block 1 is mined by Account1Addr
+		// Block 1 is mined by TestBankAddress
-		// Account1Addr creates a new contract
+		// TestBankAddress creates a new contract
 		block.SetCoinbase(TestBankAddress)
 		tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(params.GWei), ContractCode), signer, TestBankKey)
 		ContractAddr = crypto.CreateAddress(TestBankAddress, 0)
 		block.AddTx(tx)
 	case 1:
-		// Block 2 is mined by Account1Addr
+		// Block 2 is mined by TestBankAddress
-		// Account1Addr self-destructs the contract
+		// TestBankAddress self-destructs the contract
 		block.SetCoinbase(TestBankAddress)
 		data := common.Hex2Bytes("43D726D6")
 		tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.GWei), data), signer, TestBankKey)

statediff/types/types.go

@@ -39,7 +39,7 @@ type StateObject struct {
 // AccountMap is a mapping of hex encoded path => account wrapper
 type AccountMap map[string]AccountWrapper
 
-// AccountWrapper is used to temporary associate the unpacked node with its raw values
+// AccountWrapper is used to temporarily associate the unpacked node with its raw values
 type AccountWrapper struct {
 	Account *types.StateAccount
 	LeafKey []byte