diff --git a/go.mod b/go.mod index 7d44b3eaa..0be358207 100644 --- a/go.mod +++ b/go.mod @@ -40,11 +40,7 @@ require ( github.com/influxdata/influxdb v1.8.3 github.com/influxdata/influxdb-client-go/v2 v2.4.0 github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect - github.com/ipfs/go-block-format v0.0.3 github.com/ipfs/go-cid v0.2.0 - github.com/ipfs/go-ipfs-blockstore v1.2.0 - github.com/ipfs/go-ipfs-ds-help v1.1.0 - github.com/ipfs/go-ipld-format v0.4.0 github.com/jackc/pgconn v1.10.0 github.com/jackc/pgx/v4 v4.13.0 github.com/jackpal/go-nat-pmp v1.0.2 @@ -67,7 +63,7 @@ require ( github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.0 github.com/supranational/blst v0.3.8 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/thoas/go-funk v0.9.2 @@ -101,12 +97,6 @@ require ( github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/gogo/protobuf v1.3.1 // indirect - github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/go-datastore v0.5.0 // indirect - github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-log v0.0.1 // indirect - github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -114,7 +104,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.8.1 // indirect github.com/jackc/puddle v1.1.3 // indirect - github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect @@ -131,12 +120,10 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/stretchr/objx v0.2.0 // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/tklauser/numcpus v0.2.2 // indirect - github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.uber.org/atomic v1.6.0 // indirect golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect diff --git a/go.sum b/go.sum index ce34b8198..a1a93a6fb 100644 --- a/go.sum +++ b/go.sum @@ -175,8 +175,6 @@ github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -226,7 +224,6 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -237,13 +234,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= -github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -270,36 +264,8 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= -github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= -github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= -github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= -github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= -github.com/ipfs/go-datastore v0.5.0 h1:rQicVCEacWyk4JZ6G5bD9TKR7lZEG1MWcG7UdWYrFAU= -github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= -github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= -github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= -github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= -github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= -github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= -github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= -github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= -github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= -github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= -github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= -github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -374,9 +340,6 @@ github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= -github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= -github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b h1:ZGiXF8sz7PDk6RgkP+A/SFfUD0ZR/AgG6SpRNEDKZy8= github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b/go.mod h1:hQmNrgofl+IY/8L+n20H6E6PWBBTokdsv+q49j0QhsU= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= @@ -397,7 +360,6 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -412,7 +374,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -464,8 +425,6 @@ github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4f github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= -github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= @@ -476,7 +435,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -484,16 +442,11 @@ github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= -github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= -github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA= github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= -github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -583,15 +536,18 @@ github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57N github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/supranational/blst v0.3.8 h1:glwLF4oBRSJOTr05lRBgNwGQST0ndP2wg29fSeTRKCY= github.com/supranational/blst v0.3.8/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -611,8 +567,6 @@ github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhA github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= -github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -628,7 +582,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -638,13 +591,11 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -683,7 +634,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= @@ -702,7 +652,6 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -746,7 +695,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -804,7 +752,6 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -897,7 +844,6 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/statediff/builder.go b/statediff/builder.go index 3759c67cc..f6449fbc6 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -102,7 +102,7 @@ func (sdb *StateDiffBuilder) BuildStateDiffObject(args Args, params Params) (typ }, nil } -// WriteStateDiffObject writes a statediff object to output callback +// WriteStateDiffObject writes a statediff object to output sinks func (sdb *StateDiffBuilder) WriteStateDiffObject(args types2.StateRoots, params Params, output types2.StateNodeSink, ipldOutput types2.IPLDSink) error { // Load tries for old and new states diff --git a/statediff/indexer/database/file/csv_indexer_legacy_test.go b/statediff/indexer/database/file/csv_indexer_legacy_test.go index 55350a912..f16926d95 100644 --- a/statediff/indexer/database/file/csv_indexer_legacy_test.go +++ b/statediff/indexer/database/file/csv_indexer_legacy_test.go @@ -28,8 +28,8 @@ import ( "github.com/stretchr/testify/require" "github.com/ethereum/go-ethereum/statediff/indexer/database/file" - "github.com/ethereum/go-ethereum/statediff/indexer/database/file/types" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/shared/schema" "github.com/ethereum/go-ethereum/statediff/indexer/test" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" ) @@ -90,7 +90,7 @@ func resetAndDumpWatchedAddressesCSVFileData(t *testing.T) { test_helpers.TearDownDB(t, db) outputFilePath := filepath.Join(dbDirectory, file.CSVTestConfig.WatchedAddressesFilePath) - stmt := fmt.Sprintf(pgCopyStatement, types.TableWatchedAddresses.Name, outputFilePath) + stmt := fmt.Sprintf(pgCopyStatement, schema.TableWatchedAddresses.Name, outputFilePath) _, err = db.Exec(context.Background(), stmt) require.NoError(t, err) diff --git a/statediff/indexer/database/file/csv_writer.go b/statediff/indexer/database/file/csv_writer.go index 3bdcc5bc2..1d6816395 100644 --- a/statediff/indexer/database/file/csv_writer.go +++ b/statediff/indexer/database/file/csv_writer.go @@ -28,29 +28,29 @@ import ( "github.com/thoas/go-funk" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/indexer/database/file/types" "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/models" nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/shared/schema" sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) var ( - Tables = []*types.Table{ - &types.TableIPLDBlock, - &types.TableNodeInfo, - &types.TableHeader, - &types.TableStateNode, - &types.TableStorageNode, - &types.TableUncle, - &types.TableTransaction, - &types.TableReceipt, - &types.TableLog, + Tables = []*schema.Table{ + &schema.TableIPLDBlock, + &schema.TableNodeInfo, + &schema.TableHeader, + &schema.TableStateNode, + &schema.TableStorageNode, + &schema.TableUncle, + &schema.TableTransaction, + &schema.TableReceipt, + &schema.TableLog, } ) type tableRow struct { - table types.Table + table schema.Table values []interface{} } @@ -90,7 +90,7 @@ func newFileWriter(path string) (ret fileWriter, err error) { return } -func makeFileWriters(dir string, tables []*types.Table) (fileWriters, error) { +func makeFileWriters(dir string, tables []*schema.Table) (fileWriters, error) { if err := os.MkdirAll(dir, 0755); err != nil { return nil, err } @@ -105,7 +105,7 @@ func makeFileWriters(dir string, tables []*types.Table) (fileWriters, error) { return writers, nil } -func (tx fileWriters) write(tbl *types.Table, args ...interface{}) error { +func (tx fileWriters) write(tbl *schema.Table, args ...interface{}) error { row := tbl.ToCsvRow(args...) return tx[tbl.Name].Write(row) } @@ -204,13 +204,13 @@ func (csw *CSVWriter) Close() error { func (csw *CSVWriter) upsertNode(node nodeinfo.Info) { var values []interface{} values = append(values, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID) - csw.rows <- tableRow{types.TableNodeInfo, values} + csw.rows <- tableRow{schema.TableNodeInfo, values} } func (csw *CSVWriter) upsertIPLD(ipld models.IPLDModel) { var values []interface{} values = append(values, ipld.BlockNumber, ipld.Key, ipld.Data) - csw.rows <- tableRow{types.TableIPLDBlock, values} + csw.rows <- tableRow{schema.TableIPLDBlock, values} } func (csw *CSVWriter) upsertIPLDDirect(blockNumber, key string, value []byte) { @@ -234,7 +234,7 @@ func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) { values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, header.NodeIDs, header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.Coinbase) - csw.rows <- tableRow{types.TableHeader, values} + csw.rows <- tableRow{schema.TableHeader, values} indexerMetrics.blocks.Inc(1) } @@ -242,14 +242,14 @@ func (csw *CSVWriter) upsertUncleCID(uncle models.UncleModel) { var values []interface{} values = append(values, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.Index) - csw.rows <- tableRow{types.TableUncle, values} + csw.rows <- tableRow{schema.TableUncle, values} } func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) { var values []interface{} values = append(values, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index, transaction.Type, transaction.Value) - csw.rows <- tableRow{types.TableTransaction, values} + csw.rows <- tableRow{schema.TableTransaction, values} indexerMetrics.transactions.Inc(1) } @@ -257,7 +257,7 @@ func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) { var values []interface{} values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.PostState, rct.PostStatus) - csw.rows <- tableRow{types.TableReceipt, values} + csw.rows <- tableRow{schema.TableReceipt, values} indexerMetrics.receipts.Inc(1) } @@ -266,7 +266,7 @@ func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) { var values []interface{} values = append(values, l.BlockNumber, l.HeaderID, l.CID, l.ReceiptID, l.Address, l.Index, l.Topic0, l.Topic1, l.Topic2, l.Topic3) - csw.rows <- tableRow{types.TableLog, values} + csw.rows <- tableRow{schema.TableLog, values} indexerMetrics.logs.Inc(1) } } @@ -280,7 +280,7 @@ func (csw *CSVWriter) upsertStateCID(stateNode models.StateNodeModel) { var values []interface{} values = append(values, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, true, stateNode.Balance, strconv.FormatUint(stateNode.Nonce, 10), stateNode.CodeHash, stateNode.StorageRoot, stateNode.Removed) - csw.rows <- tableRow{types.TableStateNode, values} + csw.rows <- tableRow{schema.TableStateNode, values} } func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) { @@ -292,7 +292,7 @@ func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) { var values []interface{} values = append(values, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageKey, storageCID.CID, true, storageCID.Value, storageCID.Removed) - csw.rows <- tableRow{types.TableStorageNode, values} + csw.rows <- tableRow{schema.TableStorageNode, values} } // LoadWatchedAddresses loads watched addresses from a file @@ -332,7 +332,7 @@ func (csw *CSVWriter) insertWatchedAddresses(args []sdtypes.WatchAddressArg, cur var values []interface{} values = append(values, arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0") - row := types.TableWatchedAddresses.ToCsvRow(values...) + row := schema.TableWatchedAddresses.ToCsvRow(values...) // writing directly instead of using rows channel as it needs to be flushed immediately err = csw.watchedAddressesWriter.Write(row) @@ -375,7 +375,7 @@ func (csw *CSVWriter) removeWatchedAddresses(args []sdtypes.WatchAddressArg) err func (csw *CSVWriter) setWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { var rows [][]string for _, arg := range args { - row := types.TableWatchedAddresses.ToCsvRow(arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0") + row := schema.TableWatchedAddresses.ToCsvRow(arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0") rows = append(rows, row) } diff --git a/statediff/indexer/database/file/indexer.go b/statediff/indexer/database/file/indexer.go index 2586dc993..bd5241f17 100644 --- a/statediff/indexer/database/file/indexer.go +++ b/statediff/indexer/database/file/indexer.go @@ -443,7 +443,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt storageModel := models.StorageNodeModel{ BlockNumber: tx.BlockNumber, HeaderID: headerID, - StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).Hex(), + StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(), StorageKey: common.BytesToHash(storageNode.LeafKey).String(), CID: storageNode.CID, Removed: false, diff --git a/statediff/indexer/database/file/types/schema.go b/statediff/indexer/database/file/types/schema.go deleted file mode 100644 index a7e2823fc..000000000 --- a/statediff/indexer/database/file/types/schema.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package types - -var TableIPLDBlock = Table{ - `ipld.blocks`, - []column{ - {name: "block_number", dbType: bigint}, - {name: "key", dbType: text}, - {name: "data", dbType: bytea}, - }, -} - -var TableNodeInfo = Table{ - Name: `public.nodes`, - Columns: []column{ - {name: "genesis_block", dbType: varchar}, - {name: "network_id", dbType: varchar}, - {name: "node_id", dbType: varchar}, - {name: "client_name", dbType: varchar}, - {name: "chain_id", dbType: integer}, - }, -} - -var TableHeader = Table{ - "eth.header_cids", - []column{ - {name: "block_number", dbType: bigint}, - {name: "block_hash", dbType: varchar}, - {name: "parent_hash", dbType: varchar}, - {name: "cid", dbType: text}, - {name: "td", dbType: numeric}, - {name: "node_ids", dbType: varchar, isArray: true}, - {name: "reward", dbType: numeric}, - {name: "state_root", dbType: varchar}, - {name: "tx_root", dbType: varchar}, - {name: "receipt_root", dbType: varchar}, - {name: "uncles_hash", dbType: varchar}, - {name: "bloom", dbType: bytea}, - {name: "timestamp", dbType: numeric}, - {name: "coinbase", dbType: varchar}, - }, -} - -var TableStateNode = Table{ - "eth.state_cids", - []column{ - {name: "block_number", dbType: bigint}, - {name: "header_id", dbType: varchar}, - {name: "state_leaf_key", dbType: varchar}, - {name: "cid", dbType: text}, - {name: "removed", dbType: boolean}, - {name: "diff", dbType: boolean}, - {name: "balance", dbType: numeric}, - {name: "nonce", dbType: bigint}, - {name: "code_hash", dbType: varchar}, - {name: "storage_root", dbType: varchar}, - }, -} - -var TableStorageNode = Table{ - "eth.storage_cids", - []column{ - {name: "block_number", dbType: bigint}, - {name: "header_id", dbType: varchar}, - {name: "state_leaf_key", dbType: varchar}, - {name: "storage_leaf_key", dbType: varchar}, - {name: "cid", dbType: text}, - {name: "removed", dbType: boolean}, - {name: "diff", dbType: boolean}, - {name: "val", dbType: bytea}, - }, -} - -var TableUncle = Table{ - "eth.uncle_cids", - []column{ - {name: "block_number", dbType: bigint}, - {name: "block_hash", dbType: varchar}, - {name: "header_id", dbType: varchar}, - {name: "parent_hash", dbType: varchar}, - {name: "cid", dbType: text}, - {name: "reward", dbType: numeric}, - {name: "index", dbType: integer}, - }, -} - -var TableTransaction = Table{ - "eth.transaction_cids", - []column{ - {name: "block_number", dbType: bigint}, - {name: "header_id", dbType: varchar}, - {name: "tx_hash", dbType: varchar}, - {name: "cid", dbType: text}, - {name: "dst", dbType: varchar}, - {name: "src", dbType: varchar}, - {name: "index", dbType: integer}, - {name: "tx_type", dbType: integer}, - {name: "value", dbType: numeric}, - }, -} - -var TableReceipt = Table{ - "eth.receipt_cids", - []column{ - {name: "block_number", dbType: bigint}, - {name: "header_id", dbType: varchar}, - {name: "tx_id", dbType: varchar}, - {name: "cid", dbType: text}, - {name: "contract", dbType: varchar}, - {name: "post_state", dbType: varchar}, - {name: "post_status", dbType: integer}, - }, -} - -var TableLog = Table{ - "eth.log_cids", - []column{ - {name: "block_number", dbType: bigint}, - {name: "header_id", dbType: varchar}, - {name: "cid", dbType: text}, - {name: "rct_id", dbType: varchar}, - {name: "address", dbType: varchar}, - {name: "index", dbType: integer}, - {name: "topic0", dbType: varchar}, - {name: "topic1", dbType: varchar}, - {name: "topic2", dbType: varchar}, - {name: "topic3", dbType: varchar}, - }, -} - -var TableWatchedAddresses = Table{ - "eth_meta.watched_addresses", - []column{ - {name: "address", dbType: varchar}, - {name: "created_at", dbType: bigint}, - {name: "watched_at", dbType: bigint}, - {name: "last_filled_at", dbType: bigint}, - }, -} diff --git a/statediff/indexer/database/file/types/table.go b/statediff/indexer/database/file/types/table.go deleted file mode 100644 index d7fd5af6c..000000000 --- a/statediff/indexer/database/file/types/table.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package types - -import ( - "fmt" - "strings" - - "github.com/thoas/go-funk" -) - -type colType int - -const ( - integer colType = iota - boolean - bigint - numeric - bytea - varchar - text -) - -type column struct { - name string - dbType colType - isArray bool -} -type Table struct { - Name string - Columns []column -} - -func (tbl *Table) ToCsvRow(args ...interface{}) []string { - var row []string - for i, col := range tbl.Columns { - value := col.dbType.formatter()(args[i]) - - if col.isArray { - valueList := funk.Map(args[i], col.dbType.formatter()).([]string) - value = fmt.Sprintf("{%s}", strings.Join(valueList, ",")) - } - - row = append(row, value) - } - return row -} - -func (tbl *Table) VarcharColumns() []string { - columns := funk.Filter(tbl.Columns, func(col column) bool { - return col.dbType == varchar - }).([]column) - - columnNames := funk.Map(columns, func(col column) string { - return col.name - }).([]string) - - return columnNames -} - -type colfmt = func(interface{}) string - -func sprintf(f string) colfmt { - return func(x interface{}) string { return fmt.Sprintf(f, x) } -} - -func (typ colType) formatter() colfmt { - switch typ { - case integer: - return sprintf("%d") - case boolean: - return func(x interface{}) string { - if x.(bool) { - return "t" - } - return "f" - } - case bigint: - return sprintf("%s") - case numeric: - return sprintf("%s") - case bytea: - return sprintf(`\x%x`) - case varchar: - return sprintf("%s") - case text: - return sprintf("%s") - } - panic("unreachable") -} diff --git a/statediff/indexer/database/sql/interfaces.go b/statediff/indexer/database/sql/interfaces.go index 4056f7dbb..3fee858d6 100644 --- a/statediff/indexer/database/sql/interfaces.go +++ b/statediff/indexer/database/sql/interfaces.go @@ -52,7 +52,6 @@ type Statements interface { InsertStorageStm() string InsertIPLDStm() string InsertIPLDsStm() string - InsertKnownGapsStm() string } // Tx interface to accommodate different concrete SQL transaction types diff --git a/statediff/indexer/database/sql/postgres/config.go b/statediff/indexer/database/sql/postgres/config.go index b5cdc02ab..2038bf1f5 100644 --- a/statediff/indexer/database/sql/postgres/config.go +++ b/statediff/indexer/database/sql/postgres/config.go @@ -18,6 +18,8 @@ package postgres import ( "fmt" + "os" + "strconv" "strings" "time" @@ -33,6 +35,15 @@ const ( Unknown DriverType = "Unknown" ) +// Env variables +const ( + DATABASE_NAME = "DATABASE_NAME" + DATABASE_HOSTNAME = "DATABASE_HOSTNAME" + DATABASE_PORT = "DATABASE_PORT" + DATABASE_USER = "DATABASE_USER" + DATABASE_PASSWORD = "DATABASE_PASSWORD" +) + // ResolveDriverType resolves a DriverType from a provided string func ResolveDriverType(str string) (DriverType, error) { switch strings.ToLower(str) { @@ -49,7 +60,7 @@ func ResolveDriverType(str string) (DriverType, error) { var DefaultConfig = Config{ Hostname: "localhost", Port: 8077, - DatabaseName: "vulcanize_testing", + DatabaseName: "cerc_testing", Username: "vdbm", Password: "password", } @@ -100,3 +111,26 @@ func (c Config) DbConnectionString() string { } return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName) } + +func (c Config) WithEnv() (Config, error) { + if val := os.Getenv(DATABASE_NAME); val != "" { + c.DatabaseName = val + } + if val := os.Getenv(DATABASE_HOSTNAME); val != "" { + c.Hostname = val + } + if val := os.Getenv(DATABASE_PORT); val != "" { + port, err := strconv.Atoi(val) + if err != nil { + return c, err + } + c.Port = port + } + if val := os.Getenv(DATABASE_USER); val != "" { + c.Username = val + } + if val := os.Getenv(DATABASE_PASSWORD); val != "" { + c.Password = val + } + return c, nil +} diff --git a/statediff/indexer/database/sql/postgres/database.go b/statediff/indexer/database/sql/postgres/database.go index 600b34fcc..d19e35abf 100644 --- a/statediff/indexer/database/sql/postgres/database.go +++ b/statediff/indexer/database/sql/postgres/database.go @@ -16,7 +16,10 @@ package postgres -import "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" +import ( + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/shared/schema" +) var _ sql.Database = &DB{} @@ -39,73 +42,51 @@ type DB struct { // InsertHeaderStm satisfies the sql.Statements interface // Stm == Statement func (db *DB) InsertHeaderStm() string { - if db.upsert { - return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) - ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)` - } - return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) - ON CONFLICT (block_hash, block_number) DO NOTHING` + return schema.TableHeader.ToInsertStatement(db.upsert) + } // InsertUncleStm satisfies the sql.Statements interface func (db *DB) InsertUncleStm() string { - return `INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, index) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (block_hash, block_number, index) DO NOTHING` + return schema.TableUncle.ToInsertStatement(db.upsert) } // InsertTxStm satisfies the sql.Statements interface func (db *DB) InsertTxStm() string { - return `INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING` + return schema.TableTransaction.ToInsertStatement(db.upsert) + } // InsertRctStm satisfies the sql.Statements interface func (db *DB) InsertRctStm() string { - return `INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, post_status) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (tx_id, header_id, block_number) DO NOTHING` + return schema.TableReceipt.ToInsertStatement(db.upsert) + } // InsertLogStm satisfies the sql.Statements interface func (db *DB) InsertLogStm() string { - return `INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, topic3) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING` + return schema.TableLog.ToInsertStatement(db.upsert) + } // InsertStateStm satisfies the sql.Statements interface func (db *DB) InsertStateStm() string { - if db.upsert { - return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (header_id, state_leaf_key, block_number) DO UPDATE SET (cid, removed, diff, balance, nonce, code_hash, storage_root) = ($4, $5, $6, $7, $8, $9, $10)` - } - return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING` + return schema.TableStateNode.ToInsertStatement(db.upsert) + } // InsertStorageStm satisfies the sql.Statements interface func (db *DB) InsertStorageStm() string { - if db.upsert { - return `INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO UPDATE SET (cid, removed, diff, val) = ($4, $5, $6, $7, $8)` - } - return `INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHING` + return schema.TableStorageNode.ToInsertStatement(db.upsert) + } // InsertIPLDStm satisfies the sql.Statements interface func (db *DB) InsertIPLDStm() string { - return `INSERT INTO ipld.blocks (block_number, key, data) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING` + return schema.TableIPLDBlock.ToInsertStatement(db.upsert) } // InsertIPLDsStm satisfies the sql.Statements interface func (db *DB) InsertIPLDsStm() string { return `INSERT INTO ipld.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING` } - -// InsertKnownGapsStm satisfies the sql.Statements interface -func (db *DB) InsertKnownGapsStm() string { - return `INSERT INTO eth_meta.known_gaps (starting_block_number, ending_block_number, checked_out, processing_key) VALUES ($1, $2, $3, $4) - ON CONFLICT (starting_block_number) DO UPDATE SET (ending_block_number, processing_key) = ($2, $4) - WHERE eth_meta.known_gaps.ending_block_number <= $2` -} diff --git a/statediff/indexer/database/sql/postgres/errors.go b/statediff/indexer/database/sql/postgres/errors.go index effa74aa1..1fcd9598f 100644 --- a/statediff/indexer/database/sql/postgres/errors.go +++ b/statediff/indexer/database/sql/postgres/errors.go @@ -26,13 +26,13 @@ const ( ) func ErrDBConnectionFailed(connectErr error) error { - return formatError(DbConnectionFailedMsg, connectErr.Error()) + return formatError(DbConnectionFailedMsg, connectErr) } func ErrUnableToSetNode(setErr error) error { - return formatError(SettingNodeFailedMsg, setErr.Error()) + return formatError(SettingNodeFailedMsg, setErr) } -func formatError(msg, err string) error { - return fmt.Errorf("%s: %s", msg, err) +func formatError(msg string, err error) error { + return fmt.Errorf("%s: %w", msg, err) } diff --git a/statediff/indexer/database/sql/postgres/pgx.go b/statediff/indexer/database/sql/postgres/pgx.go index 9f1c4d571..073d92744 100644 --- a/statediff/indexer/database/sql/postgres/pgx.go +++ b/statediff/indexer/database/sql/postgres/pgx.go @@ -39,14 +39,19 @@ type PGXDriver struct { nodeID string } -// NewPGXDriver returns a new pgx driver -// it initializes the connection pool and creates the node info table -func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) { +// ConnectPGX initializes and returns a PGX connection pool +func ConnectPGX(ctx context.Context, config Config) (*pgxpool.Pool, error) { pgConf, err := MakeConfig(config) if err != nil { return nil, err } - dbPool, err := pgxpool.ConnectConfig(ctx, pgConf) + return pgxpool.ConnectConfig(ctx, pgConf) +} + +// NewPGXDriver returns a new pgx driver +// it initializes the connection pool and creates the node info table +func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) { + dbPool, err := ConnectPGX(ctx, config) if err != nil { return nil, ErrDBConnectionFailed(err) } diff --git a/statediff/indexer/database/sql/postgres/sqlx.go b/statediff/indexer/database/sql/postgres/sqlx.go index 9f1753e67..c41d39828 100644 --- a/statediff/indexer/database/sql/postgres/sqlx.go +++ b/statediff/indexer/database/sql/postgres/sqlx.go @@ -35,12 +35,11 @@ type SQLXDriver struct { nodeID string } -// NewSQLXDriver returns a new sqlx driver for Postgres -// it initializes the connection pool and creates the node info table -func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) { +// ConnectSQLX initializes and returns a SQLX connection pool for postgres +func ConnectSQLX(ctx context.Context, config Config) (*sqlx.DB, error) { db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString()) if err != nil { - return &SQLXDriver{}, ErrDBConnectionFailed(err) + return nil, ErrDBConnectionFailed(err) } if config.MaxConns > 0 { db.SetMaxOpenConns(config.MaxConns) @@ -49,9 +48,19 @@ func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDri db.SetConnMaxLifetime(config.MaxConnLifetime) } db.SetMaxIdleConns(config.MaxIdle) + return db, nil +} + +// NewSQLXDriver returns a new sqlx driver for Postgres +// it initializes the connection pool and creates the node info table +func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) { + db, err := ConnectSQLX(ctx, config) + if err != nil { + return nil, err + } driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node} if err := driver.createNode(); err != nil { - return &SQLXDriver{}, ErrUnableToSetNode(err) + return nil, err } return driver, nil } @@ -59,8 +68,10 @@ func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDri func (driver *SQLXDriver) createNode() error { _, err := driver.db.Exec( createNodeStm, - driver.nodeInfo.GenesisBlock, driver.nodeInfo.NetworkID, - driver.nodeInfo.ID, driver.nodeInfo.ClientName, + driver.nodeInfo.GenesisBlock, + driver.nodeInfo.NetworkID, + driver.nodeInfo.ID, + driver.nodeInfo.ClientName, driver.nodeInfo.ChainID) if err != nil { return ErrUnableToSetNode(err) diff --git a/statediff/indexer/database/sql/writer.go b/statediff/indexer/database/sql/writer.go index 74bf49c0d..bd6fb5b67 100644 --- a/statediff/indexer/database/sql/writer.go +++ b/statediff/indexer/database/sql/writer.go @@ -47,10 +47,22 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT (block_hash, block_number) DO NOTHING */ func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error { + nodeIDs := pq.StringArray([]string{w.db.NodeID()}) _, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(), - header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, pq.StringArray([]string{w.db.NodeID()}), - header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UnclesHash, header.Bloom, - header.Timestamp, header.Coinbase) + header.BlockNumber, + header.BlockHash, + header.ParentHash, + header.CID, + header.TotalDifficulty, + nodeIDs, + header.Reward, + header.StateRoot, + header.TxRoot, + header.RctRoot, + header.UnclesHash, + header.Bloom, + header.Timestamp, + header.Coinbase) if err != nil { return insertError{"eth.header_cids", err, w.db.InsertHeaderStm(), header} } @@ -64,7 +76,13 @@ ON CONFLICT (block_hash, block_number) DO NOTHING */ func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error { _, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(), - uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.Index) + uncle.BlockNumber, + uncle.BlockHash, + uncle.HeaderID, + uncle.ParentHash, + uncle.CID, + uncle.Reward, + uncle.Index) if err != nil { return insertError{"eth.uncle_cids", err, w.db.InsertUncleStm(), uncle} } @@ -77,8 +95,15 @@ ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING */ func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error { _, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(), - transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, - transaction.Index, transaction.Type, transaction.Value) + transaction.BlockNumber, + transaction.HeaderID, + transaction.TxHash, + transaction.CID, + transaction.Dst, + transaction.Src, + transaction.Index, + transaction.Type, + transaction.Value) if err != nil { return insertError{"eth.transaction_cids", err, w.db.InsertTxStm(), transaction} } @@ -92,7 +117,12 @@ ON CONFLICT (tx_id, header_id, block_number) DO NOTHING */ func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error { _, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(), - rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.PostState, + rct.BlockNumber, + rct.HeaderID, + rct.TxID, + rct.CID, + rct.Contract, + rct.PostState, rct.PostStatus) if err != nil { return insertError{"eth.receipt_cids", err, w.db.InsertRctStm(), *rct} @@ -108,8 +138,16 @@ ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error { for _, log := range logs { _, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(), - log.BlockNumber, log.HeaderID, log.CID, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1, - log.Topic2, log.Topic3) + log.BlockNumber, + log.HeaderID, + log.CID, + log.ReceiptID, + log.Address, + log.Index, + log.Topic0, + log.Topic1, + log.Topic2, + log.Topic3) if err != nil { return insertError{"eth.log_cids", err, w.db.InsertLogStm(), *log} } @@ -123,20 +161,24 @@ INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, remove ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING */ func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error { + balance := stateNode.Balance if stateNode.Removed { - _, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(), - stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Removed, true, - "0", stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot) - if err != nil { - return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode} - } - } else { - _, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(), - stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Removed, true, - stateNode.Balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot) - if err != nil { - return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode} - } + balance = "0" + } + _, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(), + stateNode.BlockNumber, + stateNode.HeaderID, + stateNode.StateKey, + stateNode.CID, + true, + balance, + stateNode.Nonce, + stateNode.CodeHash, + stateNode.StorageRoot, + stateNode.Removed, + ) + if err != nil { + return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode} } return nil } @@ -147,8 +189,15 @@ ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHI */ func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error { _, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(), - storageCID.BlockNumber, storageCID.HeaderID, storageCID.StateKey, storageCID.StorageKey, storageCID.CID, - storageCID.Removed, true, storageCID.Value) + storageCID.BlockNumber, + storageCID.HeaderID, + storageCID.StateKey, + storageCID.StorageKey, + storageCID.CID, + true, + storageCID.Value, + storageCID.Removed, + ) if err != nil { return insertError{"eth.storage_cids", err, w.db.InsertStorageStm(), storageCID} } diff --git a/statediff/indexer/shared/schema/schema.go b/statediff/indexer/shared/schema/schema.go new file mode 100644 index 000000000..151672790 --- /dev/null +++ b/statediff/indexer/shared/schema/schema.go @@ -0,0 +1,173 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package schema + +var TableIPLDBlock = Table{ + Name: `ipld.blocks`, + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "key", Type: Dtext}, + {Name: "data", Type: Dbytea}, + }, +} + +var TableNodeInfo = Table{ + Name: `public.nodes`, + Columns: []Column{ + {Name: "genesis_block", Type: Dvarchar}, + {Name: "network_id", Type: Dvarchar}, + {Name: "node_id", Type: Dvarchar}, + {Name: "client_name", Type: Dvarchar}, + {Name: "chain_id", Type: Dinteger}, + }, +} + +var TableHeader = Table{ + Name: "eth.header_cids", + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "block_hash", Type: Dvarchar}, + {Name: "parent_hash", Type: Dvarchar}, + {Name: "cid", Type: Dtext}, + {Name: "td", Type: Dnumeric}, + {Name: "node_ids", Type: Dvarchar, Array: true}, + {Name: "reward", Type: Dnumeric}, + {Name: "state_root", Type: Dvarchar}, + {Name: "tx_root", Type: Dvarchar}, + {Name: "receipt_root", Type: Dvarchar}, + {Name: "uncles_hash", Type: Dvarchar}, + {Name: "bloom", Type: Dbytea}, + {Name: "timestamp", Type: Dnumeric}, + {Name: "coinbase", Type: Dvarchar}, + }, + UpsertClause: OnConflict("block_number", "block_hash").Set( + "parent_hash", + "cid", + "td", + "node_ids", + "reward", + "state_root", + "tx_root", + "receipt_root", + "uncles_hash", + "bloom", + "timestamp", + "coinbase", + )} + +var TableStateNode = Table{ + Name: "eth.state_cids", + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "header_id", Type: Dvarchar}, + {Name: "state_leaf_key", Type: Dvarchar}, + {Name: "cid", Type: Dtext}, + {Name: "diff", Type: Dboolean}, + {Name: "balance", Type: Dnumeric}, + {Name: "nonce", Type: Dbigint}, + {Name: "code_hash", Type: Dvarchar}, + {Name: "storage_root", Type: Dvarchar}, + {Name: "removed", Type: Dboolean}, + }, + UpsertClause: OnConflict("block_number", "header_id", "state_leaf_key"), +} + +var TableStorageNode = Table{ + Name: "eth.storage_cids", + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "header_id", Type: Dvarchar}, + {Name: "state_leaf_key", Type: Dvarchar}, + {Name: "storage_leaf_key", Type: Dvarchar}, + {Name: "cid", Type: Dtext}, + {Name: "diff", Type: Dboolean}, + {Name: "val", Type: Dbytea}, + {Name: "removed", Type: Dboolean}, + }, + UpsertClause: OnConflict("block_number", "header_id", "state_leaf_key", "storage_leaf_key"), +} + +var TableUncle = Table{ + Name: "eth.uncle_cids", + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "block_hash", Type: Dvarchar}, + {Name: "header_id", Type: Dvarchar}, + {Name: "parent_hash", Type: Dvarchar}, + {Name: "cid", Type: Dtext}, + {Name: "reward", Type: Dnumeric}, + {Name: "index", Type: Dinteger}, + }, + UpsertClause: OnConflict("block_number", "block_hash"), +} + +var TableTransaction = Table{ + Name: "eth.transaction_cids", + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "header_id", Type: Dvarchar}, + {Name: "tx_hash", Type: Dvarchar}, + {Name: "cid", Type: Dtext}, + {Name: "dst", Type: Dvarchar}, + {Name: "src", Type: Dvarchar}, + {Name: "index", Type: Dinteger}, + {Name: "tx_type", Type: Dinteger}, + {Name: "value", Type: Dnumeric}, + }, + UpsertClause: OnConflict("block_number", "header_id", "tx_hash"), +} + +var TableReceipt = Table{ + Name: "eth.receipt_cids", + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "header_id", Type: Dvarchar}, + {Name: "tx_id", Type: Dvarchar}, + {Name: "cid", Type: Dtext}, + {Name: "contract", Type: Dvarchar}, + {Name: "post_state", Type: Dvarchar}, + {Name: "post_status", Type: Dinteger}, + }, + UpsertClause: OnConflict("block_number", "header_id", "tx_id"), +} + +var TableLog = Table{ + Name: "eth.log_cids", + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "header_id", Type: Dvarchar}, + {Name: "cid", Type: Dtext}, + {Name: "rct_id", Type: Dvarchar}, + {Name: "address", Type: Dvarchar}, + {Name: "index", Type: Dinteger}, + {Name: "topic0", Type: Dvarchar}, + {Name: "topic1", Type: Dvarchar}, + {Name: "topic2", Type: Dvarchar}, + {Name: "topic3", Type: Dvarchar}, + }, + UpsertClause: OnConflict("block_number", "header_id", "rct_id", "index"), +} + +var TableWatchedAddresses = Table{ + Name: "eth_meta.watched_addresses", + Columns: []Column{ + {Name: "address", Type: Dvarchar}, + {Name: "created_at", Type: Dbigint}, + {Name: "watched_at", Type: Dbigint}, + {Name: "last_filled_at", Type: Dbigint}, + }, +} diff --git a/statediff/indexer/shared/schema/table.go b/statediff/indexer/shared/schema/table.go new file mode 100644 index 000000000..9bc19ac3d --- /dev/null +++ b/statediff/indexer/shared/schema/table.go @@ -0,0 +1,147 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package schema + +import ( + "fmt" + "strings" + + "github.com/thoas/go-funk" +) + +type colType int + +const ( + Dinteger colType = iota + Dboolean + Dbigint + Dnumeric + Dbytea + Dvarchar + Dtext +) + +type ConflictClause struct { + Target []string + Update []string +} + +type Column struct { + Name string + Type colType + Array bool +} +type Table struct { + Name string + Columns []Column + + UpsertClause ConflictClause +} + +type colfmt = func(interface{}) string + +func (tbl *Table) ToCsvRow(args ...interface{}) []string { + var row []string + for i, col := range tbl.Columns { + value := col.Type.formatter()(args[i]) + + if col.Array { + valueList := funk.Map(args[i], col.Type.formatter()).([]string) + value = fmt.Sprintf("{%s}", strings.Join(valueList, ",")) + } + + row = append(row, value) + } + return row +} + +func (tbl *Table) VarcharColumns() []string { + columns := funk.Filter(tbl.Columns, func(col Column) bool { + return col.Type == Dvarchar + }).([]Column) + + columnNames := funk.Map(columns, func(col Column) string { + return col.Name + }).([]string) + return columnNames +} + +func OnConflict(target ...string) ConflictClause { + return ConflictClause{Target: target} +} +func (c ConflictClause) Set(fields ...string) ConflictClause { + c.Update = fields + return c +} + +// ToInsertStatement returns a Postgres-compatible SQL insert statement for the table +// using positional placeholders +func (tbl *Table) ToInsertStatement(upsert bool) string { + var colnames, placeholders []string + for i, col := range tbl.Columns { + colnames = append(colnames, col.Name) + placeholders = append(placeholders, fmt.Sprintf("$%d", i+1)) + } + suffix := fmt.Sprintf("ON CONFLICT (%s)", strings.Join(tbl.UpsertClause.Target, ", ")) + if upsert && len(tbl.UpsertClause.Update) != 0 { + var update_placeholders []string + for _, name := range tbl.UpsertClause.Update { + i := funk.IndexOf(tbl.Columns, func(col Column) bool { return col.Name == name }) + update_placeholders = append(update_placeholders, fmt.Sprintf("$%d", i+1)) + } + suffix += fmt.Sprintf( + " DO UPDATE SET (%s) = (%s)", + strings.Join(tbl.UpsertClause.Update, ", "), strings.Join(update_placeholders, ", "), + ) + } else { + suffix += " DO NOTHING" + } + + return fmt.Sprintf( + "INSERT INTO %s (%s) VALUES (%s) %s", + tbl.Name, strings.Join(colnames, ", "), strings.Join(placeholders, ", "), suffix, + ) +} + +func sprintf(f string) colfmt { + return func(x interface{}) string { return fmt.Sprintf(f, x) } +} + +func (typ colType) formatter() colfmt { + switch typ { + case Dinteger: + return sprintf("%d") + case Dboolean: + return func(x interface{}) string { + if x.(bool) { + return "t" + } + return "f" + } + case Dbigint: + return sprintf("%s") + case Dnumeric: + return sprintf("%s") + case Dbytea: + return sprintf(`\x%x`) + case Dvarchar: + return sprintf("%s") + case Dtext: + return sprintf("%s") + } + panic("unreachable") +} diff --git a/statediff/indexer/shared/schema/table_test.go b/statediff/indexer/shared/schema/table_test.go new file mode 100644 index 000000000..b38ef6e07 --- /dev/null +++ b/statediff/indexer/shared/schema/table_test.go @@ -0,0 +1,56 @@ +package schema_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + . "github.com/ethereum/go-ethereum/statediff/indexer/shared/schema" +) + +var testHeaderTable = Table{ + Name: "eth.header_cids", + Columns: []Column{ + {Name: "block_number", Type: Dbigint}, + {Name: "block_hash", Type: Dvarchar}, + {Name: "parent_hash", Type: Dvarchar}, + {Name: "cid", Type: Dtext}, + {Name: "td", Type: Dnumeric}, + {Name: "node_id", Type: Dvarchar}, + {Name: "reward", Type: Dnumeric}, + {Name: "state_root", Type: Dvarchar}, + {Name: "tx_root", Type: Dvarchar}, + {Name: "receipt_root", Type: Dvarchar}, + {Name: "uncle_root", Type: Dvarchar}, + {Name: "bloom", Type: Dbytea}, + {Name: "timestamp", Type: Dnumeric}, + {Name: "mh_key", Type: Dtext}, + {Name: "times_validated", Type: Dinteger}, + {Name: "coinbase", Type: Dvarchar}, + }, + UpsertClause: OnConflict("block_hash", "block_number").Set( + "parent_hash", + "cid", + "td", + "node_id", + "reward", + "state_root", + "tx_root", + "receipt_root", + "uncle_root", + "bloom", + "timestamp", + "mh_key", + "times_validated", + "coinbase", + )} + +func TestTable(t *testing.T) { + + headerUpsert := `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)` + + headerNoUpsert := `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT (block_hash, block_number) DO NOTHING` + + require.Equal(t, headerNoUpsert, testHeaderTable.ToInsertStatement(false)) + require.Equal(t, headerUpsert, testHeaderTable.ToInsertStatement(true)) +} diff --git a/statediff/test_helpers/helpers.go b/statediff/test_helpers/helpers.go index 64d5b72cc..5e3823be2 100644 --- a/statediff/test_helpers/helpers.go +++ b/statediff/test_helpers/helpers.go @@ -50,15 +50,15 @@ func TestSelfDestructChainGen(i int, block *core.BlockGen) { signer := types.HomesteadSigner{} switch i { case 0: - // Block 1 is mined by Account1Addr - // Account1Addr creates a new contract + // Block 1 is mined by TestBankAddress + // TestBankAddress creates a new contract block.SetCoinbase(TestBankAddress) tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(params.GWei), ContractCode), signer, TestBankKey) ContractAddr = crypto.CreateAddress(TestBankAddress, 0) block.AddTx(tx) case 1: - // Block 2 is mined by Account1Addr - // Account1Addr self-destructs the contract + // Block 2 is mined by TestBankAddress + // TestBankAddress self-destructs the contract block.SetCoinbase(TestBankAddress) data := common.Hex2Bytes("43D726D6") tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.GWei), data), signer, TestBankKey) diff --git a/statediff/types/types.go b/statediff/types/types.go index 9374e1d30..11287cd1b 100644 --- a/statediff/types/types.go +++ b/statediff/types/types.go @@ -39,7 +39,7 @@ type StateObject struct { // AccountMap is a mapping of hex encoded path => account wrapper type AccountMap map[string]AccountWrapper -// AccountWrapper is used to temporary associate the unpacked node with its raw values +// AccountWrapper is used to temporarily associate the unpacked node with its raw values type AccountWrapper struct { Account *types.StateAccount LeafKey []byte