diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 0770355a..c139e829 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -1,7 +1,6 @@ name: Docker Build on: [pull_request] - jobs: build: name: Run docker build @@ -14,25 +13,49 @@ jobs: name: Run unit tests env: GOPATH: /tmp/go - strategy: - matrix: - go-version: [1.16.x, 1.17.x] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + # To run the unit tests you need to add secrets to your repository. + BUILD_HOSTNAME: ${{ secrets.BUILD_HOSTNAME }} + BUILD_USERNAME: ${{ secrets.BUILD_USERNAME }} + BUILD_KEY: ${{ secrets.BUILD_KEY }} + #strategy: + # matrix: + # go-version: [1.16.x, 1.17.x] + runs-on: ubuntu-latest steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - uses: actions/checkout@v2 - - name: Run database - run: docker-compose up -d ipld-eth-db - - name: Test + + # Passed experience with GHA has taught me to store variables in files instead of passing them as variables. + - name: Output variables to files run: | - sleep 10 - PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test + echo $GITHUB_REPOSITORY > /tmp/git_repository + echo $GITHUB_HEAD_REF > /tmp/git_head_ref + echo "-----BEGIN OPENSSH PRIVATE KEY-----" >> /tmp/key + echo ${{ env.BUILD_KEY }} >> /tmp/key + echo "-----END OPENSSH PRIVATE KEY-----" >> /tmp/key + chmod 400 /tmp/key + cat /tmp/git_repository + cat /tmp/git_head_ref + + - name: Raw SCP + run: | + scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_repository ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_repository + scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_head_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_head_ref + scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key .github/workflows/run_unit_test.sh ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/run_unit_test.sh + + - name: Trigger Unit Test + run: | + ssh -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }} chmod +x /tmp/run_unit_test.sh /tmp/run_unit_test.sh + ssh -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }} /tmp/run_unit_test.sh + + - name: Get the logs and cat them + run: | + scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/test.log . + cat ./test.log + + - name: Check Error Code + run: | + scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/return_test.txt . + [ $(cat ./return_test.txt) -eq 0 ] integrationtest: name: Run integration tests @@ -73,8 +96,8 @@ jobs: ETH_HTTP_PATH: "dapptools:8545" strategy: matrix: - go-version: [ 1.16.x ] - os: [ ubuntu-latest ] + go-version: [1.16.x] + os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: - name: Create GOPATH diff --git a/.github/workflows/run_unit_test.sh b/.github/workflows/run_unit_test.sh new file mode 100755 index 00000000..8b80c5ab --- /dev/null +++ b/.github/workflows/run_unit_test.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +# Set up repo +start_dir=$(pwd) +temp_dir=$(mktemp -d) +cd $temp_dir +git clone -b $(cat /tmp/git_head_ref) "https://github.com/$(cat /tmp/git_repository).git" +cd ipld-eth-server + +## Remove the branch and github related info. This way future runs wont be confused. +rm -f /tmp/git_head_ref /tmp/git_repository + +# Spin up DB +docker-compose up -d ipld-eth-db +trap "docker-compose down --remove-orphans; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR +sleep 10 + +# Remove old logs so there's no confusion, then run test +rm -f /tmp/test.log /tmp/return_test.txt +PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test > /tmp/test.log +echo $? > /tmp/return_test.txt + +# Clean up +docker-compose down -v --remove-orphans +cd $start_dir +rm -fr $temp_dir + diff --git a/cmd/validate.go b/cmd/validate.go index 1fcd5a82..4b61eeb7 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -57,7 +57,7 @@ func validate() { stateRoot := common.HexToHash(stateRootStr) cacheSize := viper.GetInt("cacheSize") - ethDB := ipfsethdb.NewDatabase(config.DB.DB, ipfsethdb.CacheConfig{ + ethDB := ipfsethdb.NewDatabase(config.DB, ipfsethdb.CacheConfig{ Name: GroupName, Size: cacheSize * 1024 * 1024, ExpiryDuration: time.Minute * time.Duration(CacheExpiryInMins), diff --git a/docker-compose.yml b/docker-compose.yml index fe0baff1..cef19e6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: restart: unless-stopped depends_on: - ipld-eth-db - image: vulcanize/dapptools:v0.30.0-v1.10.14-statediff-0.0.29 + image: vulcanize/dapptools:v0.30.0-v1.10.16-statediff-3.0.2a environment: DB_USER: vdbm DB_NAME: vulcanize_testing @@ -13,13 +13,15 @@ services: DB_PORT: 5432 DB_PASSWORD: password DB_WRITE: $DB_WRITE + DB_TYPE: postgres + DB_DRIVER: sqlx ports: - "127.0.0.1:8545:8545" - "127.0.0.1:8546:8546" ipld-eth-db: restart: always - image: vulcanize/ipld-eth-db:v0.2.0 + image: vulcanize/ipld-eth-db:v3.0.7 environment: POSTGRES_USER: "vdbm" POSTGRES_DB: "vulcanize_testing" diff --git a/go.mod b/go.mod index 1fdcfb72..4f17ee96 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module github.com/vulcanize/ipld-eth-server go 1.15 require ( - github.com/ethereum/go-ethereum v1.10.15 + github.com/ethereum/go-ethereum v1.10.16 github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 + github.com/graph-gophers/graphql-go v1.3.0 github.com/ipfs/go-block-format v0.0.3 github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-ipfs-blockstore v1.0.1 @@ -18,7 +18,6 @@ require ( github.com/machinebox/graphql v0.2.2 github.com/mailgun/groupcache/v2 v2.2.1 github.com/matryer/is v1.4.0 // indirect - github.com/mattn/go-sqlite3 v1.14.9 // indirect github.com/multiformats/go-multihash v0.0.15 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.13.0 @@ -36,4 +35,4 @@ require ( golang.org/x/tools v0.1.8 // indirect ) -replace github.com/ethereum/go-ethereum v1.10.15 => github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0 +replace github.com/ethereum/go-ethereum v1.10.16 => github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2 diff --git a/go.sum b/go.sum index c009ba5f..b72e93a6 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -183,6 +185,10 @@ github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3h github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cockroach-go/v2 v2.0.3 h1:ZA346ACHIZctef6trOTwBAEvPVm1k0uLm/bb2Atc+S8= +github.com/cockroachdb/cockroach-go/v2 v2.0.3/go.mod h1:hAuDgiVgDVkfirP9JnhXEfcXEPRKBpYdGz+l7mvYSzw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= @@ -198,6 +204,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -220,12 +227,14 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= @@ -266,6 +275,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/ethereum/go-ethereum v1.9.11/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ= github.com/ethereum/go-ethereum v1.10.14/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= @@ -296,6 +306,8 @@ github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5 github.com/gabriel-vasile/mimetype v1.1.2/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/georgysavva/scany v0.2.9 h1:Xt6rjYpHnMClTm/g+oZTnoSxUwiln5GqMNU+QeLNHQU= +github.com/georgysavva/scany v0.2.9/go.mod h1:yeOeC1BdIdl6hOwy8uefL2WNSlseFzbhlG/frrh65SA= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -334,7 +346,10 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8Wd github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -343,6 +358,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -447,8 +463,9 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graphql-go/graphql v0.7.9 h1:5Va/Rt4l5g3YjwDnid3vFfn43faaQBq7rMcIZ0VnV34= github.com/graphql-go/graphql v0.7.9/go.mod h1:k6yrAYQaSP59DC5UVxbgxESlmVyojThKdORUqGDGmrI= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -728,6 +745,78 @@ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvB github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.12.2 h1:StIquYvKIRuSEAtjJDr39fyzBtziioHPwVC75tBiXzo= github.com/ipld/go-ipld-prime v0.12.2/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78= +github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7JMeA= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -751,6 +840,9 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/copier v0.2.4 h1:dT3tI+8GzU8DjJFCj9mLYtjfRtUmK7edauduQdcZCpI= github.com/jinzhu/copier v0.2.4/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro= +github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -782,6 +874,7 @@ github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -799,6 +892,7 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -811,6 +905,7 @@ github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -819,6 +914,11 @@ github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4F github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= @@ -1184,6 +1284,7 @@ github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1193,6 +1294,7 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -1206,9 +1308,8 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= -github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= +github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -1470,11 +1571,15 @@ github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9Ac github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= @@ -1484,6 +1589,11 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc= github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v0.0.0-20200419222939-1884f454f8ea/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -1508,6 +1618,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= @@ -1561,6 +1672,8 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1606,8 +1719,8 @@ github.com/vulcanize/eth-ipfs-state-validator v0.1.0 h1:ZB54GOUrxQeSYvOmqk8jMgtG github.com/vulcanize/eth-ipfs-state-validator v0.1.0/go.mod h1:QtmfhqDjN29UHHk2Fb+ouzO4j/lbhM7GuiEmxHIKdzk= github.com/vulcanize/gap-filler v0.3.1 h1:N5d+jCJo/VTWFvBSbTD7biRhK/OqDZzi1tgA85SIBKs= github.com/vulcanize/gap-filler v0.3.1/go.mod h1:qowG1cgshVpBqMokiWro/1xhh0uypw7oAu8FQ42JMy4= -github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0 h1:/BiYPUHnubh46YVtASGt4MPlFR96Rc+iJuTyOI8KZa4= -github.com/vulcanize/go-ethereum v1.10.15-statediff-2.0.0/go.mod h1:9L+QY31AnWnX2/2HDOySCjQoYUdWNGBRMezFJVfH73E= +github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2 h1:H3SLHZdvTyKYbFc1CO2b8A9XF3BcakcXtvThKPbgT8k= +github.com/vulcanize/go-ethereum v1.10.16-statediff-3.0.2/go.mod h1:NI+tCVeIQBPrMfJUZvTLjhCieb7CZcmNPbJVlXbncxU= github.com/vulcanize/ipfs-ethdb v0.0.6 h1:Jl+YHtee5Zd8jD9wix2aJLYXwX/WpPt37QBzoz8pVwM= github.com/vulcanize/ipfs-ethdb v0.0.6/go.mod h1:DvZpevG89N5ST26WeaErQ+31Q1lQwGBEhn/s9wgGo3g= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= @@ -1651,6 +1764,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -1720,6 +1834,8 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1728,20 +1844,27 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1885,6 +2008,7 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1995,12 +2119,14 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2046,6 +2172,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2173,6 +2301,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index 203a7572..38c42161 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -19,7 +19,6 @@ package eth_test import ( "context" "math/big" - "os" "strconv" "github.com/ethereum/go-ethereum/common" @@ -31,10 +30,9 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff/indexer" - "github.com/ethereum/go-ethereum/statediff/indexer/node" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/jmoiron/sqlx" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -183,22 +181,9 @@ var ( } ) -// SetupDB is use to setup a db for watcher tests -func SetupDB() (*postgres.DB, error) { - port, _ := strconv.Atoi(os.Getenv("DATABASE_PORT")) - uri := postgres.DbConnectionString(postgres.ConnectionParams{ - User: os.Getenv("DATABASE_USER"), - Password: os.Getenv("DATABASE_PASSWORD"), - Hostname: os.Getenv("DATABASE_HOSTNAME"), - Name: os.Getenv("DATABASE_NAME"), - Port: port, - }) - return postgres.NewDB(uri, postgres.ConnectionConfig{}, node.Info{}) -} - var _ = Describe("API", func() { var ( - db *postgres.DB + db *sqlx.DB api *eth.PublicEthAPI chainConfig = params.TestChainConfig ) @@ -207,14 +192,12 @@ var _ = Describe("API", func() { It("test init", func() { var ( err error - tx *indexer.BlockTx + tx interfaces.Batch ) - db, err = SetupDB() - Expect(err).ToNot(HaveOccurred()) + db = eth.SetupTestDB() + indexAndPublisher := eth.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash()) - indexAndPublisher, err := indexer.NewStateDiffIndexer(chainConfig, db) - Expect(err).ToNot(HaveOccurred()) backend, err := eth.NewEthBackend(db, ð.Config{ ChainConfig: chainConfig, VMConfig: vm.Config{}, @@ -233,11 +216,6 @@ var _ = Describe("API", func() { tx, err = indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty()) Expect(err).ToNot(HaveOccurred()) - for _, node := range test_helpers.MockStateNodes { - err = indexAndPublisher.PushStateNode(tx, node) - Expect(err).ToNot(HaveOccurred()) - } - ccHash := sdtypes.CodeAndCodeHash{ Hash: test_helpers.ContractCodeHash, Code: test_helpers.ContractCode, @@ -246,7 +224,12 @@ var _ = Describe("API", func() { err = indexAndPublisher.PushCodeAndCodeHash(tx, ccHash) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + for _, node := range test_helpers.MockStateNodes { + err = indexAndPublisher.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String()) + Expect(err).ToNot(HaveOccurred()) + } + + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) uncles := test_helpers.MockBlock.Uncles() @@ -258,18 +241,17 @@ var _ = Describe("API", func() { // setting chain config to for london block chainConfig.LondonBlock = big.NewInt(2) - indexAndPublisher, err = indexer.NewStateDiffIndexer(chainConfig, db) - Expect(err).ToNot(HaveOccurred()) + indexAndPublisher = eth.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash()) tx, err = indexAndPublisher.PushBlock(test_helpers.MockLondonBlock, test_helpers.MockLondonReceipts, test_helpers.MockLondonBlock.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) }) // Single test db tear down at end of all tests - defer It("test teardown", func() { eth.TearDownDB(db) }) + defer It("test teardown", func() { eth.TearDownTestDB(db) }) /* Headers and blocks @@ -287,8 +269,6 @@ var _ = Describe("API", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) Expect(header).To(BeNil()) - _, err = api.B.DB.Beginx() - Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 3d9485a4..424c600f 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -39,14 +39,15 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" - ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" + "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/trie" + "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" validator "github.com/vulcanize/eth-ipfs-state-validator/pkg" ipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres" + ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" + "github.com/vulcanize/ipld-eth-server/pkg/shared" ) @@ -64,24 +65,24 @@ var ( const ( RetrieveCanonicalBlockHashByNumber = `SELECT block_hash FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) - WHERE id = (SELECT canonical_header_id($1))` + WHERE block_hash = (SELECT canonical_header_hash($1))` RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) - WHERE id = (SELECT canonical_header_id($1))` - RetrieveTD = `SELECT td FROM eth.header_cids + WHERE block_hash = (SELECT canonical_header_hash($1))` + RetrieveTD = `SELECT CAST(td as Text) FROM eth.header_cids WHERE header_cids.block_hash = $1` RetrieveRPCTransaction = `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids WHERE blocks.key = transaction_cids.mh_key - AND transaction_cids.header_id = header_cids.id + AND transaction_cids.header_id = header_cids.block_hash AND transaction_cids.tx_hash = $1` RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids - WHERE state_accounts.state_id = state_cids.id - AND state_cids.header_id = header_cids.id + WHERE state_accounts.header_id = state_cids.header_id AND state_accounts.state_path = state_cids.state_path + AND state_cids.header_id = header_cids.block_hash AND state_leaf_key = $1 AND block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = $2) - AND header_cids.id = (SELECT canonical_header_id(block_number)) + AND header_cids.block_hash = (SELECT canonical_header_hash(block_number)) ORDER BY block_number DESC LIMIT 1` RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` @@ -93,7 +94,7 @@ const ( type Backend struct { // underlying postgres db - DB *postgres.DB + DB *sqlx.DB // postgres db interfaces Retriever *CIDRetriever @@ -115,7 +116,7 @@ type Config struct { GroupCacheConfig *shared.GroupCacheConfig } -func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) { +func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) { gcc := c.GroupCacheConfig groupName := gcc.StateDB.Name @@ -124,7 +125,7 @@ func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) { } r := NewCIDRetriever(db) - ethDB := ipfsethdb.NewDatabase(db.DB, ipfsethdb.CacheConfig{ + ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{ Name: groupName, Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024, ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins), @@ -313,17 +314,17 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber } defer func() { if p := recover(); p != nil { - ethServerShared.Rollback(tx) + shared.Rollback(tx) panic(p) } else if err != nil { - ethServerShared.Rollback(tx) + shared.Rollback(tx) } else { err = tx.Commit() } }() // Fetch and decode the header IPLD - var headerIPLD ipfs.BlockModel + var headerIPLD models.IPLDModel headerIPLD, err = b.Fetcher.FetchHeader(tx, headerCID) if err != nil { if err == sql.ErrNoRows { @@ -337,7 +338,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber return nil, err } // Fetch and decode the uncle IPLDs - var uncleIPLDs []ipfs.BlockModel + var uncleIPLDs []models.IPLDModel uncleIPLDs, err = b.Fetcher.FetchUncles(tx, uncleCIDs) if err != nil { return nil, err @@ -352,7 +353,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber uncles = append(uncles, &uncle) } // Fetch and decode the transaction IPLDs - var txIPLDs []ipfs.BlockModel + var txIPLDs []models.IPLDModel txIPLDs, err = b.Fetcher.FetchTrxs(tx, txCIDs) if err != nil { return nil, err @@ -367,7 +368,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber transactions = append(transactions, &transaction) } // Fetch and decode the receipt IPLDs - var rctIPLDs []ipfs.BlockModel + var rctIPLDs []models.IPLDModel rctIPLDs, err = b.Fetcher.FetchRcts(tx, rctCIDs) if err != nil { return nil, err @@ -409,17 +410,17 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo } defer func() { if p := recover(); p != nil { - ethServerShared.Rollback(tx) + shared.Rollback(tx) panic(p) } else if err != nil { - ethServerShared.Rollback(tx) + shared.Rollback(tx) } else { err = tx.Commit() } }() // Fetch and decode the header IPLD - var headerIPLD ipfs.BlockModel + var headerIPLD models.IPLDModel headerIPLD, err = b.Fetcher.FetchHeader(tx, headerCID) if err != nil { if err == sql.ErrNoRows { @@ -433,7 +434,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo return nil, err } // Fetch and decode the uncle IPLDs - var uncleIPLDs []ipfs.BlockModel + var uncleIPLDs []models.IPLDModel uncleIPLDs, err = b.Fetcher.FetchUncles(tx, uncleCIDs) if err != nil { if err == sql.ErrNoRows { @@ -451,7 +452,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo uncles = append(uncles, &uncle) } // Fetch and decode the transaction IPLDs - var txIPLDs []ipfs.BlockModel + var txIPLDs []models.IPLDModel txIPLDs, err = b.Fetcher.FetchTrxs(tx, txCIDs) if err != nil { if err == sql.ErrNoRows { @@ -469,7 +470,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo transactions = append(transactions, &transaction) } // Fetch and decode the receipt IPLDs - var rctIPLDs []ipfs.BlockModel + var rctIPLDs []models.IPLDModel rctIPLDs, err = b.Fetcher.FetchRcts(tx, rctCIDs) if err != nil { if err == sql.ErrNoRows { @@ -735,10 +736,10 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has } defer func() { if p := recover(); p != nil { - ethServerShared.Rollback(tx) + shared.Rollback(tx) panic(p) } else if err != nil { - ethServerShared.Rollback(tx) + shared.Rollback(tx) } else { err = tx.Commit() } diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go index 4b3a193b..b0875382 100644 --- a/pkg/eth/cid_retriever.go +++ b/pkg/eth/cid_retriever.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/jmoiron/sqlx" "github.com/lib/pq" log "github.com/sirupsen/logrus" @@ -40,11 +39,11 @@ type Retriever interface { // CIDRetriever satisfies the CIDRetriever interface for ethereum type CIDRetriever struct { - db *postgres.DB + db *sqlx.DB } // NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface -func NewCIDRetriever(db *postgres.DB) *CIDRetriever { +func NewCIDRetriever(db *sqlx.DB) *CIDRetriever { return &CIDRetriever{ db: db, } @@ -102,7 +101,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64 if filter.HeaderFilter.Uncles { // Retrieve uncle cids for this header id var uncleCIDs []models.UncleModel - uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, header.ID) + uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, header.BlockHash) if err != nil { log.Error("uncle cid retrieval error") return nil, true, err @@ -112,7 +111,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64 } // Retrieve cached trx CIDs if !filter.TxFilter.Off { - cw.Transactions, err = ecr.RetrieveTxCIDs(tx, filter.TxFilter, header.ID) + cw.Transactions, err = ecr.RetrieveTxCIDs(tx, filter.TxFilter, header.BlockHash) if err != nil { log.Error("transaction cid retrieval error") return nil, true, err @@ -121,13 +120,13 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64 empty = false } } - trxIds := make([]int64, len(cw.Transactions)) - for j, tx := range cw.Transactions { - trxIds[j] = tx.ID + trxHashes := make([]string, len(cw.Transactions)) + for j, t := range cw.Transactions { + trxHashes[j] = t.TxHash } // Retrieve cached receipt CIDs if !filter.ReceiptFilter.Off { - cw.Receipts, err = ecr.RetrieveRctCIDsByHeaderID(tx, filter.ReceiptFilter, header.ID, trxIds) + cw.Receipts, err = ecr.RetrieveRctCIDsByHeaderID(tx, filter.ReceiptFilter, header.BlockHash, trxHashes) if err != nil { log.Error("receipt cid retrieval error") return nil, true, err @@ -138,7 +137,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64 } // Retrieve cached state CIDs if !filter.StateFilter.Off { - cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, filter.StateFilter, header.ID) + cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, filter.StateFilter, header.BlockHash) if err != nil { log.Error("state cid retrieval error") return nil, true, err @@ -149,7 +148,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64 } // Retrieve cached storage CIDs if !filter.StorageFilter.Off { - cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, filter.StorageFilter, header.ID) + cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, filter.StorageFilter, header.BlockHash) if err != nil { log.Error("storage cid retrieval error") return nil, true, err @@ -168,32 +167,33 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64 func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]models.HeaderModel, error) { log.Debug("retrieving header cids for block ", blockNumber) headers := make([]models.HeaderModel, 0) - pgStr := `SELECT * FROM eth.header_cids + pgStr := `SELECT CAST(block_number as Text), block_hash,parent_hash,cid,mh_key,CAST(td as Text),node_id, + CAST(reward as Text), state_root,uncle_root,tx_root,receipt_root,bloom,timestamp,times_validated, + coinbase FROM eth.header_cids WHERE block_number = $1` return headers, tx.Select(&headers, pgStr, blockNumber) } // RetrieveUncleCIDsByHeaderID retrieves and returns all of the uncle cids for the provided header -func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]models.UncleModel, error) { +func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID string) ([]models.UncleModel, error) { log.Debug("retrieving uncle cids for block id ", headerID) headers := make([]models.UncleModel, 0) - pgStr := `SELECT * FROM eth.uncle_cids + pgStr := `SELECT header_id,block_hash,parent_hash,cid,mh_key, CAST(reward as text) FROM eth.uncle_cids WHERE header_id = $1` return headers, tx.Select(&headers, pgStr, headerID) } // RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters // also returns the ids for the returned transaction cids -func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]models.TxModel, error) { +func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID string) ([]models.TxModel, error) { log.Debug("retrieving transaction cids for header id ", headerID) args := make([]interface{}, 0, 3) results := make([]models.TxModel, 0) id := 1 - pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id, - transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.mh_key, - transaction_cids.dst, transaction_cids.src, transaction_cids.index, transaction_cids.tx_data - FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) - WHERE header_cids.id = $%d`, id) + pgStr := fmt.Sprintf(`SELECT transaction_cids.tx_hash, transaction_cids.header_id,transaction_cids.cid, transaction_cids.mh_key, + transaction_cids.dst, transaction_cids.src, transaction_cids.index, transaction_cids.tx_data + FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) + WHERE header_cids.block_hash = $%d`, id) args = append(args, headerID) id++ if len(txFilter.Dst) > 0 { @@ -242,9 +242,9 @@ func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter Rec return pgStr, args } -func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, trxIds []int64) (string, []interface{}) { - rctCond := " AND (receipt_cids.id = ANY ( " - logQuery := "SELECT receipt_id FROM eth.log_cids WHERE" +func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) { + rctCond := " AND (receipt_cids.tx_id = ANY ( " + logQuery := "SELECT rct_id FROM eth.log_cids WHERE" if len(rctFilter.LogAddresses) > 0 { // Filter on log contract addresses if there are any pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id) @@ -258,10 +258,10 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte pgStr += ")" - // Filter on txIDs if there are any and we are matching txs - if rctFilter.MatchTxs && len(trxIds) > 0 { - pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, *id) - args = append(args, pq.Array(trxIds)) + // Filter on txHashes if there are any, and we are matching txs + if rctFilter.MatchTxs && len(txHashes) > 0 { + pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) } pgStr += ")" } else { // If there are no contract addresses to filter on @@ -270,17 +270,17 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte pgStr += rctCond + logQuery pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true) pgStr += ")" - // Filter on txIDs if there are any and we are matching txs - if rctFilter.MatchTxs && len(trxIds) > 0 { - pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, *id) - args = append(args, pq.Array(trxIds)) + // Filter on txHashes if there are any, and we are matching txs + if rctFilter.MatchTxs && len(txHashes) > 0 { + pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) } pgStr += ")" - } else if rctFilter.MatchTxs && len(trxIds) > 0 { + } else if rctFilter.MatchTxs && len(txHashes) > 0 { // If there are no contract addresses or topics to filter on, - // Filter on txIDs if there are any and we are matching txs - pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, *id) - args = append(args, pq.Array(trxIds)) + // Filter on txHashes if there are any, and we are matching txs + pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) } } @@ -289,19 +289,19 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte // RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided // filter parameters and correspond to the provided tx ids -func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]models.ReceiptModel, error) { +func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID string, trxHashes []string) ([]models.ReceiptModel, error) { log.Debug("retrieving receipt cids for header id ", headerID) args := make([]interface{}, 0, 4) - pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, + pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.contract, receipt_cids.contract_hash FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids - WHERE receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id - AND header_cids.id = $1` + WHERE receipt_cids.tx_id = transaction_cids.tx_hash + AND transaction_cids.header_id = header_cids.block_hash + AND header_cids.block_hash = $1` id := 2 args = append(args, headerID) - pgStr, args = receiptFilterConditions(&id, pgStr, args, rctFilter, trxIds) + pgStr, args = receiptFilterConditions(&id, pgStr, args, rctFilter, trxHashes) pgStr += ` ORDER BY transaction_cids.index` receiptCids := make([]models.ReceiptModel, 0) @@ -314,13 +314,13 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF log.Debug("retrieving log cids for receipt ids") args := make([]interface{}, 0, 4) id := 1 - pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id, - eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, + pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, + eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks - WHERE eth.log_cids.receipt_id = receipt_cids.id - AND receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id + WHERE eth.log_cids.rct_id = receipt_cids.tx_id + AND receipt_cids.tx_id = transaction_cids.tx_hash + AND transaction_cids.header_id = header_cids.block_hash AND log_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1` args = append(args, blockHash.String()) @@ -343,14 +343,14 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) { log.Debug("retrieving log cids for receipt ids") args := make([]interface{}, 0, 4) - pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id, - eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, - eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, - header_cids.block_hash, header_cids.block_number - FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids - WHERE eth.log_cids.receipt_id = receipt_cids.id - AND receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id` + pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, + eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, + eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, + header_cids.block_hash, CAST(header_cids.block_number as Text) + FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE eth.log_cids.rct_id = receipt_cids.tx_id + AND receipt_cids.tx_id = transaction_cids.tx_hash + AND transaction_cids.header_id = header_cids.block_hash` id := 1 if blockNumber > 0 { pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id) @@ -377,13 +377,13 @@ func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilte // RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided // filter parameters and correspond to the provided tx ids -func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]models.ReceiptModel, error) { +func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, txHashes []string) ([]models.ReceiptModel, error) { log.Debug("retrieving receipt cids for block ", blockNumber) args := make([]interface{}, 0, 5) - pgStr := `SELECT receipt_cids.id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id + pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids - WHERE receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id` + WHERE receipt_cids.tx_id = transaction_cids.tx_hash + AND transaction_cids.header_id = header_cids.block_hash` id := 1 if blockNumber > 0 { pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id) @@ -396,7 +396,7 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b id++ } - pgStr, args = receiptFilterConditions(&id, pgStr, args, rctFilter, trxIds) + pgStr, args = receiptFilterConditions(&id, pgStr, args, rctFilter, txHashes) pgStr += ` ORDER BY transaction_cids.index` receiptCids := make([]models.ReceiptModel, 0) @@ -413,13 +413,13 @@ func hasTopics(topics [][]string) bool { } // RetrieveStateCIDs retrieves and returns all of the state node cids at the provided header ID that conform to the provided filter parameters -func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]models.StateNodeModel, error) { +func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID string) ([]models.StateNodeModel, error) { log.Debug("retrieving state cids for header id ", headerID) args := make([]interface{}, 0, 2) - pgStr := `SELECT state_cids.id, state_cids.header_id, + pgStr := `SELECT state_cids.header_id, state_cids.state_leaf_key, state_cids.node_type, state_cids.cid, state_cids.mh_key, state_cids.state_path - FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE header_cids.id = $1` + FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) + WHERE header_cids.block_hash = $1` args = append(args, headerID) addrLen := len(stateFilter.Addresses) if addrLen > 0 { @@ -438,15 +438,15 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, } // RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided header id that conform to the provided filter parameters -func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]models.StorageNodeWithStateKeyModel, error) { +func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID string) ([]models.StorageNodeWithStateKeyModel, error) { log.Debug("retrieving storage cids for header id ", headerID) args := make([]interface{}, 0, 3) - pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_leaf_key, storage_cids.node_type, - storage_cids.cid, storage_cids.mh_key, storage_cids.storage_path, state_cids.state_leaf_key + pgStr := `SELECT storage_cids.header_id, storage_cids.storage_leaf_key, storage_cids.node_type, + storage_cids.cid, storage_cids.mh_key, storage_cids.storage_path, storage_cids.state_path, state_cids.state_leaf_key FROM eth.storage_cids, eth.state_cids, eth.header_cids - WHERE storage_cids.state_id = state_cids.id - AND state_cids.header_id = header_cids.id - AND header_cids.id = $1` + WHERE storage_cids.header_id = state_cids.header_id AND storage_cids.state_path = state_cids.state_path + AND state_cids.header_id = header_cids.block_hash + AND header_cids.block_hash = $1` args = append(args, headerID) id := 2 addrLen := len(storageFilter.Addresses) @@ -497,23 +497,23 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (models.Head return models.HeaderModel{}, nil, nil, nil, err } var uncleCIDs []models.UncleModel - uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID) + uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.BlockHash) if err != nil { log.Error("uncle cid retrieval error") return models.HeaderModel{}, nil, nil, nil, err } var txCIDs []models.TxModel - txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID) + txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.BlockHash) if err != nil { log.Error("tx cid retrieval error") return models.HeaderModel{}, nil, nil, nil, err } - txIDs := make([]int64, len(txCIDs)) + txHashes := make([]string, len(txCIDs)) for i, txCID := range txCIDs { - txIDs[i] = txCID.ID + txHashes[i] = txCID.TxHash } var rctCIDs []models.ReceiptModel - rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs) + rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txHashes) if err != nil { log.Error("rct cid retrieval error") } @@ -550,23 +550,23 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (models.Header return models.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber) } var uncleCIDs []models.UncleModel - uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].ID) + uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].BlockHash) if err != nil { log.Error("uncle cid retrieval error") return models.HeaderModel{}, nil, nil, nil, err } var txCIDs []models.TxModel - txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID) + txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].BlockHash) if err != nil { log.Error("tx cid retrieval error") return models.HeaderModel{}, nil, nil, nil, err } - txIDs := make([]int64, len(txCIDs)) + txHashes := make([]string, len(txCIDs)) for i, txCID := range txCIDs { - txIDs[i] = txCID.ID + txHashes[i] = txCID.TxHash } var rctCIDs []models.ReceiptModel - rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs) + rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txHashes) if err != nil { log.Error("rct cid retrieval error") } @@ -576,14 +576,14 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (models.Header // RetrieveHeaderCIDByHash returns the header for the given block hash func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (models.HeaderModel, error) { log.Debug("retrieving header cids for block hash ", blockHash.String()) - pgStr := `SELECT * FROM eth.header_cids + pgStr := `SELECT block_hash,cid,mh_key FROM eth.header_cids WHERE block_hash = $1` var headerCID models.HeaderModel return headerCID, tx.Get(&headerCID, pgStr, blockHash.String()) } // RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id -func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]models.TxModel, error) { +func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID string) ([]models.TxModel, error) { log.Debug("retrieving tx cids for block id ", headerID) pgStr := `SELECT * FROM eth.transaction_cids WHERE header_id = $1 @@ -593,14 +593,14 @@ func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ( } // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs -func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]models.ReceiptModel, error) { - log.Debugf("retrieving receipt cids for tx ids %v", txIDs) - pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, +func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txHashes []string) ([]models.ReceiptModel, error) { + log.Debugf("retrieving receipt cids for tx hashes %v", txHashes) + pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.contract, receipt_cids.contract_hash FROM eth.receipt_cids, eth.transaction_cids - WHERE tx_id = ANY($1::INTEGER[]) - AND receipt_cids.tx_id = transaction_cids.id + WHERE tx_id = ANY($1) + AND receipt_cids.tx_id = transaction_cids.tx_hash ORDER BY transaction_cids.index` var rctCIDs []models.ReceiptModel - return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs)) + return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txHashes)) } diff --git a/pkg/eth/cid_retriever_test.go b/pkg/eth/cid_retriever_test.go index f6b74b99..ff8d0698 100644 --- a/pkg/eth/cid_retriever_test.go +++ b/pkg/eth/cid_retriever_test.go @@ -19,14 +19,13 @@ package eth_test import ( "math/big" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/statediff/indexer" - "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/trie" + "github.com/jmoiron/sqlx" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -211,21 +210,18 @@ var ( var _ = Describe("Retriever", func() { var ( - db *postgres.DB - diffIndexer *indexer.StateDiffIndexer + db *sqlx.DB + diffIndexer interfaces.StateDiffIndexer retriever *eth.CIDRetriever ) BeforeEach(func() { - var err error - db, err = SetupDB() - Expect(err).ToNot(HaveOccurred()) - diffIndexer, err = indexer.NewStateDiffIndexer(params.TestChainConfig, db) - Expect(err).ToNot(HaveOccurred()) + db = eth.SetupTestDB() + diffIndexer = eth.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash()) retriever = eth.NewCIDRetriever(db) }) AfterEach(func() { - eth.TearDownDB(db) + eth.TearDownTestDB(db) }) Describe("Retrieve", func() { @@ -233,11 +229,11 @@ var _ = Describe("Retriever", func() { tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty()) Expect(err).ToNot(HaveOccurred()) for _, node := range test_helpers.MockStateNodes { - err = diffIndexer.PushStateNode(tx, node) + err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String()) Expect(err).ToNot(HaveOccurred()) } - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) }) It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { @@ -247,8 +243,8 @@ var _ = Describe("Retriever", func() { } expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0) pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids - WHERE receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id + WHERE receipt_cids.tx_id = transaction_cids.tx_hash + AND transaction_cids.header_id = header_cids.block_hash AND header_cids.block_number = $1 ORDER BY transaction_cids.index` err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64()) @@ -259,7 +255,7 @@ var _ = Describe("Retriever", func() { Expect(cids[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber)) expectedHeaderCID := test_helpers.MockCIDWrapper.Header - expectedHeaderCID.ID = cids[0].Header.ID + expectedHeaderCID.BlockHash = cids[0].Header.BlockHash expectedHeaderCID.NodeID = cids[0].Header.NodeID Expect(cids[0].Header).To(Equal(expectedHeaderCID)) Expect(len(cids[0].Transactions)).To(Equal(4)) @@ -286,8 +282,8 @@ var _ = Describe("Retriever", func() { } Expect(len(cids[0].StorageNodes)).To(Equal(1)) expectedStorageNodeCIDs := test_helpers.MockCIDWrapper.StorageNodes - expectedStorageNodeCIDs[0].ID = cids[0].StorageNodes[0].ID - expectedStorageNodeCIDs[0].StateID = cids[0].StorageNodes[0].StateID + expectedStorageNodeCIDs[0].HeaderID = cids[0].StorageNodes[0].HeaderID + expectedStorageNodeCIDs[0].StatePath = cids[0].StorageNodes[0].StatePath Expect(cids[0].StorageNodes).To(Equal(expectedStorageNodeCIDs)) }) @@ -298,8 +294,8 @@ var _ = Describe("Retriever", func() { } expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0) pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids - WHERE receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id + WHERE receipt_cids.tx_id = transaction_cids.tx_hash + AND transaction_cids.header_id = header_cids.block_hash AND header_cids.block_number = $1 ORDER BY transaction_cids.index` err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64()) @@ -314,7 +310,6 @@ var _ = Describe("Retriever", func() { Expect(len(cids1[0].StorageNodes)).To(Equal(0)) Expect(len(cids1[0].Receipts)).To(Equal(1)) expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0] - expectedReceiptCID.ID = cids1[0].Receipts[0].ID expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey @@ -331,7 +326,6 @@ var _ = Describe("Retriever", func() { Expect(len(cids2[0].StorageNodes)).To(Equal(0)) Expect(len(cids2[0].Receipts)).To(Equal(1)) expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0] - expectedReceiptCID.ID = cids2[0].Receipts[0].ID expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey @@ -348,7 +342,6 @@ var _ = Describe("Retriever", func() { Expect(len(cids3[0].StorageNodes)).To(Equal(0)) Expect(len(cids3[0].Receipts)).To(Equal(1)) expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0] - expectedReceiptCID.ID = cids3[0].Receipts[0].ID expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey @@ -365,7 +358,6 @@ var _ = Describe("Retriever", func() { Expect(len(cids4[0].StorageNodes)).To(Equal(0)) Expect(len(cids4[0].Receipts)).To(Equal(1)) expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1] - expectedReceiptCID.ID = cids4[0].Receipts[0].ID expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey @@ -396,14 +388,13 @@ var _ = Describe("Retriever", func() { Expect(cids6[0].Header).To(Equal(models.HeaderModel{})) Expect(len(cids6[0].Transactions)).To(Equal(1)) expectedTxCID := test_helpers.MockCIDWrapper.Transactions[1] - expectedTxCID.ID = cids6[0].Transactions[0].ID + expectedTxCID.TxHash = cids6[0].Transactions[0].TxHash expectedTxCID.HeaderID = cids6[0].Transactions[0].HeaderID Expect(cids6[0].Transactions[0]).To(Equal(expectedTxCID)) Expect(len(cids6[0].StateNodes)).To(Equal(0)) Expect(len(cids6[0].StorageNodes)).To(Equal(0)) Expect(len(cids6[0].Receipts)).To(Equal(1)) expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1] - expectedReceiptCID.ID = cids6[0].Receipts[0].ID expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey @@ -420,7 +411,6 @@ var _ = Describe("Retriever", func() { Expect(len(cids7[0].StorageNodes)).To(Equal(0)) Expect(len(cids7[0].StateNodes)).To(Equal(1)) Expect(cids7[0].StateNodes[0]).To(Equal(models.StateNodeModel{ - ID: cids7[0].StateNodes[0].ID, HeaderID: cids7[0].StateNodes[0].HeaderID, NodeType: 2, StateKey: common.BytesToHash(test_helpers.AccountLeafKey).Hex(), @@ -444,7 +434,7 @@ var _ = Describe("Retriever", func() { tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) num, err := retriever.RetrieveFirstBlockNumber() @@ -458,7 +448,7 @@ var _ = Describe("Retriever", func() { tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) num, err := retriever.RetrieveFirstBlockNumber() @@ -473,12 +463,12 @@ var _ = Describe("Retriever", func() { payload2.Block = newMockBlock(5) tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) num, err := retriever.RetrieveFirstBlockNumber() @@ -495,7 +485,7 @@ var _ = Describe("Retriever", func() { It("Gets the number of the latest block that has data in the database", func() { tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) num, err := retriever.RetrieveLastBlockNumber() @@ -509,7 +499,7 @@ var _ = Describe("Retriever", func() { tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) num, err := retriever.RetrieveLastBlockNumber() @@ -524,12 +514,12 @@ var _ = Describe("Retriever", func() { payload2.Block = newMockBlock(5) tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) num, err := retriever.RetrieveLastBlockNumber() diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 581c0628..17d0434f 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -21,6 +21,7 @@ import ( "context" "io/ioutil" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -31,9 +32,8 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/indexer" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/jmoiron/sqlx" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -64,7 +64,7 @@ var _ = Describe("eth state reading tests", func() { blocks []*types.Block receipts []types.Receipts chain *core.BlockChain - db *postgres.DB + db *sqlx.DB api *eth.PublicEthAPI backend *eth.Backend chainConfig = params.TestChainConfig @@ -74,11 +74,8 @@ var _ = Describe("eth state reading tests", func() { It("test init", func() { // db and type initializations var err error - db, err = SetupDB() - Expect(err).ToNot(HaveOccurred()) - - transformer, err := indexer.NewStateDiffIndexer(chainConfig, db) - Expect(err).ToNot(HaveOccurred()) + db = eth.SetupTestDB() + transformer := eth.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash()) backend, err = eth.NewEthBackend(db, ð.Config{ ChainConfig: chainConfig, @@ -150,21 +147,20 @@ var _ = Describe("eth state reading tests", func() { Expect(err).ToNot(HaveOccurred()) for _, node := range diff.Nodes { - err = transformer.PushStateNode(tx, node) + err = transformer.PushStateNode(tx, node, block.Hash().String()) Expect(err).ToNot(HaveOccurred()) } - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) } // Insert some non-canonical data into the database so that we test our ability to discern canonicity - indexAndPublisher, err := indexer.NewStateDiffIndexer(chainConfig, db) - Expect(err).ToNot(HaveOccurred()) + indexAndPublisher := eth.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash()) tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) // The non-canonical header has a child @@ -179,11 +175,13 @@ var _ = Describe("eth state reading tests", func() { err = indexAndPublisher.PushCodeAndCodeHash(tx, hash) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + // wait for tx batch process to complete. + time.Sleep(10000 * time.Millisecond) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) }) defer It("test teardown", func() { - eth.TearDownDB(db) + eth.TearDownTestDB(db) chain.Stop() }) diff --git a/pkg/eth/filterer.go b/pkg/eth/filterer.go index 3f258e37..3fee2209 100644 --- a/pkg/eth/filterer.go +++ b/pkg/eth/filterer.go @@ -23,12 +23,11 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" - - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" ) // Filterer interface for substituing mocks in tests @@ -82,12 +81,12 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP if err != nil { return err } - response.Header = ipfs.BlockModel{ + response.Header = models.IPLDModel{ Data: headerRLP, - CID: cid.String(), + Key: cid.String(), } if headerFilter.Uncles { - response.Uncles = make([]ipfs.BlockModel, len(payload.Block.Body().Uncles)) + response.Uncles = make([]models.IPLDModel, len(payload.Block.Body().Uncles)) for i, uncle := range payload.Block.Body().Uncles { uncleRlp, err := rlp.EncodeToBytes(uncle) if err != nil { @@ -97,9 +96,9 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP if err != nil { return err } - response.Uncles[i] = ipfs.BlockModel{ + response.Uncles[i] = models.IPLDModel{ Data: uncleRlp, - CID: cid.String(), + Key: cid.String(), } } } @@ -119,7 +118,7 @@ func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLD if !trxFilter.Off { trxLen := len(payload.Block.Body().Transactions) trxHashes = make([]common.Hash, 0, trxLen) - response.Transactions = make([]ipfs.BlockModel, 0, trxLen) + response.Transactions = make([]models.IPLDModel, 0, trxLen) for i, trx := range payload.Block.Body().Transactions { // TODO: check if want corresponding receipt and if we do we must include this transaction if checkTransactionAddrs(trxFilter.Src, trxFilter.Dst, payload.TxMetaData[i].Src, payload.TxMetaData[i].Dst) { @@ -132,9 +131,9 @@ func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLD if err != nil { return nil, err } - response.Transactions = append(response.Transactions, ipfs.BlockModel{ + response.Transactions = append(response.Transactions, models.IPLDModel{ Data: data, - CID: cid.String(), + Key: cid.String(), }) trxHashes = append(trxHashes, trx.Hash()) } @@ -164,7 +163,7 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error { if !receiptFilter.Off { - response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts)) + response.Receipts = make([]models.IPLDModel, 0, len(payload.Receipts)) rctLeafCID, rctIPLDData, err := GetRctLeafNodeData(payload.Receipts) if err != nil { return err @@ -183,9 +182,9 @@ func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response * // TODO: Verify this filter logic. if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) { - response.Receipts = append(response.Receipts, ipfs.BlockModel{ + response.Receipts = append(response.Receipts, models.IPLDModel{ Data: rctIPLDData[idx], - CID: rctLeafCID[idx].String(), + Key: rctLeafCID[idx].String(), }) } } @@ -282,9 +281,9 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag response.StateNodes = append(response.StateNodes, StateNode{ StateLeafKey: common.BytesToHash(stateNode.LeafKey), Path: stateNode.Path, - IPLD: ipfs.BlockModel{ + IPLD: models.IPLDModel{ Data: stateNode.NodeValue, - CID: cid.String(), + Key: cid.String(), }, Type: stateNode.NodeType, }) @@ -300,9 +299,9 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag response.StorageNodes = append(response.StorageNodes, StorageNode{ StateLeafKey: common.BytesToHash(stateNode.LeafKey), StorageLeafKey: common.BytesToHash(storageNode.LeafKey), - IPLD: ipfs.BlockModel{ + IPLD: models.IPLDModel{ Data: storageNode.NodeValue, - CID: cid.String(), + Key: cid.String(), }, Type: storageNode.NodeType, Path: storageNode.Path, diff --git a/pkg/eth/filterer_test.go b/pkg/eth/filterer_test.go index 84f4de44..6894f99c 100644 --- a/pkg/eth/filterer_test.go +++ b/pkg/eth/filterer_test.go @@ -19,7 +19,7 @@ package eth_test import ( "bytes" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" + "github.com/ethereum/go-ethereum/statediff/indexer/models" sdtypes "github.com/ethereum/go-ethereum/statediff/types" . "github.com/onsi/ginkgo" @@ -46,7 +46,7 @@ var _ = Describe("Filterer", func() { Expect(iplds).ToNot(BeNil()) Expect(iplds.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) Expect(iplds.Header).To(Equal(test_helpers.MockIPLDs.Header)) - var expectedEmptyUncles []ipfs.BlockModel + var expectedEmptyUncles []models.IPLDModel Expect(iplds.Uncles).To(Equal(expectedEmptyUncles)) Expect(len(iplds.Transactions)).To(Equal(4)) Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx1)).To(BeTrue()) @@ -60,15 +60,15 @@ var _ = Describe("Filterer", func() { for _, stateNode := range iplds.StateNodes { Expect(stateNode.Type).To(Equal(sdtypes.Leaf)) if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.AccountLeafKey) { - Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{ + Expect(stateNode.IPLD).To(Equal(models.IPLDModel{ Data: test_helpers.State2IPLD.RawData(), - CID: test_helpers.State2IPLD.Cid().String(), + Key: test_helpers.State2IPLD.Cid().String(), })) } if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.ContractLeafKey) { - Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{ + Expect(stateNode.IPLD).To(Equal(models.IPLDModel{ Data: test_helpers.State1IPLD.RawData(), - CID: test_helpers.State1IPLD.Cid().String(), + Key: test_helpers.State1IPLD.Cid().String(), })) } } @@ -80,67 +80,67 @@ var _ = Describe("Filterer", func() { Expect(err).ToNot(HaveOccurred()) Expect(iplds1).ToNot(BeNil()) Expect(iplds1.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) - Expect(iplds1.Header).To(Equal(ipfs.BlockModel{})) + Expect(iplds1.Header).To(Equal(models.IPLDModel{})) Expect(len(iplds1.Uncles)).To(Equal(0)) Expect(len(iplds1.Transactions)).To(Equal(0)) Expect(len(iplds1.StorageNodes)).To(Equal(0)) Expect(len(iplds1.StateNodes)).To(Equal(0)) Expect(len(iplds1.Receipts)).To(Equal(1)) - Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{ + Expect(iplds1.Receipts[0]).To(Equal(models.IPLDModel{ Data: test_helpers.Rct1IPLD, - CID: test_helpers.Rct1CID.String(), + Key: test_helpers.Rct1CID.String(), })) iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload) Expect(err).ToNot(HaveOccurred()) Expect(iplds2).ToNot(BeNil()) Expect(iplds2.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) - Expect(iplds2.Header).To(Equal(ipfs.BlockModel{})) + Expect(iplds2.Header).To(Equal(models.IPLDModel{})) Expect(len(iplds2.Uncles)).To(Equal(0)) Expect(len(iplds2.Transactions)).To(Equal(0)) Expect(len(iplds2.StorageNodes)).To(Equal(0)) Expect(len(iplds2.StateNodes)).To(Equal(0)) Expect(len(iplds2.Receipts)).To(Equal(1)) - Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{ + Expect(iplds2.Receipts[0]).To(Equal(models.IPLDModel{ Data: test_helpers.Rct1IPLD, - CID: test_helpers.Rct1CID.String(), + Key: test_helpers.Rct1CID.String(), })) iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload) Expect(err).ToNot(HaveOccurred()) Expect(iplds3).ToNot(BeNil()) Expect(iplds3.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) - Expect(iplds3.Header).To(Equal(ipfs.BlockModel{})) + Expect(iplds3.Header).To(Equal(models.IPLDModel{})) Expect(len(iplds3.Uncles)).To(Equal(0)) Expect(len(iplds3.Transactions)).To(Equal(0)) Expect(len(iplds3.StorageNodes)).To(Equal(0)) Expect(len(iplds3.StateNodes)).To(Equal(0)) Expect(len(iplds3.Receipts)).To(Equal(1)) - Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{ + Expect(iplds3.Receipts[0]).To(Equal(models.IPLDModel{ Data: test_helpers.Rct1IPLD, - CID: test_helpers.Rct1CID.String(), + Key: test_helpers.Rct1CID.String(), })) iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload) Expect(err).ToNot(HaveOccurred()) Expect(iplds4).ToNot(BeNil()) Expect(iplds4.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) - Expect(iplds4.Header).To(Equal(ipfs.BlockModel{})) + Expect(iplds4.Header).To(Equal(models.IPLDModel{})) Expect(len(iplds4.Uncles)).To(Equal(0)) Expect(len(iplds4.Transactions)).To(Equal(0)) Expect(len(iplds4.StorageNodes)).To(Equal(0)) Expect(len(iplds4.StateNodes)).To(Equal(0)) Expect(len(iplds4.Receipts)).To(Equal(1)) - Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{ + Expect(iplds4.Receipts[0]).To(Equal(models.IPLDModel{ Data: test_helpers.Rct2IPLD, - CID: test_helpers.Rct2CID.String(), + Key: test_helpers.Rct2CID.String(), })) iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload) Expect(err).ToNot(HaveOccurred()) Expect(iplds5).ToNot(BeNil()) Expect(iplds5.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) - Expect(iplds5.Header).To(Equal(ipfs.BlockModel{})) + Expect(iplds5.Header).To(Equal(models.IPLDModel{})) Expect(len(iplds5.Uncles)).To(Equal(0)) Expect(len(iplds5.Transactions)).To(Equal(4)) Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx1)).To(BeTrue()) @@ -157,39 +157,39 @@ var _ = Describe("Filterer", func() { Expect(err).ToNot(HaveOccurred()) Expect(iplds6).ToNot(BeNil()) Expect(iplds6.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) - Expect(iplds6.Header).To(Equal(ipfs.BlockModel{})) + Expect(iplds6.Header).To(Equal(models.IPLDModel{})) Expect(len(iplds6.Uncles)).To(Equal(0)) Expect(len(iplds6.Transactions)).To(Equal(1)) Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx2)).To(BeTrue()) Expect(len(iplds6.StorageNodes)).To(Equal(0)) Expect(len(iplds6.StateNodes)).To(Equal(0)) Expect(len(iplds6.Receipts)).To(Equal(1)) - Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{ + Expect(iplds4.Receipts[0]).To(Equal(models.IPLDModel{ Data: test_helpers.Rct2IPLD, - CID: test_helpers.Rct2CID.String(), + Key: test_helpers.Rct2CID.String(), })) iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload) Expect(err).ToNot(HaveOccurred()) Expect(iplds7).ToNot(BeNil()) Expect(iplds7.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) - Expect(iplds7.Header).To(Equal(ipfs.BlockModel{})) + Expect(iplds7.Header).To(Equal(models.IPLDModel{})) Expect(len(iplds7.Uncles)).To(Equal(0)) Expect(len(iplds7.Transactions)).To(Equal(0)) Expect(len(iplds7.StorageNodes)).To(Equal(0)) Expect(len(iplds7.Receipts)).To(Equal(0)) Expect(len(iplds7.StateNodes)).To(Equal(1)) Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(test_helpers.AccountLeafKey)) - Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{ + Expect(iplds7.StateNodes[0].IPLD).To(Equal(models.IPLDModel{ Data: test_helpers.State2IPLD.RawData(), - CID: test_helpers.State2IPLD.Cid().String(), + Key: test_helpers.State2IPLD.Cid().String(), })) iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, test_helpers.MockConvertedPayload) Expect(err).ToNot(HaveOccurred()) Expect(iplds8).ToNot(BeNil()) Expect(iplds8.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64())) - Expect(iplds8.Header).To(Equal(ipfs.BlockModel{})) + Expect(iplds8.Header).To(Equal(models.IPLDModel{})) Expect(len(iplds8.Uncles)).To(Equal(0)) Expect(len(iplds8.Transactions)).To(Equal(0)) Expect(len(iplds8.StorageNodes)).To(Equal(0)) diff --git a/pkg/eth/ipld_fetcher.go b/pkg/eth/ipld_fetcher.go index 94047004..fbef10bd 100644 --- a/pkg/eth/ipld_fetcher.go +++ b/pkg/eth/ipld_fetcher.go @@ -22,9 +22,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" "github.com/vulcanize/ipld-eth-server/pkg/shared" @@ -38,11 +36,11 @@ type Fetcher interface { // IPLDFetcher satisfies the IPLDFetcher interface for ethereum // It interfaces directly with PG-IPFS type IPLDFetcher struct { - db *postgres.DB + db *sqlx.DB } // NewIPLDFetcher creates a pointer to a new IPLDFetcher -func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher { +func NewIPLDFetcher(db *sqlx.DB) *IPLDFetcher { return &IPLDFetcher{ db: db, } @@ -102,65 +100,65 @@ func (f *IPLDFetcher) Fetch(cids CIDWrapper) (*IPLDs, error) { } // FetchHeaders fetches headers -func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c models.HeaderModel) (ipfs.BlockModel, error) { +func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c models.HeaderModel) (models.IPLDModel, error) { log.Debug("fetching header ipld") headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey) if err != nil { - return ipfs.BlockModel{}, err + return models.IPLDModel{}, err } - return ipfs.BlockModel{ + return models.IPLDModel{ Data: headerBytes, - CID: c.CID, + Key: c.CID, }, nil } // FetchUncles fetches uncles -func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []models.UncleModel) ([]ipfs.BlockModel, error) { +func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []models.UncleModel) ([]models.IPLDModel, error) { log.Debug("fetching uncle iplds") - uncleIPLDs := make([]ipfs.BlockModel, len(cids)) + uncleIPLDs := make([]models.IPLDModel, len(cids)) for i, c := range cids { uncleBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey) if err != nil { return nil, err } - uncleIPLDs[i] = ipfs.BlockModel{ + uncleIPLDs[i] = models.IPLDModel{ Data: uncleBytes, - CID: c.CID, + Key: c.CID, } } return uncleIPLDs, nil } // FetchTrxs fetches transactions -func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []models.TxModel) ([]ipfs.BlockModel, error) { +func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []models.TxModel) ([]models.IPLDModel, error) { log.Debug("fetching transaction iplds") - trxIPLDs := make([]ipfs.BlockModel, len(cids)) + trxIPLDs := make([]models.IPLDModel, len(cids)) for i, c := range cids { txBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey) if err != nil { return nil, err } - trxIPLDs[i] = ipfs.BlockModel{ + trxIPLDs[i] = models.IPLDModel{ Data: txBytes, - CID: c.CID, + Key: c.CID, } } return trxIPLDs, nil } // FetchRcts fetches receipts -func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]ipfs.BlockModel, error) { +func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]models.IPLDModel, error) { log.Debug("fetching receipt iplds") - rctIPLDs := make([]ipfs.BlockModel, len(cids)) + rctIPLDs := make([]models.IPLDModel, len(cids)) for i, c := range cids { rctBytes, err := shared.FetchIPLDByMhKey(tx, c.LeafMhKey) if err != nil { return nil, err } //nodeVal, err := DecodeLeafNode(rctBytes) - rctIPLDs[i] = ipfs.BlockModel{ + rctIPLDs[i] = models.IPLDModel{ Data: rctBytes, - CID: c.LeafCID, + Key: c.LeafCID, } } return rctIPLDs, nil @@ -179,9 +177,9 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []models.StateNodeModel) ([]S return nil, err } stateNodes = append(stateNodes, StateNode{ - IPLD: ipfs.BlockModel{ + IPLD: models.IPLDModel{ Data: stateBytes, - CID: stateNode.CID, + Key: stateNode.CID, }, StateLeafKey: common.HexToHash(stateNode.StateKey), Type: ResolveToNodeType(stateNode.NodeType), @@ -204,9 +202,9 @@ func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []models.StorageNodeWithSta return nil, err } storageNodes = append(storageNodes, StorageNode{ - IPLD: ipfs.BlockModel{ + IPLD: models.IPLDModel{ Data: storageBytes, - CID: storageNode.CID, + Key: storageNode.CID, }, StateLeafKey: common.HexToHash(storageNode.StateKey), StorageLeafKey: common.HexToHash(storageNode.StorageKey), diff --git a/pkg/eth/ipld_fetcher_test.go b/pkg/eth/ipld_fetcher_test.go index c6699a09..f5521188 100644 --- a/pkg/eth/ipld_fetcher_test.go +++ b/pkg/eth/ipld_fetcher_test.go @@ -18,8 +18,8 @@ package eth_test import ( "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/statediff/indexer" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/jmoiron/sqlx" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,34 +29,32 @@ import ( var _ = Describe("IPLDFetcher", func() { var ( - db *postgres.DB - pubAndIndexer *indexer.StateDiffIndexer + db *sqlx.DB + pubAndIndexer interfaces.StateDiffIndexer fetcher *eth.IPLDFetcher ) Describe("Fetch", func() { BeforeEach(func() { var ( err error - tx *indexer.BlockTx + tx interfaces.Batch ) - db, err = SetupDB() - Expect(err).ToNot(HaveOccurred()) - pubAndIndexer, err = indexer.NewStateDiffIndexer(params.TestChainConfig, db) - Expect(err).ToNot(HaveOccurred()) + db = eth.SetupTestDB() + pubAndIndexer = eth.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash()) tx, err = pubAndIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty()) for _, node := range test_helpers.MockStateNodes { - err = pubAndIndexer.PushStateNode(tx, node) + err = pubAndIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String()) Expect(err).ToNot(HaveOccurred()) } - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) fetcher = eth.NewIPLDFetcher(db) }) AfterEach(func() { - eth.TearDownDB(db) + eth.TearDownTestDB(db) }) It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() { diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 08e11884..3b6d4073 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -19,13 +19,13 @@ package eth import ( "fmt" - "github.com/ethereum/go-ethereum/statediff/trie" + "github.com/ethereum/go-ethereum/statediff/trie_helpers" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/jmoiron/sqlx" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/lib/pq" ) @@ -52,12 +52,12 @@ const ( WHERE block_hash = ANY($1::VARCHAR(66)[])` RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids - INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id) + INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = $1` RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids - INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id) + INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_number = $1` RetrieveUncleByHashPgStr = `SELECT cid, data @@ -70,13 +70,13 @@ const ( WHERE tx_hash = ANY($1::VARCHAR(66)[])` RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids - INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids - INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE block_number = $1 ORDER BY eth.transaction_cids.index ASC` @@ -86,42 +86,42 @@ const ( WHERE tx_hash = $1` RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash) INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) WHERE tx_hash = ANY($1::VARCHAR(66)[])` RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) - INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash) + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) - INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash) + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) WHERE block_number = $1 ORDER BY eth.transaction_cids.index ASC` RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash) INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) WHERE tx_hash = $1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) WHERE state_leaf_key = $1 AND block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = $2) - AND header_cids.id = (SELECT canonical_header_id(block_number)) + AND header_cids.block_hash = (SELECT canonical_header_hash(block_number)) ORDER BY block_number DESC LIMIT 1` RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.node_type FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) WHERE state_leaf_key = $1 AND block_number <= $2 @@ -129,8 +129,11 @@ const ( LIMIT 1` RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed($1, $3) AS state_leaf_removed FROM eth.storage_cids - INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN eth.state_cids ON ( + storage_cids.header_id = state_cids.header_id + AND storage_cids.state_path = state_cids.state_path + ) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key) WHERE state_leaf_key = $1 AND storage_leaf_key = $2 @@ -139,15 +142,18 @@ const ( LIMIT 1` RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed($1, $3) AS state_leaf_removed FROM eth.storage_cids - INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN eth.state_cids ON ( + storage_cids.header_id = state_cids.header_id + AND storage_cids.state_path = state_cids.state_path + ) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key) WHERE state_leaf_key = $1 AND storage_leaf_key = $2 AND block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = $3) - AND header_cids.id = (SELECT canonical_header_id(block_number)) + AND header_cids.block_hash = (SELECT canonical_header_hash(block_number)) ORDER BY block_number DESC LIMIT 1` ) @@ -167,10 +173,10 @@ type ipldResult struct { } type IPLDRetriever struct { - db *postgres.DB + db *sqlx.DB } -func NewIPLDRetriever(db *postgres.DB) *IPLDRetriever { +func NewIPLDRetriever(db *sqlx.DB) *IPLDRetriever { return &IPLDRetriever{ db: db, } @@ -333,7 +339,7 @@ func DecodeLeafNode(node []byte) ([]byte, error) { if err := rlp.DecodeBytes(node, &nodeElements); err != nil { return nil, err } - ty, err := trie.CheckKeyType(nodeElements) + ty, err := trie_helpers.CheckKeyType(nodeElements) if err != nil { return nil, err } diff --git a/pkg/eth/test_helpers.go b/pkg/eth/test_helpers.go index 8c4557db..621ea6fd 100644 --- a/pkg/eth/test_helpers.go +++ b/pkg/eth/test_helpers.go @@ -17,17 +17,34 @@ package eth import ( + "context" + "os" + "strconv" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff/indexer" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/jmoiron/sqlx" . "github.com/onsi/gomega" + "github.com/vulcanize/ipld-eth-server/pkg/shared" ) -// TearDownDB is used to tear down the watcher dbs after tests -func TearDownDB(db *postgres.DB) { - tx, err := db.Beginx() +func SetupTestDB() *sqlx.DB { + config := getTestDBConfig() + + db, err := shared.NewDB(config.DbConnectionString(), config) Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM eth.header_cids`) + return db +} + +// TearDownTestDB is used to tear down the watcher dbs after tests +func TearDownTestDB(db *sqlx.DB) { + tx, err := db.Beginx() Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM eth.transaction_cids`) Expect(err).NotTo(HaveOccurred()) @@ -46,6 +63,21 @@ func TearDownDB(db *postgres.DB) { Expect(err).NotTo(HaveOccurred()) } +func SetupTestStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, genHash common.Hash) interfaces.StateDiffIndexer { + testInfo := node.Info{ + GenesisBlock: genHash.String(), + NetworkID: "1", + ID: "1", + ClientName: "geth", + ChainID: params.TestChainConfig.ChainID.Uint64(), + } + + stateDiffIndexer, err := indexer.NewStateDiffIndexer(ctx, chainConfig, testInfo, getTestDBConfig()) + Expect(err).NotTo(HaveOccurred()) + + return stateDiffIndexer +} + // TxModelsContainsCID used to check if a list of TxModels contains a specific cid string func TxModelsContainsCID(txs []models.TxModel, cid string) bool { for _, tx := range txs { @@ -65,3 +97,15 @@ func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool { } return false } + +func getTestDBConfig() postgres.Config { + port, _ := strconv.Atoi(os.Getenv("DATABASE_PORT")) + return postgres.Config{ + Hostname: os.Getenv("DATABASE_HOSTNAME"), + DatabaseName: os.Getenv("DATABASE_NAME"), + Username: os.Getenv("DATABASE_USER"), + Password: os.Getenv("DATABASE_PASSWORD"), + Port: port, + Driver: postgres.PGX, + } +} diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index fff0806b..0c7e46dd 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -28,13 +28,10 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff/indexer" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/shared" - "github.com/ethereum/go-ethereum/statediff/testhelpers" + testhelpers "github.com/ethereum/go-ethereum/statediff/test_helpers" sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" blocks "github.com/ipfs/go-block-format" @@ -47,7 +44,6 @@ import ( // Test variables var ( // block data - db *postgres.DB BlockNumber = big.NewInt(1) MockHeader = types.Header{ Time: 0, @@ -439,8 +435,7 @@ var ( StateNodes: MockStateNodes, } - Reward = indexer.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts) - + Reward = shared.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts) MockCIDWrapper = ð.CIDWrapper{ BlockNumber: new(big.Int).Set(BlockNumber), Header: models.HeaderModel{ @@ -458,6 +453,7 @@ var ( Bloom: MockBlock.Bloom().Bytes(), Timestamp: MockBlock.Time(), TimesValidated: 1, + Coinbase: "0x0000000000000000000000000000000000000000", }, Transactions: MockTrxMetaPostPublsh, Receipts: MockRctMetaPostPublish, @@ -486,62 +482,62 @@ var ( MockIPLDs = eth.IPLDs{ BlockNumber: new(big.Int).Set(BlockNumber), - Header: ipfs.BlockModel{ + Header: models.IPLDModel{ Data: HeaderIPLD.RawData(), - CID: HeaderIPLD.Cid().String(), + Key: HeaderIPLD.Cid().String(), }, - Transactions: []ipfs.BlockModel{ + Transactions: []models.IPLDModel{ { Data: Trx1IPLD.RawData(), - CID: Trx1IPLD.Cid().String(), + Key: Trx1IPLD.Cid().String(), }, { Data: Trx2IPLD.RawData(), - CID: Trx2IPLD.Cid().String(), + Key: Trx2IPLD.Cid().String(), }, { Data: Trx3IPLD.RawData(), - CID: Trx3IPLD.Cid().String(), + Key: Trx3IPLD.Cid().String(), }, { Data: Trx4IPLD.RawData(), - CID: Trx4IPLD.Cid().String(), + Key: Trx4IPLD.Cid().String(), }, }, - Receipts: []ipfs.BlockModel{ + Receipts: []models.IPLDModel{ { Data: Rct1IPLD, - CID: Rct1CID.String(), + Key: Rct1CID.String(), }, { Data: Rct2IPLD, - CID: Rct2CID.String(), + Key: Rct2CID.String(), }, { Data: Rct3IPLD, - CID: Rct3CID.String(), + Key: Rct3CID.String(), }, { Data: Rct4IPLD, - CID: Rct4CID.String(), + Key: Rct4CID.String(), }, }, StateNodes: []eth.StateNode{ { StateLeafKey: common.BytesToHash(ContractLeafKey), Type: sdtypes.Leaf, - IPLD: ipfs.BlockModel{ + IPLD: models.IPLDModel{ Data: State1IPLD.RawData(), - CID: State1IPLD.Cid().String(), + Key: State1IPLD.Cid().String(), }, Path: []byte{'\x06'}, }, { StateLeafKey: common.BytesToHash(AccountLeafKey), Type: sdtypes.Leaf, - IPLD: ipfs.BlockModel{ + IPLD: models.IPLDModel{ Data: State2IPLD.RawData(), - CID: State2IPLD.Cid().String(), + Key: State2IPLD.Cid().String(), }, Path: []byte{'\x0c'}, }, @@ -551,9 +547,9 @@ var ( StateLeafKey: common.BytesToHash(ContractLeafKey), StorageLeafKey: common.BytesToHash(StorageLeafKey), Type: sdtypes.Leaf, - IPLD: ipfs.BlockModel{ + IPLD: models.IPLDModel{ Data: StorageIPLD.RawData(), - CID: StorageIPLD.Cid().String(), + Key: StorageIPLD.Cid().String(), }, Path: []byte{}, }, diff --git a/pkg/eth/types.go b/pkg/eth/types.go index 9e371ff2..5118eda2 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" "github.com/ethereum/go-ethereum/statediff/indexer/models" sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/sirupsen/logrus" @@ -197,10 +196,10 @@ func (arg *CallArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (types.Mes type IPLDs struct { BlockNumber *big.Int TotalDifficulty *big.Int - Header ipfs.BlockModel - Uncles []ipfs.BlockModel - Transactions []ipfs.BlockModel - Receipts []ipfs.BlockModel + Header models.IPLDModel + Uncles []models.IPLDModel + Transactions []models.IPLDModel + Receipts []models.IPLDModel StateNodes []StateNode StorageNodes []StorageNode } @@ -209,7 +208,7 @@ type StateNode struct { Type sdtypes.NodeType StateLeafKey common.Hash Path []byte - IPLD ipfs.BlockModel + IPLD models.IPLDModel } type StorageNode struct { @@ -217,7 +216,7 @@ type StorageNode struct { StateLeafKey common.Hash StorageLeafKey common.Hash Path []byte - IPLD ipfs.BlockModel + IPLD models.IPLDModel } // CIDWrapper is used to direct fetching of IPLDs from IPFS @@ -249,7 +248,7 @@ type ConvertedPayload struct { // LogResult represent a log. type LogResult struct { LeafCID string `db:"leaf_cid"` - ReceiptID int64 `db:"receipt_id"` + ReceiptID string `db:"rct_id"` Address string `db:"address"` Index int64 `db:"index"` Data []byte `db:"log_data"` diff --git a/pkg/graphql/graphql_test.go b/pkg/graphql/graphql_test.go index ceb87c9d..dec0d0dd 100644 --- a/pkg/graphql/graphql_test.go +++ b/pkg/graphql/graphql_test.go @@ -20,8 +20,6 @@ import ( "context" "fmt" "math/big" - "os" - "strconv" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -32,10 +30,8 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/indexer" - "github.com/ethereum/go-ethereum/statediff/indexer/node" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/jmoiron/sqlx" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -45,19 +41,6 @@ import ( ethServerShared "github.com/vulcanize/ipld-eth-server/pkg/shared" ) -// SetupDB is use to setup a db for watcher tests -func SetupDB() (*postgres.DB, error) { - port, _ := strconv.Atoi(os.Getenv("DATABASE_PORT")) - uri := postgres.DbConnectionString(postgres.ConnectionParams{ - User: os.Getenv("DATABASE_USER"), - Password: os.Getenv("DATABASE_PASSWORD"), - Hostname: os.Getenv("DATABASE_HOSTNAME"), - Name: os.Getenv("DATABASE_NAME"), - Port: port, - }) - return postgres.NewDB(uri, postgres.ConnectionConfig{}, node.Info{}) -} - var _ = Describe("GraphQL", func() { const ( gqlEndPoint = "127.0.0.1:8083" @@ -68,7 +51,7 @@ var _ = Describe("GraphQL", func() { blocks []*types.Block receipts []types.Receipts chain *core.BlockChain - db *postgres.DB + db *sqlx.DB blockHashes []common.Hash backend *eth.Backend graphQLServer *graphql.Service @@ -82,11 +65,9 @@ var _ = Describe("GraphQL", func() { It("test init", func() { var err error - db, err = SetupDB() - Expect(err).ToNot(HaveOccurred()) + db = eth.SetupTestDB() + transformer := eth.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash()) - transformer, err := indexer.NewStateDiffIndexer(chainConfig, db) - Expect(err).ToNot(HaveOccurred()) backend, err = eth.NewEthBackend(db, ð.Config{ ChainConfig: chainConfig, VMConfig: vm.Config{}, @@ -132,7 +113,7 @@ var _ = Describe("GraphQL", func() { rcts = receipts[i-1] } - var diff statediff.StateObject + var diff sdtypes.StateObject diff, err = builder.BuildStateDiffObject(args, params) Expect(err).ToNot(HaveOccurred()) @@ -140,17 +121,16 @@ var _ = Describe("GraphQL", func() { Expect(err).ToNot(HaveOccurred()) for _, node := range diff.Nodes { - err = transformer.PushStateNode(tx, node) + err = transformer.PushStateNode(tx, node, block.Hash().String()) Expect(err).ToNot(HaveOccurred()) } - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) } // Insert some non-canonical data into the database so that we test our ability to discern canonicity - indexAndPublisher, err := indexer.NewStateDiffIndexer(chainConfig, db) - Expect(err).ToNot(HaveOccurred()) + indexAndPublisher := eth.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash()) blockHash = test_helpers.MockBlock.Hash() contractAddress = test_helpers.ContractAddr @@ -158,7 +138,7 @@ var _ = Describe("GraphQL", func() { tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty()) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) // The non-canonical header has a child @@ -173,7 +153,7 @@ var _ = Describe("GraphQL", func() { err = indexAndPublisher.PushCodeAndCodeHash(tx, ccHash) Expect(err).ToNot(HaveOccurred()) - err = tx.Close(err) + err = tx.Submit(err) Expect(err).ToNot(HaveOccurred()) graphQLServer, err = graphql.New(backend, gqlEndPoint, nil, []string{"*"}, rpc.HTTPTimeouts{}) @@ -186,7 +166,7 @@ var _ = Describe("GraphQL", func() { defer It("test teardown", func() { err := graphQLServer.Stop() Expect(err).ToNot(HaveOccurred()) - eth.TearDownDB(db) + eth.TearDownTestDB(db) chain.Stop() }) diff --git a/pkg/serve/api.go b/pkg/serve/api.go index e00877c4..8cf3d43f 100644 --- a/pkg/serve/api.go +++ b/pkg/serve/api.go @@ -20,7 +20,6 @@ import ( "context" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff/indexer/shared" log "github.com/sirupsen/logrus" "github.com/vulcanize/ipld-eth-server/pkg/eth" @@ -82,8 +81,3 @@ func (api *PublicServerAPI) Stream(ctx context.Context, params eth.SubscriptionS return rpcSub, nil } - -// Chain returns the chain type that this watcher instance supports -func (api *PublicServerAPI) Chain() shared.ChainType { - return shared.Ethereum -} diff --git a/pkg/serve/config.go b/pkg/serve/config.go index b378bf31..c971691c 100644 --- a/pkg/serve/config.go +++ b/pkg/serve/config.go @@ -22,15 +22,18 @@ import ( "math/big" "os" "path/filepath" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/jmoiron/sqlx" "github.com/spf13/viper" "github.com/vulcanize/ipld-eth-server/pkg/eth" "github.com/vulcanize/ipld-eth-server/pkg/prom" + "github.com/vulcanize/ipld-eth-server/pkg/shared" ethServerShared "github.com/vulcanize/ipld-eth-server/pkg/shared" ) @@ -57,9 +60,8 @@ const ( // Config struct type Config struct { - DB *postgres.DB - DBConfig postgres.ConnectionConfig - DBParams postgres.ConnectionParams + DB *sqlx.DB + DBConfig postgres.Config WSEnabled bool WSEndpoint string @@ -87,6 +89,7 @@ type Config struct { SupportStateDiff bool ForwardEthCalls bool ProxyOnError bool + NodeNetworkID string // Cache configuration. GroupCache *ethServerShared.GroupCacheConfig @@ -112,6 +115,7 @@ func NewConfig() (*Config, error) { ethHTTP := viper.GetString("ethereum.httpPath") ethHTTPEndpoint := fmt.Sprintf("http://%s", ethHTTP) nodeInfo, cli, err := getEthNodeAndClient(ethHTTPEndpoint) + c.NodeNetworkID = nodeInfo.NetworkID if err != nil { return nil, err } @@ -198,12 +202,12 @@ func NewConfig() (*Config, error) { c.IpldGraphqlEnabled = ipldGraphqlEnabled overrideDBConnConfig(&c.DBConfig) - serveDB, err := postgres.NewDB(postgres.DbConnectionString(c.DBParams), c.DBConfig, nodeInfo) + serveDB, err := shared.NewDB(c.DBConfig.DbConnectionString(), c.DBConfig) if err != nil { return nil, err } - prom.RegisterDBCollector(c.DBParams.Name, serveDB.DB) + prom.RegisterDBCollector(c.DBConfig.DatabaseName, serveDB) c.DB = serveDB defaultSenderStr := viper.GetString("ethereum.defaultSender") @@ -231,13 +235,13 @@ func NewConfig() (*Config, error) { return c, err } -func overrideDBConnConfig(con *postgres.ConnectionConfig) { +func overrideDBConnConfig(con *postgres.Config) { viper.BindEnv("database.server.maxIdle", SERVER_MAX_IDLE_CONNECTIONS) viper.BindEnv("database.server.maxOpen", SERVER_MAX_OPEN_CONNECTIONS) viper.BindEnv("database.server.maxLifetime", SERVER_MAX_CONN_LIFETIME) con.MaxIdle = viper.GetInt("database.server.maxIdle") - con.MaxOpen = viper.GetInt("database.server.maxOpen") - con.MaxLifetime = viper.GetInt("database.server.maxLifetime") + con.MaxConns = viper.GetInt("database.server.maxOpen") + con.MaxConnLifetime = time.Duration(viper.GetInt("database.server.maxLifetime")) } func (c *Config) dbInit() { @@ -250,14 +254,14 @@ func (c *Config) dbInit() { viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS) viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME) - c.DBParams.Name = viper.GetString("database.name") - c.DBParams.Hostname = viper.GetString("database.hostname") - c.DBParams.Port = viper.GetInt("database.port") - c.DBParams.User = viper.GetString("database.user") - c.DBParams.Password = viper.GetString("database.password") + c.DBConfig.DatabaseName = viper.GetString("database.name") + c.DBConfig.Hostname = viper.GetString("database.hostname") + c.DBConfig.Port = viper.GetInt("database.port") + c.DBConfig.Username = viper.GetString("database.user") + c.DBConfig.Password = viper.GetString("database.password") c.DBConfig.MaxIdle = viper.GetInt("database.maxIdle") - c.DBConfig.MaxOpen = viper.GetInt("database.maxOpen") - c.DBConfig.MaxLifetime = viper.GetInt("database.maxLifetime") + c.DBConfig.MaxConns = viper.GetInt("database.maxOpen") + c.DBConfig.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime")) } func (c *Config) loadGroupCacheConfig() { diff --git a/pkg/serve/service.go b/pkg/serve/service.go index b1c295f2..1704ea64 100644 --- a/pkg/serve/service.go +++ b/pkg/serve/service.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" "github.com/vulcanize/ipld-eth-server/pkg/eth" @@ -74,7 +74,7 @@ type Service struct { // A mapping of subscription params hash to the corresponding subscription params SubscriptionTypes map[common.Hash]eth.SubscriptionSettings // Underlying db - db *postgres.DB + db *sqlx.DB // wg for syncing serve processes serveWg *sync.WaitGroup // rpc client for forwarding cache misses @@ -87,6 +87,8 @@ type Service struct { forwardEthCalls bool // whether to forward all calls to proxy node if they throw an error locally proxyOnError bool + // eth node network id + nodeNetworkId string } // NewServer creates a new Server using an underlying Service struct @@ -103,6 +105,7 @@ func NewServer(settings *Config) (Server, error) { sap.supportsStateDiffing = settings.SupportStateDiff sap.forwardEthCalls = settings.ForwardEthCalls sap.proxyOnError = settings.ProxyOnError + sap.nodeNetworkId = settings.NodeNetworkID var err error sap.backend, err = eth.NewEthBackend(sap.db, ð.Config{ ChainConfig: settings.ChainConfig, @@ -121,7 +124,7 @@ func (sap *Service) Protocols() []p2p.Protocol { // APIs returns the RPC descriptors the watcher service offers func (sap *Service) APIs() []rpc.API { - networkID, _ := strconv.ParseUint(sap.db.Node.NetworkID, 10, 64) + networkID, _ := strconv.ParseUint(sap.nodeNetworkId, 10, 64) apis := []rpc.API{ { Namespace: APIName, diff --git a/pkg/shared/database.go b/pkg/shared/database.go new file mode 100644 index 00000000..6a541442 --- /dev/null +++ b/pkg/shared/database.go @@ -0,0 +1,41 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/jmoiron/sqlx" +) + +// NewDB creates a new db connection and initializes the connection pool +func NewDB(connectString string, config postgres.Config) (*sqlx.DB, error) { + db, connectErr := sqlx.Connect("postgres", connectString) + if connectErr != nil { + return nil, postgres.ErrDBConnectionFailed(connectErr) + } + if config.MaxConns > 0 { + db.SetMaxOpenConns(config.MaxConns) + } + if config.MaxIdle > 0 { + db.SetMaxIdleConns(config.MaxIdle) + } + if config.MaxConnLifetime > 0 { + db.SetConnMaxLifetime(config.MaxConnLifetime) + } + + return db, nil +} diff --git a/pkg/shared/functions.go b/pkg/shared/functions.go index aaa72234..b1ac4550 100644 --- a/pkg/shared/functions.go +++ b/pkg/shared/functions.go @@ -18,7 +18,7 @@ package shared import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" diff --git a/pkg/shared/test_helpers.go b/pkg/shared/test_helpers.go index 35b7adfa..eed5a7f0 100644 --- a/pkg/shared/test_helpers.go +++ b/pkg/shared/test_helpers.go @@ -19,11 +19,11 @@ package shared import ( "bytes" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" + "github.com/ethereum/go-ethereum/statediff/indexer/models" ) // IPLDsContainBytes used to check if a list of strings contains a particular string -func IPLDsContainBytes(iplds []ipfs.BlockModel, b []byte) bool { +func IPLDsContainBytes(iplds []models.IPLDModel, b []byte) bool { for _, ipld := range iplds { if bytes.Equal(ipld.Data, b) { return true diff --git a/test_config/test_config.go b/test_config/test_config.go index 9f48aded..cc2d0eb8 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -19,12 +19,12 @@ package test_config import ( "errors" - "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/sirupsen/logrus" "github.com/spf13/viper" ) -var DBConfig postgres.ConnectionParams +var DBConfig postgres.Config func init() { setTestConfig() @@ -52,9 +52,9 @@ func setTestConfig() { port := vip.GetInt("database.port") name := vip.GetString("database.name") - DBConfig = postgres.ConnectionParams{ - Hostname: hn, - Name: name, - Port: port, + DBConfig = postgres.Config{ + Hostname: hn, + DatabaseName: name, + Port: port, } }